fix: speed up academic degree filter

app/db.py

@@ -51,10 +51,20 @@ def _ensure_runtime_schema() -> None:
             missing_columns.append("profile_unavailable_streak INTEGER NOT NULL DEFAULT 0")
         if "last_profile_check_at" not in employee_columns:
             missing_columns.append("last_profile_check_at TIMESTAMPTZ")
+        if "has_academic_degree" not in employee_columns:
+            missing_columns.append("has_academic_degree BOOLEAN NOT NULL DEFAULT FALSE")
         if missing_columns:
             with engine.begin() as connection:
                 for column in missing_columns:
                     connection.execute(text(f"ALTER TABLE employees ADD COLUMN {column}"))
+                if engine.dialect.name == "postgresql":
+                    connection.execute(
+                        text(
+                            "UPDATE employees SET has_academic_degree = "
+                            "COALESCE(current_data::text ~* '(кандидат|доктор).{0,80}наук|ph\\.?[[:space:]]*d\\.?', FALSE)"
+                        )
+                    )
+                connection.execute(text("CREATE INDEX IF NOT EXISTS ix_employees_has_academic_degree ON employees (has_academic_degree)"))
     if "crawl_runs" not in table_names:
         return
     crawl_run_columns = {column["name"] for column in inspector.get_columns("crawl_runs")}

app/models.py

@@ -1,6 +1,6 @@
 from datetime import datetime, timezone
 
-from sqlalchemy import DateTime, ForeignKey, Index, Integer, LargeBinary, String, Text, UniqueConstraint
+from sqlalchemy import Boolean, DateTime, ForeignKey, Index, Integer, LargeBinary, String, Text, UniqueConstraint
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 from sqlalchemy.types import JSON
@@ -21,6 +21,7 @@ class Employee(Base):
         UniqueConstraint("profile_key", name="uq_employees_profile_key"),
         Index("ix_employees_full_name", "full_name"),
         Index("ix_employees_status", "status"),
+        Index("ix_employees_has_academic_degree", "has_academic_degree"),
     )
 
     id: Mapped[int] = mapped_column(Integer, primary_key=True)
@@ -37,6 +38,7 @@ class Employee(Base):
     last_profile_check_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     parser_version: Mapped[str | None] = mapped_column(String(32))
     current_data: Mapped[dict | None] = mapped_column(json_type)
+    has_academic_degree: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
     current_checksum: Mapped[str | None] = mapped_column(String(64))
     created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)
     updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow, nullable=False)

app/services/academic_degrees.py

@@ -0,0 +1,23 @@
+import re
+from typing import Any
+
+
+_PATTERN = re.compile(r"\b(?:кандидат|доктор)\s+[\w\s-]{0,80}?\s+наук\b|\bph\.?\s*d\.?\b", re.IGNORECASE)
+
+
+def academic_degrees(data: dict[str, Any] | None) -> list[str]:
+    degrees = []
+    for section in (data or {}).get("sections") or []:
+        if not isinstance(section, dict) or not re.search(r"уч[её]н.*степен|academic degree", str(section.get("title") or ""), re.IGNORECASE):
+            continue
+        values = [
+            *(entry.get("text") for entry in section.get("year_entries") or [] if isinstance(entry, dict)),
+            *(section.get("paragraphs") or []),
+            *(section.get("items") or []),
+            section.get("raw_text"),
+        ]
+        for value in values:
+            text = str(value or "").strip()
+            if text and _PATTERN.search(text) and text not in degrees:
+                degrees.append(text)
+    return degrees

app/services/admin_data.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import re
 from datetime import date, datetime, time
 from math import ceil
 from typing import Any
@@ -10,6 +9,7 @@ from sqlalchemy import Select, Text, and_, desc, func, or_, select
 from sqlalchemy.orm import Session
 
 from app.models import CrawlError, CrawlRun, CrawlRunEmployeeChange, Employee, EmployeeNewsLink
+from app.services.academic_degrees import academic_degrees
 
 EMPLOYEE_SORTS = {
     "full_name": Employee.full_name,
@@ -20,9 +20,6 @@ EMPLOYEE_SORTS = {
     "hse_start_year": Employee.current_data["hse_start_year"].as_integer(),
 }
 
-_ACADEMIC_DEGREE_PATTERN = re.compile(r"\b(?:кандидат|доктор)\s+[\w\s-]{0,80}?\s+наук\b|\bph\.?\s*d\.?\b", re.IGNORECASE)
-
-
 def employee_display_payload(employee: Employee) -> dict[str, Any]:
     data = _as_dict(employee.current_data)
     contacts = _as_dict(data.get("contacts"))
@@ -31,7 +28,7 @@ def employee_display_payload(employee: Employee) -> dict[str, Any]:
     positions = _clean_list(data.get("positions"))
     emails = _clean_list(contacts.get("emails"))
     phones = _clean_list(contacts.get("phones"))
-    academic_degrees = _academic_degrees(sections)
+    degree_values = academic_degrees(data)
     return {
         "id": employee.id,
         "full_name": employee.full_name,
@@ -46,7 +43,7 @@ def employee_display_payload(employee: Employee) -> dict[str, Any]:
         "phones": phones,
         "phone_text": ", ".join(phones),
         "address": contacts.get("address"),
-        "academic_degree_text": "; ".join(academic_degrees),
+        "academic_degree_text": "; ".join(degree_values),
         "publications_count": _count_section_items(sections, "publications"),
         "courses_count": _count_section_items(sections, "courses_by_year"),
         "news_count": len(stored_news_links) or _count_section_items(sections, "news"),
@@ -104,14 +101,7 @@ def build_employee_query(
     elif has_email is False:
         filters.append(or_(Employee.current_data.is_(None), ~Employee.current_data.cast(Text).ilike("%@%")))
     if has_academic_degree is not None:
-        data_text = Employee.current_data.cast(Text)
+        filters.append(Employee.has_academic_degree.is_(has_academic_degree))
-        degree_condition = or_(
-            and_(_json_text_contains(data_text, "кандидат"), _json_text_contains(data_text, "наук")),
-            and_(_json_text_contains(data_text, "доктор"), _json_text_contains(data_text, "наук")),
-            _json_text_contains(data_text, "phd"),
-            _json_text_contains(data_text, "ph.d."),
-        )
-        filters.append(degree_condition if has_academic_degree else or_(Employee.current_data.is_(None), ~degree_condition))
     if filters:
         stmt = stmt.where(and_(*filters))
     return stmt
@@ -235,36 +225,6 @@ def format_admin_datetime(value: Any) -> str:
     return value.strftime("%d.%m.%Y %H:%M")
 
 
-def _academic_degrees(sections: list[Any]) -> list[str]:
-    degrees = []
-    for section in sections:
-        section_data = _as_dict(section)
-        title = str(section_data.get("title") or "")
-        if not re.search(r"уч[её]н.*степен|academic degree", title, re.IGNORECASE):
-            continue
-        values = [
-            *(_as_dict(entry).get("text") for entry in _as_list(section_data.get("year_entries"))),
-            *_clean_list(section_data.get("paragraphs")),
-            *_clean_list(section_data.get("items")),
-            section_data.get("raw_text"),
-        ]
-        for value in values:
-            text = str(value or "").strip()
-            if text and _ACADEMIC_DEGREE_PATTERN.search(text) and text not in degrees:
-                degrees.append(text)
-    return degrees
-
-
-def _json_text_contains(data_text: Any, value: str) -> Any:
-    escaped = value.encode("unicode_escape").decode("ascii")
-    escaped_capitalized = value.capitalize().encode("unicode_escape").decode("ascii")
-    return or_(
-        data_text.ilike(f"%{value}%"),
-        data_text.ilike(f"%{escaped}%"),
-        data_text.ilike(f"%{escaped_capitalized}%"),
-    )
-
-
 def _employee_status_display(status: str | None) -> str:
     labels = {"active": "Работает", "verification_required": "Требует проверки", "dismissed": "Уволен"}
     return labels.get(status or "", status or "Не указано")

app/services/crawler.py

@@ -25,6 +25,7 @@ from app.models import (
 from app.parser.collector import collect_profile_links
 from app.parser.profile import parse_person_profile
 from app.parser.profile_url import profile_key
+from app.services.academic_degrees import academic_degrees
 from app.services.dataset_versions import get_or_create_current_version
 from app.services.resource_cache import ResourceCache
 
@@ -265,6 +266,7 @@ def _upsert_employee(db: Session, run: CrawlRun, parsed: dict) -> tuple[Employee
     employee.profile_unavailable_streak = 0
     employee.last_profile_check_at = now
     employee.parser_version = parser_version
+    employee.has_academic_degree = bool(academic_degrees(parsed))
     if changed:
         employee.current_data = parsed
     employee.current_checksum = checksum

app/version.py

@@ -1,3 +1,3 @@
-APP_VERSION = "0.7.3"
+APP_VERSION = "0.7.4"
-FRONTEND_VERSION = "0.7.3"
+FRONTEND_VERSION = "0.7.4"
-BACKEND_VERSION = "0.7.3"
+BACKEND_VERSION = "0.7.4"

migrations/001_init.sql

@@ -33,6 +33,7 @@ CREATE TABLE IF NOT EXISTS employees (
   dismissed_at TIMESTAMPTZ,
   parser_version VARCHAR(32),
   current_data JSONB,
+  has_academic_degree BOOLEAN NOT NULL DEFAULT FALSE,
   current_checksum VARCHAR(64),
   created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
   updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
@@ -40,6 +41,7 @@ CREATE TABLE IF NOT EXISTS employees (
 
 CREATE INDEX IF NOT EXISTS ix_employees_full_name ON employees (full_name);
 CREATE INDEX IF NOT EXISTS ix_employees_status ON employees (status);
+CREATE INDEX IF NOT EXISTS ix_employees_has_academic_degree ON employees (has_academic_degree);
 
 CREATE TABLE IF NOT EXISTS employee_snapshots (
   id SERIAL PRIMARY KEY,

migrations/009_academic_degree_filter.sql

@@ -0,0 +1,11 @@
+ALTER TABLE employees
+ADD COLUMN IF NOT EXISTS has_academic_degree BOOLEAN NOT NULL DEFAULT FALSE;
+
+UPDATE employees
+SET has_academic_degree = COALESCE(
+  current_data::text ~* '(кандидат|доктор).{0,80}наук|ph\\.?[[:space:]]*d\\.?',
+  FALSE
+);
+
+CREATE INDEX IF NOT EXISTS ix_employees_has_academic_degree
+  ON employees (has_academic_degree);

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "miem-workers"
-version = "0.7.3"
+version = "0.7.4"
 description = "MIEM employees parser, admin API, and MCP server"
 requires-python = ">=3.11"
 dependencies = [

tests/test_admin_data.py

@@ -61,6 +61,7 @@ def test_list_employees_page_filters_and_displays_academic_degrees(db_session):
                 status="active",
                 first_seen_at=datetime.now(timezone.utc),
                 last_seen_at=datetime.now(timezone.utc),
+                has_academic_degree=True,
                 current_data={
                     "sections": [
                         {
@@ -77,6 +78,7 @@ def test_list_employees_page_filters_and_displays_academic_degrees(db_session):
                 status="active",
                 first_seen_at=datetime.now(timezone.utc),
                 last_seen_at=datetime.now(timezone.utc),
+                has_academic_degree=False,
                 current_data={"sections": [{"title": "Образование", "items": ["Магистратура"]}]},
             ),
         ]
@@ -89,6 +91,10 @@ def test_list_employees_page_filters_and_displays_academic_degrees(db_session):
     assert page["employees"][0]["full_name"] == "Doctor"
     assert page["employees"][0]["academic_degree_text"] == "Доктор технических наук"
 
+    page = list_employees_page(db_session, has_academic_degree=False)
+
+    assert page["employees"][0]["full_name"] == "Master"
+
 
 def test_employee_detail_payload_normalizes_human_readable_sections(db_session):
     employee = Employee(

tests/test_crawler.py

@@ -273,7 +273,7 @@ def test_upsert_employee_increments_new_count_and_records_change_for_new_employe
     db_session.add(run)
     db_session.commit()
 
-    _upsert_employee(
+    employee, _ = _upsert_employee(
         db_session,
         run,
         {
@@ -282,7 +282,12 @@ def test_upsert_employee_increments_new_count_and_records_change_for_new_employe
             "profile_id": "newperson",
             "full_name": "New Person",
             "tabs": [],
-            "sections": [],
+            "sections": [
+                {
+                    "title": "Образование и учёные степени",
+                    "year_entries": [{"text": "Кандидат технических наук"}],
+                }
+            ],
             "parser_version": "0.2.0",
             "_html": "<html></html>",
         },
@@ -290,6 +295,7 @@ def test_upsert_employee_increments_new_count_and_records_change_for_new_employe
     db_session.commit()
 
     assert run.new_count == 1
+    assert employee.has_academic_degree is True
     change = db_session.query(CrawlRunEmployeeChange).one()
     assert change.change_type == "new"
     assert change.full_name == "New Person"

tests/test_db_schema.py

@@ -140,5 +140,5 @@ def test_runtime_schema_adds_profile_verification_fields(monkeypatch):
 
     inspector = inspect(engine)
     columns = {column["name"] for column in inspector.get_columns("employees")}
-    assert {"profile_unavailable_streak", "last_profile_check_at"}.issubset(columns)
+    assert {"profile_unavailable_streak", "last_profile_check_at", "has_academic_degree"}.issubset(columns)
     assert "employee_profile_urls" in inspector.get_table_names()