From aac909d0efefa2426f73a701855d1fa598ab679e Mon Sep 17 00:00:00 2001 From: admin Date: Wed, 26 Aug 2026 17:59:05 +0300 Subject: [PATCH 1/3] fix: speed up academic degree filter --- app/db.py | 10 +++++ app/models.py | 4 +- app/services/academic_degrees.py | 23 +++++++++++ app/services/admin_data.py | 48 ++--------------------- app/services/crawler.py | 8 ++-- app/version.py | 6 +-- migrations/001_init.sql | 2 + migrations/009_academic_degree_filter.sql | 11 ++++++ pyproject.toml | 2 +- tests/test_admin_data.py | 6 +++ tests/test_crawler.py | 12 ++++-- tests/test_db_schema.py | 2 +- 12 files changed, 78 insertions(+), 56 deletions(-) create mode 100644 app/services/academic_degrees.py create mode 100644 migrations/009_academic_degree_filter.sql diff --git a/app/db.py b/app/db.py index 8bfeb4f..103e709 100644 --- a/app/db.py +++ b/app/db.py @@ -51,10 +51,20 @@ def _ensure_runtime_schema() -> None: missing_columns.append("profile_unavailable_streak INTEGER NOT NULL DEFAULT 0") if "last_profile_check_at" not in employee_columns: missing_columns.append("last_profile_check_at TIMESTAMPTZ") + if "has_academic_degree" not in employee_columns: + missing_columns.append("has_academic_degree BOOLEAN NOT NULL DEFAULT FALSE") if missing_columns: with engine.begin() as connection: for column in missing_columns: connection.execute(text(f"ALTER TABLE employees ADD COLUMN {column}")) + if engine.dialect.name == "postgresql": + connection.execute( + text( + "UPDATE employees SET has_academic_degree = " + "COALESCE(current_data::text ~* '(кандидат|доктор).{0,80}наук|ph\\.?[[:space:]]*d\\.?', FALSE)" + ) + ) + connection.execute(text("CREATE INDEX IF NOT EXISTS ix_employees_has_academic_degree ON employees (has_academic_degree)")) if "crawl_runs" not in table_names: return crawl_run_columns = {column["name"] for column in inspector.get_columns("crawl_runs")} diff --git a/app/models.py b/app/models.py index f00cff6..2772f91 100644 --- a/app/models.py +++ b/app/models.py @@ -1,6 +1,6 @@ from datetime import datetime, timezone -from sqlalchemy import DateTime, ForeignKey, Index, Integer, LargeBinary, String, Text, UniqueConstraint +from sqlalchemy import Boolean, DateTime, ForeignKey, Index, Integer, LargeBinary, String, Text, UniqueConstraint from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import Mapped, mapped_column, relationship from sqlalchemy.types import JSON @@ -21,6 +21,7 @@ class Employee(Base): UniqueConstraint("profile_key", name="uq_employees_profile_key"), Index("ix_employees_full_name", "full_name"), Index("ix_employees_status", "status"), + Index("ix_employees_has_academic_degree", "has_academic_degree"), ) id: Mapped[int] = mapped_column(Integer, primary_key=True) @@ -37,6 +38,7 @@ class Employee(Base): last_profile_check_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) parser_version: Mapped[str | None] = mapped_column(String(32)) current_data: Mapped[dict | None] = mapped_column(json_type) + has_academic_degree: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) current_checksum: Mapped[str | None] = mapped_column(String(64)) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False) updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow, nullable=False) diff --git a/app/services/academic_degrees.py b/app/services/academic_degrees.py new file mode 100644 index 0000000..5962f9a --- /dev/null +++ b/app/services/academic_degrees.py @@ -0,0 +1,23 @@ +import re +from typing import Any + + +_PATTERN = re.compile(r"\b(?:кандидат|доктор)\s+[\w\s-]{0,80}?\s+наук\b|\bph\.?\s*d\.?\b", re.IGNORECASE) + + +def academic_degrees(data: dict[str, Any] | None) -> list[str]: + degrees = [] + for section in (data or {}).get("sections") or []: + if not isinstance(section, dict) or not re.search(r"уч[её]н.*степен|academic degree", str(section.get("title") or ""), re.IGNORECASE): + continue + values = [ + *(entry.get("text") for entry in section.get("year_entries") or [] if isinstance(entry, dict)), + *(section.get("paragraphs") or []), + *(section.get("items") or []), + section.get("raw_text"), + ] + for value in values: + text = str(value or "").strip() + if text and _PATTERN.search(text) and text not in degrees: + degrees.append(text) + return degrees diff --git a/app/services/admin_data.py b/app/services/admin_data.py index 3394b61..6464859 100644 --- a/app/services/admin_data.py +++ b/app/services/admin_data.py @@ -1,6 +1,5 @@ from __future__ import annotations -import re from datetime import date, datetime, time from math import ceil from typing import Any @@ -10,6 +9,7 @@ from sqlalchemy import Select, Text, and_, desc, func, or_, select from sqlalchemy.orm import Session from app.models import CrawlError, CrawlRun, CrawlRunEmployeeChange, Employee, EmployeeNewsLink +from app.services.academic_degrees import academic_degrees EMPLOYEE_SORTS = { "full_name": Employee.full_name, @@ -20,9 +20,6 @@ EMPLOYEE_SORTS = { "hse_start_year": Employee.current_data["hse_start_year"].as_integer(), } -_ACADEMIC_DEGREE_PATTERN = re.compile(r"\b(?:кандидат|доктор)\s+[\w\s-]{0,80}?\s+наук\b|\bph\.?\s*d\.?\b", re.IGNORECASE) - - def employee_display_payload(employee: Employee) -> dict[str, Any]: data = _as_dict(employee.current_data) contacts = _as_dict(data.get("contacts")) @@ -31,7 +28,7 @@ def employee_display_payload(employee: Employee) -> dict[str, Any]: positions = _clean_list(data.get("positions")) emails = _clean_list(contacts.get("emails")) phones = _clean_list(contacts.get("phones")) - academic_degrees = _academic_degrees(sections) + degree_values = academic_degrees(data) return { "id": employee.id, "full_name": employee.full_name, @@ -46,7 +43,7 @@ def employee_display_payload(employee: Employee) -> dict[str, Any]: "phones": phones, "phone_text": ", ".join(phones), "address": contacts.get("address"), - "academic_degree_text": "; ".join(academic_degrees), + "academic_degree_text": "; ".join(degree_values), "publications_count": _count_section_items(sections, "publications"), "courses_count": _count_section_items(sections, "courses_by_year"), "news_count": len(stored_news_links) or _count_section_items(sections, "news"), @@ -104,14 +101,7 @@ def build_employee_query( elif has_email is False: filters.append(or_(Employee.current_data.is_(None), ~Employee.current_data.cast(Text).ilike("%@%"))) if has_academic_degree is not None: - data_text = Employee.current_data.cast(Text) - degree_condition = or_( - and_(_json_text_contains(data_text, "кандидат"), _json_text_contains(data_text, "наук")), - and_(_json_text_contains(data_text, "доктор"), _json_text_contains(data_text, "наук")), - _json_text_contains(data_text, "phd"), - _json_text_contains(data_text, "ph.d."), - ) - filters.append(degree_condition if has_academic_degree else or_(Employee.current_data.is_(None), ~degree_condition)) + filters.append(Employee.has_academic_degree.is_(has_academic_degree)) if filters: stmt = stmt.where(and_(*filters)) return stmt @@ -235,36 +225,6 @@ def format_admin_datetime(value: Any) -> str: return value.strftime("%d.%m.%Y %H:%M") -def _academic_degrees(sections: list[Any]) -> list[str]: - degrees = [] - for section in sections: - section_data = _as_dict(section) - title = str(section_data.get("title") or "") - if not re.search(r"уч[её]н.*степен|academic degree", title, re.IGNORECASE): - continue - values = [ - *(_as_dict(entry).get("text") for entry in _as_list(section_data.get("year_entries"))), - *_clean_list(section_data.get("paragraphs")), - *_clean_list(section_data.get("items")), - section_data.get("raw_text"), - ] - for value in values: - text = str(value or "").strip() - if text and _ACADEMIC_DEGREE_PATTERN.search(text) and text not in degrees: - degrees.append(text) - return degrees - - -def _json_text_contains(data_text: Any, value: str) -> Any: - escaped = value.encode("unicode_escape").decode("ascii") - escaped_capitalized = value.capitalize().encode("unicode_escape").decode("ascii") - return or_( - data_text.ilike(f"%{value}%"), - data_text.ilike(f"%{escaped}%"), - data_text.ilike(f"%{escaped_capitalized}%"), - ) - - def _employee_status_display(status: str | None) -> str: labels = {"active": "Работает", "verification_required": "Требует проверки", "dismissed": "Уволен"} return labels.get(status or "", status or "Не указано") diff --git a/app/services/crawler.py b/app/services/crawler.py index 46612df..e458cb7 100644 --- a/app/services/crawler.py +++ b/app/services/crawler.py @@ -24,7 +24,8 @@ from app.models import ( ) from app.parser.collector import collect_profile_links from app.parser.profile import parse_person_profile -from app.parser.profile_url import profile_key +from app.parser.profile_url import profile_key +from app.services.academic_degrees import academic_degrees from app.services.dataset_versions import get_or_create_current_version from app.services.resource_cache import ResourceCache @@ -264,8 +265,9 @@ def _upsert_employee(db: Session, run: CrawlRun, parsed: dict) -> tuple[Employee employee.dismissed_at = None employee.profile_unavailable_streak = 0 employee.last_profile_check_at = now - employee.parser_version = parser_version - if changed: + employee.parser_version = parser_version + employee.has_academic_degree = bool(academic_degrees(parsed)) + if changed: employee.current_data = parsed employee.current_checksum = checksum db.flush() diff --git a/app/version.py b/app/version.py index 822491c..576f286 100644 --- a/app/version.py +++ b/app/version.py @@ -1,3 +1,3 @@ -APP_VERSION = "0.7.3" -FRONTEND_VERSION = "0.7.3" -BACKEND_VERSION = "0.7.3" +APP_VERSION = "0.7.4" +FRONTEND_VERSION = "0.7.4" +BACKEND_VERSION = "0.7.4" diff --git a/migrations/001_init.sql b/migrations/001_init.sql index 4326ecb..92ac995 100644 --- a/migrations/001_init.sql +++ b/migrations/001_init.sql @@ -33,6 +33,7 @@ CREATE TABLE IF NOT EXISTS employees ( dismissed_at TIMESTAMPTZ, parser_version VARCHAR(32), current_data JSONB, + has_academic_degree BOOLEAN NOT NULL DEFAULT FALSE, current_checksum VARCHAR(64), created_at TIMESTAMPTZ NOT NULL DEFAULT now(), updated_at TIMESTAMPTZ NOT NULL DEFAULT now() @@ -40,6 +41,7 @@ CREATE TABLE IF NOT EXISTS employees ( CREATE INDEX IF NOT EXISTS ix_employees_full_name ON employees (full_name); CREATE INDEX IF NOT EXISTS ix_employees_status ON employees (status); +CREATE INDEX IF NOT EXISTS ix_employees_has_academic_degree ON employees (has_academic_degree); CREATE TABLE IF NOT EXISTS employee_snapshots ( id SERIAL PRIMARY KEY, diff --git a/migrations/009_academic_degree_filter.sql b/migrations/009_academic_degree_filter.sql new file mode 100644 index 0000000..d66c321 --- /dev/null +++ b/migrations/009_academic_degree_filter.sql @@ -0,0 +1,11 @@ +ALTER TABLE employees +ADD COLUMN IF NOT EXISTS has_academic_degree BOOLEAN NOT NULL DEFAULT FALSE; + +UPDATE employees +SET has_academic_degree = COALESCE( + current_data::text ~* '(кандидат|доктор).{0,80}наук|ph\\.?[[:space:]]*d\\.?', + FALSE +); + +CREATE INDEX IF NOT EXISTS ix_employees_has_academic_degree + ON employees (has_academic_degree); diff --git a/pyproject.toml b/pyproject.toml index 582c5a0..fa132b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "miem-workers" -version = "0.7.3" +version = "0.7.4" description = "MIEM employees parser, admin API, and MCP server" requires-python = ">=3.11" dependencies = [ diff --git a/tests/test_admin_data.py b/tests/test_admin_data.py index 2bad2dc..48eead4 100644 --- a/tests/test_admin_data.py +++ b/tests/test_admin_data.py @@ -61,6 +61,7 @@ def test_list_employees_page_filters_and_displays_academic_degrees(db_session): status="active", first_seen_at=datetime.now(timezone.utc), last_seen_at=datetime.now(timezone.utc), + has_academic_degree=True, current_data={ "sections": [ { @@ -77,6 +78,7 @@ def test_list_employees_page_filters_and_displays_academic_degrees(db_session): status="active", first_seen_at=datetime.now(timezone.utc), last_seen_at=datetime.now(timezone.utc), + has_academic_degree=False, current_data={"sections": [{"title": "Образование", "items": ["Магистратура"]}]}, ), ] @@ -89,6 +91,10 @@ def test_list_employees_page_filters_and_displays_academic_degrees(db_session): assert page["employees"][0]["full_name"] == "Doctor" assert page["employees"][0]["academic_degree_text"] == "Доктор технических наук" + page = list_employees_page(db_session, has_academic_degree=False) + + assert page["employees"][0]["full_name"] == "Master" + def test_employee_detail_payload_normalizes_human_readable_sections(db_session): employee = Employee( diff --git a/tests/test_crawler.py b/tests/test_crawler.py index cd34bea..c1c2256 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -273,7 +273,7 @@ def test_upsert_employee_increments_new_count_and_records_change_for_new_employe db_session.add(run) db_session.commit() - _upsert_employee( + employee, _ = _upsert_employee( db_session, run, { @@ -282,14 +282,20 @@ def test_upsert_employee_increments_new_count_and_records_change_for_new_employe "profile_id": "newperson", "full_name": "New Person", "tabs": [], - "sections": [], + "sections": [ + { + "title": "Образование и учёные степени", + "year_entries": [{"text": "Кандидат технических наук"}], + } + ], "parser_version": "0.2.0", "_html": "", }, ) db_session.commit() - assert run.new_count == 1 + assert run.new_count == 1 + assert employee.has_academic_degree is True change = db_session.query(CrawlRunEmployeeChange).one() assert change.change_type == "new" assert change.full_name == "New Person" diff --git a/tests/test_db_schema.py b/tests/test_db_schema.py index fd16031..a8735de 100644 --- a/tests/test_db_schema.py +++ b/tests/test_db_schema.py @@ -140,5 +140,5 @@ def test_runtime_schema_adds_profile_verification_fields(monkeypatch): inspector = inspect(engine) columns = {column["name"] for column in inspector.get_columns("employees")} - assert {"profile_unavailable_streak", "last_profile_check_at"}.issubset(columns) + assert {"profile_unavailable_streak", "last_profile_check_at", "has_academic_degree"}.issubset(columns) assert "employee_profile_urls" in inspector.get_table_names() -- 2.49.1 From 9c43e55aecdaeb5247c82e79d863643a59b4cd0e Mon Sep 17 00:00:00 2001 From: admin Date: Wed, 26 Aug 2026 18:01:08 +0300 Subject: [PATCH 2/3] fix: match PhD values in degree backfill --- migrations/009_academic_degree_filter.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/009_academic_degree_filter.sql b/migrations/009_academic_degree_filter.sql index d66c321..471dfe0 100644 --- a/migrations/009_academic_degree_filter.sql +++ b/migrations/009_academic_degree_filter.sql @@ -3,7 +3,7 @@ ADD COLUMN IF NOT EXISTS has_academic_degree BOOLEAN NOT NULL DEFAULT FALSE; UPDATE employees SET has_academic_degree = COALESCE( - current_data::text ~* '(кандидат|доктор).{0,80}наук|ph\\.?[[:space:]]*d\\.?', + current_data::text ~* '(кандидат|доктор).{0,80}наук|ph\.?[[:space:]]*d\.?', FALSE ); -- 2.49.1 From 2d2ebd818d7f4c0425bb389455b63de2660c6583 Mon Sep 17 00:00:00 2001 From: admin Date: Wed, 26 Aug 2026 18:02:14 +0300 Subject: [PATCH 3/3] docs: add filter performance changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa710c6..af1c3a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.7.4 + +- Ускорена фильтрация сотрудников по учёной степени. + ## 0.7.3 - Восстановлена проверка профилей сотрудников и хранение истории URL профиля. -- 2.49.1