Merge pull request 'fix: enrich HSE profile parsing with publications and theses' (#10) from fix/hse-profile-parser-publications-vkr-pagination into main

Reviewed-on: #10

.gitignore

@@ -4,6 +4,7 @@ __pycache__/
 *.py[cod]
 *.db
 .pytest_cache/
+pytest-cache-files-*/
 .coverage
 htmlcov/
 postgres_data/

README.md

@@ -110,4 +110,4 @@ docker compose exec postgres pg_dump -U miem miem_workers > backup.sql
 docker compose down
 ```
 
-Версия сервиса: `0.2.7`. Админка всегда показывает версии backend и frontend в footer.
+Версия сервиса: `0.2.8`. Админка всегда показывает версии backend и frontend в footer.

app/admin.py

@@ -108,7 +108,7 @@ def directory(
                 "has_email": has_email or "",
                 "sort": sort,
                 "direction": direction,
-                "limit": limit,
+                "limit": page["limit"],
                 "offset": offset,
             },
         },

app/parser/profile.py

@@ -164,6 +164,7 @@ def parse_person_profile(
     header = extract_person_header(soup, normalized_url)
     tabs = extract_person_tabs(soup, normalized_url)
     sections = extract_sections(soup, normalized_url)
+    sections = enrich_sections_from_hse_widgets(session, soup, normalized_url, headers, timeout, sections)
     internal_links = [tab["href"] for tab in tabs if tab.get("href")]
 
     return {
@@ -183,6 +184,25 @@ def parse_person_profile(
     }
 
 
+def enrich_sections_from_hse_widgets(
+    session: Session,
+    soup: BeautifulSoup,
+    source_url: str,
+    headers: dict[str, str],
+    timeout: int,
+    sections: list[dict],
+) -> list[dict]:
+    enriched = list(sections)
+    publications = _load_widget_publications(session, soup, headers, timeout)
+    if publications:
+        enriched = _upsert_publications_section(enriched, publications)
+
+    theses = _load_widget_graduation_theses(session, soup, source_url, headers, timeout)
+    if theses:
+        enriched = _upsert_graduation_theses_section(enriched, theses)
+    return enriched
+
+
 def _render_with_playwright(source_url: str, fallback_html: str) -> str:
     try:
         from playwright.sync_api import sync_playwright
@@ -206,6 +226,89 @@ def _render_with_playwright(source_url: str, fallback_html: str) -> str:
         return fallback_html
 
 
+def _load_widget_publications(session: Session, soup: BeautifulSoup, headers: dict[str, str], timeout: int) -> list[dict]:
+    script = soup.select_one('script[data-widget-name="AuthorSearch"][data-author]')
+    if not script:
+        return []
+    author_id = normalize_ws(script.get("data-author"))
+    if not author_id:
+        return []
+
+    publications = []
+    page_id = 1
+    per_page = 100
+    while page_id <= 20:
+        payload = {
+            "type": "ANY",
+            "filterParams": (
+                f'"acceptLanguage":"ru"|"fullTextPublicEnabled": 1|'
+                f'"pubsAuthor": {author_id}|"widgetName": "AuthorSearch"'
+            ),
+            "paginationParams": {
+                "publsSort": ["TITLE_ASC"],
+                "publsCount": per_page,
+                "pageId": page_id,
+            },
+        }
+        try:
+            response = session.post(
+                "https://publications.hse.ru/api/searchPubs",
+                json=payload,
+                headers=headers,
+                timeout=timeout,
+            )
+            response.raise_for_status()
+            data = response.json()
+        except Exception:
+            return publications
+
+        result = data.get("result") if isinstance(data, dict) else {}
+        items = result.get("items") if isinstance(result, dict) else []
+        if not isinstance(items, list) or not items:
+            break
+        publications.extend(_normalize_publication_item(item) for item in items if isinstance(item, dict))
+
+        total = int(result.get("total") or 0)
+        if not result.get("more") and len(publications) >= total:
+            break
+        page_id += 1
+    return _dedupe_publications(publications)
+
+
+def _load_widget_graduation_theses(
+    session: Session,
+    soup: BeautifulSoup,
+    source_url: str,
+    headers: dict[str, str],
+    timeout: int,
+) -> list[dict]:
+    script = soup.select_one('script[src*="/n/stat/vkr/app.js"][data-person-id]')
+    if not script:
+        return []
+    person_id = normalize_ws(script.get("data-person-id"))
+    api_url = normalize_ws(script.get("data-api-url")) or "/n/vkr/api/"
+    if not person_id:
+        return []
+
+    request_headers = {**headers, "x-portal-language": "ru"}
+    try:
+        response = session.get(
+            urljoin(source_url, api_url),
+            params={"supervisorId": person_id},
+            headers=request_headers,
+            timeout=timeout,
+        )
+        response.raise_for_status()
+        data = response.json()
+    except Exception:
+        return []
+
+    items = data.get("data") if isinstance(data, dict) else []
+    if not isinstance(items, list):
+        return []
+    return [_normalize_vkr_item(item, source_url) for item in items if isinstance(item, dict)]
+
+
 def _collect_between_h2(start_h2: Tag) -> list[Tag | NavigableString | str]:
     nodes = []
     for sibling in start_h2.next_siblings:
@@ -353,6 +456,122 @@ def _parse_vkr_items(nodes: list) -> list[str]:
     return [item for item in dict.fromkeys(items) if item]
 
 
+def _normalize_publication_item(item: dict) -> dict:
+    publication_id = str(item.get("id") or "").strip()
+    title = _html_to_text(item.get("title"))
+    year = item.get("year")
+    publication_type = str(item.get("type") or "").strip() or None
+    description = item.get("description") if isinstance(item.get("description"), dict) else {}
+    short_description = _localized_value(description.get("short")) or _localized_value(description.get("shortLeft"))
+    text = normalize_ws(" ".join(part for part in [title, str(year or ""), short_description] if part))
+    return {
+        "id": publication_id or None,
+        "title": title or publication_id,
+        "year": year,
+        "type": publication_type,
+        "url": f"https://publications.hse.ru/view/{publication_id}" if publication_id else None,
+        "text": text or title or publication_id,
+    }
+
+
+def _normalize_vkr_item(item: dict, source_url: str) -> dict:
+    thesis_id = item.get("id")
+    program = item.get("learnProgram") if isinstance(item.get("learnProgram"), dict) else {}
+    org_unit = item.get("orgUnit") if isinstance(item.get("orgUnit"), dict) else {}
+    supervisors = []
+    for supervisor in item.get("supervisors") or []:
+        if not isinstance(supervisor, dict):
+            continue
+        name = normalize_ws(supervisor.get("name"))
+        url = normalize_ws(supervisor.get("url"))
+        if name or url:
+            supervisors.append({"name": name or url, "url": url or None})
+    return {
+        "id": thesis_id,
+        "student": normalize_ws(item.get("student")),
+        "title": normalize_ws(item.get("title")),
+        "defense_year": item.get("year"),
+        "level": normalize_ws(item.get("level")),
+        "rating": item.get("rating"),
+        "project_url": urljoin(source_url, f"/edu/vkr/{thesis_id}") if thesis_id else None,
+        "program": normalize_ws(program.get("title")),
+        "program_url": urljoin(source_url, program.get("url")) if program.get("url") else None,
+        "org_unit": normalize_ws(org_unit.get("title")),
+        "org_unit_url": urljoin(source_url, org_unit.get("url")) if org_unit.get("url") else None,
+        "supervisors": supervisors,
+        "text": normalize_ws(" ".join(str(part) for part in [item.get("student"), item.get("title"), item.get("year")] if part)),
+    }
+
+
+def _upsert_publications_section(sections: list[dict], publications: list[dict]) -> list[dict]:
+    merged = []
+    inserted = False
+    for section in sections:
+        if section.get("type") != "publications":
+            merged.append(section)
+            continue
+        existing = section.get("publications") or []
+        section = {
+            **section,
+            "publications_count": max(section.get("publications_count") or 0, len(publications)),
+            "publications": _dedupe_publications([*existing, *publications]),
+        }
+        section["items"] = [item["text"] for item in section["publications"] if item.get("text")]
+        merged.append(section)
+        inserted = True
+    if not inserted:
+        merged.append(
+            {
+                "title": "Публикации и исследования",
+                "slug": "publikacii_i_issledovaniya",
+                "type": "publications",
+                "raw_text": "",
+                "paragraphs": [],
+                "items": [item["text"] for item in publications if item.get("text")],
+                "links": [],
+                "publications_count": len(publications),
+                "publications": publications,
+            }
+        )
+    return merged
+
+
+def _upsert_graduation_theses_section(sections: list[dict], theses: list[dict]) -> list[dict]:
+    section = {
+        "title": "Выпускные квалификационные работы студентов НИУ ВШЭ",
+        "slug": "vypusknye_kvalifikacionnye_raboty_studentov_niu_vshe",
+        "type": "graduation_theses",
+        "raw_text": "",
+        "paragraphs": [],
+        "items": [item["text"] for item in theses if item.get("text")],
+        "links": [{"text": item["title"], "url": item["project_url"]} for item in theses if item.get("title") and item.get("project_url")],
+        "theses_count": len(theses),
+        "theses": theses,
+    }
+    return [item for item in sections if item.get("type") != "graduation_theses"] + [section]
+
+
+def _dedupe_publications(items: list[dict]) -> list[dict]:
+    seen = set()
+    unique = []
+    for item in items:
+        key = item.get("id") or item.get("url") or item.get("title")
+        if key and key not in seen:
+            seen.add(key)
+            unique.append(item)
+    return unique
+
+
+def _html_to_text(value: object) -> str:
+    return normalize_ws(BeautifulSoup(str(value or ""), "html.parser").get_text(" ", strip=True))
+
+
+def _localized_value(value: object) -> str:
+    if isinstance(value, dict):
+        return normalize_ws(value.get("ru") or value.get("publ") or value.get("en"))
+    return normalize_ws(str(value or ""))
+
+
 def _slugify(value: str) -> str:
     cleaned = re.sub(r"[^\w\s-]", "", value.lower(), flags=re.UNICODE)
     return re.sub(r"[-\s]+", "_", cleaned).strip("_") or "section"

app/services/admin_data.py

@@ -112,7 +112,7 @@ def list_employees_page(
     limit: int = 50,
     offset: int = 0,
 ) -> dict[str, Any]:
-    limit = max(1, min(limit, 200))
+    limit = limit if limit in {25, 50, 100} else 50
     offset = max(0, offset)
     base_stmt = build_employee_query(
         status=status,
@@ -281,6 +281,8 @@ def _normalize_section(section: Any) -> dict[str, Any]:
         "year_entries": _normalize_year_entries(section.get("year_entries")),
         "publications": _normalize_publications(section.get("publications")),
         "publications_count": section.get("publications_count"),
+        "theses": _normalize_theses(section.get("theses")),
+        "theses_count": section.get("theses_count"),
         "academic_year": section.get("academic_year"),
         "courses": _normalize_courses(section.get("courses")),
         "table": _normalize_table(section.get("table")),
@@ -349,6 +351,35 @@ def _normalize_courses(items: Any) -> list[dict[str, str | None]]:
     return normalized
 
 
+def _normalize_theses(items: Any) -> list[dict[str, Any]]:
+    normalized = []
+    if not isinstance(items, list):
+        return normalized
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        title = str(item.get("title") or "").strip()
+        student = str(item.get("student") or "").strip()
+        if not title and not student:
+            continue
+        normalized.append(
+            {
+                "id": item.get("id"),
+                "student": student,
+                "title": title,
+                "defense_year": item.get("defense_year") or item.get("year"),
+                "level": str(item.get("level") or "").strip(),
+                "rating": item.get("rating"),
+                "project_url": str(item.get("project_url") or "").strip() or None,
+                "program": str(item.get("program") or "").strip(),
+                "program_url": str(item.get("program_url") or "").strip() or None,
+                "org_unit": str(item.get("org_unit") or "").strip(),
+                "org_unit_url": str(item.get("org_unit_url") or "").strip() or None,
+            }
+        )
+    return normalized
+
+
 def _normalize_table(table: Any) -> dict[str, Any] | None:
     if not isinstance(table, dict):
         return None

app/static/admin.css

@@ -270,6 +270,18 @@
   line-height: 1.55;
 }
 
+.employee-section__meta {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 8px 12px;
+  color: #4b5563;
+  font-size: 13px;
+}
+
+.employee-section__meta-item {
+  line-height: 1.4;
+}
+
 .employee-section__table-wrap {
   overflow-x: auto;
 }

app/templates/directory.html

@@ -33,6 +33,12 @@
       <option value="asc" {% if filters.direction == "asc" %}selected{% endif %}>По возрастанию</option>
       <option value="desc" {% if filters.direction == "desc" %}selected{% endif %}>По убыванию</option>
     </select>
+    <select class="directory__input" name="limit" onchange="this.form.offset.value = 0; this.form.submit()">
+      {% for value in [25, 50, 100] %}
+      <option value="{{ value }}" {% if filters.limit == value %}selected{% endif %}>На странице: {{ value }}</option>
+      {% endfor %}
+    </select>
+    <input type="hidden" name="offset" value="{{ filters.offset }}">
     <button class="button" type="submit">Применить</button>
   </form>
 

app/templates/employee_detail.html

@@ -138,6 +138,34 @@
           </li>
           {% endfor %}
         </ul>
+        {% elif section.type == "graduation_theses" and section.theses %}
+        {% if section.theses_count %}<p class="employee-section__note">Всего: {{ section.theses_count }}</p>{% endif %}
+        <ul class="employee-card__list">
+          {% for thesis in section.theses %}
+          <li class="employee-card__list-item">
+            {% if thesis.student %}<strong>{{ thesis.student }}</strong>{% endif %}
+            {% if thesis.title %}
+            <div class="employee-section__text">
+              {% if thesis.project_url %}
+              <a class="admin__link" href="{{ thesis.project_url }}">{{ thesis.title }}</a>
+              {% else %}
+              {{ thesis.title }}
+              {% endif %}
+            </div>
+            {% endif %}
+            <div class="employee-section__meta">
+              {% if thesis.defense_year %}<span class="employee-section__meta-item">Год защиты: {{ thesis.defense_year }}</span>{% endif %}
+              {% if thesis.level %}<span class="employee-section__meta-item">{{ thesis.level }}</span>{% endif %}
+              {% if thesis.rating is not none %}<span class="employee-section__meta-item">Оценка: {{ thesis.rating }}</span>{% endif %}
+              {% if thesis.program %}
+              <span class="employee-section__meta-item">
+                {% if thesis.program_url %}<a class="admin__link" href="{{ thesis.program_url }}">{{ thesis.program }}</a>{% else %}{{ thesis.program }}{% endif %}
+              </span>
+              {% endif %}
+            </div>
+          </li>
+          {% endfor %}
+        </ul>
         {% elif section.type == "table" and section.table %}
         <div class="employee-section__table-wrap">
           <table class="employee-section__table">
@@ -170,7 +198,7 @@
         {% endif %}
         {% endif %}
 
-        {% if section.links and section.type not in ["courses_by_year"] %}
+        {% if section.links and section.type not in ["courses_by_year", "graduation_theses"] %}
         <div class="employee-section__links">
           {% for link in section.links %}
           <a class="employee-section__link" href="{{ link.url }}">{{ link.text }}</a>

app/version.py

@@ -1,3 +1,3 @@
-APP_VERSION = "0.2.7"
+APP_VERSION = "0.2.8"
-FRONTEND_VERSION = "0.2.7"
+FRONTEND_VERSION = "0.2.8"
-BACKEND_VERSION = "0.2.7"
+BACKEND_VERSION = "0.2.8"

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "miem-workers"
-version = "0.2.6"
+version = "0.2.8"
 description = "MIEM employees parser, admin API, and MCP server"
 requires-python = ">=3.11"
 dependencies = [

tests/test_admin_data.py

@@ -85,6 +85,19 @@ def test_employee_detail_payload_normalizes_human_readable_sections(db_session):
                     "academic_year": "2025/2026",
                     "courses": [{"title": "Course", "url": "https://example.test/course"}],
                 },
+                {
+                    "title": "ВКР",
+                    "type": "graduation_theses",
+                    "theses_count": 1,
+                    "theses": [
+                        {
+                            "student": "Student Name",
+                            "title": "Thesis title",
+                            "defense_year": 2025,
+                            "project_url": "https://www.hse.ru/edu/vkr/1",
+                        }
+                    ],
+                },
                 {
                     "title": "Fallback",
                     "type": "generic",
@@ -102,7 +115,8 @@ def test_employee_detail_payload_normalizes_human_readable_sections(db_session):
     assert payload["sections"][0]["year_entries"][0]["text"] == "Master degree"
     assert payload["sections"][1]["publications"][0]["title"] == "Paper"
     assert payload["sections"][2]["courses"][0]["title"] == "Course"
-    assert payload["sections"][3]["paragraphs"] == ["Fallback text"]
+    assert payload["sections"][3]["theses"][0]["student"] == "Student Name"
+    assert payload["sections"][4]["paragraphs"] == ["Fallback text"]
 
 
 def test_employee_payloads_tolerate_malformed_current_data(db_session):
@@ -155,6 +169,7 @@ def test_list_employees_page_filters_sorts_and_paginates(db_session):
 
     assert page["total"] == 1
     assert page["employees"][0]["full_name"] == "Alpha"
+    assert page["limit"] == 50
 
 
 def test_stats_payload_uses_latest_run_new_count(db_session):

tests/test_admin_templates.py

@@ -18,6 +18,8 @@ def test_directory_template_is_russian_and_uses_display_dates():
     assert "Сотрудники" in template
     assert "Колонки" in template
     assert "Применить" in template
+    assert "На странице: {{ value }}" in template
+    assert "{% for value in [25, 50, 100] %}" in template
     assert "Найдено:" in template
     assert "employee.first_seen_display" in template
     assert "employee.last_seen_display" in template

tests/test_api_mcp.py

@@ -18,7 +18,7 @@ def test_health_returns_versions():
     response = client.get("/api/health")
 
     assert response.status_code == 200
-    assert response.json()["backend_version"] == "0.2.7"
+    assert response.json()["backend_version"] == "0.2.8"
 
 
 def test_mcp_requires_token_and_lists_tools():

tests/test_employee_detail_template.py

@@ -14,6 +14,8 @@ def test_employee_detail_template_is_human_readable():
     assert "Основная информация" in template
     assert "Контакты" in template
     assert "Разделы профиля" in template
+    assert "graduation_theses" in template
+    assert "Год защиты" in template
     assert "Parser version" not in template
     assert "First seen" not in template
     assert "Last seen" not in template

tests/test_parser.py

@@ -1,9 +1,69 @@
 from bs4 import BeautifulSoup
 
-from app.parser.profile import extract_person_tabs
+from app.parser.profile import enrich_sections_from_hse_widgets, extract_person_tabs
 from app.parser.profile_url import normalize_profile_url, parse_profile_identity
 
 
+class FakeResponse:
+    def __init__(self, payload):
+        self.payload = payload
+
+    def raise_for_status(self):
+        return None
+
+    def json(self):
+        return self.payload
+
+
+class FakeSession:
+    def __init__(self):
+        self.posts = []
+        self.gets = []
+
+    def post(self, url, **kwargs):
+        self.posts.append((url, kwargs))
+        return FakeResponse(
+            {
+                "status": "ok",
+                "result": {
+                    "more": False,
+                    "total": 1,
+                    "items": [
+                        {
+                            "id": "888959076",
+                            "type": "ARTICLE",
+                            "title": "Дублирование пакетов",
+                            "year": 2023,
+                            "description": {"short": {"ru": "Информационные процессы. 2023."}},
+                        }
+                    ],
+                },
+            }
+        )
+
+    def get(self, url, **kwargs):
+        self.gets.append((url, kwargs))
+        return FakeResponse(
+            {
+                "lang": "ru",
+                "success": True,
+                "data": [
+                    {
+                        "id": 1045750164,
+                        "year": 2025,
+                        "level": "Бакалавриат",
+                        "title": "Аппаратно-программный комплекс защиты сети",
+                        "rating": 8,
+                        "student": "Лесняк Владислав Евгеньевич",
+                        "learnProgram": {"title": "Информатика и вычислительная техника", "url": "https://hse.ru/ba/isct/"},
+                        "orgUnit": {"title": "МИЭМ", "url": "https://www.hse.ru/org/url/59315150"},
+                        "supervisors": [{"url": "https://www.hse.ru/org/persons/803294906", "name": "Борисов Сергей Петрович"}],
+                    }
+                ],
+            }
+        )
+
+
 def test_normalize_profile_url_supports_staff_and_org_persons():
     assert normalize_profile_url("/staff/avsergeev#sci") == "https://www.hse.ru/staff/avsergeev"
     assert normalize_profile_url("https://www.hse.ru/org/persons/123/") == "https://www.hse.ru/org/persons/123"
@@ -26,3 +86,34 @@ def test_extract_person_tabs_prefers_person_menu_addition():
 
     assert [tab["title"] for tab in tabs] == ["Домашняя страница", "Публикации"]
     assert tabs[1]["href"] == "https://www.hse.ru/staff/avsergeev#sci"
+
+
+def test_enrich_sections_from_hse_widgets_loads_publications_and_vkr():
+    soup = BeautifulSoup(
+        """
+        <script src="/n/stat/publications/dist-w/publs.js" data-author="568398853" data-widget-name="AuthorSearch"></script>
+        <script src="/n/stat/vkr/app.js" data-api-url="/n/vkr/api/" data-person-id="803294906"></script>
+        """,
+        "html.parser",
+    )
+    session = FakeSession()
+
+    sections = enrich_sections_from_hse_widgets(
+        session,
+        soup,
+        "https://www.hse.ru/org/persons/803294906",
+        {"User-Agent": "test"},
+        10,
+        [],
+    )
+
+    publications = next(section for section in sections if section["type"] == "publications")
+    theses = next(section for section in sections if section["type"] == "graduation_theses")
+
+    assert publications["publications_count"] == 1
+    assert publications["publications"][0]["url"] == "https://publications.hse.ru/view/888959076"
+    assert theses["theses_count"] == 1
+    assert theses["theses"][0]["student"] == "Лесняк Владислав Евгеньевич"
+    assert theses["theses"][0]["project_url"] == "https://www.hse.ru/edu/vkr/1045750164"
+    assert session.posts[0][0] == "https://publications.hse.ru/api/searchPubs"
+    assert session.gets[0][1]["params"] == {"supervisorId": "803294906"}