Compare commits

...
Author SHA1 Message Date
codex 65d851bb92 Parse mixed education records and certificates, flag unresolved DPO
CI / test (pull_request) Successful in 12m17s
CI / Fingerprint diagnostics (production deployment temporarily disabled) (pull_request) Skipped
2026-11-20 16:02:16 +00:00
agent 8dfd700bd5 Merge pull request 'Показывать загрузку и блокировать фильтры справочника' (#80) from fix/directory-loading-state into main
CI / test (push) Successful in 12m21s
CI / Fingerprint diagnostics (production deployment temporarily disabled) (push) Skipped
2026-11-20 09:11:13 +00:00
18 changed files with 335 additions and 14 deletions

No files matched your search

+4
View File
@@ -1,5 +1,9 @@
# Changelog
## 2.3.0
- Раздел ДПО объединяет сведения из профиля и документов. Справочник показывает отдельный счётчик ДПО, а предупреждение ведёт к нераспознанным фрагментам в карточке.
## 2.2.1
- При применении фильтров справочника, переключении ППС, сортировки и размера страницы появляется заглушка «Обновляем список…». Поля, кнопки и ссылки справочника заблокированы до завершения запроса.
+1 -1
View File
@@ -6,7 +6,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
WORKDIR /app
RUN apt-get update \
&& apt-get install -y --no-install-recommends gcc libpq-dev \
&& apt-get install -y --no-install-recommends gcc libpq-dev poppler-utils tesseract-ocr tesseract-ocr-rus \
&& rm -rf /var/lib/apt/lists/*
COPY requirements.txt .
+65
View File
@@ -0,0 +1,65 @@
"""Assemble additional education events from the public profile's mixed markup."""
import re
YEAR = re.compile(r"\b(?:19|20)\d{2}\b")
_QUOTE = re.compile(r'«([^»]+)»|"([^"]+)"')
_EVENT = re.compile(r"стажиров|стипенди|исследовательск|обучени|квалификаци|переподготовк|прош[её]л|прошла|семинар|сертификаци|программ[аеуы]|курсы?\b", re.I)
_CONTINUES = re.compile(r"(?:по программе|присвоением степени|на тему|по курсу|по теме|по специальности)[:\s]*$", re.I)
_CAPTION = re.compile(r"^(?:удостоверение|сертификат|свидетельство)(?: о повышении квалификации)?[.:]?$", re.I)
def _key(text):
return " ".join(re.findall(r"\w+", text.casefold().replace("ё", "е")))
def _provider(text):
text = _key(text)
for variants in (("вшэ", "высшая школа экономики", "высшей школе экономики"),
("мифи", "московский инженерно физический институт")):
if any(v in text.split() if " " not in v else v in text for v in variants):
return variants[0]
return text
def assemble(rows):
events = []
unresolved = []
pending = []
for text, year, structured, list_item in rows:
if _CAPTION.fullmatch(text):
continue
if (pending and not structured and not list_item and not YEAR.match(text)
and (_CONTINUES.search(pending[-1][0]) or
(pending[-1][3] and not _EVENT.search(text[:20]) and not YEAR.search(text)) or
(pending[-1][3] and text.startswith("Курс ") and not YEAR.search(text)))):
pending[-1][0] += " " + text
pending[-1][3] = False
else:
pending.append([text, year, structured, list_item])
for text, year, structured, _ in pending:
if not structured and (len(_key(text).split()) < 2 or not (_EVENT.search(text) or YEAR.match(text))):
unresolved.append(text)
continue
quotes = [a or b for a, b in _QUOTE.findall(text)]
title = _key(quotes[-1]) if quotes else ""
if structured and text.lstrip("0123456789 ").startswith("«"):
title = _key(text.split("«", 1)[1].rsplit("»", 1)[0])
provider = _provider(text.rsplit("»", 1)[-1] if structured and "»" in text else text)
if not year:
match = YEAR.search(_QUOTE.sub("", text))
year = int(match[0]) if match else None
matches = [e for e in events if title and e["title"] == title and e["provider"] == provider
and (e["year"] == year or not year or not e["year"])]
if len(matches) == 1:
target = matches[0]
hours = re.search(r"\b\d+\s+час[а-я]*", text, re.I)
if hours and hours[0] not in target["text"]:
target["text"] += " (" + hours[0] + ")"
continue
if any(e["text"] == text for e in events):
continue
events.append(dict(text=text, year=year, structured=structured, title=title, provider=provider))
return events, unresolved
+70
View File
@@ -0,0 +1,70 @@
"""Read public HSE education documents. An inconclusive extraction stays visible for review."""
import hashlib
import re
import subprocess
from pathlib import Path
from tempfile import TemporaryDirectory
from urllib.parse import urlsplit
import requests
_MAX_BYTES = 12 * 1024 * 1024
def _run(args):
return subprocess.run(args, check=True, capture_output=True, timeout=30).stdout.decode("utf-8")
def _certificate(text):
text = " ".join(text.split())
title = re.search(r'(?:по (?:дополнительной профессиональной )?программе|по теме|на тему)\s*[:«"\s]*([^»"]{5,300})[»"]', text, re.I)
if not title:
title = re.search(r'[«"]([^«»"]{8,250})[»"]\s*[,.(]?\s*(?:в объ[её]ме|\d+ час)', text, re.I)
years = re.findall(r"\b(?:19|20)\d{2}\b", text)
if not title or not years or not re.search(r"квалификаци|сертификат|удостоверение", text, re.I):
return None
hours = re.search(r"(\d+)\s*(?:академических\s+)?час", text, re.I)
provider = ""
if re.search(r"высш[а-я]+ школ[а-я]+ экономик|НИУ ВШЭ", text, re.I):
provider = ", НИУ ВШЭ"
elif re.search("МИСИС", text, re.I):
provider = ", НИТУ МИСИС"
return f'{years[-1]} «{title[1].strip()}», повышение квалификации{provider}' + (f", {hours[1]} часов" if hours else "")
def _recognize(data):
with TemporaryDirectory() as tmp:
src = Path(tmp) / "document"
src.write_bytes(data)
images = [src]
if data.startswith(b"%PDF"):
direct = _certificate(_run(["pdftotext", "-layout", str(src), "-"]))
if direct:
return direct
_run(["pdftoppm", "-f", "1", "-l", "2", "-scale-to", "2400", "-png", str(src), str(src)])
images = sorted(Path(tmp).glob("document-*.png"))
elif not data.startswith((b"\x89PNG", b"\xff\xd8\xff")):
return None
return _certificate(" ".join(_run(["tesseract", str(img), "stdout", "-l", "rus+eng"]) for img in images))
def read_document(session, url, timeout, cache, profile_key):
parts = urlsplit(url)
if parts.scheme != "https" or parts.netloc != "www.hse.ru" or not parts.path.startswith("/mirror/pubs/share/"):
return None
try:
with session.get(url, stream=True, timeout=timeout, allow_redirects=False) as response:
if response.status_code != 200:
return None
data = bytearray()
for chunk in response.iter_content(65536):
data.extend(chunk)
if len(data) > _MAX_BYTES:
return None
digest = hashlib.sha256(data).hexdigest()
if cache:
return cache.education_document(profile_key, url, digest, lambda: _recognize(data))
return _recognize(data)
except (OSError, ValueError, subprocess.SubprocessError, requests.RequestException):
return None
+81 -3
View File
@@ -1,12 +1,15 @@
import hashlib
import json
import re
from copy import deepcopy
from datetime import datetime, timezone
from urllib.parse import urljoin
from bs4 import BeautifulSoup, NavigableString, Tag
from bs4 import BeautifulSoup, Comment, NavigableString, Tag
from requests import Session
from app.parser.education import assemble
from app.parser.education_document import read_document
from app.parser.profile_url import normalize_profile_url, parse_profile_identity
from app.version import BACKEND_VERSION
@@ -99,7 +102,7 @@ def extract_person_header(soup: BeautifulSoup, source_url: str) -> dict:
}
def extract_sections(soup: BeautifulSoup, source_url: str) -> list[dict]:
def extract_sections(soup: BeautifulSoup, source_url: str, document_reader=None) -> list[dict]:
sections = []
for h2 in soup.select("h2"):
if h2.find_parent(class_="post") or h2.find_parent(attrs={"data-tab": "press_links_news"}):
@@ -108,6 +111,11 @@ def extract_sections(soup: BeautifulSoup, source_url: str) -> list[dict]:
if not title or "расписание занятий" in title.lower():
continue
nodes = _collect_between_h2(h2)
education_titles = {part.strip().rstrip(":") for part in title.casefold().split("/")}
if (h2.find_parent(attrs={"tab-node": "additional_education"})
or education_titles & {"дополнительное образование", "повышение квалификации", "курсы повышения квалификации"}):
sections.append(_parse_additional_education(title, nodes, source_url, document_reader))
continue
raw_text = _nodes_raw_text(nodes)
paragraphs = _nodes_paragraphs(nodes)
items = _nodes_list_items(nodes)
@@ -193,7 +201,7 @@ def parse_person_profile(
soup = BeautifulSoup(html, "html.parser")
header = extract_person_header(soup, normalized_url)
tabs = extract_person_tabs(soup, normalized_url)
sections = extract_sections(soup, normalized_url)
sections = extract_sections(soup, normalized_url, lambda url: read_document(session, url, timeout, resource_cache, cache_profile_key))
sections = enrich_sections_from_hse_widgets(
session,
soup,
@@ -219,6 +227,7 @@ def parse_person_profile(
"tabs": tabs,
"sections": sections,
"employee_internal_links": internal_links,
"education_parsed": True,
"parser_version": BACKEND_VERSION,
"_html": html,
"_resource_manifest": resource_manifest,
@@ -585,6 +594,75 @@ def _parse_year_entries(nodes: list, source_url: str) -> list[dict]:
return entries
_FILE_FORMAT = r"(?:pdf|jpe?g|png|docx?|pptx?|xlsx?|zip|rar)"
_FILE_SIZE = re.compile(rf"\(\s*{_FILE_FORMAT}\s*,\s*[\d.,\s]+[кмгkmg]?\s*[бb](?:айт[а-я]*)?\s*\)", re.I)
_FILE_CAPTION = re.compile(rf"\b(?:файл|сертификат|свидетельство|удостоверение)(?:_\w+)?\s*{_FILE_SIZE.pattern}", re.I)
_FILE_URL = re.compile(rf"/mirror/pubs/share/|\.{_FILE_FORMAT}(?:[?#]|$)", re.I)
_INITIAL_YEAR = re.compile(r"^(?:19|20)\d{2}\b")
def _education_blocks(node):
"""Yield whole course rows or paragraphs once, in document order."""
if not isinstance(node, Tag):
if not isinstance(node, Comment) and normalize_ws(str(node)):
yield node
return
if node.name == "tr" and node.find("th"):
return
if (node.name in {"li", "p", "tr"} or node.find(class_="person-list-hangover", recursive=False)
or not node.find(["div", "ul", "ol", "li", "p", "table", "tr"])):
yield node
return
for child in node.children:
yield from _education_blocks(child)
def _parse_additional_education(title: str, nodes: list, source_url: str, document_reader=None) -> dict:
rows = []
documents = {}
unresolved_documents = []
links = []
for original in nodes:
node = deepcopy(original)
if isinstance(node, Tag):
for anchor in node.select("a[href]"):
following = anchor.next_sibling
suffix = following.get_text(strip=True) if isinstance(following, Tag) else str(following or "")
if (_FILE_URL.search(anchor["href"]) or anchor.has_attr("download")
or _FILE_SIZE.match(normalize_ws(suffix))):
documents[urljoin(source_url, anchor["href"])] = normalize_ws(anchor.get_text(" ", strip=True)) or "Документ"
anchor.decompose()
for block in _education_blocks(node):
is_tag = isinstance(block, Tag)
content = block.get_text(" ", strip=True) if is_tag else str(block)
text = normalize_ws(_FILE_SIZE.sub("", _FILE_CAPTION.sub("", content)))
text = re.sub(r"\s+([,.;:])", r"\1", text)
text = re.sub(r"(?<=[.!?])\.$", "", text)
if not re.search(r"\w", text):
continue
block_links = _extract_links(block, source_url) if is_tag else []
links.extend(block_links)
year_node = block.find(class_="person-list-hangover", recursive=False) if is_tag else None
year_match = _INITIAL_YEAR.match(year_node.get_text(strip=True)) if year_node else None
rows.append((text, int(year_match[0]) if year_match else None, bool(year_node), is_tag and block.name == "li"))
for url, caption in documents.items():
recognized = document_reader(url) if document_reader else None
if recognized:
rows.append((recognized, None, False, False))
elif document_reader:
unresolved_documents.append({"text": caption, "url": url})
events, unresolved = assemble(rows)
items = [event["text"] for event in events]
year_entries = [{"year": event["year"], "text": event["text"], "links": []}
for event in events if event["structured"]]
return {
"title": title, "slug": _slugify(title), "type": "additional_education",
"items": items, "paragraphs": [], "year_entries": year_entries,
"unresolved": unresolved, "unresolved_documents": unresolved_documents,
"raw_text": "\n".join(items), "links": links,
}
def _parse_vkr_items(nodes: list) -> list[str]:
items = []
for node in nodes:
+11
View File
@@ -38,6 +38,7 @@ DIRECTORY_COLUMNS = (
("last_seen_at", "Последний раз найден", "last_seen_display"),
("dismissed_at", "Дата увольнения", "dismissed_display"),
("profile", "Профиль", "canonical_url"),
("dpo_count", "Количество ДПО", "dpo_count"),
)
_TRAINING_LABEL = "повышение квалификации"
@@ -57,16 +58,20 @@ def _has_training(data: dict[str, Any], since_year: int | None) -> bool:
dedicated = title in {_TRAINING_LABEL, "курсы повышения квалификации"}
if not dedicated and _TRAINING_LABEL not in (part.strip() for part in title.split("/")):
continue
structured_texts = set()
for value in _as_list(section.get("year_entries")):
entry = _as_dict(value)
year = entry.get("year")
text = str(entry.get("text") or "").strip()
structured_texts.add(text)
if since_year is not None and (type(year) is not int or year < since_year):
continue
if text and text != str(year) and (dedicated or _TRAINING_KIND.search(_QUOTED_TEXT.sub("", text))):
return True
# Profiles may also list completed courses in manually entered paragraphs or lists.
for text in _clean_list(section.get("paragraphs")) + _clean_list(section.get("items")):
if text in structured_texts:
continue
description = _QUOTED_TEXT.sub("", text).strip()
if not dedicated and not _TRAINING_MENTION.search(description):
continue
@@ -87,6 +92,8 @@ def employee_display_payload(employee: Employee) -> dict[str, Any]:
emails = _clean_list(contacts.get("emails"))
phones = _clean_list(contacts.get("phones"))
degree_values = academic_degrees(data)
education = [s for s in sections if isinstance(s, dict) and s.get("type") == "additional_education"]
parsed_education = bool(education) or data.get("education_parsed", False)
return {
"id": employee.id,
"full_name": employee.full_name,
@@ -103,6 +110,8 @@ def employee_display_payload(employee: Employee) -> dict[str, Any]:
"address": contacts.get("address"),
"academic_degree_text": "; ".join(degree_values),
"publications_count": _count_section_items(sections, "publications"),
"dpo_count": sum(len(s.get("items", [])) for s in education) if parsed_education else None,
"dpo_uncertain": any(s.get("unresolved") or s.get("unresolved_documents") for s in education),
"courses_count": _count_section_items(sections, "courses_by_year"),
"news_count": len(stored_news_links) or _count_section_items(sections, "news"),
"first_seen_at": employee.first_seen_at.isoformat() if employee.first_seen_at else None,
@@ -456,6 +465,8 @@ def _normalize_section(section: Any) -> dict[str, Any]:
"raw_text": raw_text,
"paragraphs": paragraphs,
"list_items": items,
"unresolved": _clean_list(section.get("unresolved")),
"unresolved_documents": _normalize_links(section.get("unresolved_documents")),
"links": _normalize_links(section.get("links")),
"year_entries": _normalize_year_entries(section.get("year_entries")),
"publications": _normalize_publications(section.get("publications")),
+1 -1
View File
@@ -19,7 +19,7 @@ def directory_xlsx(employees: list[dict]) -> bytes:
(employee[key] or "Без имени") if key == "full_name" else employee[key]
for _, _, key in DIRECTORY_COLUMNS
])
for cell, width in zip(sheet[1], (34, 20, 48, 14, 28, 22, 34, 30, 14, 10, 10, 22, 24, 22, 46)):
for cell, width in zip(sheet[1], (34, 20, 48, 14, 28, 22, 34, 30, 14, 10, 10, 22, 24, 22, 46, 18)):
cell.font = Font(bold=True)
sheet.column_dimensions[get_column_letter(cell.column)].width = width
sheet.freeze_panes = "A2"
+18
View File
@@ -27,6 +27,24 @@ class ResourceCache:
def __init__(self, db: Session):
self.db = db
def education_document(self, profile_key, url, digest, recognize):
key = "education-document-v1"
existing = self.db.scalar(select(ParseResourceCache).where(
ParseResourceCache.profile_key == profile_key, ParseResourceCache.resource_key == key,
ParseResourceCache.request_fingerprint == digest))
if existing:
return gzip.decompress(existing.body_snapshot).decode() or None
result = recognize()
from sqlalchemy.exc import IntegrityError
try:
with self.db.begin_nested():
self.db.add(ParseResourceCache(profile_key=profile_key, resource_key=key, method="OCR", url=url,
request_fingerprint=digest, body_hash=digest, body_snapshot=gzip.compress((result or "").encode()),
parser_version=BACKEND_VERSION))
except IntegrityError:
pass # Another crawl cached the same document.
return result
def fetch_text(
self,
session: requests.Session,
+3
View File
@@ -1172,3 +1172,6 @@
.guide__fields dd { margin: 4px 0 0; overflow-wrap: anywhere; }
.guide--page { max-width: 75ch; }
.guide--page .guide__step + .guide__step { margin-block-start: 32px; }
.dpo-warning { color: #ab202b; text-decoration: none; }
#unresolved-dpo:target { outline: 2px solid #ab202b; outline-offset: 3px; }
+1 -1
View File
@@ -10,7 +10,7 @@
const columnPresets = {
review: ["full_name", "status", "academic_degree", "last_seen_at", "profile"],
contacts: ["full_name", "status", "email", "phone", "address", "profile"],
full: ["full_name", "status", "positions", "hse_start_year", "email", "phone", "address", "academic_degree", "publications_count", "courses_count", "news_count", "first_seen_at", "last_seen_at", "dismissed_at", "profile"],
full: ["full_name", "status", "positions", "hse_start_year", "email", "phone", "address", "academic_degree", "publications_count", "courses_count", "news_count", "first_seen_at", "last_seen_at", "dismissed_at", "profile", "dpo_count"],
};
function readColumns() {
+1
View File
@@ -123,6 +123,7 @@
<td class="directory-table__cell" data-column="last_seen_at">{{ employee.last_seen_display }}</td>
<td class="directory-table__cell" data-column="dismissed_at">{{ employee.dismissed_display }}</td>
<td class="directory-table__cell" data-column="profile"><a class="admin__link" href="{{ employee.canonical_url }}" target="_blank" rel="noopener">Внешний профиль</a></td>
<td class="directory-table__cell" data-column="dpo_count">{{ employee.dpo_count if employee.dpo_count is not none else "—" }}{% if employee.dpo_uncertain %} <a class="dpo-warning" href="/admin/employees/{{ employee.id }}#unresolved-dpo" title="Часть сведений о ДПО не распознана. Количество может быть неполным." aria-label="Часть сведений о ДПО не распознана. Количество может быть неполным."><span aria-hidden="true">⚠︎</span></a>{% endif %}</td>
</tr>
{% else %}
<tr><td class="directory-table__empty" colspan="{{ directory_columns|length }}">
+12 -1
View File
@@ -4,7 +4,7 @@
<section class="employee-card">
<div class="employee-card__header">
<div class="employee-card__identity">
<h2 class="employee-card__title">{{ employee_view.full_name or employee.profile_key }}</h2>
<h2 class="employee-card__title">{{ employee_view.full_name or employee.profile_key }}{% if employee_view.dpo_uncertain %} <a class="dpo-warning" href="#unresolved-dpo" title="Часть сведений о ДПО не распознана. Количество может быть неполным." aria-label="Часть сведений о ДПО не распознана. Количество может быть неполным."><span aria-hidden="true">⚠︎</span></a>{% endif %}</h2>
<span class="badge {% if employee_view.status == "dismissed" %}badge--dismissed{% endif %}">{{ employee_view.status_display }}</span>
</div>
<div class="employee-card__actions">
@@ -215,6 +215,9 @@
</table>
</div>
{% else %}
{% if section.type == "additional_education" and section.list_items %}
<p class="employee-section__note">Всего: {{ section.list_items | length }}</p>
{% endif %}
{% if section.paragraphs %}
{% for paragraph in section.paragraphs %}
<p class="employee-section__text">{{ paragraph }}</p>
@@ -238,6 +241,14 @@
{% endif %}
</article>
{% endfor %}
{% if employee_view.dpo_uncertain %}
<div id="unresolved-dpo" tabindex="-1">
{% for section in profile_sections %}
{% for text in section.unresolved %}<p>{{ text }}</p>{% endfor %}
{% for doc in section.unresolved_documents %}<p>{% if doc.url.startswith(('https://', 'http://')) %}<a href="{{ doc.url }}">{{ doc.text }}</a>{% else %}{{ doc.text }}{% endif %}</p>{% endfor %}
{% endfor %}
</div>
{% endif %}
</div>
{% else %}
<p class="employee-section__text">Разделы профиля не найдены.</p>
+3 -3
View File
@@ -1,3 +1,3 @@
APP_VERSION = "2.2.1"
FRONTEND_VERSION = "2.2.1"
BACKEND_VERSION = "2.2.1"
APP_VERSION = "2.3.0"
FRONTEND_VERSION = "2.3.0"
BACKEND_VERSION = "2.3.0"
+1 -1
View File
@@ -1,6 +1,6 @@
[project]
name = "miem-workers"
version = "2.2.1"
version = "2.3.0"
description = "MIEM employees parser, admin API, and web admin"
requires-python = ">=3.11"
dependencies = [
+2 -2
View File
@@ -22,7 +22,7 @@ def test_health_returns_versions():
response = TestClient(app).get("/api/health")
assert response.status_code == 200
assert response.json()["backend_version"] == "2.2.1"
assert response.json()["backend_version"] == "2.3.0"
def test_api_employees_and_stats_require_admin_session():
@@ -202,7 +202,7 @@ def test_directory_export_uses_applied_filters_and_sort_across_pages(db_session)
assert [cell.value for cell in sheet[1]] == [
cell.get_text(strip=True) for cell in page.select(".directory-table th")
]
assert sheet.max_column == 15
assert sheet.max_column == 16
assert [cell.value for cell in sheet["A"][1:]] == [f"Export {number:02d}" for number in range(28)]
assert [cell.value for cell in sheet["A"][-3:]] == [
cell.get_text(strip=True) for cell in page.select("tbody [data-column='full_name']")
+1 -1
View File
@@ -230,7 +230,7 @@ def test_public_page_data_minimization_and_safe_links(db_session, export):
assert "javascript:" not in fragment.text and "private@example.org" not in fragment.text
assert "javascript:" not in response.text and "private@example.org" not in response.text
assert "/admin/employees/" not in response.text and "/admin/logout" not in response.text
assert "Backend 2.2.1 · Frontend 2.2.1" in response.text
assert "Backend 2.3.0 · Frontend 2.3.0" in response.text
assert "Вход в систему" in response.text
assert client.get("/admin/directory", follow_redirects=False).status_code in (302, 303, 401, 403)
assert "Сотрудники не найдены" in client.get("/admin/council?q=missing").text
+29
View File
@@ -0,0 +1,29 @@
from bs4 import BeautifulSoup
from app.parser.education_document import _certificate
from app.parser.profile import extract_sections
from app.services.resource_cache import ResourceCache
def test_education_events_preserve_ambiguities_and_separate_periods():
row = '<div><div class="person-list-hangover">{}</div><span>«Первая помощь»</span>, повышение квалификации, {}</div>'
html = ('<h2>Дополнительное образование</h2>' + row.format(2025, "НИУ ВШЭ") +
row.format(2025, "Высшая школа экономики") + row.format(2023, "НИУ ВШЭ") +
'<p>Прошел тест с присвоением степени</p><p>«Инженер»</p>'
'<p>Дополнительные сведения</p><p><a href="/mirror/pubs/share/1">Документ</a></p>')
section, = extract_sections(BeautifulSoup(html, "html.parser"), "https://www.hse.ru", lambda url: None)
assert len(section["items"]) == 3
assert section["items"][-1] == "Прошел тест с присвоением степени «Инженер»"
assert section["unresolved"] == ["Дополнительные сведения"]
assert section["unresolved_documents"][0]["url"] == "https://www.hse.ru/mirror/pubs/share/1"
def test_document_read_and_cache(db_session):
text = 'НИУ ВШЭ. Удостоверение о повышении квалификации по программе «Деловой английский» в объеме 24 часов. 2025 год'
result = _certificate(text)
assert result == "2025 «Деловой английский», повышение квалификации, НИУ ВШЭ, 24 часов"
assert _certificate("Сертификат 2025") is None
cache = ResourceCache(db_session)
assert cache.education_document("staff:test", "https://www.hse.ru/test", "1" * 64, lambda: result) == result
assert cache.education_document("staff:test", "https://www.hse.ru/test", "1" * 64, lambda: 1 / 0) == result
+31
View File
@@ -4,6 +4,37 @@ from app.parser.profile import enrich_sections_from_hse_widgets, extract_person_
from app.parser.profile_url import normalize_profile_url, parse_profile_identity
def test_mixed_additional_education_keeps_all_courses_and_omits_attachments():
structured = "".join(
f'<div class="g-ul"><div class="with-indent"><div class="person-list-hangover">{year}</div>'
f'<span>{title}</span><span>, {provider}</span></div></div>'
for year, title, provider in [(2025, "Экспортный контроль", "МИФИ"),
(2024, "Smart LMS", "ВШЭ"), (2018, "Методическая работа", "Дирекция")]
)
html = f'''<div tab-node="additional_education"><h2 class="c_14">Дополнительное образование</h2>
{structured}<div class="with-indent"><ul><li>2020 г. НИУ ВШЭ, организация учебного процесса</li></ul>
<p><a href="/f.pdf">Файл</a> (PDF, 407 Кб)</p>
<ul><li>2017 г. МГУ, методика <a href="/mirror/pubs/share/123">Сертификат</a></li>
<li>2016 г. КФУ, образовательные стандарты</li><li>2015 г. ЮФУ, ФГОС</li>
<li><strong>2015 г</strong>. МГТУ МИРЭА.</li></ul>
<p class="text">Направления совершенствования практической подготовки.</p>
<p><a href="/cert.jpg">Сертификат_МИРЭА</a> (JPG, 7,28 Мб).</p>
<ul><li>2014 г. АИОР, аккредитация.</li></ul>
<p><a href="/certificate">Свидетельство</a> (PDF, 9,36 Мб).</p>
<ul><li>2011 г. РАНХиГС.</li></ul>
<p>Курс <a href="/programme">Современные подходы</a>.</p>
<p><a href="/cert.jpg">Удостоверение</a> (JPG, 6,69 Мб).</p></div></div>'''
section, = extract_sections(BeautifulSoup(html, "html.parser"), "https://www.hse.ru/")
assert len(section["items"]) == 10
assert [row["year"] for row in section["year_entries"]] == [2025, 2024, 2018]
assert section["items"][0] == "2025 Экспортный контроль, МИФИ"
assert section["items"][7] == "2015 г. МГТУ МИРЭА. Направления совершенствования практической подготовки."
assert section["items"][9].startswith("2011 г. РАНХиГС. Курс Современные подходы")
assert section["links"] == [{"text": "Современные подходы", "url": "https://www.hse.ru/programme"}]
for fragment in ("Сертификат", "Файл", "Свидетельство", "Удостоверение", "PDF", "JPG"):
assert fragment not in str(section)
class FakeResponse:
def __init__(self, payload):
self.payload = payload