Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
65d851bb92 | ||
|
|
8dfd700bd5 |
No files matched your search
@@ -1,5 +1,9 @@
|
||||
# Changelog
|
||||
|
||||
## 2.3.0
|
||||
|
||||
- Раздел ДПО объединяет сведения из профиля и документов. Справочник показывает отдельный счётчик ДПО, а предупреждение ведёт к нераспознанным фрагментам в карточке.
|
||||
|
||||
## 2.2.1
|
||||
|
||||
- При применении фильтров справочника, переключении ППС, сортировки и размера страницы появляется заглушка «Обновляем список…». Поля, кнопки и ссылки справочника заблокированы до завершения запроса.
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
WORKDIR /app
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends gcc libpq-dev \
|
||||
&& apt-get install -y --no-install-recommends gcc libpq-dev poppler-utils tesseract-ocr tesseract-ocr-rus \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY requirements.txt .
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Assemble additional education events from the public profile's mixed markup."""
|
||||
import re
|
||||
|
||||
|
||||
YEAR = re.compile(r"\b(?:19|20)\d{2}\b")
|
||||
_QUOTE = re.compile(r'«([^»]+)»|"([^"]+)"')
|
||||
_EVENT = re.compile(r"стажиров|стипенди|исследовательск|обучени|квалификаци|переподготовк|прош[её]л|прошла|семинар|сертификаци|программ[аеуы]|курсы?\b", re.I)
|
||||
_CONTINUES = re.compile(r"(?:по программе|присвоением степени|на тему|по курсу|по теме|по специальности)[:\s]*$", re.I)
|
||||
_CAPTION = re.compile(r"^(?:удостоверение|сертификат|свидетельство)(?: о повышении квалификации)?[.:]?$", re.I)
|
||||
|
||||
|
||||
def _key(text):
|
||||
return " ".join(re.findall(r"\w+", text.casefold().replace("ё", "е")))
|
||||
|
||||
|
||||
def _provider(text):
|
||||
text = _key(text)
|
||||
for variants in (("вшэ", "высшая школа экономики", "высшей школе экономики"),
|
||||
("мифи", "московский инженерно физический институт")):
|
||||
if any(v in text.split() if " " not in v else v in text for v in variants):
|
||||
return variants[0]
|
||||
return text
|
||||
|
||||
|
||||
def assemble(rows):
|
||||
events = []
|
||||
unresolved = []
|
||||
pending = []
|
||||
for text, year, structured, list_item in rows:
|
||||
if _CAPTION.fullmatch(text):
|
||||
continue
|
||||
if (pending and not structured and not list_item and not YEAR.match(text)
|
||||
and (_CONTINUES.search(pending[-1][0]) or
|
||||
(pending[-1][3] and not _EVENT.search(text[:20]) and not YEAR.search(text)) or
|
||||
(pending[-1][3] and text.startswith("Курс ") and not YEAR.search(text)))):
|
||||
pending[-1][0] += " " + text
|
||||
pending[-1][3] = False
|
||||
else:
|
||||
pending.append([text, year, structured, list_item])
|
||||
|
||||
for text, year, structured, _ in pending:
|
||||
if not structured and (len(_key(text).split()) < 2 or not (_EVENT.search(text) or YEAR.match(text))):
|
||||
unresolved.append(text)
|
||||
continue
|
||||
quotes = [a or b for a, b in _QUOTE.findall(text)]
|
||||
title = _key(quotes[-1]) if quotes else ""
|
||||
if structured and text.lstrip("0123456789 ").startswith("«"):
|
||||
title = _key(text.split("«", 1)[1].rsplit("»", 1)[0])
|
||||
provider = _provider(text.rsplit("»", 1)[-1] if structured and "»" in text else text)
|
||||
if not year:
|
||||
match = YEAR.search(_QUOTE.sub("", text))
|
||||
year = int(match[0]) if match else None
|
||||
matches = [e for e in events if title and e["title"] == title and e["provider"] == provider
|
||||
and (e["year"] == year or not year or not e["year"])]
|
||||
if len(matches) == 1:
|
||||
target = matches[0]
|
||||
hours = re.search(r"\b\d+\s+час[а-я]*", text, re.I)
|
||||
if hours and hours[0] not in target["text"]:
|
||||
target["text"] += " (" + hours[0] + ")"
|
||||
continue
|
||||
if any(e["text"] == text for e in events):
|
||||
continue
|
||||
events.append(dict(text=text, year=year, structured=structured, title=title, provider=provider))
|
||||
return events, unresolved
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Read public HSE education documents. An inconclusive extraction stays visible for review."""
|
||||
import hashlib
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import requests
|
||||
|
||||
_MAX_BYTES = 12 * 1024 * 1024
|
||||
|
||||
|
||||
def _run(args):
|
||||
return subprocess.run(args, check=True, capture_output=True, timeout=30).stdout.decode("utf-8")
|
||||
|
||||
|
||||
def _certificate(text):
|
||||
text = " ".join(text.split())
|
||||
title = re.search(r'(?:по (?:дополнительной профессиональной )?программе|по теме|на тему)\s*[:«"\s]*([^»"]{5,300})[»"]', text, re.I)
|
||||
if not title:
|
||||
title = re.search(r'[«"]([^«»"]{8,250})[»"]\s*[,.(]?\s*(?:в объ[её]ме|\d+ час)', text, re.I)
|
||||
years = re.findall(r"\b(?:19|20)\d{2}\b", text)
|
||||
if not title or not years or not re.search(r"квалификаци|сертификат|удостоверение", text, re.I):
|
||||
return None
|
||||
hours = re.search(r"(\d+)\s*(?:академических\s+)?час", text, re.I)
|
||||
provider = ""
|
||||
if re.search(r"высш[а-я]+ школ[а-я]+ экономик|НИУ ВШЭ", text, re.I):
|
||||
provider = ", НИУ ВШЭ"
|
||||
elif re.search("МИСИС", text, re.I):
|
||||
provider = ", НИТУ МИСИС"
|
||||
return f'{years[-1]} «{title[1].strip()}», повышение квалификации{provider}' + (f", {hours[1]} часов" if hours else "")
|
||||
|
||||
|
||||
def _recognize(data):
|
||||
with TemporaryDirectory() as tmp:
|
||||
src = Path(tmp) / "document"
|
||||
src.write_bytes(data)
|
||||
images = [src]
|
||||
if data.startswith(b"%PDF"):
|
||||
direct = _certificate(_run(["pdftotext", "-layout", str(src), "-"]))
|
||||
if direct:
|
||||
return direct
|
||||
_run(["pdftoppm", "-f", "1", "-l", "2", "-scale-to", "2400", "-png", str(src), str(src)])
|
||||
images = sorted(Path(tmp).glob("document-*.png"))
|
||||
elif not data.startswith((b"\x89PNG", b"\xff\xd8\xff")):
|
||||
return None
|
||||
return _certificate(" ".join(_run(["tesseract", str(img), "stdout", "-l", "rus+eng"]) for img in images))
|
||||
|
||||
|
||||
def read_document(session, url, timeout, cache, profile_key):
|
||||
parts = urlsplit(url)
|
||||
if parts.scheme != "https" or parts.netloc != "www.hse.ru" or not parts.path.startswith("/mirror/pubs/share/"):
|
||||
return None
|
||||
try:
|
||||
with session.get(url, stream=True, timeout=timeout, allow_redirects=False) as response:
|
||||
if response.status_code != 200:
|
||||
return None
|
||||
data = bytearray()
|
||||
for chunk in response.iter_content(65536):
|
||||
data.extend(chunk)
|
||||
if len(data) > _MAX_BYTES:
|
||||
return None
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
if cache:
|
||||
return cache.education_document(profile_key, url, digest, lambda: _recognize(data))
|
||||
return _recognize(data)
|
||||
except (OSError, ValueError, subprocess.SubprocessError, requests.RequestException):
|
||||
return None
|
||||
|
||||
+81
-3
@@ -1,12 +1,15 @@
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from copy import deepcopy
|
||||
from datetime import datetime, timezone
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from bs4 import BeautifulSoup, NavigableString, Tag
|
||||
from bs4 import BeautifulSoup, Comment, NavigableString, Tag
|
||||
from requests import Session
|
||||
|
||||
from app.parser.education import assemble
|
||||
from app.parser.education_document import read_document
|
||||
from app.parser.profile_url import normalize_profile_url, parse_profile_identity
|
||||
from app.version import BACKEND_VERSION
|
||||
|
||||
@@ -99,7 +102,7 @@ def extract_person_header(soup: BeautifulSoup, source_url: str) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def extract_sections(soup: BeautifulSoup, source_url: str) -> list[dict]:
|
||||
def extract_sections(soup: BeautifulSoup, source_url: str, document_reader=None) -> list[dict]:
|
||||
sections = []
|
||||
for h2 in soup.select("h2"):
|
||||
if h2.find_parent(class_="post") or h2.find_parent(attrs={"data-tab": "press_links_news"}):
|
||||
@@ -108,6 +111,11 @@ def extract_sections(soup: BeautifulSoup, source_url: str) -> list[dict]:
|
||||
if not title or "расписание занятий" in title.lower():
|
||||
continue
|
||||
nodes = _collect_between_h2(h2)
|
||||
education_titles = {part.strip().rstrip(":") for part in title.casefold().split("/")}
|
||||
if (h2.find_parent(attrs={"tab-node": "additional_education"})
|
||||
or education_titles & {"дополнительное образование", "повышение квалификации", "курсы повышения квалификации"}):
|
||||
sections.append(_parse_additional_education(title, nodes, source_url, document_reader))
|
||||
continue
|
||||
raw_text = _nodes_raw_text(nodes)
|
||||
paragraphs = _nodes_paragraphs(nodes)
|
||||
items = _nodes_list_items(nodes)
|
||||
@@ -193,7 +201,7 @@ def parse_person_profile(
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
header = extract_person_header(soup, normalized_url)
|
||||
tabs = extract_person_tabs(soup, normalized_url)
|
||||
sections = extract_sections(soup, normalized_url)
|
||||
sections = extract_sections(soup, normalized_url, lambda url: read_document(session, url, timeout, resource_cache, cache_profile_key))
|
||||
sections = enrich_sections_from_hse_widgets(
|
||||
session,
|
||||
soup,
|
||||
@@ -219,6 +227,7 @@ def parse_person_profile(
|
||||
"tabs": tabs,
|
||||
"sections": sections,
|
||||
"employee_internal_links": internal_links,
|
||||
"education_parsed": True,
|
||||
"parser_version": BACKEND_VERSION,
|
||||
"_html": html,
|
||||
"_resource_manifest": resource_manifest,
|
||||
@@ -585,6 +594,75 @@ def _parse_year_entries(nodes: list, source_url: str) -> list[dict]:
|
||||
return entries
|
||||
|
||||
|
||||
_FILE_FORMAT = r"(?:pdf|jpe?g|png|docx?|pptx?|xlsx?|zip|rar)"
|
||||
_FILE_SIZE = re.compile(rf"\(\s*{_FILE_FORMAT}\s*,\s*[\d.,\s]+[кмгkmg]?\s*[бb](?:айт[а-я]*)?\s*\)", re.I)
|
||||
_FILE_CAPTION = re.compile(rf"\b(?:файл|сертификат|свидетельство|удостоверение)(?:_\w+)?\s*{_FILE_SIZE.pattern}", re.I)
|
||||
_FILE_URL = re.compile(rf"/mirror/pubs/share/|\.{_FILE_FORMAT}(?:[?#]|$)", re.I)
|
||||
_INITIAL_YEAR = re.compile(r"^(?:19|20)\d{2}\b")
|
||||
|
||||
|
||||
def _education_blocks(node):
|
||||
"""Yield whole course rows or paragraphs once, in document order."""
|
||||
if not isinstance(node, Tag):
|
||||
if not isinstance(node, Comment) and normalize_ws(str(node)):
|
||||
yield node
|
||||
return
|
||||
if node.name == "tr" and node.find("th"):
|
||||
return
|
||||
if (node.name in {"li", "p", "tr"} or node.find(class_="person-list-hangover", recursive=False)
|
||||
or not node.find(["div", "ul", "ol", "li", "p", "table", "tr"])):
|
||||
yield node
|
||||
return
|
||||
for child in node.children:
|
||||
yield from _education_blocks(child)
|
||||
|
||||
|
||||
def _parse_additional_education(title: str, nodes: list, source_url: str, document_reader=None) -> dict:
|
||||
rows = []
|
||||
documents = {}
|
||||
unresolved_documents = []
|
||||
links = []
|
||||
for original in nodes:
|
||||
node = deepcopy(original)
|
||||
if isinstance(node, Tag):
|
||||
for anchor in node.select("a[href]"):
|
||||
following = anchor.next_sibling
|
||||
suffix = following.get_text(strip=True) if isinstance(following, Tag) else str(following or "")
|
||||
if (_FILE_URL.search(anchor["href"]) or anchor.has_attr("download")
|
||||
or _FILE_SIZE.match(normalize_ws(suffix))):
|
||||
documents[urljoin(source_url, anchor["href"])] = normalize_ws(anchor.get_text(" ", strip=True)) or "Документ"
|
||||
anchor.decompose()
|
||||
for block in _education_blocks(node):
|
||||
is_tag = isinstance(block, Tag)
|
||||
content = block.get_text(" ", strip=True) if is_tag else str(block)
|
||||
text = normalize_ws(_FILE_SIZE.sub("", _FILE_CAPTION.sub("", content)))
|
||||
text = re.sub(r"\s+([,.;:])", r"\1", text)
|
||||
text = re.sub(r"(?<=[.!?])\.$", "", text)
|
||||
if not re.search(r"\w", text):
|
||||
continue
|
||||
block_links = _extract_links(block, source_url) if is_tag else []
|
||||
links.extend(block_links)
|
||||
year_node = block.find(class_="person-list-hangover", recursive=False) if is_tag else None
|
||||
year_match = _INITIAL_YEAR.match(year_node.get_text(strip=True)) if year_node else None
|
||||
rows.append((text, int(year_match[0]) if year_match else None, bool(year_node), is_tag and block.name == "li"))
|
||||
for url, caption in documents.items():
|
||||
recognized = document_reader(url) if document_reader else None
|
||||
if recognized:
|
||||
rows.append((recognized, None, False, False))
|
||||
elif document_reader:
|
||||
unresolved_documents.append({"text": caption, "url": url})
|
||||
events, unresolved = assemble(rows)
|
||||
items = [event["text"] for event in events]
|
||||
year_entries = [{"year": event["year"], "text": event["text"], "links": []}
|
||||
for event in events if event["structured"]]
|
||||
return {
|
||||
"title": title, "slug": _slugify(title), "type": "additional_education",
|
||||
"items": items, "paragraphs": [], "year_entries": year_entries,
|
||||
"unresolved": unresolved, "unresolved_documents": unresolved_documents,
|
||||
"raw_text": "\n".join(items), "links": links,
|
||||
}
|
||||
|
||||
|
||||
def _parse_vkr_items(nodes: list) -> list[str]:
|
||||
items = []
|
||||
for node in nodes:
|
||||
|
||||
@@ -38,6 +38,7 @@ DIRECTORY_COLUMNS = (
|
||||
("last_seen_at", "Последний раз найден", "last_seen_display"),
|
||||
("dismissed_at", "Дата увольнения", "dismissed_display"),
|
||||
("profile", "Профиль", "canonical_url"),
|
||||
("dpo_count", "Количество ДПО", "dpo_count"),
|
||||
)
|
||||
|
||||
_TRAINING_LABEL = "повышение квалификации"
|
||||
@@ -57,16 +58,20 @@ def _has_training(data: dict[str, Any], since_year: int | None) -> bool:
|
||||
dedicated = title in {_TRAINING_LABEL, "курсы повышения квалификации"}
|
||||
if not dedicated and _TRAINING_LABEL not in (part.strip() for part in title.split("/")):
|
||||
continue
|
||||
structured_texts = set()
|
||||
for value in _as_list(section.get("year_entries")):
|
||||
entry = _as_dict(value)
|
||||
year = entry.get("year")
|
||||
text = str(entry.get("text") or "").strip()
|
||||
structured_texts.add(text)
|
||||
if since_year is not None and (type(year) is not int or year < since_year):
|
||||
continue
|
||||
if text and text != str(year) and (dedicated or _TRAINING_KIND.search(_QUOTED_TEXT.sub("", text))):
|
||||
return True
|
||||
# Profiles may also list completed courses in manually entered paragraphs or lists.
|
||||
for text in _clean_list(section.get("paragraphs")) + _clean_list(section.get("items")):
|
||||
if text in structured_texts:
|
||||
continue
|
||||
description = _QUOTED_TEXT.sub("", text).strip()
|
||||
if not dedicated and not _TRAINING_MENTION.search(description):
|
||||
continue
|
||||
@@ -87,6 +92,8 @@ def employee_display_payload(employee: Employee) -> dict[str, Any]:
|
||||
emails = _clean_list(contacts.get("emails"))
|
||||
phones = _clean_list(contacts.get("phones"))
|
||||
degree_values = academic_degrees(data)
|
||||
education = [s for s in sections if isinstance(s, dict) and s.get("type") == "additional_education"]
|
||||
parsed_education = bool(education) or data.get("education_parsed", False)
|
||||
return {
|
||||
"id": employee.id,
|
||||
"full_name": employee.full_name,
|
||||
@@ -103,6 +110,8 @@ def employee_display_payload(employee: Employee) -> dict[str, Any]:
|
||||
"address": contacts.get("address"),
|
||||
"academic_degree_text": "; ".join(degree_values),
|
||||
"publications_count": _count_section_items(sections, "publications"),
|
||||
"dpo_count": sum(len(s.get("items", [])) for s in education) if parsed_education else None,
|
||||
"dpo_uncertain": any(s.get("unresolved") or s.get("unresolved_documents") for s in education),
|
||||
"courses_count": _count_section_items(sections, "courses_by_year"),
|
||||
"news_count": len(stored_news_links) or _count_section_items(sections, "news"),
|
||||
"first_seen_at": employee.first_seen_at.isoformat() if employee.first_seen_at else None,
|
||||
@@ -456,6 +465,8 @@ def _normalize_section(section: Any) -> dict[str, Any]:
|
||||
"raw_text": raw_text,
|
||||
"paragraphs": paragraphs,
|
||||
"list_items": items,
|
||||
"unresolved": _clean_list(section.get("unresolved")),
|
||||
"unresolved_documents": _normalize_links(section.get("unresolved_documents")),
|
||||
"links": _normalize_links(section.get("links")),
|
||||
"year_entries": _normalize_year_entries(section.get("year_entries")),
|
||||
"publications": _normalize_publications(section.get("publications")),
|
||||
|
||||
@@ -19,7 +19,7 @@ def directory_xlsx(employees: list[dict]) -> bytes:
|
||||
(employee[key] or "Без имени") if key == "full_name" else employee[key]
|
||||
for _, _, key in DIRECTORY_COLUMNS
|
||||
])
|
||||
for cell, width in zip(sheet[1], (34, 20, 48, 14, 28, 22, 34, 30, 14, 10, 10, 22, 24, 22, 46)):
|
||||
for cell, width in zip(sheet[1], (34, 20, 48, 14, 28, 22, 34, 30, 14, 10, 10, 22, 24, 22, 46, 18)):
|
||||
cell.font = Font(bold=True)
|
||||
sheet.column_dimensions[get_column_letter(cell.column)].width = width
|
||||
sheet.freeze_panes = "A2"
|
||||
|
||||
@@ -27,6 +27,24 @@ class ResourceCache:
|
||||
def __init__(self, db: Session):
|
||||
self.db = db
|
||||
|
||||
def education_document(self, profile_key, url, digest, recognize):
|
||||
key = "education-document-v1"
|
||||
existing = self.db.scalar(select(ParseResourceCache).where(
|
||||
ParseResourceCache.profile_key == profile_key, ParseResourceCache.resource_key == key,
|
||||
ParseResourceCache.request_fingerprint == digest))
|
||||
if existing:
|
||||
return gzip.decompress(existing.body_snapshot).decode() or None
|
||||
result = recognize()
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
try:
|
||||
with self.db.begin_nested():
|
||||
self.db.add(ParseResourceCache(profile_key=profile_key, resource_key=key, method="OCR", url=url,
|
||||
request_fingerprint=digest, body_hash=digest, body_snapshot=gzip.compress((result or "").encode()),
|
||||
parser_version=BACKEND_VERSION))
|
||||
except IntegrityError:
|
||||
pass # Another crawl cached the same document.
|
||||
return result
|
||||
|
||||
def fetch_text(
|
||||
self,
|
||||
session: requests.Session,
|
||||
|
||||
@@ -1172,3 +1172,6 @@
|
||||
.guide__fields dd { margin: 4px 0 0; overflow-wrap: anywhere; }
|
||||
.guide--page { max-width: 75ch; }
|
||||
.guide--page .guide__step + .guide__step { margin-block-start: 32px; }
|
||||
|
||||
.dpo-warning { color: #ab202b; text-decoration: none; }
|
||||
#unresolved-dpo:target { outline: 2px solid #ab202b; outline-offset: 3px; }
|
||||
+1
-1
@@ -10,7 +10,7 @@
|
||||
const columnPresets = {
|
||||
review: ["full_name", "status", "academic_degree", "last_seen_at", "profile"],
|
||||
contacts: ["full_name", "status", "email", "phone", "address", "profile"],
|
||||
full: ["full_name", "status", "positions", "hse_start_year", "email", "phone", "address", "academic_degree", "publications_count", "courses_count", "news_count", "first_seen_at", "last_seen_at", "dismissed_at", "profile"],
|
||||
full: ["full_name", "status", "positions", "hse_start_year", "email", "phone", "address", "academic_degree", "publications_count", "courses_count", "news_count", "first_seen_at", "last_seen_at", "dismissed_at", "profile", "dpo_count"],
|
||||
};
|
||||
|
||||
function readColumns() {
|
||||
|
||||
@@ -123,6 +123,7 @@
|
||||
<td class="directory-table__cell" data-column="last_seen_at">{{ employee.last_seen_display }}</td>
|
||||
<td class="directory-table__cell" data-column="dismissed_at">{{ employee.dismissed_display }}</td>
|
||||
<td class="directory-table__cell" data-column="profile"><a class="admin__link" href="{{ employee.canonical_url }}" target="_blank" rel="noopener">Внешний профиль</a></td>
|
||||
<td class="directory-table__cell" data-column="dpo_count">{{ employee.dpo_count if employee.dpo_count is not none else "—" }}{% if employee.dpo_uncertain %} <a class="dpo-warning" href="/admin/employees/{{ employee.id }}#unresolved-dpo" title="Часть сведений о ДПО не распознана. Количество может быть неполным." aria-label="Часть сведений о ДПО не распознана. Количество может быть неполным."><span aria-hidden="true">⚠︎</span></a>{% endif %}</td>
|
||||
</tr>
|
||||
{% else %}
|
||||
<tr><td class="directory-table__empty" colspan="{{ directory_columns|length }}">
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<section class="employee-card">
|
||||
<div class="employee-card__header">
|
||||
<div class="employee-card__identity">
|
||||
<h2 class="employee-card__title">{{ employee_view.full_name or employee.profile_key }}</h2>
|
||||
<h2 class="employee-card__title">{{ employee_view.full_name or employee.profile_key }}{% if employee_view.dpo_uncertain %} <a class="dpo-warning" href="#unresolved-dpo" title="Часть сведений о ДПО не распознана. Количество может быть неполным." aria-label="Часть сведений о ДПО не распознана. Количество может быть неполным."><span aria-hidden="true">⚠︎</span></a>{% endif %}</h2>
|
||||
<span class="badge {% if employee_view.status == "dismissed" %}badge--dismissed{% endif %}">{{ employee_view.status_display }}</span>
|
||||
</div>
|
||||
<div class="employee-card__actions">
|
||||
@@ -215,6 +215,9 @@
|
||||
</table>
|
||||
</div>
|
||||
{% else %}
|
||||
{% if section.type == "additional_education" and section.list_items %}
|
||||
<p class="employee-section__note">Всего: {{ section.list_items | length }}</p>
|
||||
{% endif %}
|
||||
{% if section.paragraphs %}
|
||||
{% for paragraph in section.paragraphs %}
|
||||
<p class="employee-section__text">{{ paragraph }}</p>
|
||||
@@ -238,6 +241,14 @@
|
||||
{% endif %}
|
||||
</article>
|
||||
{% endfor %}
|
||||
{% if employee_view.dpo_uncertain %}
|
||||
<div id="unresolved-dpo" tabindex="-1">
|
||||
{% for section in profile_sections %}
|
||||
{% for text in section.unresolved %}<p>{{ text }}</p>{% endfor %}
|
||||
{% for doc in section.unresolved_documents %}<p>{% if doc.url.startswith(('https://', 'http://')) %}<a href="{{ doc.url }}">{{ doc.text }}</a>{% else %}{{ doc.text }}{% endif %}</p>{% endfor %}
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% else %}
|
||||
<p class="employee-section__text">Разделы профиля не найдены.</p>
|
||||
|
||||
+3
-3
@@ -1,3 +1,3 @@
|
||||
APP_VERSION = "2.2.1"
|
||||
FRONTEND_VERSION = "2.2.1"
|
||||
BACKEND_VERSION = "2.2.1"
|
||||
APP_VERSION = "2.3.0"
|
||||
FRONTEND_VERSION = "2.3.0"
|
||||
BACKEND_VERSION = "2.3.0"
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "miem-workers"
|
||||
version = "2.2.1"
|
||||
version = "2.3.0"
|
||||
description = "MIEM employees parser, admin API, and web admin"
|
||||
requires-python = ">=3.11"
|
||||
dependencies = [
|
||||
|
||||
+2
-2
@@ -22,7 +22,7 @@ def test_health_returns_versions():
|
||||
response = TestClient(app).get("/api/health")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["backend_version"] == "2.2.1"
|
||||
assert response.json()["backend_version"] == "2.3.0"
|
||||
|
||||
|
||||
def test_api_employees_and_stats_require_admin_session():
|
||||
@@ -202,7 +202,7 @@ def test_directory_export_uses_applied_filters_and_sort_across_pages(db_session)
|
||||
assert [cell.value for cell in sheet[1]] == [
|
||||
cell.get_text(strip=True) for cell in page.select(".directory-table th")
|
||||
]
|
||||
assert sheet.max_column == 15
|
||||
assert sheet.max_column == 16
|
||||
assert [cell.value for cell in sheet["A"][1:]] == [f"Export {number:02d}" for number in range(28)]
|
||||
assert [cell.value for cell in sheet["A"][-3:]] == [
|
||||
cell.get_text(strip=True) for cell in page.select("tbody [data-column='full_name']")
|
||||
|
||||
@@ -230,7 +230,7 @@ def test_public_page_data_minimization_and_safe_links(db_session, export):
|
||||
assert "javascript:" not in fragment.text and "private@example.org" not in fragment.text
|
||||
assert "javascript:" not in response.text and "private@example.org" not in response.text
|
||||
assert "/admin/employees/" not in response.text and "/admin/logout" not in response.text
|
||||
assert "Backend 2.2.1 · Frontend 2.2.1" in response.text
|
||||
assert "Backend 2.3.0 · Frontend 2.3.0" in response.text
|
||||
assert "Вход в систему" in response.text
|
||||
assert client.get("/admin/directory", follow_redirects=False).status_code in (302, 303, 401, 403)
|
||||
assert "Сотрудники не найдены" in client.get("/admin/council?q=missing").text
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from app.parser.education_document import _certificate
|
||||
from app.parser.profile import extract_sections
|
||||
from app.services.resource_cache import ResourceCache
|
||||
|
||||
|
||||
def test_education_events_preserve_ambiguities_and_separate_periods():
|
||||
row = '<div><div class="person-list-hangover">{}</div><span>«Первая помощь»</span>, повышение квалификации, {}</div>'
|
||||
html = ('<h2>Дополнительное образование</h2>' + row.format(2025, "НИУ ВШЭ") +
|
||||
row.format(2025, "Высшая школа экономики") + row.format(2023, "НИУ ВШЭ") +
|
||||
'<p>Прошел тест с присвоением степени</p><p>«Инженер»</p>'
|
||||
'<p>Дополнительные сведения</p><p><a href="/mirror/pubs/share/1">Документ</a></p>')
|
||||
section, = extract_sections(BeautifulSoup(html, "html.parser"), "https://www.hse.ru", lambda url: None)
|
||||
assert len(section["items"]) == 3
|
||||
assert section["items"][-1] == "Прошел тест с присвоением степени «Инженер»"
|
||||
assert section["unresolved"] == ["Дополнительные сведения"]
|
||||
assert section["unresolved_documents"][0]["url"] == "https://www.hse.ru/mirror/pubs/share/1"
|
||||
|
||||
|
||||
def test_document_read_and_cache(db_session):
|
||||
text = 'НИУ ВШЭ. Удостоверение о повышении квалификации по программе «Деловой английский» в объеме 24 часов. 2025 год'
|
||||
result = _certificate(text)
|
||||
assert result == "2025 «Деловой английский», повышение квалификации, НИУ ВШЭ, 24 часов"
|
||||
assert _certificate("Сертификат 2025") is None
|
||||
cache = ResourceCache(db_session)
|
||||
assert cache.education_document("staff:test", "https://www.hse.ru/test", "1" * 64, lambda: result) == result
|
||||
assert cache.education_document("staff:test", "https://www.hse.ru/test", "1" * 64, lambda: 1 / 0) == result
|
||||
|
||||
@@ -4,6 +4,37 @@ from app.parser.profile import enrich_sections_from_hse_widgets, extract_person_
|
||||
from app.parser.profile_url import normalize_profile_url, parse_profile_identity
|
||||
|
||||
|
||||
def test_mixed_additional_education_keeps_all_courses_and_omits_attachments():
|
||||
structured = "".join(
|
||||
f'<div class="g-ul"><div class="with-indent"><div class="person-list-hangover">{year}</div>'
|
||||
f'<span>{title}</span><span>, {provider}</span></div></div>'
|
||||
for year, title, provider in [(2025, "Экспортный контроль", "МИФИ"),
|
||||
(2024, "Smart LMS", "ВШЭ"), (2018, "Методическая работа", "Дирекция")]
|
||||
)
|
||||
html = f'''<div tab-node="additional_education"><h2 class="c_14">Дополнительное образование</h2>
|
||||
{structured}<div class="with-indent"><ul><li>2020 г. НИУ ВШЭ, организация учебного процесса</li></ul>
|
||||
<p><a href="/f.pdf">Файл</a> (PDF, 407 Кб)</p>
|
||||
<ul><li>2017 г. МГУ, методика <a href="/mirror/pubs/share/123">Сертификат</a></li>
|
||||
<li>2016 г. КФУ, образовательные стандарты</li><li>2015 г. ЮФУ, ФГОС</li>
|
||||
<li><strong>2015 г</strong>. МГТУ МИРЭА.</li></ul>
|
||||
<p class="text">Направления совершенствования практической подготовки.</p>
|
||||
<p><a href="/cert.jpg">Сертификат_МИРЭА</a> (JPG, 7,28 Мб).</p>
|
||||
<ul><li>2014 г. АИОР, аккредитация.</li></ul>
|
||||
<p><a href="/certificate">Свидетельство</a> (PDF, 9,36 Мб).</p>
|
||||
<ul><li>2011 г. РАНХиГС.</li></ul>
|
||||
<p>Курс <a href="/programme">Современные подходы</a>.</p>
|
||||
<p><a href="/cert.jpg">Удостоверение</a> (JPG, 6,69 Мб).</p></div></div>'''
|
||||
section, = extract_sections(BeautifulSoup(html, "html.parser"), "https://www.hse.ru/")
|
||||
assert len(section["items"]) == 10
|
||||
assert [row["year"] for row in section["year_entries"]] == [2025, 2024, 2018]
|
||||
assert section["items"][0] == "2025 Экспортный контроль, МИФИ"
|
||||
assert section["items"][7] == "2015 г. МГТУ МИРЭА. Направления совершенствования практической подготовки."
|
||||
assert section["items"][9].startswith("2011 г. РАНХиГС. Курс Современные подходы")
|
||||
assert section["links"] == [{"text": "Современные подходы", "url": "https://www.hse.ru/programme"}]
|
||||
for fragment in ("Сертификат", "Файл", "Свидетельство", "Удостоверение", "PDF", "JPG"):
|
||||
assert fragment not in str(section)
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(self, payload):
|
||||
self.payload = payload
|
||||
|
||||
Reference in new issue
Block a user