Files
miem_workers/app/parser/education.py
T

66 lines
3.3 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Assemble additional education events from the public profile's mixed markup."""
import re
YEAR = re.compile(r"\b(?:19|20)\d{2}\b")
_QUOTE = re.compile(r'«([^»]+)»|"([^"]+)"')
_EVENT = re.compile(r"стажиров|стипенди|исследовательск|обучени|квалификаци|переподготовк|прош[её]л|прошла|семинар|сертификаци|программ[аеуы]|курсы?\b", re.I)
_CONTINUES = re.compile(r"(?:по программе|присвоением степени|на тему|по курсу|по теме|по специальности)[:\s]*$", re.I)
_CAPTION = re.compile(r"^(?:удостоверение|сертификат|свидетельство)(?: о повышении квалификации)?[.:]?$", re.I)
def _key(text):
return " ".join(re.findall(r"\w+", text.casefold().replace("ё", "е")))
def _provider(text):
text = _key(text)
for variants in (("вшэ", "высшая школа экономики", "высшей школе экономики"),
("мифи", "московский инженерно физический институт")):
if any(v in text.split() if " " not in v else v in text for v in variants):
return variants[0]
return text
def assemble(rows):
events = []
unresolved = []
pending = []
for text, year, structured, list_item in rows:
if _CAPTION.fullmatch(text):
continue
if (pending and not structured and not list_item and not YEAR.match(text)
and (_CONTINUES.search(pending[-1][0]) or
(pending[-1][3] and not _EVENT.search(text[:20]) and not YEAR.search(text)) or
(pending[-1][3] and text.startswith("Курс ") and not YEAR.search(text)))):
pending[-1][0] += " " + text
pending[-1][3] = False
else:
pending.append([text, year, structured, list_item])
for text, year, structured, _ in pending:
if not structured and (len(_key(text).split()) < 2 or not (_EVENT.search(text) or YEAR.match(text))):
unresolved.append(text)
continue
quotes = [a or b for a, b in _QUOTE.findall(text)]
title = _key(quotes[-1]) if quotes else ""
if structured and text.lstrip("0123456789 ").startswith("«"):
title = _key(text.split("«", 1)[1].rsplit("»", 1)[0])
provider = _provider(text.rsplit("»", 1)[-1] if structured and "»" in text else text)
if not year:
match = YEAR.search(_QUOTE.sub("", text))
year = int(match[0]) if match else None
matches = [e for e in events if title and e["title"] == title and e["provider"] == provider
and (e["year"] == year or not year or not e["year"])]
if len(matches) == 1:
target = matches[0]
hours = re.search(r"\b\d+\s+час[а-я]*", text, re.I)
if hours and hours[0] not in target["text"]:
target["text"] += " (" + hours[0] + ")"
continue
if any(e["text"] == text for e in events):
continue
events.append(dict(text=text, year=year, structured=structured, title=title, provider=provider))
return events, unresolved