Files
miem_workers/app/parser/education_document.py
T

71 lines
3.0 KiB
Python

"""Read public HSE education documents. An inconclusive extraction stays visible for review."""
import hashlib
import re
import subprocess
from pathlib import Path
from tempfile import TemporaryDirectory
from urllib.parse import urlsplit
import requests
_MAX_BYTES = 12 * 1024 * 1024
def _run(args):
return subprocess.run(args, check=True, capture_output=True, timeout=30).stdout.decode("utf-8")
def _certificate(text):
text = " ".join(text.split())
title = re.search(r'(?:по (?:дополнительной профессиональной )?программе|по теме|на тему)\s*[:«"\s]*([^»"]{5,300})[»"]', text, re.I)
if not title:
title = re.search(r'[«"]([^«»"]{8,250})[»"]\s*[,.(]?\s*(?:в объ[её]ме|\d+ час)', text, re.I)
years = re.findall(r"\b(?:19|20)\d{2}\b", text)
if not title or not years or not re.search(r"квалификаци|сертификат|удостоверение", text, re.I):
return None
hours = re.search(r"(\d+)\s*(?:академических\s+)?час", text, re.I)
provider = ""
if re.search(r"высш[а-я]+ школ[а-я]+ экономик|НИУ ВШЭ", text, re.I):
provider = ", НИУ ВШЭ"
elif re.search("МИСИС", text, re.I):
provider = ", НИТУ МИСИС"
return f'{years[-1]} «{title[1].strip()}», повышение квалификации{provider}' + (f", {hours[1]} часов" if hours else "")
def _recognize(data):
with TemporaryDirectory() as tmp:
src = Path(tmp) / "document"
src.write_bytes(data)
images = [src]
if data.startswith(b"%PDF"):
direct = _certificate(_run(["pdftotext", "-layout", str(src), "-"]))
if direct:
return direct
_run(["pdftoppm", "-f", "1", "-l", "2", "-scale-to", "2400", "-png", str(src), str(src)])
images = sorted(Path(tmp).glob("document-*.png"))
elif not data.startswith((b"\x89PNG", b"\xff\xd8\xff")):
return None
return _certificate(" ".join(_run(["tesseract", str(img), "stdout", "-l", "rus+eng"]) for img in images))
def read_document(session, url, timeout, cache, profile_key):
parts = urlsplit(url)
if parts.scheme != "https" or parts.netloc != "www.hse.ru" or not parts.path.startswith("/mirror/pubs/share/"):
return None
try:
with session.get(url, stream=True, timeout=timeout, allow_redirects=False) as response:
if response.status_code != 200:
return None
data = bytearray()
for chunk in response.iter_content(65536):
data.extend(chunk)
if len(data) > _MAX_BYTES:
return None
digest = hashlib.sha256(data).hexdigest()
if cache:
return cache.education_document(profile_key, url, digest, lambda: _recognize(data))
return _recognize(data)
except (OSError, ValueError, subprocess.SubprocessError, requests.RequestException):
return None