forked from admin/miem_workers
71 lines
3.0 KiB
Python
71 lines
3.0 KiB
Python
"""Read public HSE education documents. An inconclusive extraction stays visible for review."""
|
|
import hashlib
|
|
import re
|
|
import subprocess
|
|
from pathlib import Path
|
|
from tempfile import TemporaryDirectory
|
|
from urllib.parse import urlsplit
|
|
|
|
import requests
|
|
|
|
_MAX_BYTES = 12 * 1024 * 1024
|
|
|
|
|
|
def _run(args):
|
|
return subprocess.run(args, check=True, capture_output=True, timeout=30).stdout.decode("utf-8")
|
|
|
|
|
|
def _certificate(text):
|
|
text = " ".join(text.split())
|
|
title = re.search(r'(?:по (?:дополнительной профессиональной )?программе|по теме|на тему)\s*[:«"\s]*([^»"]{5,300})[»"]', text, re.I)
|
|
if not title:
|
|
title = re.search(r'[«"]([^«»"]{8,250})[»"]\s*[,.(]?\s*(?:в объ[её]ме|\d+ час)', text, re.I)
|
|
years = re.findall(r"\b(?:19|20)\d{2}\b", text)
|
|
if not title or not years or not re.search(r"квалификаци|сертификат|удостоверение", text, re.I):
|
|
return None
|
|
hours = re.search(r"(\d+)\s*(?:академических\s+)?час", text, re.I)
|
|
provider = ""
|
|
if re.search(r"высш[а-я]+ школ[а-я]+ экономик|НИУ ВШЭ", text, re.I):
|
|
provider = ", НИУ ВШЭ"
|
|
elif re.search("МИСИС", text, re.I):
|
|
provider = ", НИТУ МИСИС"
|
|
return f'{years[-1]} «{title[1].strip()}», повышение квалификации{provider}' + (f", {hours[1]} часов" if hours else "")
|
|
|
|
|
|
def _recognize(data):
|
|
with TemporaryDirectory() as tmp:
|
|
src = Path(tmp) / "document"
|
|
src.write_bytes(data)
|
|
images = [src]
|
|
if data.startswith(b"%PDF"):
|
|
direct = _certificate(_run(["pdftotext", "-layout", str(src), "-"]))
|
|
if direct:
|
|
return direct
|
|
_run(["pdftoppm", "-f", "1", "-l", "2", "-scale-to", "2400", "-png", str(src), str(src)])
|
|
images = sorted(Path(tmp).glob("document-*.png"))
|
|
elif not data.startswith((b"\x89PNG", b"\xff\xd8\xff")):
|
|
return None
|
|
return _certificate(" ".join(_run(["tesseract", str(img), "stdout", "-l", "rus+eng"]) for img in images))
|
|
|
|
|
|
def read_document(session, url, timeout, cache, profile_key):
|
|
parts = urlsplit(url)
|
|
if parts.scheme != "https" or parts.netloc != "www.hse.ru" or not parts.path.startswith("/mirror/pubs/share/"):
|
|
return None
|
|
try:
|
|
with session.get(url, stream=True, timeout=timeout, allow_redirects=False) as response:
|
|
if response.status_code != 200:
|
|
return None
|
|
data = bytearray()
|
|
for chunk in response.iter_content(65536):
|
|
data.extend(chunk)
|
|
if len(data) > _MAX_BYTES:
|
|
return None
|
|
digest = hashlib.sha256(data).hexdigest()
|
|
if cache:
|
|
return cache.education_document(profile_key, url, digest, lambda: _recognize(data))
|
|
return _recognize(data)
|
|
except (OSError, ValueError, subprocess.SubprocessError, requests.RequestException):
|
|
return None
|
|
|