Files
codex e08c30848f
CI / test (pull_request) Successful in 9m30s
CI / Fingerprint diagnostics (production deployment temporarily disabled) (pull_request) Has been skipped
perf: load council publications on demand and reduce database payload
2026-10-09 16:23:55 +03:00

121 lines
6.4 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Prepared council publication data and shared source normalization."""
import csv
import hashlib
import json
from io import StringIO
from collections import defaultdict
from pathlib import Path
import re
import unicodedata
DOI = re.compile(r"10\.\d{4,9}/\S+", re.IGNORECASE)
def _name(value: str) -> str:
return " ".join(unicodedata.normalize("NFKC", value).casefold().replace("ё", "е").split())
def _doi(value: str) -> str:
value = re.sub(r"^(?:https?://(?:dx\.)?doi\.org/|doi:\s*)", "", value.strip(), flags=re.I).lower()
return value if DOI.fullmatch(value) else ""
def _quartiles(row: dict) -> list[str]:
return [f"{label} {str(row.get(field) or '').strip().upper()}"
for label, field in (("Scopus", "Квартиль scopus"), ("Web of Science", "Квартиль wos"))
if str(row.get(field) or '').strip().upper() in {"Q1", "Q2"}]
def _journal_names(data: dict) -> list[str]:
names = []
for field in ("Название журнала", "Журнал", "journal"):
value = data.get(field)
if isinstance(value, dict):
value = value.get("name")
if isinstance(value, str) and value.strip():
names.append(value.strip())
return names
def _description_journals(description: dict) -> list[str]:
from bs4 import BeautifulSoup
names = []
for field in ("short", "shortLeft"):
value = description.get(field)
values = value.values() if isinstance(value, dict) else [value]
for text in values:
if isinstance(text, str):
text = BeautifulSoup(text, "html.parser").get_text(" ", strip=True) if "<" in text or "&" in text else text.strip()
# HSE short descriptions start with the journal, followed by the year.
names.append(re.split(r"\b(?:19|20)\d{2}\b", text, maxsplit=1)[0].strip(" .,:;"))
return names
def prepare_reference(source: Path) -> dict:
publications = []
by_author = defaultdict(list)
# Git stores this text export with LF; canonicalize Windows CRLF before
# parsing and hashing so the prepared artifact is identical on both hosts.
source_bytes = source.read_bytes().replace(b"\r\n", b"\n")
with StringIO(source_bytes.decode("utf-8-sig"), newline="") as stream:
reader = csv.DictReader(stream, delimiter=";")
required = {"Название публикации", "Год", "DOI", "Авторы НИУ ВШЭ", "Квартиль scopus", "Квартиль wos"}
if not required <= set(reader.fieldnames or []):
raise ValueError("Missing council CSV headers")
if len(reader.fieldnames) != len(set(reader.fieldnames)):
raise ValueError("Duplicate council CSV headers")
for row_number, row in enumerate(reader, 2):
if None in row or any(value is None for value in row.values()):
raise ValueError("Malformed council CSV record")
authors = list(dict.fromkeys(value.strip() for value in row["Авторы НИУ ВШЭ"].split(";") if value.strip()))
record = {"source_row": row_number, "title": row["Название публикации"].strip(),
"year": int(row["Год"]) if row["Год"].strip().isdigit() else None,
"doi": _doi(row["DOI"]), "authors": authors, "quartiles": _quartiles(row),
"journals": _journal_names(row),
"metadata": {key: value for key, value in row.items()
if key not in {"Название публикации", "Год", "DOI", "Авторы НИУ ВШЭ"}}}
index = len(publications)
publications.append(record)
if record["title"]:
for author in sorted({_name(value) for value in authors} - {""}):
by_author[author].append(index)
return {"schema_version": 1, "source_file": source.name,
"source_sha256": hashlib.sha256(source_bytes).hexdigest(), "source_hash_eol": "LF",
"source_rows": len(publications), "publications": publications, "by_author": dict(by_author)}
def load_reference(path: Path) -> dict[str, list[dict]]:
data = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(data, dict) or type(data.get("schema_version")) is not int or data.get("schema_version") != 1:
raise ValueError("Unsupported council reference schema")
publications = data.get("publications")
by_author = data.get("by_author")
if not isinstance(publications, list) or not isinstance(by_author, dict) or data.get("source_rows") != len(publications):
raise ValueError("Invalid council reference records")
articles = []
expected = defaultdict(list)
for index, record in enumerate(publications):
if not isinstance(record, dict) or not isinstance(record.get("title"), str) or not isinstance(record.get("doi"), str):
raise ValueError("Invalid council publication text")
if record["doi"] != _doi(record["doi"]) or (record.get("year") is not None and type(record["year"]) is not int):
raise ValueError("Invalid council publication DOI/year")
if record.get("source_row") != index + 2:
raise ValueError("Invalid council source row")
for field in ("authors", "quartiles", "journals"):
if not isinstance(record.get(field), list) or not all(isinstance(value, str) for value in record[field]):
raise ValueError("Invalid council publication list")
if not set(record["quartiles"]) <= {"Scopus Q1", "Scopus Q2", "Web of Science Q1", "Web of Science Q2"}:
raise ValueError("Invalid council quartiles")
if record["title"]:
for author in sorted({_name(value) for value in record["authors"]} - {""}):
expected[author].append(index)
articles.append({"title": record["title"], "year": record["year"], "doi": record["doi"],
"types": [], "quartiles": record["quartiles"], "journals": record["journals"],
"origins": ["Данные ВШЭ"], "rows": [record["source_row"]],
"in_csv": True, "in_db": False, "url": None})
if by_author != dict(expected) or any(type(i) is not int for indexes in by_author.values() for i in indexes):
raise ValueError("Council author index disagrees with publication authors")
return {author: [articles[index] for index in indexes] for author, indexes in by_author.items()}