"""Prepared council publication data and shared source normalization.""" import csv import hashlib import json from io import StringIO from collections import defaultdict from pathlib import Path import re import unicodedata DOI = re.compile(r"10\.\d{4,9}/\S+", re.IGNORECASE) def _name(value: str) -> str: return " ".join(unicodedata.normalize("NFKC", value).casefold().replace("ё", "е").split()) def _doi(value: str) -> str: value = re.sub(r"^(?:https?://(?:dx\.)?doi\.org/|doi:\s*)", "", value.strip(), flags=re.I).lower() return value if DOI.fullmatch(value) else "" def _quartiles(row: dict) -> list[str]: return [f"{label} {str(row.get(field) or '').strip().upper()}" for label, field in (("Scopus", "Квартиль scopus"), ("Web of Science", "Квартиль wos")) if str(row.get(field) or '').strip().upper() in {"Q1", "Q2"}] def _journal_names(data: dict) -> list[str]: names = [] for field in ("Название журнала", "Журнал", "journal"): value = data.get(field) if isinstance(value, dict): value = value.get("name") if isinstance(value, str) and value.strip(): names.append(value.strip()) return names def _description_journals(description: dict) -> list[str]: from bs4 import BeautifulSoup names = [] for field in ("short", "shortLeft"): value = description.get(field) values = value.values() if isinstance(value, dict) else [value] for text in values: if isinstance(text, str): text = BeautifulSoup(text, "html.parser").get_text(" ", strip=True) if "<" in text or "&" in text else text.strip() # HSE short descriptions start with the journal, followed by the year. names.append(re.split(r"\b(?:19|20)\d{2}\b", text, maxsplit=1)[0].strip(" .,:;")) return names def prepare_reference(source: Path) -> dict: publications = [] by_author = defaultdict(list) # Git stores this text export with LF; canonicalize Windows CRLF before # parsing and hashing so the prepared artifact is identical on both hosts. source_bytes = source.read_bytes().replace(b"\r\n", b"\n") with StringIO(source_bytes.decode("utf-8-sig"), newline="") as stream: reader = csv.DictReader(stream, delimiter=";") required = {"Название публикации", "Год", "DOI", "Авторы НИУ ВШЭ", "Квартиль scopus", "Квартиль wos"} if not required <= set(reader.fieldnames or []): raise ValueError("Missing council CSV headers") if len(reader.fieldnames) != len(set(reader.fieldnames)): raise ValueError("Duplicate council CSV headers") for row_number, row in enumerate(reader, 2): if None in row or any(value is None for value in row.values()): raise ValueError("Malformed council CSV record") authors = list(dict.fromkeys(value.strip() for value in row["Авторы НИУ ВШЭ"].split(";") if value.strip())) record = {"source_row": row_number, "title": row["Название публикации"].strip(), "year": int(row["Год"]) if row["Год"].strip().isdigit() else None, "doi": _doi(row["DOI"]), "authors": authors, "quartiles": _quartiles(row), "journals": _journal_names(row), "metadata": {key: value for key, value in row.items() if key not in {"Название публикации", "Год", "DOI", "Авторы НИУ ВШЭ"}}} index = len(publications) publications.append(record) if record["title"]: for author in sorted({_name(value) for value in authors} - {""}): by_author[author].append(index) return {"schema_version": 1, "source_file": source.name, "source_sha256": hashlib.sha256(source_bytes).hexdigest(), "source_hash_eol": "LF", "source_rows": len(publications), "publications": publications, "by_author": dict(by_author)} def load_reference(path: Path) -> dict[str, list[dict]]: data = json.loads(path.read_text(encoding="utf-8")) if not isinstance(data, dict) or type(data.get("schema_version")) is not int or data.get("schema_version") != 1: raise ValueError("Unsupported council reference schema") publications = data.get("publications") by_author = data.get("by_author") if not isinstance(publications, list) or not isinstance(by_author, dict) or data.get("source_rows") != len(publications): raise ValueError("Invalid council reference records") articles = [] expected = defaultdict(list) for index, record in enumerate(publications): if not isinstance(record, dict) or not isinstance(record.get("title"), str) or not isinstance(record.get("doi"), str): raise ValueError("Invalid council publication text") if record["doi"] != _doi(record["doi"]) or (record.get("year") is not None and type(record["year"]) is not int): raise ValueError("Invalid council publication DOI/year") if record.get("source_row") != index + 2: raise ValueError("Invalid council source row") for field in ("authors", "quartiles", "journals"): if not isinstance(record.get(field), list) or not all(isinstance(value, str) for value in record[field]): raise ValueError("Invalid council publication list") if not set(record["quartiles"]) <= {"Scopus Q1", "Scopus Q2", "Web of Science Q1", "Web of Science Q2"}: raise ValueError("Invalid council quartiles") if record["title"]: for author in sorted({_name(value) for value in record["authors"]} - {""}): expected[author].append(index) articles.append({"title": record["title"], "year": record["year"], "doi": record["doi"], "types": [], "quartiles": record["quartiles"], "journals": record["journals"], "origins": ["Данные ВШЭ"], "rows": [record["source_row"]], "in_csv": True, "in_db": False, "url": None}) if by_author != dict(expected) or any(type(i) is not int for indexes in by_author.values() for i in indexes): raise ValueError("Council author index disagrees with publication authors") return {author: [articles[index] for index in indexes] for author, indexes in by_author.items()}