121 lines
6.4 KiB
Python
121 lines
6.4 KiB
Python
"""Prepared council publication data and shared source normalization."""
|
||
|
||
import csv
|
||
import hashlib
|
||
import json
|
||
from io import StringIO
|
||
from collections import defaultdict
|
||
from pathlib import Path
|
||
import re
|
||
import unicodedata
|
||
|
||
DOI = re.compile(r"10\.\d{4,9}/\S+", re.IGNORECASE)
|
||
|
||
|
||
def _name(value: str) -> str:
|
||
return " ".join(unicodedata.normalize("NFKC", value).casefold().replace("ё", "е").split())
|
||
|
||
|
||
def _doi(value: str) -> str:
|
||
value = re.sub(r"^(?:https?://(?:dx\.)?doi\.org/|doi:\s*)", "", value.strip(), flags=re.I).lower()
|
||
return value if DOI.fullmatch(value) else ""
|
||
|
||
|
||
def _quartiles(row: dict) -> list[str]:
|
||
return [f"{label} {str(row.get(field) or '').strip().upper()}"
|
||
for label, field in (("Scopus", "Квартиль scopus"), ("Web of Science", "Квартиль wos"))
|
||
if str(row.get(field) or '').strip().upper() in {"Q1", "Q2"}]
|
||
|
||
|
||
def _journal_names(data: dict) -> list[str]:
|
||
names = []
|
||
for field in ("Название журнала", "Журнал", "journal"):
|
||
value = data.get(field)
|
||
if isinstance(value, dict):
|
||
value = value.get("name")
|
||
if isinstance(value, str) and value.strip():
|
||
names.append(value.strip())
|
||
return names
|
||
|
||
|
||
def _description_journals(description: dict) -> list[str]:
|
||
from bs4 import BeautifulSoup
|
||
|
||
names = []
|
||
for field in ("short", "shortLeft"):
|
||
value = description.get(field)
|
||
values = value.values() if isinstance(value, dict) else [value]
|
||
for text in values:
|
||
if isinstance(text, str):
|
||
text = BeautifulSoup(text, "html.parser").get_text(" ", strip=True) if "<" in text or "&" in text else text.strip()
|
||
# HSE short descriptions start with the journal, followed by the year.
|
||
names.append(re.split(r"\b(?:19|20)\d{2}\b", text, maxsplit=1)[0].strip(" .,:;"))
|
||
return names
|
||
|
||
|
||
def prepare_reference(source: Path) -> dict:
|
||
publications = []
|
||
by_author = defaultdict(list)
|
||
# Git stores this text export with LF; canonicalize Windows CRLF before
|
||
# parsing and hashing so the prepared artifact is identical on both hosts.
|
||
source_bytes = source.read_bytes().replace(b"\r\n", b"\n")
|
||
with StringIO(source_bytes.decode("utf-8-sig"), newline="") as stream:
|
||
reader = csv.DictReader(stream, delimiter=";")
|
||
required = {"Название публикации", "Год", "DOI", "Авторы НИУ ВШЭ", "Квартиль scopus", "Квартиль wos"}
|
||
if not required <= set(reader.fieldnames or []):
|
||
raise ValueError("Missing council CSV headers")
|
||
if len(reader.fieldnames) != len(set(reader.fieldnames)):
|
||
raise ValueError("Duplicate council CSV headers")
|
||
for row_number, row in enumerate(reader, 2):
|
||
if None in row or any(value is None for value in row.values()):
|
||
raise ValueError("Malformed council CSV record")
|
||
authors = list(dict.fromkeys(value.strip() for value in row["Авторы НИУ ВШЭ"].split(";") if value.strip()))
|
||
record = {"source_row": row_number, "title": row["Название публикации"].strip(),
|
||
"year": int(row["Год"]) if row["Год"].strip().isdigit() else None,
|
||
"doi": _doi(row["DOI"]), "authors": authors, "quartiles": _quartiles(row),
|
||
"journals": _journal_names(row),
|
||
"metadata": {key: value for key, value in row.items()
|
||
if key not in {"Название публикации", "Год", "DOI", "Авторы НИУ ВШЭ"}}}
|
||
index = len(publications)
|
||
publications.append(record)
|
||
if record["title"]:
|
||
for author in sorted({_name(value) for value in authors} - {""}):
|
||
by_author[author].append(index)
|
||
return {"schema_version": 1, "source_file": source.name,
|
||
"source_sha256": hashlib.sha256(source_bytes).hexdigest(), "source_hash_eol": "LF",
|
||
"source_rows": len(publications), "publications": publications, "by_author": dict(by_author)}
|
||
|
||
|
||
def load_reference(path: Path) -> dict[str, list[dict]]:
|
||
data = json.loads(path.read_text(encoding="utf-8"))
|
||
if not isinstance(data, dict) or type(data.get("schema_version")) is not int or data.get("schema_version") != 1:
|
||
raise ValueError("Unsupported council reference schema")
|
||
publications = data.get("publications")
|
||
by_author = data.get("by_author")
|
||
if not isinstance(publications, list) or not isinstance(by_author, dict) or data.get("source_rows") != len(publications):
|
||
raise ValueError("Invalid council reference records")
|
||
articles = []
|
||
expected = defaultdict(list)
|
||
for index, record in enumerate(publications):
|
||
if not isinstance(record, dict) or not isinstance(record.get("title"), str) or not isinstance(record.get("doi"), str):
|
||
raise ValueError("Invalid council publication text")
|
||
if record["doi"] != _doi(record["doi"]) or (record.get("year") is not None and type(record["year"]) is not int):
|
||
raise ValueError("Invalid council publication DOI/year")
|
||
if record.get("source_row") != index + 2:
|
||
raise ValueError("Invalid council source row")
|
||
for field in ("authors", "quartiles", "journals"):
|
||
if not isinstance(record.get(field), list) or not all(isinstance(value, str) for value in record[field]):
|
||
raise ValueError("Invalid council publication list")
|
||
if not set(record["quartiles"]) <= {"Scopus Q1", "Scopus Q2", "Web of Science Q1", "Web of Science Q2"}:
|
||
raise ValueError("Invalid council quartiles")
|
||
if record["title"]:
|
||
for author in sorted({_name(value) for value in record["authors"]} - {""}):
|
||
expected[author].append(index)
|
||
articles.append({"title": record["title"], "year": record["year"], "doi": record["doi"],
|
||
"types": [], "quartiles": record["quartiles"], "journals": record["journals"],
|
||
"origins": ["Данные ВШЭ"], "rows": [record["source_row"]],
|
||
"in_csv": True, "in_db": False, "url": None})
|
||
if by_author != dict(expected) or any(type(i) is not int for indexes in by_author.values() for i in indexes):
|
||
raise ValueError("Council author index disagrees with publication authors")
|
||
return {author: [articles[index] for index in indexes] for author, indexes in by_author.items()}
|