Files
miem_workers/app/services/council.py
T

286 lines
15 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Public publication evidence from the council export and employee database."""
import calendar
import csv
import json
import re
import unicodedata
from collections import Counter, defaultdict
from datetime import date, datetime
from functools import lru_cache
from pathlib import Path
from urllib.parse import quote, urlsplit
from zoneinfo import ZoneInfo
from bs4 import BeautifulSoup
from sqlalchemy import select
from sqlalchemy.orm import Session, selectinload
from app.models import Employee
from app.services.academic_degrees import academic_degrees
EXPORT = Path(__file__).resolve().parents[1] / "data" / "council_articles.csv"
K1_JOURNALS = EXPORT.with_name("k1_journals.json")
K1_SPECIALTIES = EXPORT.with_name("k1_specialties.json")
DOCTOR = re.compile(r"^доктор\b.+\bнаук$", re.IGNORECASE)
DOI = re.compile(r"10\.\d{4,9}/\S+", re.IGNORECASE)
TYPES = {"ARTICLE": "Статья", "BOOK": "Книга", "CHAPTER": "Глава книги",
"BOOK_CHAPTER": "Глава книги", "REPORT": "Доклад", "PATENT": "Патент",
"PREPRINT": "Препринт", "CONFERENCE_PAPER": "Материал конференции"}
def _name(value: str) -> str:
return " ".join(unicodedata.normalize("NFKC", value).casefold().replace("ё", "е").split())
def _doi(value: str) -> str:
value = re.sub(r"^(?:https?://(?:dx\.)?doi\.org/|doi:\s*)", "", value.strip(), flags=re.I).lower()
return value if DOI.fullmatch(value) else ""
def _safe_url(value: str | None) -> str | None:
try:
url = urlsplit(value or "")
if url.scheme in {"http", "https"} and url.hostname and not url.username and not url.password:
return value
except ValueError:
pass
return None
def period(today: date) -> tuple[date, date]:
year = today.year - 5
return date(year, today.month, min(today.day, calendar.monthrange(year, today.month)[1])), today
def _quartiles(row: dict) -> list[str]:
return [f"{label} {str(row.get(field) or '').strip().upper()}"
for label, field in (("Scopus", "Квартиль scopus"), ("Web of Science", "Квартиль wos"))
if str(row.get(field) or '').strip().upper() in {"Q1", "Q2"}]
def _journal_key(value: str) -> str:
return " ".join(re.findall(r"\w+", _name(value)))
@lru_cache(maxsize=1)
def _k1_journals() -> dict[str, str]:
data = json.loads(K1_JOURNALS.read_text(encoding="utf-8"))
journals = {}
for journal in data["journals"]:
for name in [journal["name"], *journal.get("aliases", [])]:
key = _journal_key(name)
if key in journals and journals[key] != journal["name"]:
raise ValueError("Ambiguous K1 journal alias")
journals[key] = journal["name"]
return journals
@lru_cache(maxsize=1)
def _k1_specialties() -> dict:
data = json.loads(K1_SPECIALTIES.read_text(encoding="utf-8"))
return {**data, "by_journal": {j["name"]: j["specialty_codes"] for j in data["journals"]}}
def _journal_names(data: dict) -> list[str]:
names = []
for field in ("Название журнала", "Журнал", "journal"):
value = data.get(field)
if isinstance(value, dict):
value = value.get("name")
if isinstance(value, str) and value.strip():
names.append(value.strip())
return names
def _description_journals(description: dict) -> list[str]:
names = []
for field in ("short", "shortLeft"):
value = description.get(field)
values = value.values() if isinstance(value, dict) else [value]
for text in values:
if isinstance(text, str):
text = BeautifulSoup(text, "html.parser").get_text(" ", strip=True)
# HSE short descriptions start with the journal, followed by the year.
names.append(re.split(r"\b(?:19|20)\d{2}\b", text, maxsplit=1)[0].strip(" .,:;"))
return names
@lru_cache(maxsize=1)
def _articles_by_author() -> dict[str, list[dict]]:
articles = defaultdict(list)
with EXPORT.open(encoding="utf-8-sig", newline="") as source:
reader = csv.DictReader(source, delimiter=";")
required = {"Название публикации", "Год", "DOI", "Авторы НИУ ВШЭ", "Квартиль scopus", "Квартиль wos"}
if not required <= set(reader.fieldnames or []):
raise ValueError("Missing council CSV headers")
for row_number, row in enumerate(reader, 2):
if None in row or any(value is None for value in row.values()):
raise ValueError("Malformed council CSV record")
title = row["Название публикации"].strip()
if not title:
continue
year = int(row["Год"]) if row["Год"].strip().isdigit() else None
article = {"title": title, "year": year, "doi": _doi(row["DOI"]),
"types": [], "quartiles": _quartiles(row), "journals": _journal_names(row), "origins": ["Файл CSV"],
"rows": [row_number], "in_csv": True, "in_db": False, "url": None}
for author in set(_name(value) for value in row["Авторы НИУ ВШЭ"].split(";")) - {""}:
articles[author].append(article)
return articles
def _database_article(publication) -> dict:
raw = publication.raw_data if isinstance(publication.raw_data, dict) else {}
imported = raw.get("csv_enrichment")
from_site = bool(publication.publication_id or publication.url or any(key != "csv_enrichment" for key in raw))
origins = ["БД miem-workers · сайт ВШЭ"] if from_site else []
if imported:
origins.append("БД miem-workers · импорт CSV")
kind = publication.publication_type
description = publication.description if isinstance(publication.description, dict) else raw.get("description")
journals = _journal_names(raw)
if isinstance(imported, dict):
journals.extend(_journal_names(imported))
if isinstance(description, dict):
journals.extend(_description_journals(description))
return {"title": publication.title, "year": publication.year, "doi": _doi(publication.doi_url or ""),
"types": [TYPES.get(kind.upper(), f"Другой тип: {kind}")] if kind else [],
"quartiles": _quartiles(imported) if isinstance(imported, dict) else [],
"journals": journals,
"origins": origins or ["БД miem-workers"], "rows": [], "in_csv": False, "in_db": True,
"url": _safe_url(publication.url)}
def _unique_articles(articles: list[dict], start: date, end: date) -> list[dict]:
k1_journals = _k1_journals()
specialties = _k1_specialties()["by_journal"]
title_years = defaultdict(set)
for article in articles:
if article["year"] is not None:
title_years[_name(article["title"])].add(article["year"])
def title_key(article):
title = _name(article["title"])
years = title_years[title]
year = article["year"]
return title, next(iter(years)) if year is None and len(years) == 1 else year
# A title/year fallback is safe only when it identifies at most one DOI.
title_dois = defaultdict(set)
for article in articles:
if article["doi"]:
title_dois[title_key(article)].add(article["doi"])
unique = {}
for article in articles:
fallback = title_key(article)
identifiers = title_dois[fallback]
identifier = article["doi"] or (next(iter(identifiers)) if len(identifiers) == 1 else "")
key = identifier or fallback
if key not in unique:
unique[key] = {**article, "doi": identifier,
**{field: list(article[field]) for field in ("types", "quartiles", "journals", "origins", "rows")},
"evidence": []}
merged = unique[key]
merged["evidence"].append(article)
for field in ("types", "quartiles", "journals", "origins", "rows"):
merged[field] = list(dict.fromkeys([*merged[field], *article[field]]))
merged["in_csv"] |= article["in_csv"]
merged["in_db"] |= article["in_db"]
merged["url"] = merged["url"] or article["url"]
for article in unique.values():
evidence = article.pop("evidence")
years = sorted({item["year"] for item in evidence if item["year"] is not None})
# Never pick a conflicting source year merely because that row came first.
year = article["year"] = years[0] if len(years) == 1 else None
article["criteria"] = list(article["quartiles"])
matches = list(dict.fromkeys(k1_journals[key] for name in article["journals"]
if (key := _journal_key(name)) in k1_journals))
article["criteria"].extend(f"К1 · {name} (справочник ВАК)" for name in matches)
eligible_type = not article["types"] or article["types"] == ["Статья"]
article["q1"] = article["q2"] = article["k1"] = False
article["specialty_codes"] = []
for item in evidence:
source_year = item["year"] if item["year"] is not None else year
if not eligible_type or source_year is None or not start.year <= source_year <= end.year:
continue
article["q1"] |= any(q.endswith(" Q1") for q in item["quartiles"])
article["q2"] |= any(q.endswith(" Q2") for q in item["quartiles"])
item_journals = {k1_journals[key] for name in item["journals"]
if (key := _journal_key(name)) in k1_journals}
article["k1"] |= bool(item_journals)
article["specialty_codes"] = sorted(set(article["specialty_codes"]).union(
*(specialties.get(name, []) for name in item_journals)))
article["qualifies"] = article["q1"] or article["q2"] or article["k1"]
if len(years) > 1:
article["criteria"].append("Годы источников расходятся: " + ", ".join(map(str, years))
+ "; критерии учитываются по году каждой исходной записи")
elif year is None:
article["criteria"].append("Год не указан — период не подтверждён")
elif start.year <= year <= end.year:
article["criteria"].append("В пределах пяти лет по году; точную дату нужно проверить" if year in {start.year, end.year}
else "В пределах пяти лет")
else:
article["criteria"].append("Вне пятилетнего периода")
if not article["quartiles"] and not matches:
article["criteria"].append("Q1/Q2 и К1 не подтверждены доступными данными")
article["doi_url"] = f"https://doi.org/{quote(article['doi'], safe='/')}" if article["doi"] else None
article["type_label"] = " / ".join(article["types"]) or "Тип не указан — статья не подтверждена"
if article["types"] and article["types"] != ["Статья"]:
article["criteria"].append("Тип материала не подтверждён как статья — не включён в счётчики Q1/Q2/К1")
return sorted(unique.values(), key=lambda item: (not item["qualifies"], -(item["year"] or 0), item["title"].casefold()))
def council_candidates(db: Session, today: date | None = None, q: str = "", offset: int = 0,
sort: str = "total_count", direction: str = "desc", paginate: bool = True, specialty: str = "") -> dict:
today = today or datetime.now(ZoneInfo("Europe/Moscow")).date()
start, end = period(today)
specialties = _k1_specialties()["specialties"]
specialty = specialty if specialty in {s["code"] for s in specialties} else ""
specialty_name = next((s["name"] for s in specialties if s["code"] == specialty), "")
employees = db.scalars(select(Employee).options(selectinload(Employee.publications))).all()
name_counts = Counter(_name(employee.full_name or "") for employee in employees)
by_author = _articles_by_author()
candidates = []
for employee in employees:
author = _name(employee.full_name or "")
ambiguous = bool(author and name_counts[author] > 1 and by_author.get(author))
csv_articles = by_author.get(author, []) if author and name_counts[author] == 1 else []
articles = _unique_articles([*csv_articles, *(_database_article(p) for p in employee.publications)], start, end)
if not articles:
continue
data = employee.current_data if isinstance(employee.current_data, dict) else {}
try:
degrees = academic_degrees(data)
except (AttributeError, TypeError):
degrees = []
if not any(DOCTOR.fullmatch(degree) for degree in degrees):
continue
specialty_count = sum(specialty in a["specialty_codes"] for a in articles) if specialty else 0
for article in articles:
article["matches_specialty"] = bool(specialty and specialty in article["specialty_codes"])
if article["matches_specialty"]:
article["criteria"].append(f"По специальности {specialty}: {specialty_name} (журнал К1, Перечень ВАК)")
candidates.append({"specialty_count": specialty_count,"name": employee.full_name or "ФИО не указано", "degree": "; ".join(degrees) or "Учёная степень не указана",
"count": len(articles),
**{f"{key}_count": sum(a[key] for a in articles) for key in ("q1", "q2", "k1")},
"total_count": sum(a["qualifies"] for a in articles),
"db_count": sum(a["in_db"] for a in articles), "articles": articles,
"profile_url": _safe_url(employee.canonical_url), "ambiguous": ambiguous,
"dismissed": employee.status == "dismissed"})
sort = sort if sort in {"full_name", "q1_count", "q2_count", "k1_count", "total_count", "db_count", "count"} else "total_count"
direction = direction if direction in {"asc", "desc"} else "desc"
candidates.sort(key=lambda item: _name(item["name"]), reverse=sort == "full_name" and direction == "desc")
if sort != "full_name":
candidates.sort(key=lambda item: (item[sort], item["count"]), reverse=direction == "desc")
total = len(candidates)
query = _name(q)
candidates = [item for item in candidates if query in _name(item["name"])
and (not specialty or item["specialty_count"] > 0)]
matched = len(candidates)
limit = 25
offset = min(max(0, offset), ((matched - 1) // limit) * limit) if matched else 0
return {"candidates": candidates[offset:offset + limit] if paginate else candidates,
"start": start, "end": end, "total": total,
"specialties": specialties, "specialty": specialty, "specialty_name": specialty_name,
"matched": matched, "q": q, "offset": offset, "limit": limit, "sort": sort, "direction": direction}