forked from admin/miem_workers
286 lines
15 KiB
Python
286 lines
15 KiB
Python
"""Public publication evidence from the council export and employee database."""
|
||
|
||
import calendar
|
||
import csv
|
||
import json
|
||
import re
|
||
import unicodedata
|
||
from collections import Counter, defaultdict
|
||
from datetime import date, datetime
|
||
from functools import lru_cache
|
||
from pathlib import Path
|
||
from urllib.parse import quote, urlsplit
|
||
from zoneinfo import ZoneInfo
|
||
|
||
from bs4 import BeautifulSoup
|
||
from sqlalchemy import select
|
||
from sqlalchemy.orm import Session, selectinload
|
||
|
||
from app.models import Employee
|
||
from app.services.academic_degrees import academic_degrees
|
||
|
||
|
||
EXPORT = Path(__file__).resolve().parents[1] / "data" / "council_articles.csv"
|
||
K1_JOURNALS = EXPORT.with_name("k1_journals.json")
|
||
K1_SPECIALTIES = EXPORT.with_name("k1_specialties.json")
|
||
DOCTOR = re.compile(r"^доктор\b.+\bнаук$", re.IGNORECASE)
|
||
DOI = re.compile(r"10\.\d{4,9}/\S+", re.IGNORECASE)
|
||
TYPES = {"ARTICLE": "Статья", "BOOK": "Книга", "CHAPTER": "Глава книги",
|
||
"BOOK_CHAPTER": "Глава книги", "REPORT": "Доклад", "PATENT": "Патент",
|
||
"PREPRINT": "Препринт", "CONFERENCE_PAPER": "Материал конференции"}
|
||
|
||
|
||
def _name(value: str) -> str:
|
||
return " ".join(unicodedata.normalize("NFKC", value).casefold().replace("ё", "е").split())
|
||
|
||
|
||
def _doi(value: str) -> str:
|
||
value = re.sub(r"^(?:https?://(?:dx\.)?doi\.org/|doi:\s*)", "", value.strip(), flags=re.I).lower()
|
||
return value if DOI.fullmatch(value) else ""
|
||
|
||
|
||
def _safe_url(value: str | None) -> str | None:
|
||
try:
|
||
url = urlsplit(value or "")
|
||
if url.scheme in {"http", "https"} and url.hostname and not url.username and not url.password:
|
||
return value
|
||
except ValueError:
|
||
pass
|
||
return None
|
||
|
||
|
||
def period(today: date) -> tuple[date, date]:
|
||
year = today.year - 5
|
||
return date(year, today.month, min(today.day, calendar.monthrange(year, today.month)[1])), today
|
||
|
||
|
||
def _quartiles(row: dict) -> list[str]:
|
||
return [f"{label} {str(row.get(field) or '').strip().upper()}"
|
||
for label, field in (("Scopus", "Квартиль scopus"), ("Web of Science", "Квартиль wos"))
|
||
if str(row.get(field) or '').strip().upper() in {"Q1", "Q2"}]
|
||
|
||
|
||
def _journal_key(value: str) -> str:
|
||
return " ".join(re.findall(r"\w+", _name(value)))
|
||
|
||
|
||
@lru_cache(maxsize=1)
|
||
def _k1_journals() -> dict[str, str]:
|
||
data = json.loads(K1_JOURNALS.read_text(encoding="utf-8"))
|
||
journals = {}
|
||
for journal in data["journals"]:
|
||
for name in [journal["name"], *journal.get("aliases", [])]:
|
||
key = _journal_key(name)
|
||
if key in journals and journals[key] != journal["name"]:
|
||
raise ValueError("Ambiguous K1 journal alias")
|
||
journals[key] = journal["name"]
|
||
return journals
|
||
|
||
|
||
@lru_cache(maxsize=1)
|
||
def _k1_specialties() -> dict:
|
||
data = json.loads(K1_SPECIALTIES.read_text(encoding="utf-8"))
|
||
return {**data, "by_journal": {j["name"]: j["specialty_codes"] for j in data["journals"]}}
|
||
|
||
|
||
def _journal_names(data: dict) -> list[str]:
|
||
names = []
|
||
for field in ("Название журнала", "Журнал", "journal"):
|
||
value = data.get(field)
|
||
if isinstance(value, dict):
|
||
value = value.get("name")
|
||
if isinstance(value, str) and value.strip():
|
||
names.append(value.strip())
|
||
return names
|
||
|
||
|
||
def _description_journals(description: dict) -> list[str]:
|
||
names = []
|
||
for field in ("short", "shortLeft"):
|
||
value = description.get(field)
|
||
values = value.values() if isinstance(value, dict) else [value]
|
||
for text in values:
|
||
if isinstance(text, str):
|
||
text = BeautifulSoup(text, "html.parser").get_text(" ", strip=True)
|
||
# HSE short descriptions start with the journal, followed by the year.
|
||
names.append(re.split(r"\b(?:19|20)\d{2}\b", text, maxsplit=1)[0].strip(" .,:;"))
|
||
return names
|
||
|
||
|
||
@lru_cache(maxsize=1)
|
||
def _articles_by_author() -> dict[str, list[dict]]:
|
||
articles = defaultdict(list)
|
||
with EXPORT.open(encoding="utf-8-sig", newline="") as source:
|
||
reader = csv.DictReader(source, delimiter=";")
|
||
required = {"Название публикации", "Год", "DOI", "Авторы НИУ ВШЭ", "Квартиль scopus", "Квартиль wos"}
|
||
if not required <= set(reader.fieldnames or []):
|
||
raise ValueError("Missing council CSV headers")
|
||
for row_number, row in enumerate(reader, 2):
|
||
if None in row or any(value is None for value in row.values()):
|
||
raise ValueError("Malformed council CSV record")
|
||
title = row["Название публикации"].strip()
|
||
if not title:
|
||
continue
|
||
year = int(row["Год"]) if row["Год"].strip().isdigit() else None
|
||
article = {"title": title, "year": year, "doi": _doi(row["DOI"]),
|
||
"types": [], "quartiles": _quartiles(row), "journals": _journal_names(row), "origins": ["Файл CSV"],
|
||
"rows": [row_number], "in_csv": True, "in_db": False, "url": None}
|
||
for author in set(_name(value) for value in row["Авторы НИУ ВШЭ"].split(";")) - {""}:
|
||
articles[author].append(article)
|
||
return articles
|
||
|
||
|
||
def _database_article(publication) -> dict:
|
||
raw = publication.raw_data if isinstance(publication.raw_data, dict) else {}
|
||
imported = raw.get("csv_enrichment")
|
||
from_site = bool(publication.publication_id or publication.url or any(key != "csv_enrichment" for key in raw))
|
||
origins = ["БД miem-workers · сайт ВШЭ"] if from_site else []
|
||
if imported:
|
||
origins.append("БД miem-workers · импорт CSV")
|
||
kind = publication.publication_type
|
||
description = publication.description if isinstance(publication.description, dict) else raw.get("description")
|
||
journals = _journal_names(raw)
|
||
if isinstance(imported, dict):
|
||
journals.extend(_journal_names(imported))
|
||
if isinstance(description, dict):
|
||
journals.extend(_description_journals(description))
|
||
return {"title": publication.title, "year": publication.year, "doi": _doi(publication.doi_url or ""),
|
||
"types": [TYPES.get(kind.upper(), f"Другой тип: {kind}")] if kind else [],
|
||
"quartiles": _quartiles(imported) if isinstance(imported, dict) else [],
|
||
"journals": journals,
|
||
"origins": origins or ["БД miem-workers"], "rows": [], "in_csv": False, "in_db": True,
|
||
"url": _safe_url(publication.url)}
|
||
|
||
|
||
def _unique_articles(articles: list[dict], start: date, end: date) -> list[dict]:
|
||
k1_journals = _k1_journals()
|
||
specialties = _k1_specialties()["by_journal"]
|
||
title_years = defaultdict(set)
|
||
for article in articles:
|
||
if article["year"] is not None:
|
||
title_years[_name(article["title"])].add(article["year"])
|
||
def title_key(article):
|
||
title = _name(article["title"])
|
||
years = title_years[title]
|
||
year = article["year"]
|
||
return title, next(iter(years)) if year is None and len(years) == 1 else year
|
||
|
||
# A title/year fallback is safe only when it identifies at most one DOI.
|
||
title_dois = defaultdict(set)
|
||
for article in articles:
|
||
if article["doi"]:
|
||
title_dois[title_key(article)].add(article["doi"])
|
||
unique = {}
|
||
for article in articles:
|
||
fallback = title_key(article)
|
||
identifiers = title_dois[fallback]
|
||
identifier = article["doi"] or (next(iter(identifiers)) if len(identifiers) == 1 else "")
|
||
key = identifier or fallback
|
||
if key not in unique:
|
||
unique[key] = {**article, "doi": identifier,
|
||
**{field: list(article[field]) for field in ("types", "quartiles", "journals", "origins", "rows")},
|
||
"evidence": []}
|
||
merged = unique[key]
|
||
merged["evidence"].append(article)
|
||
for field in ("types", "quartiles", "journals", "origins", "rows"):
|
||
merged[field] = list(dict.fromkeys([*merged[field], *article[field]]))
|
||
merged["in_csv"] |= article["in_csv"]
|
||
merged["in_db"] |= article["in_db"]
|
||
merged["url"] = merged["url"] or article["url"]
|
||
for article in unique.values():
|
||
evidence = article.pop("evidence")
|
||
years = sorted({item["year"] for item in evidence if item["year"] is not None})
|
||
# Never pick a conflicting source year merely because that row came first.
|
||
year = article["year"] = years[0] if len(years) == 1 else None
|
||
article["criteria"] = list(article["quartiles"])
|
||
matches = list(dict.fromkeys(k1_journals[key] for name in article["journals"]
|
||
if (key := _journal_key(name)) in k1_journals))
|
||
article["criteria"].extend(f"К1 · {name} (справочник ВАК)" for name in matches)
|
||
eligible_type = not article["types"] or article["types"] == ["Статья"]
|
||
article["q1"] = article["q2"] = article["k1"] = False
|
||
article["specialty_codes"] = []
|
||
for item in evidence:
|
||
source_year = item["year"] if item["year"] is not None else year
|
||
if not eligible_type or source_year is None or not start.year <= source_year <= end.year:
|
||
continue
|
||
article["q1"] |= any(q.endswith(" Q1") for q in item["quartiles"])
|
||
article["q2"] |= any(q.endswith(" Q2") for q in item["quartiles"])
|
||
item_journals = {k1_journals[key] for name in item["journals"]
|
||
if (key := _journal_key(name)) in k1_journals}
|
||
article["k1"] |= bool(item_journals)
|
||
article["specialty_codes"] = sorted(set(article["specialty_codes"]).union(
|
||
*(specialties.get(name, []) for name in item_journals)))
|
||
article["qualifies"] = article["q1"] or article["q2"] or article["k1"]
|
||
if len(years) > 1:
|
||
article["criteria"].append("Годы источников расходятся: " + ", ".join(map(str, years))
|
||
+ "; критерии учитываются по году каждой исходной записи")
|
||
elif year is None:
|
||
article["criteria"].append("Год не указан — период не подтверждён")
|
||
elif start.year <= year <= end.year:
|
||
article["criteria"].append("В пределах пяти лет по году; точную дату нужно проверить" if year in {start.year, end.year}
|
||
else "В пределах пяти лет")
|
||
else:
|
||
article["criteria"].append("Вне пятилетнего периода")
|
||
if not article["quartiles"] and not matches:
|
||
article["criteria"].append("Q1/Q2 и К1 не подтверждены доступными данными")
|
||
article["doi_url"] = f"https://doi.org/{quote(article['doi'], safe='/')}" if article["doi"] else None
|
||
article["type_label"] = " / ".join(article["types"]) or "Тип не указан — статья не подтверждена"
|
||
if article["types"] and article["types"] != ["Статья"]:
|
||
article["criteria"].append("Тип материала не подтверждён как статья — не включён в счётчики Q1/Q2/К1")
|
||
return sorted(unique.values(), key=lambda item: (not item["qualifies"], -(item["year"] or 0), item["title"].casefold()))
|
||
|
||
|
||
def council_candidates(db: Session, today: date | None = None, q: str = "", offset: int = 0,
|
||
sort: str = "total_count", direction: str = "desc", paginate: bool = True, specialty: str = "") -> dict:
|
||
today = today or datetime.now(ZoneInfo("Europe/Moscow")).date()
|
||
start, end = period(today)
|
||
specialties = _k1_specialties()["specialties"]
|
||
specialty = specialty if specialty in {s["code"] for s in specialties} else ""
|
||
specialty_name = next((s["name"] for s in specialties if s["code"] == specialty), "")
|
||
employees = db.scalars(select(Employee).options(selectinload(Employee.publications))).all()
|
||
name_counts = Counter(_name(employee.full_name or "") for employee in employees)
|
||
by_author = _articles_by_author()
|
||
candidates = []
|
||
for employee in employees:
|
||
author = _name(employee.full_name or "")
|
||
ambiguous = bool(author and name_counts[author] > 1 and by_author.get(author))
|
||
csv_articles = by_author.get(author, []) if author and name_counts[author] == 1 else []
|
||
articles = _unique_articles([*csv_articles, *(_database_article(p) for p in employee.publications)], start, end)
|
||
if not articles:
|
||
continue
|
||
data = employee.current_data if isinstance(employee.current_data, dict) else {}
|
||
try:
|
||
degrees = academic_degrees(data)
|
||
except (AttributeError, TypeError):
|
||
degrees = []
|
||
if not any(DOCTOR.fullmatch(degree) for degree in degrees):
|
||
continue
|
||
specialty_count = sum(specialty in a["specialty_codes"] for a in articles) if specialty else 0
|
||
for article in articles:
|
||
article["matches_specialty"] = bool(specialty and specialty in article["specialty_codes"])
|
||
if article["matches_specialty"]:
|
||
article["criteria"].append(f"По специальности {specialty}: {specialty_name} (журнал К1, Перечень ВАК)")
|
||
candidates.append({"specialty_count": specialty_count,"name": employee.full_name or "ФИО не указано", "degree": "; ".join(degrees) or "Учёная степень не указана",
|
||
"count": len(articles),
|
||
**{f"{key}_count": sum(a[key] for a in articles) for key in ("q1", "q2", "k1")},
|
||
"total_count": sum(a["qualifies"] for a in articles),
|
||
"db_count": sum(a["in_db"] for a in articles), "articles": articles,
|
||
"profile_url": _safe_url(employee.canonical_url), "ambiguous": ambiguous,
|
||
"dismissed": employee.status == "dismissed"})
|
||
sort = sort if sort in {"full_name", "q1_count", "q2_count", "k1_count", "total_count", "db_count", "count"} else "total_count"
|
||
direction = direction if direction in {"asc", "desc"} else "desc"
|
||
candidates.sort(key=lambda item: _name(item["name"]), reverse=sort == "full_name" and direction == "desc")
|
||
if sort != "full_name":
|
||
candidates.sort(key=lambda item: (item[sort], item["count"]), reverse=direction == "desc")
|
||
total = len(candidates)
|
||
query = _name(q)
|
||
candidates = [item for item in candidates if query in _name(item["name"])
|
||
and (not specialty or item["specialty_count"] > 0)]
|
||
matched = len(candidates)
|
||
limit = 25
|
||
offset = min(max(0, offset), ((matched - 1) // limit) * limit) if matched else 0
|
||
return {"candidates": candidates[offset:offset + limit] if paginate else candidates,
|
||
"start": start, "end": end, "total": total,
|
||
"specialties": specialties, "specialty": specialty, "specialty_name": specialty_name,
|
||
"matched": matched, "q": q, "offset": offset, "limit": limit, "sort": sort, "direction": direction}
|