Files
miem_workers/app/services/publication_quartiles.py
codex e08c30848f
CI / test (pull_request) Successful in 9m30s
CI / Fingerprint diagnostics (production deployment temporarily disabled) (pull_request) Has been skipped
perf: load council publications on demand and reduce database payload
2026-10-09 16:23:55 +03:00

223 lines
12 KiB
Python

"""Publication quartiles with source, system and year kept separately."""
from datetime import date
from collections import defaultdict
import re
from app.services.council_catalog import publication_catalog, safe_url, issn
from app.services.council_source import _name, _doi
SYSTEM_FIELDS = {"Scopus": "Квартиль scopus", "Web of Science": "Квартиль wos"}
QUARTILES = {"Q1", "Q2", "Q3", "Q4"}
def entries(payload):
if payload is None:
return []
if (not isinstance(payload, dict) or type(payload.get("schema_version")) is not int or payload["schema_version"] != 1
or not isinstance(payload.get("entries"), list)
or (payload.get("lookup") is not None and not isinstance(payload["lookup"], dict))
or (isinstance(payload.get("lookup"), dict) and payload["lookup"].get("doi") is not None
and not isinstance(payload["lookup"]["doi"], str))
or not all(isinstance(item, dict) and item.get("quartile") in QUARTILES
and item.get("system") in {*SYSTEM_FIELDS, "SCImago"}
and (item.get("doi") is None or isinstance(item["doi"], str))
and (item.get("journal") is None or isinstance(item["journal"], str))
and isinstance(item.get("issns", []), list)
and all(isinstance(value, str) for value in item.get("issns", []))
for item in payload["entries"])):
raise ValueError("Invalid publication quartiles")
return payload["entries"]
def hse_entries(raw, publication_year):
metadata = raw.get("csv_enrichment", {}) if isinstance(raw, dict) else {}
if not isinstance(metadata, dict):
return []
source_year = str(metadata.get("Год") or "").strip()
applicable_year = int(source_year) if source_year.isdigit() else publication_year
output = []
for system, field in SYSTEM_FIELDS.items():
quartile = str(metadata.get(field) or "").strip().upper()
if quartile in QUARTILES:
output.append({"system": system, "quartile": quartile, "metric": None,
"doi": _doi(str(metadata.get("DOI") or "")) or None,
"metric_year": None, "publication_year": applicable_year,
"category": None, "journal": metadata.get("Название журнала") or None,
"issns": [], "source": "Данные ВШЭ", "source_url": None,
"checked_on": date.today().isoformat()})
return output
def catalog_entries(doi_metadata, publication_year, allow_latest=False):
"""Only a journal identified by DOI/ISSN and the exact requested metric year."""
if doi_metadata.get("type") != "journal-article":
return [], "not_journal_article"
if publication_year is None:
return [], "missing_publication_year"
if publication_year not in doi_metadata.get("years", []):
return [], "publication_year_conflict"
if not doi_metadata.get("issns"):
return [], "missing_journal_issn"
records, issues = publication_catalog(doi_metadata["issns"], [])
if issues:
return [], "journal_identity_conflict"
metrics = [(record, metric) for record in records for metric in record["metrics"]]
selected_years = {}
for system in SYSTEM_FIELDS:
years = {metric["year"] for _, metric in metrics if metric["system"] == system}
selected_years[system] = publication_year if publication_year in years else max(years, default=None) if allow_latest else None
output = []
for record in records:
for metric in record["metrics"]:
if metric["year"] == selected_years.get(metric["system"]) and metric["system"] in SYSTEM_FIELDS:
output.append({"system": metric["system"], "quartile": metric["quartile"],
"doi": doi_metadata.get("doi"),
"metric": metric["metric"], "metric_year": metric["year"],
"publication_year": publication_year, "category": metric["category"],
"journal": record["name"], "issns": doi_metadata["issns"],
"source": metric["source"], "source_url": safe_url(metric["source_url"]),
"checked_on": date.today().isoformat()})
return output, ("found_latest" if any(item["metric_year"] != publication_year for item in output) else "found") if output else "historical_quartile_unavailable"
def merge(existing, additions, lookup=None):
current = entries(existing)
previous_doi = _doi(((existing or {}).get("lookup") or {}).get("doi") or "") or None
# Freeze the previous lookup's identity on legacy evidence before replacing it.
output = [{**item, "doi": previous_doi} if "doi" not in item else dict(item) for item in current]
# Repeating a lookup must not duplicate evidence merely because its date changed.
def key(item):
return tuple(item.get(field) for field in ("system", "quartile", "metric", "metric_year",
"publication_year", "category", "journal", "source", "source_url", "doi"))
seen = {key(item) for item in output}
for item in additions:
if key(item) not in seen:
output.append(item)
seen.add(key(item))
result = {**(existing or {}), "schema_version": 1, "entries": output}
if lookup is not None:
result["lookup"] = lookup
return result
def binding_issue(item, payload, doi="", journals=(), identifiers=()):
value = item.get("doi") if "doi" in item else ((payload or {}).get("lookup") or {}).get("doi")
source_doi = _doi(value or "")
if value and not source_doi:
return "DOI сохранённого квартиля некорректен — требуется перепроверка"
current_doi = _doi(doi or "")
if source_doi and source_doi != current_doi:
return "Сохранённый квартиль относится к другому DOI — требуется перепроверка"
source_ids = {issn(value) for value in item.get("issns", [])} - {""}
current_ids = {issn(value) for value in identifiers} - {""}
ids_match = bool(source_ids & current_ids)
if source_ids and current_ids and not ids_match:
return "ISSN сохранённого квартиля и публикации расходятся — требуется перепроверка"
source_name = _name(item.get("journal") or "")
names_match = bool(source_name and source_name in {_name(name) for name in journals if name})
if source_name and journals and not names_match and not ids_match:
return f"Квартиль сохранён для журнала «{item.get('journal')}», а статья отнесена к «{' / '.join(journals)}». Совпадение по ISSN не подтверждено. Уточните журнал статьи и источник квартиля"
if not (source_doi and source_doi == current_doi) and not ids_match and not names_match:
return "Привязка сохранённого квартиля к публикации не подтверждена — требуется перепроверка"
return None
def identity_issues(payload, doi="", journals=(), identifiers=()):
return list(dict.fromkeys(issue for item in entries(payload)
if (issue := binding_issue(item, payload, doi, journals, identifiers))))
def qualifying_entries(payload, publication_year, doi="", journals=(), identifiers=()):
return [item for item in entries(payload)
if item["quartile"] in {"Q1", "Q2"}
and item["system"] in SYSTEM_FIELDS
and item.get("publication_year") == publication_year
and item.get("metric_year") in {None, publication_year}
and not binding_issue(item, payload, doi, journals, identifiers)]
def labels(payload, publication_year, doi="", journals=(), identifiers=()):
return list(dict.fromkeys(f"{item['system']} {item['quartile']}" for item in
qualifying_entries(payload, publication_year, doi, journals, identifiers)))
def index_sjr(rankings):
from app.services.council_catalog import issn
indexed = {}
for year, records in rankings.items():
by_issn = defaultdict(list)
for record in records:
for identifier in set(map(issn, record["issns"])) - {""}:
by_issn[identifier].append(record)
indexed[int(year)] = dict(by_issn)
return indexed
def with_title_history(metadata, by_issn, by_title, source_url):
"""Use explicit Scopus title succession, never a fuzzy journal-name match."""
from app.services.council_catalog import issn
if not safe_url(source_url):
return metadata
extra = set()
names = {_name(name) for name in metadata.get("journals", [])}
for identifier in metadata.get("issns", []):
for old in by_issn.get(issn(identifier), []):
if old["history"] not in {"Continued as", "Continues"}:
continue
for related in old["related"]:
candidates = by_title.get(_name(related), [])
if _name(related) in names and len(candidates) == 1:
extra.update(candidates[0]["issns"])
if not extra:
return metadata
return {**metadata, "ranking_issns": sorted(set(metadata["issns"]) | extra),
"journal_identity_source_url": source_url}
def sjr_entries(metadata, publication_year, rankings, allow_latest=False):
from app.services.council_catalog import issn
if metadata.get("type") != "journal-article":
return [], "not_journal_article"
if publication_year is None or publication_year not in metadata.get("years", []):
return [], "publication_year_conflict"
matches = {}
identifiers = set(map(issn, metadata.get("ranking_issns", metadata.get("issns", [])))) - {""}
for year, by_issn in rankings.items():
journals = {record["source_url"]: record for identifier in identifiers for record in by_issn.get(identifier, [])}
if journals:
matches[int(year)] = journals
year = publication_year if publication_year in matches else max(matches, default=None) if allow_latest else None
if year is None:
return [], "historical_quartile_unavailable"
if len(matches[year]) != 1:
return [], "journal_identity_conflict"
record = next(iter(matches[year].values()))
if record.get("conflict"):
return [], "ranking_category_conflict"
categories = []
for category in record.get("categories", "").split(";"):
if not category.strip():
continue
match = re.fullmatch(r"\s*(.+?)\s*\((Q[1-4])\)\s*", category)
if match:
categories.append((match[1].strip(), match[2]))
else:
return [], "ranking_category_conflict"
if categories and min(quartile for _, quartile in categories) != record["quartile"]:
return [], "ranking_category_conflict"
if not categories and record.get("quartile") in QUARTILES:
categories = [(None, record["quartile"])]
result = [{"system": "SCImago", "quartile": quartile, "metric": "SJR" if category else "SJR best quartile",
"doi": metadata.get("doi"),
"metric_year": year, "publication_year": publication_year, "category": category,
"journal": record["journal"], "issns": record["issns"],
"source": "SCImago Journal & Country Rank · Universidad de Extremadura",
"source_url": record["source_url"], "checked_on": date.today().isoformat()}
for category, quartile in categories]
if metadata.get("journal_identity_source_url"):
for item in result:
item["journal_identity_source_url"] = metadata["journal_identity_source_url"]
return result, "found" if year == publication_year else "found_latest"