feat: store and enrich publication quartiles with provenance

This commit is contained in:
codex committed 2026-10-08 23:37:53 +03:00
1 parent b777bb9e16
commit 5cc7a2dc52
14 files changed
+835 -8

No files matched your search

+299
View File
@@ -0,0 +1,299 @@
"""Backfill HSE quartiles and resolve remaining DOI journals. No .env or DDL."""
import argparse
from collections import Counter, defaultdict
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import date, datetime, timezone
import hashlib
import json
import os
from pathlib import Path
import re
import sys
import time
from urllib.parse import quote
import requests
from sqlalchemy import MetaData, Table, create_engine, select, update
from openpyxl import load_workbook
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from app.services.council_source import _doi, _name
from app.services.council_catalog import issn
from app.services.publication_quartiles import catalog_entries, entries, hse_entries, merge, sjr_entries, index_sjr, with_title_history
SOURCE_TITLES_URL = "https://downloads.ctfassets.net/o78em1y1w4i4/7xtaTxNiNcWRTeZkV86eNy/69cf2d506c905dc299531fdc93049dbb/ext_list_Aug_2026.xlsx"
SOURCE_TITLES_SHA256 = "11e81f686401c89fbef28de31d1880388bb7122f35c89e09db6e1848237e9afb"
def load_title_history(path):
if hashlib.sha256(path.read_bytes()).hexdigest() != SOURCE_TITLES_SHA256:
raise ValueError("Scopus title-list snapshot checksum mismatch")
by_issn, by_title = defaultdict(list), defaultdict(list)
workbook = load_workbook(path, read_only=True, data_only=True)
try:
rows = workbook.worksheets[0].iter_rows(values_only=True)
columns = next(rows)
required = {"Source Title", "ISSN", "EISSN", "Source Type", "Title History Indication", "Related Title 1"}
if not required <= set(columns):
raise ValueError("Invalid Scopus title list")
for row in rows:
data = dict(zip(columns, row))
if data["Source Type"] != "Journal":
continue
identifiers = {issn(data.get(key)) for key in ("ISSN", "EISSN")} - {""}
identifiers = {identifier[:4] + "-" + identifier[4:] for identifier in identifiers if len(identifier) == 8}
record = {"title": str(data["Source Title"] or ""), "issns": identifiers,
"history": data["Title History Indication"],
"related": [str(value) for key, value in data.items() if key and "Related Title" in key and value]}
by_title[_name(record["title"])].append(record)
for identifier in identifiers:
by_issn[issn(identifier)].append(record)
finally:
workbook.close()
return dict(by_issn), dict(by_title)
def load_sjr(identifiers, years, cache):
"""Read the university's licensed SCImago dataset for the requested ISSNs."""
rankings = {}
failures = []
for year in sorted(set(years) & set(range(2009, 2026))):
records = {}
for offset in range(0, len(identifiers), 200):
batch = identifiers[offset:offset + 200]
digest = hashlib.sha256(json.dumps([year, batch]).encode()).hexdigest()
path = cache / f"sjr-{digest}.json"
if path.exists():
bindings = json.loads(path.read_text(encoding="utf8"))
else:
values = " ".join(json.dumps(value) for value in batch)
query = f'''PREFIX bibo: <http://purl.org/ontology/bibo/>
PREFIX dct: <http://purl.org/dc/terms/>
PREFIX ou: <https://opendata.unex.es/def/ontouniversidad#>
SELECT DISTINCT ?uri ?title ?issn ?eissn ?quartile ?categories WHERE {{
VALUES ?wanted {{ {values} }}
{{ ?uri bibo:issn ?wanted }} UNION {{ ?uri bibo:eissn ?wanted }}
?uri a bibo:Journal; dct:title ?title; dct:identifier ?id; ou:sjrMejorCuartil ?quartile .
FILTER(STRSTARTS(STR(?id), "{year}-"))
OPTIONAL {{ ?uri bibo:issn ?issn }} OPTIONAL {{ ?uri bibo:eissn ?eissn }}
OPTIONAL {{ ?uri ou:categoriaSJR ?categories }}
}} LIMIT 10001'''
try:
response = requests.get("https://opendata.unex.es/sparql/", timeout=45,
params={"query": query, "format": "application/sparql-results+json"},
headers={"Accept": "application/sparql-results+json"})
response.raise_for_status()
bindings = response.json()["results"]["bindings"]
if len(bindings) >= 10000:
raise ValueError("Truncated ranking query")
path.write_text(json.dumps(bindings, ensure_ascii=False), encoding="utf8")
except (requests.RequestException, ValueError, KeyError, TypeError):
failures.append({"year": year, "batch": offset, "status": "ranking_source_unavailable"})
continue
for item in bindings:
value = lambda field: item.get(field, {}).get("value", "")
url = value("uri")
if not url.startswith(f"https://opendata.unex.es/recurso/ciencia-tecnologia/investigacion/revistas/Revista/{year}-"):
raise ValueError("Ranking year/identity mismatch")
record = {"journal": value("title"), "issns": sorted({value("issn"), value("eissn")} - {""}),
"quartile": value("quartile"), "categories": value("categories"), "source_url": url}
if record["quartile"] not in {"Q1", "Q2", "Q3", "Q4"}:
continue
previous = records.get(url)
if previous:
record["journal"] = min(previous["journal"], record["journal"])
record["issns"] = sorted(set(previous["issns"]) | set(record["issns"]))
record["categories"] = "; ".join(sorted({category.strip() for value in
(previous["categories"], record["categories"]) for category in value.split(";") if category.strip()}))
record["conflict"] = previous.get("conflict", False) or previous["quartile"] != record["quartile"]
records[url] = record
rankings[year] = list(records.values())
print(f"SJR {year}: {len(records)} matching journals", flush=True)
return index_sjr(rankings), failures
def resolve_datacite(doi, timeout):
url = "https://api.datacite.org/dois/" + quote(doi, safe="")
result = {"doi": doi, "status": "doi_lookup_failed", "source_url": url, "datacite_checked": True}
for attempt in range(3):
try:
response = requests.get(url, timeout=timeout, allow_redirects=False,
headers={"User-Agent": "MIEM-workers/0.17 publication-quartiles", "Accept": "application/json"})
if response.status_code == 404:
return {**result, "status": "doi_not_found"}
if response.status_code == 429 or response.status_code >= 500:
time.sleep(min(10, 2 ** attempt))
continue
response.raise_for_status()
metadata = response.json()["data"]["attributes"]
if _doi(metadata.get("doi", "")) != doi:
return {**result, "status": "doi_identity_conflict"}
types = metadata.get("types", {})
general = types.get("resourceTypeGeneral")
journal_article = general == "JournalArticle" or (general in {None, "Text"} and types.get("citeproc") == "article-journal")
container = metadata.get("container") or {}
identifiers = []
if container.get("identifierType") == "ISSN":
identifiers.append(container.get("identifier", ""))
for related in metadata.get("relatedIdentifiers", []):
if related.get("relatedIdentifierType") == "ISSN" and related.get("relationType") in {"IsPartOf", "IsPublishedIn"}:
identifiers.append(related.get("relatedIdentifier", ""))
identifiers = sorted({value[:4] + "-" + value[-4:] for value in identifiers if isinstance(value, str)
and re.fullmatch(r"\d{4}-?\d{3}[\dX]", value, re.I)})
year = metadata.get("publicationYear")
years = [int(year)] if str(year).isdigit() else []
return {**result, "status": "resolved", "type": "journal-article" if journal_article else "non-journal-material",
"issns": identifiers, "journals": [container["title"]] if isinstance(container.get("title"), str) else [], "years": years}
except (requests.RequestException, ValueError, KeyError, TypeError, AttributeError):
if attempt < 2:
time.sleep(2 ** attempt)
return result
def resolve_doi(doi, cache, timeout=20):
path = cache / (hashlib.sha256(doi.encode()).hexdigest() + ".json")
if path.exists():
result = json.loads(path.read_text(encoding="utf8"))
if result.get("doi") == doi and (result.get("status") == "resolved"
or (result.get("status") == "doi_not_found" and result.get("datacite_checked"))):
return result
url = "https://api.crossref.org/works/" + quote(doi, safe="")
result = {"doi": doi, "status": "doi_lookup_failed", "source_url": url}
for attempt in range(3):
try:
response = requests.get(url, timeout=timeout, allow_redirects=False,
headers={"User-Agent": "MIEM-workers/0.17 publication-quartiles",
"Accept": "application/json"})
if response.status_code == 404:
result = resolve_datacite(doi, timeout)
break
if response.status_code == 429 or response.status_code >= 500:
time.sleep(min(10, 2 ** attempt))
continue
response.raise_for_status()
message = response.json()["message"]
if _doi(message.get("DOI", "")) != doi:
result["status"] = "doi_identity_conflict"
break
years = set()
for field in ("published", "published-print", "published-online", "issued"):
for parts in message.get(field, {}).get("date-parts", []):
if parts and type(parts[0]) is int:
years.add(parts[0])
result.update(status="resolved", type=message.get("type"),
journals=[v for v in message.get("container-title", []) if isinstance(v, str)],
issns=[v for v in message.get("ISSN", []) if isinstance(v, str)
and re.fullmatch(r"\d{4}-\d{3}[\dX]", v, re.I)], years=sorted(years))
break
except (requests.RequestException, ValueError, KeyError, TypeError, AttributeError):
if attempt < 2:
time.sleep(2 ** attempt)
if result["status"] in {"resolved", "doi_not_found"}:
temporary = path.with_suffix(".tmp")
temporary.write_text(json.dumps(result, ensure_ascii=False), encoding="utf8")
temporary.replace(path)
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--apply", action="store_true", help="Commit changes; default is read-only")
parser.add_argument("--lookup-doi", action="store_true")
parser.add_argument("--allow-latest", action="store_true", help="Use latest available metric if the publication year is unavailable")
parser.add_argument("--sjr-history", action="store_true", help="Check UNEx/SCImago historical quartiles by DOI ISSNs")
parser.add_argument("--scopus-title-history", type=Path, help="Official Scopus August 2026 source-title XLSX for explicit ISSN succession")
parser.add_argument("--cache-dir", type=Path, required=True)
parser.add_argument("--report", type=Path, required=True)
parser.add_argument("--workers", type=int, choices=range(1, 9), default=4)
args = parser.parse_args()
pending = args.report.with_suffix(args.report.suffix + ".pending")
if args.scopus_title_history and args.scopus_title_history.resolve() in {args.report.resolve(), pending.resolve()}:
raise ValueError("Report must not overwrite the source title list")
args.cache_dir.mkdir(parents=True, exist_ok=True)
args.report.parent.mkdir(parents=True, exist_ok=True)
# Verify report destination before changing any data.
pending.write_text("", encoding="utf8")
engine = create_engine(os.environ["DATABASE_URL"])
table = Table("employee_publications", MetaData(), autoload_with=engine)
if "quartiles" not in table.c:
raise ValueError("Apply migration 010 before enrichment")
columns = [table.c[field] for field in ("id", "year", "doi_url", "raw_data", "quartiles")]
with engine.connect() as connection:
rows = [dict(row) for row in connection.execute(select(*columns)).mappings()]
def needs_lookup(row):
return not entries(row["quartiles"]) or row["quartiles"].get("lookup", {}).get("status") == "found_latest"
identifiers = sorted({_doi(row["doi_url"] or "") for row in rows
if needs_lookup(row) and not hse_entries(row["raw_data"], row["year"])} - {""})
resolved = {}
if args.lookup_doi:
with ThreadPoolExecutor(max_workers=args.workers) as pool:
tasks = {pool.submit(resolve_doi, doi, args.cache_dir): doi for doi in identifiers}
for task in as_completed(tasks):
resolved[tasks[task]] = task.result()
if len(resolved) % 100 == 0 or len(resolved) == len(identifiers):
print(f"DOI checked: {len(resolved)}/{len(identifiers)}", flush=True)
rankings, ranking_failures = {}, []
if args.scopus_title_history:
by_issn, by_title = load_title_history(args.scopus_title_history)
resolved = {doi: with_title_history(metadata, by_issn, by_title, SOURCE_TITLES_URL)
for doi, metadata in resolved.items()}
if args.lookup_doi and args.sjr_history:
issns = sorted({value for metadata in resolved.values() if metadata.get("type") == "journal-article"
for value in metadata.get("ranking_issns", metadata.get("issns", []))})
years = {row["year"] for row in rows if needs_lookup(row) and row["year"] is not None}
if args.allow_latest:
years.add(2025)
rankings, ranking_failures = load_sjr(issns, years, args.cache_dir)
counts = Counter()
report = []
for row in rows:
additions = hse_entries(row["raw_data"], row["year"])
payload = merge(row["quartiles"], additions)
if additions:
counts["hse_quartiles"] += 1
if args.lookup_doi and (not entries(payload) or payload.get("lookup", {}).get("status") == "found_latest"):
doi = _doi(row["doi_url"] or "")
metadata = resolved.get(doi, {"status": "invalid_doi" if row["doi_url"] else "missing_doi"})
status = metadata["status"]
if status == "resolved":
found, status = catalog_entries(metadata, row["year"])
if not found and status == "historical_quartile_unavailable" and rankings:
found, status = sjr_entries(metadata, row["year"], rankings)
if not found and status == "historical_quartile_unavailable" and args.allow_latest:
found, status = catalog_entries(metadata, row["year"], True)
if not found and rankings:
found, status = sjr_entries(metadata, row["year"], rankings, True)
payload = merge(payload, found)
payload["lookup"] = {**metadata, "status": status, "requested_year": row["year"],
"checked_on": date.today().isoformat()}
counts[status] += 1
counts["with_quartile" if entries(payload) else "without_quartile"] += 1
if payload == row["quartiles"]:
continue
if args.apply:
# Only this field is changed; parser-owned data and existing evidence survive.
with engine.begin() as connection:
current = dict(connection.execute(select(*columns).where(table.c.id == row["id"])
.with_for_update()).mappings().one())
if any(current[field] != row[field] for field in ("year", "doi_url", "raw_data", "quartiles")):
counts["concurrent_change_skipped"] += 1
continue
connection.execute(update(table).where(table.c.id == row["id"]).values(
quartiles=payload, updated_at=datetime.now(timezone.utc)))
counts["changed"] += 1
report.append({"id": row["id"], "quartiles": payload})
summary = {"mode": "apply" if args.apply else "dry-run", "publications": len(rows),
"unique_doi_checked": len(resolved), "counts": dict(counts), "ranking_failures": ranking_failures}
pending.write_text(json.dumps({"summary": summary, "records": report}, ensure_ascii=False, indent=2), encoding="utf8")
pending.replace(args.report)
print(json.dumps(summary, ensure_ascii=False), flush=True)
if __name__ == "__main__":
try:
main()
except Exception as error:
# Connection errors can contain credentials; do not print their text.
print(f"Enrichment failed ({type(error).__name__}); completed records are retained; retry is safe.", flush=True)
raise SystemExit(1)