Files
miem_workers/app/services/council_catalog.py
T

177 lines
10 KiB
Python

"""Read the supplied thematic snapshots; never infer article approval from them."""
import hashlib
import json
import re
from collections import defaultdict
from functools import lru_cache
from pathlib import Path
from urllib.parse import urlsplit
DATA = Path(__file__).resolve().parents[1] / "data"
CATALOGS = (DATA / "q1-journals/catalog.json", DATA / "q2-journals/catalog.json")
def safe_url(value):
if not isinstance(value, str):
return None
try:
parsed = urlsplit(value)
if parsed.scheme in {"http", "https"} and parsed.hostname and not parsed.username and not parsed.password:
return value
except ValueError:
pass
return None
def issn(value):
return re.sub(r"[^0-9X]", "", str(value or "").upper())
def citescore_quartile(percentile):
if type(percentile) not in {int, float} or not 0 <= percentile <= 100:
raise ValueError("Invalid CiteScore percentile")
return "Q1" if percentile >= 75 else "Q2" if percentile >= 50 else "Q3" if percentile >= 25 else "Q4"
def _unique(records, key):
result = {}
for record in records:
identifier = key(record)
if identifier in result:
raise ValueError("Duplicate catalog identifier")
result[identifier] = record
return result
@lru_cache(maxsize=1)
def catalog():
journals = defaultdict(list)
passports = {}
metadata = []
issues = []
for path in CATALOGS:
try:
data = json.loads(path.read_text(encoding="utf-8"))
meta = data["metadata"]
if meta["schema_version"] not in {"1.0.0", "1.1.0"}:
raise ValueError("Unsupported catalog schema")
rule = meta.get("selection_rule") or meta.get("q1_rule") or meta["q2_rule"]
sources = _unique(data["sources"], lambda r: r["id"])
specialties = _unique(data["specialties"], lambda r: r["code"])
local_passports = {}
for specialty in specialties.values():
for passport in specialty["passports"]:
pdf = (path.parent / passport["local_pdf"]).resolve()
if not pdf.is_relative_to(DATA.resolve()):
raise ValueError("Passport path outside data directory")
if hashlib.sha256(pdf.read_bytes()).hexdigest() != passport["sha256"]:
raise ValueError("Passport checksum mismatch")
entry = {**passport, "specialty_code": specialty["code"]}
if passport["id"] in local_passports:
raise ValueError("Duplicate passport")
if passport["id"] in passports and passports[passport["id"]] != entry:
raise ValueError("Conflicting passport versions")
local_passports[passport["id"]] = entry
local_journals = _unique(data["journals"], lambda r: r["id"])
links = _unique(data["specialty_links"], lambda r: (r["journal_id"], r["specialty_code"]))
assessments = _unique(data["assessments"], lambda r: (r["journal_id"], r["specialty_code"]))
for (journal_id, code), assessment in assessments.items():
if journal_id not in local_journals or code not in specialties:
raise ValueError("Unresolved assessment")
if assessment["assessment"] not in {"direct", "conditional", "not_established"}:
raise ValueError("Invalid assessment level")
for (journal_id, code), link in links.items():
if journal_id not in local_journals or code not in specialties:
raise ValueError("Unresolved specialty link")
if link["match_level"] not in {"direct", "conditional"}:
raise ValueError("Invalid specialty level")
for field in ("rationales", "conditions"):
if not isinstance(link[field], list) or not all(isinstance(v, str) for v in link[field]):
raise ValueError("Invalid specialty explanations")
if assessments.get((journal_id, code), {}).get("assessment") != link["match_level"]:
raise ValueError("Conflicting assessment")
for reference in link["passport_references"]:
passport = local_passports[reference["passport_id"]]
if passport["specialty_code"] != code or not set(reference["point_numbers"]) <= {
point["number"] for point in passport["directions"]
}:
raise ValueError("Invalid passport points")
prepared = []
for journal in local_journals.values():
if journal["id"] != f"scopus:{journal['scopus_source_id']}":
raise ValueError("Conflicting Scopus journal identifier")
metrics = []
for system, field in (("Scopus", "scopus"), ("Web of Science", "wos")):
metric = journal[field]
source = sources[metric["source_id"]]
year = metric["year"] if field == "scopus" else metric["metric_year"]
if source["metric_year"] != year:
raise ValueError("Conflicting metric year")
for category in metric["categories"]:
quartile = citescore_quartile(category["percentile"]) if field == "scopus" else category["quartile"]
if quartile not in {"Q1", "Q2", "Q3", "Q4"} or category.get("quartile", quartile) != quartile:
raise ValueError("Conflicting quartile")
metrics.append({"system": system, "metric": metric["metric"], "year": year,
"release_year": metric.get("release_year"), "category": category["category"],
"edition": category.get("edition"), "quartile": quartile,
"source": source["title"], "source_url": safe_url(source.get("url")),
"source_row": category.get("source_row"), "source_pdf_page": category.get("source_pdf_page")})
prepared.append({**journal, "metrics": metrics, "catalog": path.parent.name,
"links": {code: link for (jid, code), link in links.items() if jid == journal["id"]},
"assessments": {code: value["assessment"] for (jid, code), value in assessments.items() if jid == journal["id"]}})
# Commit only a fully validated snapshot. Broken data cannot supply evidence.
passports.update(local_passports)
for journal in prepared:
journals[journal["id"]].append(journal)
metadata.append({"catalog": path.parent.name, "schema_version": meta["schema_version"],
"rule": rule, "metric_years": meta["metric_years"], "generated_on": meta["generated_on"]})
except (OSError, ValueError, KeyError, TypeError, AttributeError, IndexError):
issues.append(f"{path.parent.name}: целостность каталога не подтверждена")
by_issn = defaultdict(set)
for identifier, records in journals.items():
for record in records:
for value in (record.get("issn"), record.get("eissn")):
if value:
by_issn[issn(value)].add(identifier)
return {"journals": dict(journals), "by_issn": dict(by_issn), "passports": passports,
"metadata": metadata, "issues": issues}
def publication_catalog(identifiers, names, specialty=""):
data = catalog()
matches = set().union(*(data["by_issn"].get(issn(value), set()) for value in identifiers))
# Snapshot failures are reported once in the assessment context. Only
# problems with this journal may prevent use of its validated metrics.
issues = []
if len(matches) > 1:
return [], [*issues, "ISSN указывают на разные журналы — требуется проверка"]
records = data["journals"].get(next(iter(matches)), []) if matches else []
if records and names and not any(str(name).casefold().strip() == r["name"].casefold() for name in names for r in records):
issues.append("Название журнала в публикации и запись по ISSN расходятся. Проверьте, относятся ли они к одному журналу")
output = []
for record in records:
link = record["links"].get(specialty)
thematic = []
labels = {"direct": "прямая", "conditional": "условная", "not_established": "не установлена"}
if link:
thematic.extend([f"Аналитическая связь журнала: {labels[link['match_level']]}", *link["rationales"], *link["conditions"]])
for ref in link["passport_references"]:
passport = data["passports"][ref["passport_id"]]
thematic.append(f"Паспорт {ref['passport_id']} ({passport['branch']}), SHA-256 {passport['sha256']}; пункты: "
+ ", ".join(map(str, ref["point_numbers"])))
elif specialty:
thematic.append("Связь журнала: " + labels.get(record["assessments"].get(specialty), "не оценена"))
output.append({"id": record["id"], "name": record["name"], "issn": record.get("issn"), "eissn": record.get("eissn"),
"catalog": record["catalog"], "metrics": record["metrics"], "thematic": thematic,
"match_level": link["match_level"] if link else None,
"specialty_codes": list(record["links"]),
"passport_urls": [safe_url(data["passports"][r["passport_id"]]["source_url"]) for r in link["passport_references"]] if link else []})
# A conflict is preserved; it never resolves by load order.
if len(records) > 1:
for field in ("name", "issn", "eissn", "metrics", "links", "assessments"):
if any(record[field] != records[0][field] for record in records[1:]):
issues.append("Сведения каталогов расходятся: " + field)
return output, issues