691 lines
26 KiB
Python
691 lines
26 KiB
Python
import gzip
|
|
import pytest
|
|
from datetime import datetime, timezone
|
|
|
|
from app.models import (
|
|
CrawlError,
|
|
CrawlRun,
|
|
CrawlRunEmployeeChange,
|
|
Employee,
|
|
EmployeeNewsLink,
|
|
EmployeePublication,
|
|
EmployeeSnapshot,
|
|
ParseResourceCache,
|
|
)
|
|
from app.config import Settings
|
|
from app.services.crawler import _checksum, _mark_dismissed, _upsert_employee, refresh_dismissed_status
|
|
from app.services.resource_cache import ResourceCache
|
|
|
|
|
|
class FakeResponse:
|
|
def __init__(self, status_code):
|
|
self.status_code = status_code
|
|
|
|
|
|
class FakeSession:
|
|
def __init__(self, statuses):
|
|
self.statuses = statuses
|
|
|
|
def get(self, url, **_kwargs):
|
|
return FakeResponse(self.statuses[url])
|
|
|
|
|
|
class ConditionalResponse:
|
|
def __init__(self, status_code, text="", headers=None):
|
|
self.status_code = status_code
|
|
self._text = text
|
|
self.headers = headers or {}
|
|
self.text_read = False
|
|
|
|
@property
|
|
def text(self):
|
|
self.text_read = True
|
|
return self._text
|
|
|
|
def raise_for_status(self):
|
|
return None
|
|
|
|
|
|
class ConditionalSession:
|
|
def __init__(self):
|
|
self.requests = []
|
|
self.not_modified_response = ConditionalResponse(304)
|
|
|
|
def get(self, url, **kwargs):
|
|
self.requests.append((url, kwargs))
|
|
if kwargs["headers"].get("If-None-Match") == '"cached"':
|
|
return self.not_modified_response
|
|
return ConditionalResponse(200, "fresh", {"ETag": '"fresh"'})
|
|
|
|
|
|
def test_refresh_dismissed_status_reactivates_only_profiles_in_source(monkeypatch, db_session):
|
|
now = datetime.now(timezone.utc)
|
|
found = Employee(
|
|
profile_key="staff:returned",
|
|
canonical_url="https://www.hse.ru/staff/returned",
|
|
status="dismissed",
|
|
dismissed_at=now,
|
|
first_seen_at=now,
|
|
last_seen_at=now,
|
|
)
|
|
still_dismissed = Employee(
|
|
profile_key="staff:gone",
|
|
canonical_url="https://www.hse.ru/staff/gone",
|
|
status="dismissed",
|
|
dismissed_at=now,
|
|
first_seen_at=now,
|
|
last_seen_at=now,
|
|
)
|
|
db_session.add_all([found, still_dismissed])
|
|
db_session.commit()
|
|
monkeypatch.setattr(
|
|
"app.services.crawler.collect_profile_links",
|
|
lambda *_args, **_kwargs: ["https://www.hse.ru/staff/returned"],
|
|
)
|
|
|
|
run = refresh_dismissed_status(db_session, Settings())
|
|
|
|
assert run.status == "completed"
|
|
assert run.parsed_count == 1
|
|
assert run.skipped_count == 1
|
|
assert found.status == "active"
|
|
assert found.dismissed_at is None
|
|
assert still_dismissed.status == "dismissed"
|
|
|
|
|
|
def test_mark_dismissed_records_missing_source_when_profile_is_available(db_session):
|
|
run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
|
|
db_session.add(run)
|
|
db_session.add(
|
|
Employee(
|
|
profile_key="staff:kept",
|
|
canonical_url="https://www.hse.ru/staff/kept",
|
|
status="active",
|
|
first_seen_at=datetime.now(timezone.utc),
|
|
last_seen_at=datetime.now(timezone.utc),
|
|
)
|
|
)
|
|
db_session.add(
|
|
Employee(
|
|
profile_key="staff:missing",
|
|
canonical_url="https://www.hse.ru/staff/missing",
|
|
status="active",
|
|
first_seen_at=datetime.now(timezone.utc),
|
|
last_seen_at=datetime.now(timezone.utc),
|
|
)
|
|
)
|
|
db_session.commit()
|
|
|
|
dismissed = _mark_dismissed(
|
|
db_session,
|
|
run,
|
|
{"staff:kept"},
|
|
FakeSession({"https://www.hse.ru/staff/missing": 200}),
|
|
30,
|
|
)
|
|
|
|
assert dismissed == 0
|
|
assert db_session.query(Employee).filter_by(profile_key="staff:kept").one().status == "active"
|
|
missing = db_session.query(Employee).filter_by(profile_key="staff:missing").one()
|
|
assert missing.status == "active"
|
|
assert missing.dismissed_at is None
|
|
change = db_session.query(CrawlRunEmployeeChange).one()
|
|
assert change.change_type == "missing_from_source"
|
|
assert change.profile_available is True
|
|
|
|
|
|
def test_mark_dismissed_requires_consecutive_unavailable_checks(db_session):
|
|
run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
|
|
employee = Employee(
|
|
profile_key="staff:gone",
|
|
canonical_url="https://www.hse.ru/staff/gone",
|
|
status="active",
|
|
first_seen_at=datetime.now(timezone.utc),
|
|
last_seen_at=datetime.now(timezone.utc),
|
|
)
|
|
db_session.add_all([run, employee])
|
|
db_session.commit()
|
|
|
|
first_check = _mark_dismissed(
|
|
db_session,
|
|
run,
|
|
set(),
|
|
FakeSession({"https://www.hse.ru/staff/gone": 404}),
|
|
30,
|
|
confirmation_runs=2,
|
|
)
|
|
|
|
assert first_check == 0
|
|
assert employee.status == "verification_required"
|
|
assert employee.dismissed_at is None
|
|
assert employee.profile_unavailable_streak == 1
|
|
assert db_session.query(CrawlRunEmployeeChange).one().change_type == "verification_required"
|
|
|
|
second_check = _mark_dismissed(
|
|
db_session,
|
|
run,
|
|
set(),
|
|
FakeSession({"https://www.hse.ru/staff/gone": 404}),
|
|
30,
|
|
confirmation_runs=2,
|
|
)
|
|
|
|
assert second_check == 1
|
|
assert employee.status == "dismissed"
|
|
assert employee.dismissed_at is not None
|
|
assert employee.profile_unavailable_streak == 2
|
|
change = db_session.query(CrawlRunEmployeeChange).order_by(CrawlRunEmployeeChange.id).all()[-1]
|
|
assert change.change_type == "dismissed"
|
|
assert change.profile_available is False
|
|
|
|
|
|
def test_mark_dismissed_does_not_dismiss_on_server_error(db_session):
|
|
run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
|
|
employee = Employee(
|
|
profile_key="staff:temporary-error",
|
|
canonical_url="https://www.hse.ru/staff/temporary-error",
|
|
status="active",
|
|
first_seen_at=datetime.now(timezone.utc),
|
|
last_seen_at=datetime.now(timezone.utc),
|
|
)
|
|
db_session.add_all([run, employee])
|
|
db_session.commit()
|
|
|
|
dismissed = _mark_dismissed(
|
|
db_session,
|
|
run,
|
|
set(),
|
|
FakeSession({"https://www.hse.ru/staff/temporary-error": 503}),
|
|
30,
|
|
confirmation_runs=1,
|
|
)
|
|
|
|
assert dismissed == 0
|
|
assert employee.status == "active"
|
|
assert employee.profile_unavailable_streak == 0
|
|
assert db_session.query(CrawlError).one().error_type == "ProfileAvailabilityCheckError"
|
|
|
|
|
|
def test_available_profile_resets_verification_streak(db_session):
|
|
run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
|
|
employee = Employee(
|
|
profile_key="staff:restored",
|
|
canonical_url="https://www.hse.ru/staff/restored",
|
|
status="active",
|
|
first_seen_at=datetime.now(timezone.utc),
|
|
last_seen_at=datetime.now(timezone.utc),
|
|
)
|
|
db_session.add_all([run, employee])
|
|
db_session.commit()
|
|
|
|
_mark_dismissed(
|
|
db_session,
|
|
run,
|
|
set(),
|
|
FakeSession({"https://www.hse.ru/staff/restored": 404}),
|
|
30,
|
|
confirmation_runs=3,
|
|
)
|
|
_mark_dismissed(
|
|
db_session,
|
|
run,
|
|
set(),
|
|
FakeSession({"https://www.hse.ru/staff/restored": 200}),
|
|
30,
|
|
confirmation_runs=3,
|
|
)
|
|
|
|
assert employee.status == "active"
|
|
assert employee.profile_unavailable_streak == 0
|
|
|
|
|
|
def test_mark_dismissed_blocks_mass_auto_dismissals(db_session):
|
|
run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
|
|
employees = [
|
|
Employee(
|
|
profile_key=f"staff:gone-{index}",
|
|
canonical_url=f"https://www.hse.ru/staff/gone-{index}",
|
|
status="active",
|
|
first_seen_at=datetime.now(timezone.utc),
|
|
last_seen_at=datetime.now(timezone.utc),
|
|
)
|
|
for index in range(2)
|
|
]
|
|
db_session.add_all([run, *employees])
|
|
db_session.commit()
|
|
|
|
dismissed = _mark_dismissed(
|
|
db_session,
|
|
run,
|
|
set(),
|
|
FakeSession({employee.canonical_url: 404 for employee in employees}),
|
|
30,
|
|
confirmation_runs=1,
|
|
max_auto_dismissals=1,
|
|
)
|
|
|
|
assert dismissed == 0
|
|
assert {employee.status for employee in employees} == {"verification_required"}
|
|
assert "приостановлено" in run.message
|
|
|
|
|
|
def test_upsert_employee_increments_new_count_and_records_change_for_new_employee(db_session):
|
|
run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
|
|
db_session.add(run)
|
|
db_session.commit()
|
|
|
|
employee, _ = _upsert_employee(
|
|
db_session,
|
|
run,
|
|
{
|
|
"source_url": "https://www.hse.ru/staff/newperson",
|
|
"profile_type": "staff",
|
|
"profile_id": "newperson",
|
|
"full_name": "New Person",
|
|
"tabs": [],
|
|
"sections": [
|
|
{
|
|
"title": "Образование и учёные степени",
|
|
"year_entries": [{"text": "Кандидат технических наук"}],
|
|
}
|
|
],
|
|
"parser_version": "0.2.0",
|
|
"_html": "<html></html>",
|
|
},
|
|
)
|
|
db_session.commit()
|
|
|
|
assert run.new_count == 1
|
|
assert employee.has_academic_degree is True
|
|
change = db_session.query(CrawlRunEmployeeChange).one()
|
|
assert change.change_type == "new"
|
|
assert change.full_name == "New Person"
|
|
|
|
|
|
def test_upsert_employee_reconciles_profile_moved_to_new_url(db_session):
|
|
run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
|
|
employee = Employee(
|
|
profile_key="staff:abelov",
|
|
canonical_url="https://www.hse.ru/staff/abelov",
|
|
full_name="Белов Александр Владимирович",
|
|
status="active",
|
|
first_seen_at=datetime.now(timezone.utc),
|
|
last_seen_at=datetime.now(timezone.utc),
|
|
)
|
|
db_session.add_all([run, employee])
|
|
db_session.commit()
|
|
employee_id = employee.id
|
|
|
|
updated, changed = _upsert_employee(
|
|
db_session,
|
|
run,
|
|
{
|
|
"source_url": "https://www.hse.ru/org/persons/47634735",
|
|
"profile_type": "org_person",
|
|
"profile_id": "47634735",
|
|
"full_name": "Белов Александр Владимирович",
|
|
"tabs": [],
|
|
"sections": [],
|
|
"parser_version": "0.7.0",
|
|
"_html": "<html></html>",
|
|
},
|
|
)
|
|
db_session.commit()
|
|
|
|
assert changed is True
|
|
assert updated.id == employee_id
|
|
assert updated.profile_key == "org_person:47634735"
|
|
assert updated.canonical_url == "https://www.hse.ru/org/persons/47634735"
|
|
assert updated.status == "active"
|
|
assert run.new_count == 0
|
|
assert db_session.query(Employee).count() == 1
|
|
assert {item.url for item in updated.profile_urls} == {
|
|
"https://www.hse.ru/staff/abelov",
|
|
"https://www.hse.ru/org/persons/47634735",
|
|
}
|
|
|
|
|
|
def test_resource_cache_uses_etag_and_reuses_cached_body_on_304(db_session):
|
|
db_session.add(
|
|
ParseResourceCache(
|
|
profile_key="staff:cached",
|
|
resource_key="main-html",
|
|
method="GET",
|
|
url="https://www.hse.ru/staff/cached",
|
|
request_fingerprint="020d59db7b358d9023d0f185bcbf5a9c085d3cf2bf91d92d48eee9147e8d0f01",
|
|
etag='"cached"',
|
|
body_hash="cached-hash",
|
|
body_snapshot=gzip.compress("cached body".encode("utf-8")),
|
|
parser_version="0.6.0",
|
|
)
|
|
)
|
|
db_session.commit()
|
|
session = ConditionalSession()
|
|
|
|
result = ResourceCache(db_session).fetch_text(
|
|
session,
|
|
profile_key="staff:cached",
|
|
resource_key="main-html",
|
|
method="GET",
|
|
url="https://www.hse.ru/staff/cached",
|
|
headers={"User-Agent": "test"},
|
|
timeout=10,
|
|
)
|
|
|
|
assert session.requests[0][1]["headers"]["If-None-Match"] == '"cached"'
|
|
assert result.text == "cached body"
|
|
assert result.from_cache is True
|
|
assert session.not_modified_response.text_read is False
|
|
|
|
|
|
def test_upsert_employee_skips_snapshot_when_checksum_is_unchanged(db_session):
|
|
first_run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
|
|
second_run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
|
|
db_session.add_all([first_run, second_run])
|
|
db_session.commit()
|
|
|
|
_, first_changed = _upsert_employee(db_session, first_run, _parsed_employee("same"))
|
|
_, second_changed = _upsert_employee(db_session, second_run, _parsed_employee("same"))
|
|
db_session.commit()
|
|
|
|
assert first_changed is True
|
|
assert second_changed is False
|
|
assert db_session.query(EmployeeSnapshot).count() == 1
|
|
|
|
|
|
def test_upsert_employee_saves_publications_and_reuses_existing_rows(db_session):
|
|
first_run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
|
|
second_run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
|
|
db_session.add_all([first_run, second_run])
|
|
db_session.commit()
|
|
|
|
parsed = _parsed_employee("published")
|
|
parsed["sections"] = [
|
|
{
|
|
"type": "publications",
|
|
"publications": [
|
|
{
|
|
"id": "888959076",
|
|
"publication_id": "888959076",
|
|
"title": "Detailed Publication",
|
|
"year": 2023,
|
|
"publication_type": "ARTICLE",
|
|
"language": "ru",
|
|
"status": 1,
|
|
"url": "https://publications.hse.ru/view/888959076",
|
|
"doi_url": "https://doi.org/10.1/test",
|
|
"citation_text": "Detailed citation",
|
|
"annotation": {"ru": "Аннотация"},
|
|
"description": {"main": "Detailed citation"},
|
|
"authors": [{"id": "1", "title_ru": "Автор"}],
|
|
"raw_data": {"id": "888959076", "title": "Detailed Publication"},
|
|
}
|
|
],
|
|
}
|
|
]
|
|
|
|
employee, _ = _upsert_employee(db_session, first_run, parsed)
|
|
db_session.commit()
|
|
_upsert_employee(db_session, second_run, _parsed_employee_with_publication("published"))
|
|
db_session.commit()
|
|
|
|
publications = db_session.query(EmployeePublication).filter_by(employee_id=employee.id).all()
|
|
assert len(publications) == 1
|
|
assert publications[0].doi_url == "https://doi.org/10.1/test"
|
|
assert publications[0].authors == [{"id": "1", "title_ru": "Автор"}]
|
|
|
|
|
|
def test_upsert_employee_records_publication_errors_without_failing_employee(monkeypatch, db_session):
|
|
run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
|
|
db_session.add(run)
|
|
db_session.commit()
|
|
|
|
def broken_sync(*_args, **_kwargs):
|
|
raise RuntimeError("boom")
|
|
|
|
monkeypatch.setattr("app.services.crawler._sync_employee_publications", broken_sync)
|
|
|
|
employee, changed = _upsert_employee(db_session, run, _parsed_employee_with_publication("error-safe"))
|
|
db_session.commit()
|
|
|
|
assert changed is True
|
|
assert employee.full_name == "Same Person"
|
|
assert db_session.query(Employee).filter_by(profile_key="staff:error-safe").one()
|
|
error = db_session.query(CrawlError).one()
|
|
assert "публикации" in error.message.lower()
|
|
|
|
|
|
def test_upsert_employee_saves_news_links_and_reuses_existing_rows(db_session):
|
|
first_run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
|
|
second_run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
|
|
db_session.add_all([first_run, second_run])
|
|
db_session.commit()
|
|
|
|
employee, _ = _upsert_employee(db_session, first_run, _parsed_employee_with_news("news-person"))
|
|
db_session.commit()
|
|
_upsert_employee(db_session, second_run, _parsed_employee_with_news("news-person"))
|
|
db_session.commit()
|
|
|
|
news_links = db_session.query(EmployeeNewsLink).filter_by(employee_id=employee.id).all()
|
|
assert len(news_links) == 1
|
|
assert news_links[0].title == "News Title"
|
|
assert news_links[0].url == "https://www.hse.ru/news/1.html"
|
|
assert news_links[0].published_year == 2026
|
|
|
|
|
|
def test_upsert_employee_records_news_errors_without_failing_employee(monkeypatch, db_session):
|
|
run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
|
|
db_session.add(run)
|
|
db_session.commit()
|
|
|
|
def broken_sync(*_args, **_kwargs):
|
|
raise RuntimeError("boom")
|
|
|
|
monkeypatch.setattr("app.services.crawler._sync_employee_news_links", broken_sync)
|
|
|
|
employee, changed = _upsert_employee(db_session, run, _parsed_employee_with_news("news-error-safe"))
|
|
db_session.commit()
|
|
|
|
assert changed is True
|
|
assert employee.full_name == "Same Person"
|
|
assert db_session.query(Employee).filter_by(profile_key="staff:news-error-safe").one()
|
|
error = db_session.query(CrawlError).one()
|
|
assert "новости" in error.message.lower()
|
|
|
|
|
|
def test_checksum_changes_when_widget_data_changes():
|
|
base = _parsed_employee("widgets")
|
|
changed = _parsed_employee("widgets")
|
|
changed["sections"] = [
|
|
{
|
|
"type": "publications",
|
|
"publications": [{"id": "1", "title": "New publication"}],
|
|
}
|
|
]
|
|
|
|
assert _checksum(base) != _checksum(changed)
|
|
|
|
|
|
def test_checksum_ignores_date_dependent_experience_text():
|
|
first = _parsed_employee("experience")
|
|
second = _parsed_employee("experience")
|
|
first["sections"] = [{"raw_text": "Стаж работы в НИУ ВШЭ: 5 лет"}]
|
|
second["sections"] = [{"raw_text": "Стаж работы в НИУ ВШЭ: 6 лет"}]
|
|
|
|
assert _checksum(first) == _checksum(second)
|
|
|
|
|
|
def _parsed_employee(profile_id: str) -> dict:
|
|
return {
|
|
"source_url": f"https://www.hse.ru/staff/{profile_id}",
|
|
"profile_type": "staff",
|
|
"profile_id": profile_id,
|
|
"full_name": "Same Person",
|
|
"tabs": [],
|
|
"sections": [],
|
|
"parser_version": "0.6.0",
|
|
"_html": "<html></html>",
|
|
}
|
|
|
|
|
|
def _parsed_employee_with_publication(profile_id: str) -> dict:
|
|
parsed = _parsed_employee(profile_id)
|
|
parsed["sections"] = [
|
|
{
|
|
"type": "publications",
|
|
"publications": [
|
|
{
|
|
"id": "888959076",
|
|
"publication_id": "888959076",
|
|
"title": "Detailed Publication",
|
|
"year": 2023,
|
|
"publication_type": "ARTICLE",
|
|
"language": "ru",
|
|
"status": 1,
|
|
"url": "https://publications.hse.ru/view/888959076",
|
|
"doi_url": "https://doi.org/10.1/test",
|
|
"citation_text": "Detailed citation",
|
|
"annotation": {"ru": "Аннотация"},
|
|
"description": {"main": "Detailed citation"},
|
|
"authors": [{"id": "1", "title_ru": "Автор"}],
|
|
"raw_data": {"id": "888959076", "title": "Detailed Publication"},
|
|
}
|
|
],
|
|
}
|
|
]
|
|
return parsed
|
|
|
|
|
|
def _parsed_employee_with_news(profile_id: str) -> dict:
|
|
parsed = _parsed_employee(profile_id)
|
|
parsed["sections"] = [
|
|
{
|
|
"type": "news",
|
|
"news_links": [
|
|
{
|
|
"title": "News Title",
|
|
"url": "https://www.hse.ru/news/1.html",
|
|
"summary": "News summary",
|
|
"published_at": "2026-04-28T00:00:00+00:00",
|
|
"published_year": 2026,
|
|
"raw_data": {"title": "News Title", "url": "https://www.hse.ru/news/1.html"},
|
|
}
|
|
],
|
|
}
|
|
]
|
|
return parsed
|
|
|
|
|
|
def test_csv_enrichment_survives_repeated_crawl_without_duplicate(db_session):
|
|
from scripts.import_publications import plan
|
|
from app.services.crawler import _sync_employee_publications
|
|
|
|
run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
|
|
db_session.add(run)
|
|
db_session.commit()
|
|
employee, _ = _upsert_employee(db_session, run, _parsed_employee_with_publication("csv-preserved"))
|
|
db_session.commit()
|
|
existing = db_session.query(EmployeePublication).filter_by(employee_id=employee.id).one()
|
|
rows = [(number, {"Название публикации": title, "Год": str(year), "DOI": identifier,
|
|
"Авторы НИУ ВШЭ": employee.full_name, "Квартиль scopus": "Q1"})
|
|
for number, title, year, identifier in (
|
|
(2, existing.title, existing.year, ""),
|
|
(3, "CSV only", 2024, "10.1234/csv-only"),
|
|
(4, "Later HSE", 2024, "10.1234/later"))]
|
|
actions, _ = plan(rows, [{"id": employee.id, "full_name": employee.full_name}],
|
|
[{field: getattr(existing, field) for field in
|
|
("id", "employee_id", "title", "year", "doi_url", "raw_data")}])
|
|
for action in actions:
|
|
if action["result"] == "insert":
|
|
db_session.add(EmployeePublication(**action["values"]))
|
|
else:
|
|
for field, value in action["values"].items():
|
|
setattr(existing, field, value)
|
|
db_session.add(EmployeePublication(employee_id=employee.id, title="Obsolete HSE",
|
|
source_hash="obsolete", raw_data={"id": "obsolete"}))
|
|
db_session.commit()
|
|
original_id = existing.id
|
|
imported_ids = {p.title: p.id for p in db_session.query(EmployeePublication).all()}
|
|
|
|
parsed = _parsed_employee_with_publication("csv-preserved")
|
|
publication = parsed["sections"][0]["publications"][0]
|
|
publication.pop("year")
|
|
publication.pop("doi_url")
|
|
publication["raw_data"]["fresh"] = True
|
|
for _ in range(2):
|
|
_sync_employee_publications(db_session, employee, parsed)
|
|
db_session.commit()
|
|
remaining = db_session.query(EmployeePublication).filter_by(employee_id=employee.id).all()
|
|
assert len(remaining) == 3
|
|
assert all(p.raw_data["csv_enrichment"]["Квартиль scopus"] == "Q1" for p in remaining)
|
|
refreshed = db_session.get(EmployeePublication, original_id)
|
|
assert refreshed.raw_data["fresh"] is True
|
|
assert refreshed.year == 2023 and refreshed.doi_url == "https://doi.org/10.1/test"
|
|
assert db_session.get(EmployeePublication, imported_ids["CSV only"]) is not None
|
|
|
|
parsed["sections"][0]["publications"].append({
|
|
"publication_id": "later-id", "title": "Later HSE", "year": 2024,
|
|
"doi_url": "http://dx.doi.org/10.1234/LATER", "raw_data": {"id": "later-id"}})
|
|
for _ in range(2):
|
|
_sync_employee_publications(db_session, employee, parsed)
|
|
db_session.commit()
|
|
assert db_session.query(EmployeePublication).filter_by(employee_id=employee.id).count() == 3
|
|
adopted = db_session.get(EmployeePublication, imported_ids["Later HSE"])
|
|
assert adopted.publication_id == "later-id"
|
|
assert adopted.raw_data["csv_enrichment"]["Квартиль scopus"] == "Q1"
|
|
|
|
|
|
def test_ambiguous_csv_match_rolls_back_publication_sync(db_session):
|
|
from app.services.crawler import _try_sync_employee_publications
|
|
run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
|
|
db_session.add(run)
|
|
db_session.commit()
|
|
employee, _ = _upsert_employee(db_session, run, _parsed_employee("csv-ambiguous"))
|
|
for index in range(2):
|
|
db_session.add(EmployeePublication(employee_id=employee.id, title="Same", year=2024,
|
|
doi_url="https://doi.org/10.1234/same", source_hash=f"csv-{index}",
|
|
raw_data={"csv_enrichment": {"Год": "2024"}}))
|
|
db_session.commit()
|
|
parsed = {"sections": [{"type": "publications", "publications": [
|
|
{"publication_id": "new", "title": "New"},
|
|
{"publication_id": "same", "title": "Same", "doi_url": "https://doi.org/10.1234/same", "year": 2024}]}]}
|
|
_try_sync_employee_publications(db_session, run, employee, parsed)
|
|
db_session.commit()
|
|
assert db_session.query(EmployeePublication).filter_by(employee_id=employee.id).count() == 2
|
|
assert db_session.query(CrawlError).filter_by(crawl_run_id=run.id, error_type="ValueError").count() == 1
|
|
|
|
|
|
@pytest.mark.parametrize("bound_id", [None, "hse-A"])
|
|
@pytest.mark.parametrize("incoming_ids", [("hse-A", "hse-B"), ("hse-B", "hse-A")])
|
|
def test_two_hse_ids_cannot_overwrite_one_csv_record(db_session, bound_id, incoming_ids):
|
|
from app.services.crawler import _try_sync_employee_publications
|
|
from app.services.admin_data import run_detail_payload
|
|
run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
|
|
db_session.add(run)
|
|
db_session.commit()
|
|
employee, _ = _upsert_employee(db_session, run, _parsed_employee("csv-hse-id-conflict"))
|
|
original_raw = {"csv_enrichment": {"Год": "2024"}, "original": "keep"}
|
|
original_authors = [{"title_ru": "Synthetic original author"}]
|
|
record = EmployeePublication(employee_id=employee.id, title="Same", year=2024,
|
|
publication_id=bound_id, source_hash="csv-original", raw_data=original_raw,
|
|
authors=original_authors)
|
|
db_session.add(record)
|
|
db_session.commit()
|
|
original_id = record.id
|
|
parsed = {"sections": [{"type": "publications", "publications": [
|
|
{"publication_id": identifier, "title": "Same", "year": 2024,
|
|
"authors": [{"id": identifier}], "raw_data": {"id": identifier}}
|
|
for identifier in incoming_ids]}]}
|
|
_try_sync_employee_publications(db_session, run, employee, parsed)
|
|
db_session.commit()
|
|
db_session.expire_all()
|
|
kept = db_session.get(EmployeePublication, original_id)
|
|
assert db_session.query(EmployeePublication).filter_by(employee_id=employee.id).count() == 1
|
|
assert kept.publication_id == bound_id and kept.source_hash == "csv-original"
|
|
assert kept.raw_data == original_raw and kept.authors == original_authors
|
|
errors = run_detail_payload(db_session, run)["errors"]
|
|
assert len(errors) == 1
|
|
assert errors[0]["error_type"] == "ValueError"
|
|
assert "Ambiguous HSE IDs" in errors[0]["message"]
|
|
assert "hse-A" in errors[0]["message"] and "hse-B" in errors[0]["message"]
|