admin/miem_workers
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add dismissed employee status refresh #29

Merged
admin merged 1 commits from feature/check-dismissed-employees into main 2026-08-17 16:11:41 +00:00
Conversation 0 Commits 1 Files Changed 11 +3062 -2602
11 changed files with 3062 additions and 2602 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md Normal file
View File

@@ -0,0 +1,5 @@
# Changelog
## 0.7.1
- Добавлена кнопка «Проверить уволенных» для принудительной сверки статуса уволенных сотрудников с текущим списком источника.

12
README.md
View File

@@ -28,6 +28,8 @@ cp .env.example .env
- `ADMIN_USERNAME`, `ADMIN_PASSWORD`: логин и пароль админки.
- `SESSION_SECRET`: секрет подписи cookie.
- `PARSER_USE_PLAYWRIGHT`: включение Playwright-рендера динамических вкладок.
- `DISMISSAL_CONFIRMATION_RUNS`: сколько последовательных проверок недоступности нужно для увольнения, по умолчанию `3`.
- `MAX_AUTO_DISMISSALS_PER_RUN`: защитный лимит массовых автоматических увольнений за один запуск, по умолчанию `25`.
## Локальный запуск
@@ -92,10 +94,16 @@ curl -X POST http://localhost:8000/api/crawl-runs --cookie "miem_admin_session=.
- найденные сотрудники получают статус `active` и обновленный `last_seen_at`;
- новые сотрудники добавляются в `employees`;
- если профиль перенесен на другой URL, он сопоставляется с прежней записью по единственному точному совпадению ФИО;
- старые URL сохраняются в истории `employee_profile_urls`;
- количество новых сотрудников за запуск сохраняется в `crawl_runs.new_count`;
- публикации из HSE Publications записываются в `employee_publications`, а краткий список остается в JSON профиля;
- новости из блока «В новостях» записываются в `employee_news_links`, а краткий список остается в JSON профиля;
- активные сотрудники, исчезнувшие из текущего списка источника, получают статус `dismissed` и `dismissed_at`;
- один `404` старого профиля переводит сотрудника в статус `verification_required`, а не в `dismissed`;
- статус `dismissed` устанавливается только после нескольких последовательных проверок `404`/`410`;
- сетевые ошибки и ответы `5xx` не считаются подтверждением увольнения;
- если число кандидатов на увольнение превышает защитный лимит, автоматическое увольнение приостанавливается;
- кнопка «Проверить уволенных» сверяет только их profile_key с текущим списком источника и возвращает найденных сотрудников в `active` без обновления содержимого профиля;
- каждый успешный новый или измененный разбор сохраняет запись в `employee_snapshots`;
- неизмененные профили учитываются в `crawl_runs.skipped_count` и не получают новый snapshot.
@@ -139,4 +147,4 @@ docker compose exec postgres pg_dump -U miem miem_workers > backup.sql
docker compose down
```
Версия сервиса: `0.7.0`. Админка всегда показывает версии backend и frontend в footer.
Версия сервиса: `0.7.1`. Админка всегда показывает версии backend и frontend в footer.

21
app/admin.py
View File

@@ -17,7 +17,7 @@ from app.services.admin_data import (
stats_payload,
)
from app.services.crawl_control import get_running_run, run_crawl_if_idle
from app.services.crawler import refresh_employee
from app.services.crawler import refresh_dismissed_status, refresh_employee
from app.version import BACKEND_VERSION, FRONTEND_VERSION
router = APIRouter(prefix="/admin")
@@ -231,6 +231,25 @@ def crawl_now(
return RedirectResponse("/admin", status_code=303)
@router.post("/dismissed/refresh")
def refresh_dismissed(
request: Request,
background_tasks: BackgroundTasks,
db: Session = Depends(get_db),
settings: Settings = Depends(get_settings),
):
require_admin(request, settings)
if get_running_run(db):
return RedirectResponse("/admin", status_code=303)
def _refresh() -> None:
with SessionLocal() as db:
refresh_dismissed_status(db, settings)
background_tasks.add_task(_refresh)
return RedirectResponse("/admin", status_code=303)
def _render(request: Request, template: str, context: dict, status_code: int = 200) -> HTMLResponse:
payload = {
"request": request,

228
app/services/crawler.py
View File

@@ -17,6 +17,7 @@ from app.models import (
Employee,
EmployeeNewsLink,
EmployeePublication,
EmployeeProfileUrl,
EmployeeSnapshot,
ParserSource,
ProfileTab,
@@ -66,7 +67,9 @@ def run_crawl(db: Session, settings: Settings) -> CrawlRun:
)
if not parsed:
continue
_, changed = _upsert_employee(db, run, parsed)
employee, changed = _upsert_employee(db, run, parsed)
if employee.profile_key:
found_keys.add(employee.profile_key)
if changed:
parsed_count += 1
else:
@@ -88,7 +91,15 @@ def run_crawl(db: Session, settings: Settings) -> CrawlRun:
finally:
time.sleep(settings.request_delay_seconds)
run.dismissed_count = _mark_dismissed(db, run, found_keys, session, settings.request_timeout)
run.dismissed_count = _mark_dismissed(
db,
run,
found_keys,
session,
settings.request_timeout,
confirmation_runs=settings.dismissal_confirmation_runs,
max_auto_dismissals=settings.max_auto_dismissals_per_run,
)
run.status = "completed"
get_or_create_current_version(db, crawl_run_id=run.id)
except Exception as exc:
@@ -101,6 +112,59 @@ def run_crawl(db: Session, settings: Settings) -> CrawlRun:
return run
def refresh_dismissed_status(db: Session, settings: Settings) -> CrawlRun:
source = _ensure_source(db, settings.source_url)
run = CrawlRun(source_url=source.source_url, status="running")
db.add(run)
db.commit()
db.refresh(run)
try:
employees = db.scalars(select(Employee).where(Employee.status == "dismissed")).all()
run.found_count = len(employees)
with requests.Session() as session:
urls = collect_profile_links(session, source.source_url, HEADERS, settings.request_timeout)
source_keys = {key for url in urls if (key := profile_key(url))}
now = datetime.now(timezone.utc)
for employee in employees:
if employee.profile_key not in source_keys:
run.skipped_count += 1
continue
employee.status = "active"
employee.dismissed_at = None
employee.last_seen_at = now
employee.profile_unavailable_streak = 0
employee.last_profile_check_at = now
_record_employee_change(
db,
run,
employee,
"reactivated",
profile_available=True,
message="Сотрудник снова найден в исходном списке.",
)
run.parsed_count += 1
run.status = "completed"
get_or_create_current_version(db, crawl_run_id=run.id)
except Exception as exc:
run.status = "failed"
run.error_count = 1
run.message = str(exc)
db.add(
CrawlError(
crawl_run_id=run.id,
profile_url=source.source_url,
error_type=type(exc).__name__,
message=str(exc),
)
)
finally:
run.finished_at = datetime.now(timezone.utc)
db.commit()
db.refresh(run)
return run
def refresh_employee(db: Session, employee: Employee, settings: Settings) -> CrawlRun:
run = CrawlRun(source_url=employee.canonical_url, status="running", found_count=1)
db.add(run)
@@ -170,6 +234,8 @@ def _upsert_employee(db: Session, run: CrawlRun, parsed: dict) -> tuple[Employee
checksum = _checksum(parsed)
key = _parsed_profile_key(parsed)
employee = db.scalar(select(Employee).where(Employee.profile_key == key))
if not employee:
employee = _find_employee_with_moved_profile(db, parsed)
now = datetime.now(timezone.utc)
if not employee:
employee = Employee(
@@ -187,15 +253,23 @@ def _upsert_employee(db: Session, run: CrawlRun, parsed: dict) -> tuple[Employee
parser_version = parsed.get("parser_version")
changed = is_new or employee.current_checksum != checksum or employee.parser_version != parser_version
previous_url = employee.canonical_url if employee.canonical_url != parsed["source_url"] else None
employee.profile_key = key
employee.profile_type = parsed.get("profile_type")
employee.profile_id = parsed.get("profile_id")
employee.canonical_url = parsed["source_url"]
employee.full_name = parsed.get("full_name")
employee.status = "active"
employee.last_seen_at = now
employee.dismissed_at = None
employee.profile_unavailable_streak = 0
employee.last_profile_check_at = now
employee.parser_version = parser_version
if changed:
employee.current_data = parsed
employee.current_checksum = checksum
db.flush()
_sync_profile_url_history(db, employee, previous_url, employee.canonical_url, now)
if is_new:
_record_employee_change(
@@ -235,6 +309,74 @@ def _upsert_employee(db: Session, run: CrawlRun, parsed: dict) -> tuple[Employee
return employee, changed
def _find_employee_with_moved_profile(db: Session, parsed: dict) -> Employee | None:
full_name = parsed.get("full_name")
if not full_name:
return None
candidates = db.scalars(select(Employee).where(Employee.full_name == full_name)).all()
if len(candidates) == 1:
return candidates[0]
parsed_identity = _profile_identity_values(parsed)
if not parsed_identity:
return None
matches = [candidate for candidate in candidates if parsed_identity & _employee_identity_values(candidate)]
return matches[0] if len(matches) == 1 else None
def _profile_identity_values(profile: dict) -> set[str]:
if not isinstance(profile, dict):
return set()
values = set()
contacts = profile.get("contacts") or {}
if not isinstance(contacts, dict):
contacts = {}
for email in contacts.get("emails") or []:
normalized = str(email).strip().lower()
if normalized:
values.add(f"email:{normalized}")
for item in profile.get("external_ids") or []:
if not isinstance(item, dict):
continue
system = str(item.get("system") or "").strip().lower()
value = str(item.get("value") or "").strip().lower()
if system and value:
values.add(f"external:{system}:{value}")
return values
def _employee_identity_values(employee: Employee) -> set[str]:
return _profile_identity_values(employee.current_data or {})
def _sync_profile_url_history(
db: Session,
employee: Employee,
previous_url: str | None,
current_url: str,
seen_at: datetime,
) -> None:
urls = {url for url in (previous_url, current_url) if url}
for url in urls:
history = db.scalar(
select(EmployeeProfileUrl).where(
EmployeeProfileUrl.employee_id == employee.id,
EmployeeProfileUrl.url == url,
)
)
if history:
history.last_seen_at = seen_at
else:
db.add(
EmployeeProfileUrl(
employee_id=employee.id,
url=url,
first_seen_at=seen_at,
last_seen_at=seen_at,
)
)
def _try_sync_employee_publications(db: Session, run: CrawlRun, employee: Employee, parsed: dict) -> None:
try:
if not _publication_payloads(parsed):
@@ -453,15 +595,41 @@ def _datetime_or_none(value: object) -> datetime | None:
return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)
def _mark_dismissed(db: Session, run: CrawlRun, found_keys: set[str], session: requests.Session, timeout: int) -> int:
def _mark_dismissed(
db: Session,
run: CrawlRun,
found_keys: set[str],
session: requests.Session,
timeout: int,
*,
confirmation_runs: int = 3,
max_auto_dismissals: int | None = 25,
) -> int:
dismissed = 0
active = db.scalars(select(Employee).where(Employee.status == "active")).all()
candidates = db.scalars(
select(Employee).where(Employee.status.in_(("active", "verification_required")))
).all()
now = datetime.now(timezone.utc)
for employee in active:
unavailable = []
for employee in candidates:
if employee.profile_key in found_keys:
continue
profile_available = _profile_is_available(session, employee.canonical_url, timeout)
profile_available = _profile_check(session, employee.canonical_url, timeout)
employee.last_profile_check_at = now
if profile_available is None:
db.add(
CrawlError(
crawl_run_id=run.id,
profile_url=employee.canonical_url,
error_type="ProfileAvailabilityCheckError",
message="Не удалось надёжно проверить доступность профиля; статус сотрудника не изменён.",
)
)
continue
if profile_available:
employee.profile_unavailable_streak = 0
if employee.status == "verification_required":
employee.status = "active"
_record_employee_change(
db,
run,
@@ -471,6 +639,37 @@ def _mark_dismissed(db: Session, run: CrawlRun, found_keys: set[str], session: r
message="Профиль доступен, но ссылка отсутствует в исходном списке.",
)
continue
next_streak = employee.profile_unavailable_streak + 1
unavailable.append((employee, next_streak))
dismissal_blocked = bool(
max_auto_dismissals is not None and len(unavailable) > max_auto_dismissals
)
if dismissal_blocked:
run.message = (
f"Автоматическое увольнение приостановлено: {len(unavailable)} профилей "
f"одновременно не подтвердились (лимит {max_auto_dismissals})."
)
for employee, next_streak in unavailable:
employee.profile_unavailable_streak = next_streak
if next_streak < confirmation_runs or dismissal_blocked:
employee.status = "verification_required"
_record_employee_change(
db,
run,
employee,
"verification_required",
profile_available=False,
message=(
"Профиль не подтвердился. Автоматическое увольнение отложено до "
f"{confirmation_runs} последовательных проверок."
if not dismissal_blocked
else "Автоматическое увольнение отложено из-за массовой ошибки проверки профилей."
),
)
continue
employee.status = "dismissed"
employee.dismissed_at = now
_record_employee_change(
@@ -479,7 +678,10 @@ def _mark_dismissed(db: Session, run: CrawlRun, found_keys: set[str], session: r
employee,
"dismissed",
profile_available=False,
message="Сотрудник отсутствует в исходном списке, профиль не подтвердился как доступный.",
message=(
"Сотрудник отсутствует в исходном списке, профиль не подтвердился "
f"{confirmation_runs} раза подряд."
),
)
dismissed += 1
db.commit()
@@ -487,11 +689,19 @@ def _mark_dismissed(db: Session, run: CrawlRun, found_keys: set[str], session: r
def _profile_is_available(session: requests.Session, url: str, timeout: int) -> bool:
return _profile_check(session, url, timeout) is True
def _profile_check(session: requests.Session, url: str, timeout: int) -> bool | None:
try:
response = session.get(url, headers=HEADERS, timeout=timeout, allow_redirects=True)
return response.status_code < 400
if response.status_code < 400:
return True
if response.status_code in {404, 410}:
return False
return None
except requests.RequestException:
return False
return None
def _record_employee_change(

16
app/static/admin.css
View File

@@ -137,6 +137,11 @@
color: #991b1b;
}
.badge--verification {
background: #fef3c7;
color: #92400e;
}
.form {
display: grid;
gap: 12px;
@@ -428,6 +433,17 @@
gap: 16px;
}
.progress-panel__actions {
display: flex;
flex-wrap: wrap;
gap: 8px;
}
.button--secondary {
color: #0f766e;
background: #ccfbf1;
}
.progress-panel__body {
display: grid;
gap: 10px;

12
app/templates/dashboard.html
View File

@@ -4,6 +4,7 @@
<section class="admin__grid">
<a class="metric metric--link" href="/admin/directory"><span class="metric__label">Всего в базе</span><span class="metric__value">{{ counts.total }}</span></a>
<a class="metric metric--link" href="/admin/directory?status=active"><span class="metric__label">Работают</span><span class="metric__value">{{ counts.active }}</span></a>
<a class="metric metric--link" href="/admin/directory?status=verification_required"><span class="metric__label">Требуют проверки</span><span class="metric__value">{{ counts.verification_required }}</span></a>
<a class="metric metric--link" href="{% if latest_run %}/admin/runs/{{ latest_run.id }}#new-employees{% else %}/admin/runs{% endif %}"><span class="metric__label">Новые за запуск</span><span class="metric__value">{{ counts.new_in_last_run }}</span></a>
<a class="metric metric--link" href="/admin/directory?status=dismissed"><span class="metric__label">Уволены</span><span class="metric__value">{{ counts.dismissed }}</span></a>
</section>
@@ -28,9 +29,14 @@
<section class="panel progress-panel" data-progress-panel>
<div class="progress-panel__header">
<h2 class="panel__title">Прогресс парсинга</h2>
<form method="post" action="/admin/crawl-now">
<button class="button" type="submit">Запустить парсинг</button>
</form>
<div class="progress-panel__actions">
<form method="post" action="/admin/crawl-now">
<button class="button" type="submit">Запустить парсинг</button>
</form>
<form method="post" action="/admin/dismissed/refresh">
<button class="button button--secondary" type="submit">Проверить уволенных</button>
</form>
</div>
</div>
{% set run = counts.current_running_run or latest_run %}
<div class="progress-panel__body" data-progress-body>

6
app/version.py
View File

@@ -1,3 +1,3 @@
APP_VERSION = "0.7.0"
FRONTEND_VERSION = "0.7.0"
BACKEND_VERSION = "0.7.0"
APP_VERSION = "0.7.1"
FRONTEND_VERSION = "0.7.1"
BACKEND_VERSION = "0.7.1"

2
pyproject.toml
View File

@@ -1,6 +1,6 @@
[project]
name = "miem-workers"
version = "0.7.0"
version = "0.7.1"
description = "MIEM employees parser, admin API, and MCP server"
requires-python = ">=3.11"
dependencies = [

10
tests/test_admin_templates.py
View File

@@ -27,6 +27,7 @@ def test_directory_template_is_russian_and_uses_display_dates():
assert "employee.first_seen_display" in template
assert "employee.last_seen_display" in template
assert "employee.dismissed_display" in template
assert "verification_required" in template
assert "Directory" not in template
assert "employees found" not in template
@@ -62,6 +63,7 @@ def test_run_detail_template_extends_base_and_shows_change_groups():
assert 'id="new-employees"' in template
assert "Новые сотрудники" in template
assert "Потеряшки" in template
assert "Требуют проверки" in template
assert "Уволенные" in template
assert "Детализация сотрудников для этого запуска недоступна" in template
@@ -71,11 +73,19 @@ def test_dashboard_metric_cards_link_to_admin_targets():
assert 'href="/admin/directory"' in template
assert 'href="/admin/directory?status=active"' in template
assert 'href="/admin/directory?status=verification_required"' in template
assert '/admin/runs/{{ latest_run.id }}#new-employees' in template
assert 'href="/admin/directory?status=dismissed"' in template
assert 'href="/admin/runs"' in template
def test_dashboard_has_dismissed_status_refresh_action():
template = Path("app/templates/dashboard.html").read_text(encoding="utf-8")
assert 'action="/admin/dismissed/refresh"' in template
assert "Проверить уволенных" in template
def test_dashboard_latest_run_rows_link_to_run_detail():
template = Path("app/templates/dashboard.html").read_text(encoding="utf-8")

4
tests/test_api_mcp.py
View File

@@ -20,7 +20,7 @@ def test_health_returns_versions():
response = client.get("/api/health")
assert response.status_code == 200
assert response.json()["backend_version"] == "0.7.0"
assert response.json()["backend_version"] == "0.7.1"
def test_mcp_lists_tools_without_auth_and_ignores_auth_header():
@@ -154,7 +154,7 @@ def test_mcp_service_info_returns_tools_and_dataset_hash():
assert response.status_code == 200
payload = json.loads(response.json()["result"]["content"][0]["text"])
assert payload["service_name"] == "miem-employees"
assert payload["backend_version"] == "0.7.0"
assert payload["backend_version"] == "0.7.1"
assert payload["dataset"]["hash"]
assert any(tool["name"] == "sync_employees" for tool in payload["tools"])

196
tests/test_crawler.py
View File

@@ -11,7 +11,8 @@ from app.models import (
EmployeeSnapshot,
ParseResourceCache,
)
from app.services.crawler import _checksum, _mark_dismissed, _upsert_employee
from app.config import Settings
from app.services.crawler import _checksum, _mark_dismissed, _upsert_employee, refresh_dismissed_status
from app.services.resource_cache import ResourceCache
@@ -56,6 +57,41 @@ class ConditionalSession:
return ConditionalResponse(200, "fresh", {"ETag": '"fresh"'})
def test_refresh_dismissed_status_reactivates_only_profiles_in_source(monkeypatch, db_session):
now = datetime.now(timezone.utc)
found = Employee(
profile_key="staff:returned",
canonical_url="https://www.hse.ru/staff/returned",
status="dismissed",
dismissed_at=now,
first_seen_at=now,
last_seen_at=now,
)
still_dismissed = Employee(
profile_key="staff:gone",
canonical_url="https://www.hse.ru/staff/gone",
status="dismissed",
dismissed_at=now,
first_seen_at=now,
last_seen_at=now,
)
db_session.add_all([found, still_dismissed])
db_session.commit()
monkeypatch.setattr(
"app.services.crawler.collect_profile_links",
lambda *_args, **_kwargs: ["https://www.hse.ru/staff/returned"],
)
run = refresh_dismissed_status(db_session, Settings())
assert run.status == "completed"
assert run.parsed_count == 1
assert run.skipped_count == 1
assert found.status == "active"
assert found.dismissed_at is None
assert still_dismissed.status == "dismissed"
def test_mark_dismissed_records_missing_source_when_profile_is_available(db_session):
run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
db_session.add(run)
@@ -97,7 +133,7 @@ def test_mark_dismissed_records_missing_source_when_profile_is_available(db_sess
assert change.profile_available is True
def test_mark_dismissed_marks_missing_employee_when_profile_is_unavailable(db_session):
def test_mark_dismissed_requires_consecutive_unavailable_checks(db_session):
run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
employee = Employee(
profile_key="staff:gone",
@@ -109,22 +145,129 @@ def test_mark_dismissed_marks_missing_employee_when_profile_is_unavailable(db_se
db_session.add_all([run, employee])
db_session.commit()
dismissed = _mark_dismissed(
first_check = _mark_dismissed(
db_session,
run,
set(),
FakeSession({"https://www.hse.ru/staff/gone": 404}),
30,
confirmation_runs=2,
)
assert dismissed == 1
assert first_check == 0
assert employee.status == "verification_required"
assert employee.dismissed_at is None
assert employee.profile_unavailable_streak == 1
assert db_session.query(CrawlRunEmployeeChange).one().change_type == "verification_required"
second_check = _mark_dismissed(
db_session,
run,
set(),
FakeSession({"https://www.hse.ru/staff/gone": 404}),
30,
confirmation_runs=2,
)
assert second_check == 1
assert employee.status == "dismissed"
assert employee.dismissed_at is not None
change = db_session.query(CrawlRunEmployeeChange).one()
assert employee.profile_unavailable_streak == 2
change = db_session.query(CrawlRunEmployeeChange).order_by(CrawlRunEmployeeChange.id).all()[-1]
assert change.change_type == "dismissed"
assert change.profile_available is False
def test_mark_dismissed_does_not_dismiss_on_server_error(db_session):
run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
employee = Employee(
profile_key="staff:temporary-error",
canonical_url="https://www.hse.ru/staff/temporary-error",
status="active",
first_seen_at=datetime.now(timezone.utc),
last_seen_at=datetime.now(timezone.utc),
)
db_session.add_all([run, employee])
db_session.commit()
dismissed = _mark_dismissed(
db_session,
run,
set(),
FakeSession({"https://www.hse.ru/staff/temporary-error": 503}),
30,
confirmation_runs=1,
)
assert dismissed == 0
assert employee.status == "active"
assert employee.profile_unavailable_streak == 0
assert db_session.query(CrawlError).one().error_type == "ProfileAvailabilityCheckError"
def test_available_profile_resets_verification_streak(db_session):
run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
employee = Employee(
profile_key="staff:restored",
canonical_url="https://www.hse.ru/staff/restored",
status="active",
first_seen_at=datetime.now(timezone.utc),
last_seen_at=datetime.now(timezone.utc),
)
db_session.add_all([run, employee])
db_session.commit()
_mark_dismissed(
db_session,
run,
set(),
FakeSession({"https://www.hse.ru/staff/restored": 404}),
30,
confirmation_runs=3,
)
_mark_dismissed(
db_session,
run,
set(),
FakeSession({"https://www.hse.ru/staff/restored": 200}),
30,
confirmation_runs=3,
)
assert employee.status == "active"
assert employee.profile_unavailable_streak == 0
def test_mark_dismissed_blocks_mass_auto_dismissals(db_session):
run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
employees = [
Employee(
profile_key=f"staff:gone-{index}",
canonical_url=f"https://www.hse.ru/staff/gone-{index}",
status="active",
first_seen_at=datetime.now(timezone.utc),
last_seen_at=datetime.now(timezone.utc),
)
for index in range(2)
]
db_session.add_all([run, *employees])
db_session.commit()
dismissed = _mark_dismissed(
db_session,
run,
set(),
FakeSession({employee.canonical_url: 404 for employee in employees}),
30,
confirmation_runs=1,
max_auto_dismissals=1,
)
assert dismissed == 0
assert {employee.status for employee in employees} == {"verification_required"}
assert "приостановлено" in run.message
def test_upsert_employee_increments_new_count_and_records_change_for_new_employee(db_session):
run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
db_session.add(run)
@@ -152,6 +295,49 @@ def test_upsert_employee_increments_new_count_and_records_change_for_new_employe
assert change.full_name == "New Person"
def test_upsert_employee_reconciles_profile_moved_to_new_url(db_session):
run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
employee = Employee(
profile_key="staff:abelov",
canonical_url="https://www.hse.ru/staff/abelov",
full_name="Белов Александр Владимирович",
status="active",
first_seen_at=datetime.now(timezone.utc),
last_seen_at=datetime.now(timezone.utc),
)
db_session.add_all([run, employee])
db_session.commit()
employee_id = employee.id
updated, changed = _upsert_employee(
db_session,
run,
{
"source_url": "https://www.hse.ru/org/persons/47634735",
"profile_type": "org_person",
"profile_id": "47634735",
"full_name": "Белов Александр Владимирович",
"tabs": [],
"sections": [],
"parser_version": "0.7.0",
"_html": "<html></html>",
},
)
db_session.commit()
assert changed is True
assert updated.id == employee_id
assert updated.profile_key == "org_person:47634735"
assert updated.canonical_url == "https://www.hse.ru/org/persons/47634735"
assert updated.status == "active"
assert run.new_count == 0
assert db_session.query(Employee).count() == 1
assert {item.url for item in updated.profile_urls} == {
"https://www.hse.ru/staff/abelov",
"https://www.hse.ru/org/persons/47634735",
}
def test_resource_cache_uses_etag_and_reuses_cached_body_on_304(db_session):
db_session.add(
ParseResourceCache(