docs: update changelog for academic degree filter

Reviewed-on: #29

CHANGELOG.md

@@ -0,0 +1,9 @@
+# Changelog
+
+## 0.7.2
+
+- В каталоге сотрудников добавлены фильтр и колонка учёной степени.
+
+## 0.7.1
+
+- Добавлена кнопка «Проверить уволенных» для принудительной сверки статуса уволенных сотрудников с текущим списком источника.

README.md

@@ -28,6 +28,8 @@ cp .env.example .env
 - `ADMIN_USERNAME`, `ADMIN_PASSWORD`: логин и пароль админки.
 - `SESSION_SECRET`: секрет подписи cookie.
 - `PARSER_USE_PLAYWRIGHT`: включение Playwright-рендера динамических вкладок.
+- `DISMISSAL_CONFIRMATION_RUNS`: сколько последовательных проверок недоступности нужно для увольнения, по умолчанию `3`.
+- `MAX_AUTO_DISMISSALS_PER_RUN`: защитный лимит массовых автоматических увольнений за один запуск, по умолчанию `25`.
 
 ## Локальный запуск
 
@@ -92,10 +94,16 @@ curl -X POST http://localhost:8000/api/crawl-runs --cookie "miem_admin_session=.
 
 - найденные сотрудники получают статус `active` и обновленный `last_seen_at`;
 - новые сотрудники добавляются в `employees`;
+- если профиль перенесен на другой URL, он сопоставляется с прежней записью по единственному точному совпадению ФИО;
+- старые URL сохраняются в истории `employee_profile_urls`;
 - количество новых сотрудников за запуск сохраняется в `crawl_runs.new_count`;
 - публикации из HSE Publications записываются в `employee_publications`, а краткий список остается в JSON профиля;
 - новости из блока «В новостях» записываются в `employee_news_links`, а краткий список остается в JSON профиля;
-- активные сотрудники, исчезнувшие из текущего списка источника, получают статус `dismissed` и `dismissed_at`;
+- один `404` старого профиля переводит сотрудника в статус `verification_required`, а не в `dismissed`;
+- статус `dismissed` устанавливается только после нескольких последовательных проверок `404`/`410`;
+- сетевые ошибки и ответы `5xx` не считаются подтверждением увольнения;
+- если число кандидатов на увольнение превышает защитный лимит, автоматическое увольнение приостанавливается;
+- кнопка «Проверить уволенных» сверяет только их profile_key с текущим списком источника и возвращает найденных сотрудников в `active` без обновления содержимого профиля;
 - каждый успешный новый или измененный разбор сохраняет запись в `employee_snapshots`;
 - неизмененные профили учитываются в `crawl_runs.skipped_count` и не получают новый snapshot.
 
@@ -139,4 +147,4 @@ docker compose exec postgres pg_dump -U miem miem_workers > backup.sql
 docker compose down
 ```
 
-Версия сервиса: `0.7.0`. Админка всегда показывает версии backend и frontend в footer.
+Версия сервиса: `0.7.1`. Админка всегда показывает версии backend и frontend в footer.

app/admin.py

@@ -17,7 +17,7 @@ from app.services.admin_data import (
     stats_payload,
 )
 from app.services.crawl_control import get_running_run, run_crawl_if_idle
-from app.services.crawler import refresh_employee
+from app.services.crawler import refresh_dismissed_status, refresh_employee
 from app.version import BACKEND_VERSION, FRONTEND_VERSION
 
 router = APIRouter(prefix="/admin")
@@ -80,6 +80,7 @@ def directory(
     started_from: str | None = None,
     started_to: str | None = None,
     has_email: str | None = None,
+    has_academic_degree: str | None = None,
     sort: str = "full_name",
     direction: str = "asc",
     limit: int = 50,
@@ -91,6 +92,7 @@ def directory(
     parsed_started_from = _parse_date(started_from)
     parsed_started_to = _parse_date(started_to)
     parsed_has_email = None if has_email in (None, "") else has_email == "true"
+    parsed_has_academic_degree = None if has_academic_degree in (None, "") else has_academic_degree == "true"
     page = list_employees_page(
         db,
         status=status,
@@ -98,6 +100,7 @@ def directory(
         started_from=parsed_started_from,
         started_to=parsed_started_to,
         has_email=parsed_has_email,
+        has_academic_degree=parsed_has_academic_degree,
         sort=sort,
         direction=direction,
         limit=limit,
@@ -114,6 +117,7 @@ def directory(
                 "started_from": started_from or "",
                 "started_to": started_to or "",
                 "has_email": has_email or "",
+                "has_academic_degree": has_academic_degree or "",
                 "sort": sort,
                 "direction": direction,
                 "limit": page["limit"],
@@ -231,6 +235,25 @@ def crawl_now(
     return RedirectResponse("/admin", status_code=303)
 
 
+@router.post("/dismissed/refresh")
+def refresh_dismissed(
+    request: Request,
+    background_tasks: BackgroundTasks,
+    db: Session = Depends(get_db),
+    settings: Settings = Depends(get_settings),
+):
+    require_admin(request, settings)
+    if get_running_run(db):
+        return RedirectResponse("/admin", status_code=303)
+
+    def _refresh() -> None:
+        with SessionLocal() as db:
+            refresh_dismissed_status(db, settings)
+
+    background_tasks.add_task(_refresh)
+    return RedirectResponse("/admin", status_code=303)
+
+
 def _render(request: Request, template: str, context: dict, status_code: int = 200) -> HTMLResponse:
     payload = {
         "request": request,

app/api.py

@@ -28,6 +28,7 @@ def list_employees(
     started_from: date | None = None,
     started_to: date | None = None,
     has_email: bool | None = None,
+    has_academic_degree: bool | None = None,
     sort: str = "full_name",
     direction: str = "asc",
     limit: int = 50,
@@ -43,6 +44,7 @@ def list_employees(
         started_from=started_from,
         started_to=started_to,
         has_email=has_email,
+        has_academic_degree=has_academic_degree,
         sort=sort,
         direction=direction,
         limit=limit,

app/services/admin_data.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import re
 from datetime import date, datetime, time
 from math import ceil
 from typing import Any
@@ -19,6 +20,8 @@ EMPLOYEE_SORTS = {
     "hse_start_year": Employee.current_data["hse_start_year"].as_integer(),
 }
 
+_ACADEMIC_DEGREE_PATTERN = re.compile(r"\b(?:кандидат|доктор)\s+[\w\s-]{0,80}?\s+наук\b|\bph\.?\s*d\.?\b", re.IGNORECASE)
+
 
 def employee_display_payload(employee: Employee) -> dict[str, Any]:
     data = _as_dict(employee.current_data)
@@ -28,6 +31,7 @@ def employee_display_payload(employee: Employee) -> dict[str, Any]:
     positions = _clean_list(data.get("positions"))
     emails = _clean_list(contacts.get("emails"))
     phones = _clean_list(contacts.get("phones"))
+    academic_degrees = _academic_degrees(sections)
     return {
         "id": employee.id,
         "full_name": employee.full_name,
@@ -42,6 +46,7 @@ def employee_display_payload(employee: Employee) -> dict[str, Any]:
         "phones": phones,
         "phone_text": ", ".join(phones),
         "address": contacts.get("address"),
+        "academic_degree_text": "; ".join(academic_degrees),
         "publications_count": _count_section_items(sections, "publications"),
         "courses_count": _count_section_items(sections, "courses_by_year"),
         "news_count": len(stored_news_links) or _count_section_items(sections, "news"),
@@ -81,6 +86,7 @@ def build_employee_query(
     started_from: date | None = None,
     started_to: date | None = None,
     has_email: bool | None = None,
+    has_academic_degree: bool | None = None,
 ) -> Select[tuple[Employee]]:
     stmt = select(Employee)
     filters = []
@@ -97,6 +103,15 @@ def build_employee_query(
         filters.append(Employee.current_data.cast(Text).ilike("%@%"))
     elif has_email is False:
         filters.append(or_(Employee.current_data.is_(None), ~Employee.current_data.cast(Text).ilike("%@%")))
+    if has_academic_degree is not None:
+        data_text = Employee.current_data.cast(Text)
+        degree_condition = or_(
+            and_(_json_text_contains(data_text, "кандидат"), _json_text_contains(data_text, "наук")),
+            and_(_json_text_contains(data_text, "доктор"), _json_text_contains(data_text, "наук")),
+            _json_text_contains(data_text, "phd"),
+            _json_text_contains(data_text, "ph.d."),
+        )
+        filters.append(degree_condition if has_academic_degree else or_(Employee.current_data.is_(None), ~degree_condition))
     if filters:
         stmt = stmt.where(and_(*filters))
     return stmt
@@ -110,6 +125,7 @@ def list_employees_page(
     started_from: date | None = None,
     started_to: date | None = None,
     has_email: bool | None = None,
+    has_academic_degree: bool | None = None,
     sort: str = "full_name",
     direction: str = "asc",
     limit: int = 50,
@@ -123,6 +139,7 @@ def list_employees_page(
         started_from=started_from,
         started_to=started_to,
         has_email=has_email,
+        has_academic_degree=has_academic_degree,
     )
     total = db.scalar(select(func.count()).select_from(base_stmt.subquery())) or 0
     sort_column = EMPLOYEE_SORTS.get(sort, Employee.full_name)
@@ -214,6 +231,36 @@ def format_admin_datetime(value: Any) -> str:
     return value.strftime("%d.%m.%Y %H:%M")
 
 
+def _academic_degrees(sections: list[Any]) -> list[str]:
+    degrees = []
+    for section in sections:
+        section_data = _as_dict(section)
+        title = str(section_data.get("title") or "")
+        if not re.search(r"уч[её]н.*степен|academic degree", title, re.IGNORECASE):
+            continue
+        values = [
+            *(_as_dict(entry).get("text") for entry in _as_list(section_data.get("year_entries"))),
+            *_clean_list(section_data.get("paragraphs")),
+            *_clean_list(section_data.get("items")),
+            section_data.get("raw_text"),
+        ]
+        for value in values:
+            text = str(value or "").strip()
+            if text and _ACADEMIC_DEGREE_PATTERN.search(text) and text not in degrees:
+                degrees.append(text)
+    return degrees
+
+
+def _json_text_contains(data_text: Any, value: str) -> Any:
+    escaped = value.encode("unicode_escape").decode("ascii")
+    escaped_capitalized = value.capitalize().encode("unicode_escape").decode("ascii")
+    return or_(
+        data_text.ilike(f"%{value}%"),
+        data_text.ilike(f"%{escaped}%"),
+        data_text.ilike(f"%{escaped_capitalized}%"),
+    )
+
+
 def _employee_status_display(status: str | None) -> str:
     labels = {"active": "Работает", "dismissed": "Уволен"}
     return labels.get(status or "", status or "Не указано")

app/services/crawler.py

@@ -17,6 +17,7 @@ from app.models import (
     Employee,
     EmployeeNewsLink,
     EmployeePublication,
+    EmployeeProfileUrl,
     EmployeeSnapshot,
     ParserSource,
     ProfileTab,
@@ -66,7 +67,9 @@ def run_crawl(db: Session, settings: Settings) -> CrawlRun:
                     )
                     if not parsed:
                         continue
-                    _, changed = _upsert_employee(db, run, parsed)
+                    employee, changed = _upsert_employee(db, run, parsed)
+                    if employee.profile_key:
+                        found_keys.add(employee.profile_key)
                     if changed:
                         parsed_count += 1
                     else:
@@ -88,7 +91,15 @@ def run_crawl(db: Session, settings: Settings) -> CrawlRun:
                 finally:
                     time.sleep(settings.request_delay_seconds)
 
-            run.dismissed_count = _mark_dismissed(db, run, found_keys, session, settings.request_timeout)
+            run.dismissed_count = _mark_dismissed(
+                db,
+                run,
+                found_keys,
+                session,
+                settings.request_timeout,
+                confirmation_runs=settings.dismissal_confirmation_runs,
+                max_auto_dismissals=settings.max_auto_dismissals_per_run,
+            )
         run.status = "completed"
         get_or_create_current_version(db, crawl_run_id=run.id)
     except Exception as exc:
@@ -101,6 +112,59 @@ def run_crawl(db: Session, settings: Settings) -> CrawlRun:
     return run
 
 
+def refresh_dismissed_status(db: Session, settings: Settings) -> CrawlRun:
+    source = _ensure_source(db, settings.source_url)
+    run = CrawlRun(source_url=source.source_url, status="running")
+    db.add(run)
+    db.commit()
+    db.refresh(run)
+
+    try:
+        employees = db.scalars(select(Employee).where(Employee.status == "dismissed")).all()
+        run.found_count = len(employees)
+        with requests.Session() as session:
+            urls = collect_profile_links(session, source.source_url, HEADERS, settings.request_timeout)
+        source_keys = {key for url in urls if (key := profile_key(url))}
+        now = datetime.now(timezone.utc)
+        for employee in employees:
+            if employee.profile_key not in source_keys:
+                run.skipped_count += 1
+                continue
+            employee.status = "active"
+            employee.dismissed_at = None
+            employee.last_seen_at = now
+            employee.profile_unavailable_streak = 0
+            employee.last_profile_check_at = now
+            _record_employee_change(
+                db,
+                run,
+                employee,
+                "reactivated",
+                profile_available=True,
+                message="Сотрудник снова найден в исходном списке.",
+            )
+            run.parsed_count += 1
+        run.status = "completed"
+        get_or_create_current_version(db, crawl_run_id=run.id)
+    except Exception as exc:
+        run.status = "failed"
+        run.error_count = 1
+        run.message = str(exc)
+        db.add(
+            CrawlError(
+                crawl_run_id=run.id,
+                profile_url=source.source_url,
+                error_type=type(exc).__name__,
+                message=str(exc),
+            )
+        )
+    finally:
+        run.finished_at = datetime.now(timezone.utc)
+        db.commit()
+        db.refresh(run)
+    return run
+
+
 def refresh_employee(db: Session, employee: Employee, settings: Settings) -> CrawlRun:
     run = CrawlRun(source_url=employee.canonical_url, status="running", found_count=1)
     db.add(run)
@@ -170,6 +234,8 @@ def _upsert_employee(db: Session, run: CrawlRun, parsed: dict) -> tuple[Employee
     checksum = _checksum(parsed)
     key = _parsed_profile_key(parsed)
     employee = db.scalar(select(Employee).where(Employee.profile_key == key))
+    if not employee:
+        employee = _find_employee_with_moved_profile(db, parsed)
     now = datetime.now(timezone.utc)
     if not employee:
         employee = Employee(
@@ -187,15 +253,23 @@ def _upsert_employee(db: Session, run: CrawlRun, parsed: dict) -> tuple[Employee
 
     parser_version = parsed.get("parser_version")
     changed = is_new or employee.current_checksum != checksum or employee.parser_version != parser_version
+    previous_url = employee.canonical_url if employee.canonical_url != parsed["source_url"] else None
+    employee.profile_key = key
+    employee.profile_type = parsed.get("profile_type")
+    employee.profile_id = parsed.get("profile_id")
+    employee.canonical_url = parsed["source_url"]
     employee.full_name = parsed.get("full_name")
     employee.status = "active"
     employee.last_seen_at = now
     employee.dismissed_at = None
+    employee.profile_unavailable_streak = 0
+    employee.last_profile_check_at = now
     employee.parser_version = parser_version
     if changed:
         employee.current_data = parsed
     employee.current_checksum = checksum
     db.flush()
+    _sync_profile_url_history(db, employee, previous_url, employee.canonical_url, now)
 
     if is_new:
         _record_employee_change(
@@ -235,6 +309,74 @@ def _upsert_employee(db: Session, run: CrawlRun, parsed: dict) -> tuple[Employee
     return employee, changed
 
 
+def _find_employee_with_moved_profile(db: Session, parsed: dict) -> Employee | None:
+    full_name = parsed.get("full_name")
+    if not full_name:
+        return None
+    candidates = db.scalars(select(Employee).where(Employee.full_name == full_name)).all()
+    if len(candidates) == 1:
+        return candidates[0]
+
+    parsed_identity = _profile_identity_values(parsed)
+    if not parsed_identity:
+        return None
+    matches = [candidate for candidate in candidates if parsed_identity & _employee_identity_values(candidate)]
+    return matches[0] if len(matches) == 1 else None
+
+
+def _profile_identity_values(profile: dict) -> set[str]:
+    if not isinstance(profile, dict):
+        return set()
+    values = set()
+    contacts = profile.get("contacts") or {}
+    if not isinstance(contacts, dict):
+        contacts = {}
+    for email in contacts.get("emails") or []:
+        normalized = str(email).strip().lower()
+        if normalized:
+            values.add(f"email:{normalized}")
+    for item in profile.get("external_ids") or []:
+        if not isinstance(item, dict):
+            continue
+        system = str(item.get("system") or "").strip().lower()
+        value = str(item.get("value") or "").strip().lower()
+        if system and value:
+            values.add(f"external:{system}:{value}")
+    return values
+
+
+def _employee_identity_values(employee: Employee) -> set[str]:
+    return _profile_identity_values(employee.current_data or {})
+
+
+def _sync_profile_url_history(
+    db: Session,
+    employee: Employee,
+    previous_url: str | None,
+    current_url: str,
+    seen_at: datetime,
+) -> None:
+    urls = {url for url in (previous_url, current_url) if url}
+    for url in urls:
+        history = db.scalar(
+            select(EmployeeProfileUrl).where(
+                EmployeeProfileUrl.employee_id == employee.id,
+                EmployeeProfileUrl.url == url,
+            )
+        )
+        if history:
+            history.last_seen_at = seen_at
+        else:
+            db.add(
+                EmployeeProfileUrl(
+                    employee_id=employee.id,
+                    url=url,
+                    first_seen_at=seen_at,
+                    last_seen_at=seen_at,
+                )
+            )
+
+
 def _try_sync_employee_publications(db: Session, run: CrawlRun, employee: Employee, parsed: dict) -> None:
     try:
         if not _publication_payloads(parsed):
@@ -453,15 +595,41 @@ def _datetime_or_none(value: object) -> datetime | None:
     return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)
 
 
-def _mark_dismissed(db: Session, run: CrawlRun, found_keys: set[str], session: requests.Session, timeout: int) -> int:
+def _mark_dismissed(
+    db: Session,
+    run: CrawlRun,
+    found_keys: set[str],
+    session: requests.Session,
+    timeout: int,
+    *,
+    confirmation_runs: int = 3,
+    max_auto_dismissals: int | None = 25,
+) -> int:
     dismissed = 0
-    active = db.scalars(select(Employee).where(Employee.status == "active")).all()
+    candidates = db.scalars(
+        select(Employee).where(Employee.status.in_(("active", "verification_required")))
+    ).all()
     now = datetime.now(timezone.utc)
-    for employee in active:
+    unavailable = []
+    for employee in candidates:
         if employee.profile_key in found_keys:
             continue
-        profile_available = _profile_is_available(session, employee.canonical_url, timeout)
+        profile_available = _profile_check(session, employee.canonical_url, timeout)
+        employee.last_profile_check_at = now
+        if profile_available is None:
+            db.add(
+                CrawlError(
+                    crawl_run_id=run.id,
+                    profile_url=employee.canonical_url,
+                    error_type="ProfileAvailabilityCheckError",
+                    message="Не удалось надёжно проверить доступность профиля; статус сотрудника не изменён.",
+                )
+            )
+            continue
         if profile_available:
+            employee.profile_unavailable_streak = 0
+            if employee.status == "verification_required":
+                employee.status = "active"
             _record_employee_change(
                 db,
                 run,
@@ -471,6 +639,37 @@ def _mark_dismissed(db: Session, run: CrawlRun, found_keys: set[str], session: r
                 message="Профиль доступен, но ссылка отсутствует в исходном списке.",
             )
             continue
+        next_streak = employee.profile_unavailable_streak + 1
+        unavailable.append((employee, next_streak))
+
+    dismissal_blocked = bool(
+        max_auto_dismissals is not None and len(unavailable) > max_auto_dismissals
+    )
+    if dismissal_blocked:
+        run.message = (
+            f"Автоматическое увольнение приостановлено: {len(unavailable)} профилей "
+            f"одновременно не подтвердились (лимит {max_auto_dismissals})."
+        )
+
+    for employee, next_streak in unavailable:
+        employee.profile_unavailable_streak = next_streak
+        if next_streak < confirmation_runs or dismissal_blocked:
+            employee.status = "verification_required"
+            _record_employee_change(
+                db,
+                run,
+                employee,
+                "verification_required",
+                profile_available=False,
+                message=(
+                    "Профиль не подтвердился. Автоматическое увольнение отложено до "
+                    f"{confirmation_runs} последовательных проверок."
+                    if not dismissal_blocked
+                    else "Автоматическое увольнение отложено из-за массовой ошибки проверки профилей."
+                ),
+            )
+            continue
+
         employee.status = "dismissed"
         employee.dismissed_at = now
         _record_employee_change(
@@ -479,7 +678,10 @@ def _mark_dismissed(db: Session, run: CrawlRun, found_keys: set[str], session: r
             employee,
             "dismissed",
             profile_available=False,
-            message="Сотрудник отсутствует в исходном списке, профиль не подтвердился как доступный.",
+            message=(
+                "Сотрудник отсутствует в исходном списке, профиль не подтвердился "
+                f"{confirmation_runs} раза подряд."
+            ),
         )
         dismissed += 1
     db.commit()
@@ -487,11 +689,19 @@ def _mark_dismissed(db: Session, run: CrawlRun, found_keys: set[str], session: r
 
 
 def _profile_is_available(session: requests.Session, url: str, timeout: int) -> bool:
+    return _profile_check(session, url, timeout) is True
+
+
+def _profile_check(session: requests.Session, url: str, timeout: int) -> bool | None:
     try:
         response = session.get(url, headers=HEADERS, timeout=timeout, allow_redirects=True)
-        return response.status_code < 400
+        if response.status_code < 400:
+            return True
+        if response.status_code in {404, 410}:
+            return False
+        return None
     except requests.RequestException:
-        return False
+        return None
 
 
 def _record_employee_change(

app/static/admin.css

@@ -137,6 +137,11 @@
   color: #991b1b;
 }
 
+.badge--verification {
+  background: #fef3c7;
+  color: #92400e;
+}
+
 .form {
   display: grid;
   gap: 12px;
@@ -428,6 +433,17 @@
   gap: 16px;
 }
 
+.progress-panel__actions {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 8px;
+}
+
+.button--secondary {
+  color: #0f766e;
+  background: #ccfbf1;
+}
+
 .progress-panel__body {
   display: grid;
   gap: 10px;

app/static/admin.js

@@ -5,6 +5,7 @@
     "positions",
     "hse_start_year",
     "email",
+    "academic_degree",
     "last_seen_at",
     "dismissed_at",
     "profile",

app/templates/dashboard.html

@@ -4,6 +4,7 @@
 <section class="admin__grid">
   <a class="metric metric--link" href="/admin/directory"><span class="metric__label">Всего в базе</span><span class="metric__value">{{ counts.total }}</span></a>
   <a class="metric metric--link" href="/admin/directory?status=active"><span class="metric__label">Работают</span><span class="metric__value">{{ counts.active }}</span></a>
+  <a class="metric metric--link" href="/admin/directory?status=verification_required"><span class="metric__label">Требуют проверки</span><span class="metric__value">{{ counts.verification_required }}</span></a>
   <a class="metric metric--link" href="{% if latest_run %}/admin/runs/{{ latest_run.id }}#new-employees{% else %}/admin/runs{% endif %}"><span class="metric__label">Новые за запуск</span><span class="metric__value">{{ counts.new_in_last_run }}</span></a>
   <a class="metric metric--link" href="/admin/directory?status=dismissed"><span class="metric__label">Уволены</span><span class="metric__value">{{ counts.dismissed }}</span></a>
 </section>
@@ -28,9 +29,14 @@
 <section class="panel progress-panel" data-progress-panel>
   <div class="progress-panel__header">
     <h2 class="panel__title">Прогресс парсинга</h2>
-    <form method="post" action="/admin/crawl-now">
+    <div class="progress-panel__actions">
-      <button class="button" type="submit">Запустить парсинг</button>
+      <form method="post" action="/admin/crawl-now">
-    </form>
+        <button class="button" type="submit">Запустить парсинг</button>
+      </form>
+      <form method="post" action="/admin/dismissed/refresh">
+        <button class="button button--secondary" type="submit">Проверить уволенных</button>
+      </form>
+    </div>
   </div>
   {% set run = counts.current_running_run or latest_run %}
   <div class="progress-panel__body" data-progress-body>

app/templates/directory.html

@@ -22,6 +22,11 @@
       <option value="true" {% if filters.has_email == "true" %}selected{% endif %}>Есть email</option>
       <option value="false" {% if filters.has_email == "false" %}selected{% endif %}>Нет email</option>
     </select>
+    <select class="directory__input" name="has_academic_degree" aria-label="Учёная степень">
+      <option value="" {% if not filters.has_academic_degree %}selected{% endif %}>Любая учёная степень</option>
+      <option value="true" {% if filters.has_academic_degree == "true" %}selected{% endif %}>Есть учёная степень</option>
+      <option value="false" {% if filters.has_academic_degree == "false" %}selected{% endif %}>Нет учёной степени</option>
+    </select>
     <input class="directory__input" type="date" name="started_from" value="{{ filters.started_from }}" aria-label="Впервые найден с">
     <input class="directory__input" type="date" name="started_to" value="{{ filters.started_to }}" aria-label="Впервые найден по">
     <select class="directory__input" name="sort">
@@ -53,6 +58,7 @@
           <th class="directory-table__head" data-column="email">Email</th>
           <th class="directory-table__head" data-column="phone">Телефон</th>
           <th class="directory-table__head" data-column="address">Адрес</th>
+          <th class="directory-table__head" data-column="academic_degree">Учёная степень</th>
           <th class="directory-table__head" data-column="publications_count">Публикации</th>
           <th class="directory-table__head" data-column="courses_count">Курсы</th>
           <th class="directory-table__head" data-column="news_count">Новости</th>
@@ -72,6 +78,7 @@
           <td class="directory-table__cell" data-column="email">{{ employee.email_text }}</td>
           <td class="directory-table__cell" data-column="phone">{{ employee.phone_text }}</td>
           <td class="directory-table__cell" data-column="address">{{ employee.address or "" }}</td>
+          <td class="directory-table__cell" data-column="academic_degree">{{ employee.academic_degree_text }}</td>
           <td class="directory-table__cell" data-column="publications_count">{{ employee.publications_count }}</td>
           <td class="directory-table__cell" data-column="courses_count">{{ employee.courses_count }}</td>
           <td class="directory-table__cell" data-column="news_count">{{ employee.news_count }}</td>
@@ -81,7 +88,7 @@
           <td class="directory-table__cell" data-column="profile"><a class="admin__link" href="{{ employee.canonical_url }}">Открыть</a></td>
         </tr>
         {% else %}
-        <tr><td class="directory-table__empty" colspan="14">По этим фильтрам сотрудники не найдены.</td></tr>
+        <tr><td class="directory-table__empty" colspan="15">По этим фильтрам сотрудники не найдены.</td></tr>
         {% endfor %}
       </tbody>
     </table>
@@ -108,7 +115,7 @@
       <button class="button button--ghost" type="button" data-columns-close>Закрыть</button>
     </div>
     <div class="columns-modal__grid">
-      {% for key, label in [("full_name", "ФИО"), ("status", "Статус"), ("positions", "Должности"), ("hse_start_year", "Год начала"), ("email", "Email"), ("phone", "Телефон"), ("address", "Адрес"), ("publications_count", "Публикации"), ("courses_count", "Курсы"), ("news_count", "Новости"), ("first_seen_at", "Впервые найден"), ("last_seen_at", "Последний раз найден"), ("dismissed_at", "Дата увольнения"), ("profile", "Профиль")] %}
+      {% for key, label in [("full_name", "ФИО"), ("status", "Статус"), ("positions", "Должности"), ("hse_start_year", "Год начала"), ("email", "Email"), ("phone", "Телефон"), ("address", "Адрес"), ("academic_degree", "Учёная степень"), ("publications_count", "Публикации"), ("courses_count", "Курсы"), ("news_count", "Новости"), ("first_seen_at", "Впервые найден"), ("last_seen_at", "Последний раз найден"), ("dismissed_at", "Дата увольнения"), ("profile", "Профиль")] %}
       <label class="columns-modal__option"><input class="columns-modal__checkbox" type="checkbox" value="{{ key }}" data-column-toggle> {{ label }}</label>
       {% endfor %}
     </div>

app/version.py

@@ -1,3 +1,3 @@
-APP_VERSION = "0.7.0"
+APP_VERSION = "0.7.2"
-FRONTEND_VERSION = "0.7.0"
+FRONTEND_VERSION = "0.7.2"
-BACKEND_VERSION = "0.7.0"
+BACKEND_VERSION = "0.7.2"

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "miem-workers"
-version = "0.7.0"
+version = "0.7.2"
 description = "MIEM employees parser, admin API, and MCP server"
 requires-python = ">=3.11"
 dependencies = [

tests/test_admin_data.py

@@ -51,6 +51,45 @@ def test_employee_display_payload_extracts_common_fields(db_session):
     assert payload["first_seen_display"] != "Не указано"
 
 
+def test_list_employees_page_filters_and_displays_academic_degrees(db_session):
+    db_session.add_all(
+        [
+            Employee(
+                profile_key="staff:degree",
+                canonical_url="https://www.hse.ru/staff/degree",
+                full_name="Doctor",
+                status="active",
+                first_seen_at=datetime.now(timezone.utc),
+                last_seen_at=datetime.now(timezone.utc),
+                current_data={
+                    "sections": [
+                        {
+                            "title": "Образование и учёные степени",
+                            "year_entries": [{"year": 2020, "text": "Доктор технических наук"}],
+                        }
+                    ]
+                },
+            ),
+            Employee(
+                profile_key="staff:no-degree",
+                canonical_url="https://www.hse.ru/staff/no-degree",
+                full_name="Master",
+                status="active",
+                first_seen_at=datetime.now(timezone.utc),
+                last_seen_at=datetime.now(timezone.utc),
+                current_data={"sections": [{"title": "Образование", "items": ["Магистратура"]}]},
+            ),
+        ]
+    )
+    db_session.commit()
+
+    page = list_employees_page(db_session, has_academic_degree=True)
+
+    assert page["total"] == 1
+    assert page["employees"][0]["full_name"] == "Doctor"
+    assert page["employees"][0]["academic_degree_text"] == "Доктор технических наук"
+
+
 def test_employee_detail_payload_normalizes_human_readable_sections(db_session):
     employee = Employee(
         profile_key="staff:person",

tests/test_admin_templates.py

@@ -23,10 +23,13 @@ def test_directory_template_is_russian_and_uses_display_dates():
     assert "{% for value in [25, 50, 100] %}" in template
     assert "Найдено:" in template
     assert "Новости" in template
+    assert "Есть учёная степень" in template
+    assert 'data-column="academic_degree"' in template
     assert "employee.news_count" in template
     assert "employee.first_seen_display" in template
     assert "employee.last_seen_display" in template
     assert "employee.dismissed_display" in template
+    assert "verification_required" in template
     assert "Directory" not in template
     assert "employees found" not in template
 
@@ -62,6 +65,7 @@ def test_run_detail_template_extends_base_and_shows_change_groups():
     assert 'id="new-employees"' in template
     assert "Новые сотрудники" in template
     assert "Потеряшки" in template
+    assert "Требуют проверки" in template
     assert "Уволенные" in template
     assert "Детализация сотрудников для этого запуска недоступна" in template
 
@@ -71,11 +75,19 @@ def test_dashboard_metric_cards_link_to_admin_targets():
 
     assert 'href="/admin/directory"' in template
     assert 'href="/admin/directory?status=active"' in template
+    assert 'href="/admin/directory?status=verification_required"' in template
     assert '/admin/runs/{{ latest_run.id }}#new-employees' in template
     assert 'href="/admin/directory?status=dismissed"' in template
     assert 'href="/admin/runs"' in template
 
 
+def test_dashboard_has_dismissed_status_refresh_action():
+    template = Path("app/templates/dashboard.html").read_text(encoding="utf-8")
+
+    assert 'action="/admin/dismissed/refresh"' in template
+    assert "Проверить уволенных" in template
+
+
 def test_dashboard_latest_run_rows_link_to_run_detail():
     template = Path("app/templates/dashboard.html").read_text(encoding="utf-8")
 

tests/test_api_mcp.py

@@ -20,7 +20,7 @@ def test_health_returns_versions():
     response = client.get("/api/health")
 
     assert response.status_code == 200
-    assert response.json()["backend_version"] == "0.7.0"
+    assert response.json()["backend_version"] == "0.7.1"
 
 
 def test_mcp_lists_tools_without_auth_and_ignores_auth_header():
@@ -154,7 +154,7 @@ def test_mcp_service_info_returns_tools_and_dataset_hash():
     assert response.status_code == 200
     payload = json.loads(response.json()["result"]["content"][0]["text"])
     assert payload["service_name"] == "miem-employees"
-    assert payload["backend_version"] == "0.7.0"
+    assert payload["backend_version"] == "0.7.1"
     assert payload["dataset"]["hash"]
     assert any(tool["name"] == "sync_employees" for tool in payload["tools"])
 

tests/test_crawler.py

@@ -11,7 +11,8 @@ from app.models import (
     EmployeeSnapshot,
     ParseResourceCache,
 )
-from app.services.crawler import _checksum, _mark_dismissed, _upsert_employee
+from app.config import Settings
+from app.services.crawler import _checksum, _mark_dismissed, _upsert_employee, refresh_dismissed_status
 from app.services.resource_cache import ResourceCache
 
 
@@ -56,6 +57,41 @@ class ConditionalSession:
         return ConditionalResponse(200, "fresh", {"ETag": '"fresh"'})
 
 
+def test_refresh_dismissed_status_reactivates_only_profiles_in_source(monkeypatch, db_session):
+    now = datetime.now(timezone.utc)
+    found = Employee(
+        profile_key="staff:returned",
+        canonical_url="https://www.hse.ru/staff/returned",
+        status="dismissed",
+        dismissed_at=now,
+        first_seen_at=now,
+        last_seen_at=now,
+    )
+    still_dismissed = Employee(
+        profile_key="staff:gone",
+        canonical_url="https://www.hse.ru/staff/gone",
+        status="dismissed",
+        dismissed_at=now,
+        first_seen_at=now,
+        last_seen_at=now,
+    )
+    db_session.add_all([found, still_dismissed])
+    db_session.commit()
+    monkeypatch.setattr(
+        "app.services.crawler.collect_profile_links",
+        lambda *_args, **_kwargs: ["https://www.hse.ru/staff/returned"],
+    )
+
+    run = refresh_dismissed_status(db_session, Settings())
+
+    assert run.status == "completed"
+    assert run.parsed_count == 1
+    assert run.skipped_count == 1
+    assert found.status == "active"
+    assert found.dismissed_at is None
+    assert still_dismissed.status == "dismissed"
+
+
 def test_mark_dismissed_records_missing_source_when_profile_is_available(db_session):
     run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
     db_session.add(run)
@@ -97,7 +133,7 @@ def test_mark_dismissed_records_missing_source_when_profile_is_available(db_sess
     assert change.profile_available is True
 
 
-def test_mark_dismissed_marks_missing_employee_when_profile_is_unavailable(db_session):
+def test_mark_dismissed_requires_consecutive_unavailable_checks(db_session):
     run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
     employee = Employee(
         profile_key="staff:gone",
@@ -109,22 +145,129 @@ def test_mark_dismissed_marks_missing_employee_when_profile_is_unavailable(db_se
     db_session.add_all([run, employee])
     db_session.commit()
 
-    dismissed = _mark_dismissed(
+    first_check = _mark_dismissed(
         db_session,
         run,
         set(),
         FakeSession({"https://www.hse.ru/staff/gone": 404}),
         30,
+        confirmation_runs=2,
     )
 
-    assert dismissed == 1
+    assert first_check == 0
+    assert employee.status == "verification_required"
+    assert employee.dismissed_at is None
+    assert employee.profile_unavailable_streak == 1
+    assert db_session.query(CrawlRunEmployeeChange).one().change_type == "verification_required"
+
+    second_check = _mark_dismissed(
+        db_session,
+        run,
+        set(),
+        FakeSession({"https://www.hse.ru/staff/gone": 404}),
+        30,
+        confirmation_runs=2,
+    )
+
+    assert second_check == 1
     assert employee.status == "dismissed"
     assert employee.dismissed_at is not None
-    change = db_session.query(CrawlRunEmployeeChange).one()
+    assert employee.profile_unavailable_streak == 2
+    change = db_session.query(CrawlRunEmployeeChange).order_by(CrawlRunEmployeeChange.id).all()[-1]
     assert change.change_type == "dismissed"
     assert change.profile_available is False
 
 
+def test_mark_dismissed_does_not_dismiss_on_server_error(db_session):
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    employee = Employee(
+        profile_key="staff:temporary-error",
+        canonical_url="https://www.hse.ru/staff/temporary-error",
+        status="active",
+        first_seen_at=datetime.now(timezone.utc),
+        last_seen_at=datetime.now(timezone.utc),
+    )
+    db_session.add_all([run, employee])
+    db_session.commit()
+
+    dismissed = _mark_dismissed(
+        db_session,
+        run,
+        set(),
+        FakeSession({"https://www.hse.ru/staff/temporary-error": 503}),
+        30,
+        confirmation_runs=1,
+    )
+
+    assert dismissed == 0
+    assert employee.status == "active"
+    assert employee.profile_unavailable_streak == 0
+    assert db_session.query(CrawlError).one().error_type == "ProfileAvailabilityCheckError"
+
+
+def test_available_profile_resets_verification_streak(db_session):
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    employee = Employee(
+        profile_key="staff:restored",
+        canonical_url="https://www.hse.ru/staff/restored",
+        status="active",
+        first_seen_at=datetime.now(timezone.utc),
+        last_seen_at=datetime.now(timezone.utc),
+    )
+    db_session.add_all([run, employee])
+    db_session.commit()
+
+    _mark_dismissed(
+        db_session,
+        run,
+        set(),
+        FakeSession({"https://www.hse.ru/staff/restored": 404}),
+        30,
+        confirmation_runs=3,
+    )
+    _mark_dismissed(
+        db_session,
+        run,
+        set(),
+        FakeSession({"https://www.hse.ru/staff/restored": 200}),
+        30,
+        confirmation_runs=3,
+    )
+
+    assert employee.status == "active"
+    assert employee.profile_unavailable_streak == 0
+
+
+def test_mark_dismissed_blocks_mass_auto_dismissals(db_session):
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    employees = [
+        Employee(
+            profile_key=f"staff:gone-{index}",
+            canonical_url=f"https://www.hse.ru/staff/gone-{index}",
+            status="active",
+            first_seen_at=datetime.now(timezone.utc),
+            last_seen_at=datetime.now(timezone.utc),
+        )
+        for index in range(2)
+    ]
+    db_session.add_all([run, *employees])
+    db_session.commit()
+
+    dismissed = _mark_dismissed(
+        db_session,
+        run,
+        set(),
+        FakeSession({employee.canonical_url: 404 for employee in employees}),
+        30,
+        confirmation_runs=1,
+        max_auto_dismissals=1,
+    )
+
+    assert dismissed == 0
+    assert {employee.status for employee in employees} == {"verification_required"}
+    assert "приостановлено" in run.message
+
+
 def test_upsert_employee_increments_new_count_and_records_change_for_new_employee(db_session):
     run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
     db_session.add(run)
@@ -152,6 +295,49 @@ def test_upsert_employee_increments_new_count_and_records_change_for_new_employe
     assert change.full_name == "New Person"
 
 
+def test_upsert_employee_reconciles_profile_moved_to_new_url(db_session):
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    employee = Employee(
+        profile_key="staff:abelov",
+        canonical_url="https://www.hse.ru/staff/abelov",
+        full_name="Белов Александр Владимирович",
+        status="active",
+        first_seen_at=datetime.now(timezone.utc),
+        last_seen_at=datetime.now(timezone.utc),
+    )
+    db_session.add_all([run, employee])
+    db_session.commit()
+    employee_id = employee.id
+
+    updated, changed = _upsert_employee(
+        db_session,
+        run,
+        {
+            "source_url": "https://www.hse.ru/org/persons/47634735",
+            "profile_type": "org_person",
+            "profile_id": "47634735",
+            "full_name": "Белов Александр Владимирович",
+            "tabs": [],
+            "sections": [],
+            "parser_version": "0.7.0",
+            "_html": "<html></html>",
+        },
+    )
+    db_session.commit()
+
+    assert changed is True
+    assert updated.id == employee_id
+    assert updated.profile_key == "org_person:47634735"
+    assert updated.canonical_url == "https://www.hse.ru/org/persons/47634735"
+    assert updated.status == "active"
+    assert run.new_count == 0
+    assert db_session.query(Employee).count() == 1
+    assert {item.url for item in updated.profile_urls} == {
+        "https://www.hse.ru/staff/abelov",
+        "https://www.hse.ru/org/persons/47634735",
+    }
+
+
 def test_resource_cache_uses_etag_and_reuses_cached_body_on_304(db_session):
     db_session.add(
         ParseResourceCache(