fix: avoid repeated worksheet scans during XLSX export

Reviewed-on: #58
Reviewed-by: agent <agent@vakanaut.ru>

.env.example

@@ -10,11 +10,10 @@ CRAWL_LIMIT=
 REQUEST_TIMEOUT=30
 REQUEST_DELAY_SECONDS=1
 PARSER_USE_PLAYWRIGHT=false
+DISMISSAL_CONFIRMATION_RUNS=3
+MAX_AUTO_DISMISSALS_PER_RUN=25
 
 ADMIN_USERNAME=admin
 ADMIN_PASSWORD=change-me
 SESSION_SECRET=change-me-session-secret
-MCP_TOKEN=change-me-mcp-token
-
 API_PORT=8000
-MCP_PORT=8001

.gitea/workflows/ci.yml

@@ -0,0 +1,121 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+    inputs:
+      diagnostic_only:
+        description: 'Print key fingerprint; production deployment is temporarily disabled'
+        type: boolean
+        default: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    env:
+      DATABASE_URL: sqlite:///./ci.db
+      PYTHONUNBUFFERED: "1"
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+      - run: python -m pip install --upgrade pip
+      - run: python -m pip install -r requirements.txt
+      - run: python -m pytest -q
+      - run: node --check app/static/admin.js
+      - run: sh -n infra/postgres-backup.sh
+      - run: sh -n infra/install-postgres-backup-cron.sh
+
+  deploy:
+    # Production deployment is disabled until separate Workers deployment access is installed.
+    name: Fingerprint diagnostics (production deployment temporarily disabled)
+    if: github.event_name == 'workflow_dispatch' && github.event.inputs.diagnostic_only == 'true'
+    needs: test
+    runs-on: ubuntu-latest
+    concurrency:
+      group: production-deploy
+      cancel-in-progress: false
+    env:
+      PROD_PATH: /srv/miem_workers
+      RELEASE_NAME: miem-workers-release-${{ github.run_id }}.tar
+    steps:
+      - uses: actions/checkout@v4
+      - name: Prepare release
+        id: prepare
+        shell: bash
+        run: |
+          set +x
+          set -euo pipefail
+          backend_version="$(sed -n 's/^BACKEND_VERSION = "\(.*\)"$/\1/p' app/version.py)"
+          case "$backend_version" in
+            ''|*[!0-9.]*) exit 1 ;;
+          esac
+          echo "backend_version=$backend_version" >> "$GITHUB_OUTPUT"
+          tar --exclude=.git --exclude=.env -cf "$RUNNER_TEMP/$RELEASE_NAME" .
+          printf '%s\n' "$PROD_SSH_KEY" > "$RUNNER_TEMP/deploy_key"
+          printf '%s\n' "$PROD_KNOWN_HOSTS" > "$RUNNER_TEMP/known_hosts"
+          chmod 600 "$RUNNER_TEMP/deploy_key" "$RUNNER_TEMP/known_hosts"
+        env:
+          PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }}
+          PROD_KNOWN_HOSTS: ${{ secrets.PROD_KNOWN_HOSTS }}
+      - name: Deployment key fingerprint
+        shell: bash
+        run: |
+          set +x
+          set -euo pipefail
+          trap 'rm -f "$RUNNER_TEMP/deploy_key" "$RUNNER_TEMP/known_hosts"' EXIT
+          ssh-keygen -y -P '' -f "$RUNNER_TEMP/deploy_key" | ssh-keygen -lf - -E sha256 | awk '{print $2}'
+      - name: Upload release
+        if: ${{ false }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          scp -i "$RUNNER_TEMP/deploy_key" \
+            -o BatchMode=yes \
+            -o UserKnownHostsFile="$RUNNER_TEMP/known_hosts" \
+            "$RUNNER_TEMP/$RELEASE_NAME" \
+            "$PROD_USER@$PROD_HOST:$RELEASE_NAME"
+        env:
+          PROD_HOST: ${{ secrets.PROD_HOST }}
+          PROD_USER: ${{ secrets.PROD_USER }}
+      - name: Deploy and verify
+        if: ${{ false }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          ssh -i "$RUNNER_TEMP/deploy_key" \
+            -o BatchMode=yes \
+            -o UserKnownHostsFile="$RUNNER_TEMP/known_hosts" \
+            "$PROD_USER@$PROD_HOST" \
+            "PROD_PATH='$PROD_PATH' RELEASE_NAME='$RELEASE_NAME' EXPECTED_BACKEND_VERSION='$EXPECTED_BACKEND_VERSION' bash -s" <<'REMOTE'
+          set -euo pipefail
+          cd "$PROD_PATH"
+          release="$HOME/$RELEASE_NAME"
+          test -f "$release"
+          archive_version="$(tar -xOf "$release" ./app/version.py | sed -n 's/^BACKEND_VERSION = "\(.*\)"$/\1/p')"
+          test "$archive_version" = "$EXPECTED_BACKEND_VERSION"
+          docker run --rm \
+            -v "$release:/src/release.tar:ro" \
+            -v "$PROD_PATH":/dst \
+            alpine:3.20 sh -c 'tar -xf /src/release.tar -C /dst'
+          sh /srv/miem_workers/infra/install-postgres-backup-cron.sh
+          docker compose up -d --build
+          runtime_version="$(docker compose exec -T api python -c 'from app.version import BACKEND_VERSION; print(BACKEND_VERSION)')"
+          test "$runtime_version" = "$EXPECTED_BACKEND_VERSION"
+          health="$(curl --fail --silent --show-error http://127.0.0.1:8000/api/health)"
+          case "$health" in
+            *"\"status\":\"ok\""*"\"backend_version\":\"$EXPECTED_BACKEND_VERSION\""*) ;;
+            *) echo "Unexpected health response: $health" >&2; exit 1 ;;
+          esac
+          docker compose ps
+          rm -f "$release"
+          REMOTE
+        env:
+          PROD_HOST: ${{ secrets.PROD_HOST }}
+          PROD_USER: ${{ secrets.PROD_USER }}
+          EXPECTED_BACKEND_VERSION: ${{ steps.prepare.outputs.backend_version }}

.gitignore

@@ -4,6 +4,10 @@ __pycache__/
 *.py[cod]
 *.db
 .pytest_cache/
+pytest-cache-files-*/
 .coverage
 htmlcov/
 postgres_data/
+MCP_DESCRIPTION.md
+
+.impeccable/review/

CHANGELOG.md

@@ -0,0 +1,91 @@
+# Changelog
+
+## 0.8.15
+
+- Исправлен production deploy: cron installer запускается через `sh` и не требует executable-бита.
+
+## 0.8.14
+
+- Production CD больше не создаёт backup при каждом deploy; еженедельная задача
+  сохраняет один полный PostgreSQL backup с атомарной заменой; ошибки и
+  пропущенные из-за lock запуски пишутся в system log.
+
+## 0.8.12
+
+- Передача версии в production deploy ограничена безопасным форматом.
+
+## 0.8.11
+
+- CSS-ассеты админки получают версию в URL и обновляются после релиза без ручной очистки кэша браузера.
+
+## 0.8.10
+
+- Production deploy проверяет совпадение версии release-архива и запущенного API.
+
+## 0.8.9
+
+- Контейнеры API, worker и PostgreSQL автоматически перезапускаются после остановки или перезагрузки хоста.
+
+## 0.8.8
+
+- Исправлено вертикальное выравнивание кнопки и ссылки в действиях фильтров справочника.
+
+## 0.8.7
+
+- Production CD оставляет только последний PostgreSQL backup после успешного деплоя.
+
+## 0.8.6
+
+- Исправлена загрузка release-архива production CD в домашний каталог deploy-пользователя.
+
+## 0.8.5
+
+- Исправлена адаптивная компоновка фильтров справочника и скрытие skip-link до клавиатурного фокуса.
+
+## 0.8.4
+
+- Подключены к конфигурации ограничения массового автоматического увольнения, используемые плановым обходом.
+
+## 0.8.3
+
+- Удалены устаревшие пояснение о недоступном endpoint и проверка его отсутствия.
+
+## 0.8.2
+
+- Добавлен production CD workflow с backup PostgreSQL и проверкой healthcheck после обновления.
+
+## 0.8.1
+
+- Улучшены доступность, фильтры, состояния и адаптивность административного каталога.
+
+## 0.7.8
+
+- В колонке учёной степени отображаются все найденные степени без лишнего текста.
+
+## 0.7.7
+
+- Удалена неиспользуемая интеграция обмена данными и связанная документация и тесты.
+
+## 0.7.6
+
+- Возвращён еженедельный автоматический запуск обхода сотрудников.
+
+## 0.7.5
+
+- Production Compose запускает только API и PostgreSQL.
+
+## 0.7.4
+
+- Ускорена фильтрация сотрудников по учёной степени.
+
+## 0.7.3
+
+- Восстановлена проверка профилей сотрудников и хранение истории URL профиля.
+
+## 0.7.2
+
+- В каталоге сотрудников добавлены фильтр и колонка учёной степени.
+
+## 0.7.1
+
+- Добавлена кнопка «Проверить уволенных» для принудительной сверки статуса уволенных сотрудников с текущим списком источника.

README.md

@@ -1,113 +1,156 @@
-# MIEM Employees Server
-
-Сервис собирает сотрудников МИЭМ с сайта ВШЭ, хранит карточки и историю обновлений в Postgres, показывает минимальную админку и отдает read-only MCP endpoint для ИИ-агентов.
-
-## Архитектура
-
-- `api`: FastAPI, REST API, HTML-админка, healthcheck.
+# MIEM Employees Server
+
+Сервис собирает сотрудников МИЭМ с сайта ВШЭ, хранит карточки и историю обновлений в Postgres и показывает минимальную админку.
+
+## Архитектура
+
+- `api`: FastAPI, REST API, HTML-админка и healthcheck.
 - `worker`: weekly scheduler, который запускает парсинг по `CRAWL_CRON`.
-- `mcp`: HTTP MCP endpoint с bearer token.
-- `postgres`: основная БД.
-
-Парсер использует фиксированный источник сотрудников, по умолчанию `https://miem.hse.ru/persons`. Для каждой карточки сохраняются ФИО, должности, год начала работы, контакты, идентификаторы, вкладки профиля, секции, публикации, курсы, ВКР, JSON-снапшот и сжатый HTML-снапшот. Ссылки обходятся только из меню профиля самого сотрудника (`person-menu`), например `#sci`, `#teaching`, `#main`.
-
-## Переменные окружения
-
-Скопируйте `.env.example` в `.env` и поменяйте секреты:
-
-```bash
-cp .env.example .env
-```
-
-Основные настройки:
-
-- `DATABASE_URL`: строка подключения SQLAlchemy.
+- `postgres`: основная БД.
+
+Парсер использует фиксированный источник сотрудников, по умолчанию `https://miem.hse.ru/persons`. Для каждой карточки сохраняются ФИО, должности, год начала работы, контакты, идентификаторы, вкладки профиля, секции, публикации, курсы, ВКР, новости, JSON-снапшот и сжатый HTML-снапшот. Детальные публикации дополнительно нормализуются в отдельную таблицу `employee_publications`, а новости из блока «В новостях» — в `employee_news_links`. Ссылки обходятся только из меню профиля самого сотрудника (`person-menu`), например `#sci`, `#teaching`, `#main`.
+
+## Переменные окружения
+
+Скопируйте `.env.example` в `.env` и поменяйте секреты:
+
+```bash
+cp .env.example .env
+```
+
+Основные настройки:
+
+- `DATABASE_URL`: строка подключения SQLAlchemy.
 - `SOURCE_URL`: список сотрудников МИЭМ.
 - `CRAWL_CRON`: расписание в формате crontab, по умолчанию `0 3 * * 1`.
-- `CRAWL_LIMIT`: опциональный лимит профилей для тестового запуска.
-- `ADMIN_USERNAME`, `ADMIN_PASSWORD`: логин и пароль админки.
-- `SESSION_SECRET`: секрет подписи cookie.
-- `MCP_TOKEN`: bearer token для `/mcp`.
-- `PARSER_USE_PLAYWRIGHT`: включение Playwright-рендера динамических вкладок.
+- `CRAWL_LIMIT`: опциональный лимит профилей для тестового запуска.
+- `ADMIN_USERNAME`, `ADMIN_PASSWORD`: логин и пароль админки.
+- `SESSION_SECRET`: секрет подписи cookie.
+- `PARSER_USE_PLAYWRIGHT`: включение Playwright-рендера динамических вкладок.
+- `DISMISSAL_CONFIRMATION_RUNS`: сколько последовательных проверок недоступности нужно для увольнения, по умолчанию `3`.
+- `MAX_AUTO_DISMISSALS_PER_RUN`: защитный лимит массовых автоматических увольнений за один запуск, по умолчанию `25`.
+
+## Локальный запуск
+
+```bash
+python -m venv .venv
+.venv\Scripts\activate
+pip install -r requirements.txt
+uvicorn app.main:app --reload
+```
+
+Админка: `http://localhost:8000/admin`.
+
+В админке доступны:
+
+- «Обзор»: статистика, последний добавленный сотрудник, прогресс парсинга и ручной запуск.
+- «Сотрудники»: поиск, фильтры, сортировка, пагинация и выбор колонок. По умолчанию показаны ФИО, статус, должности, дата последнего обнаружения и внешний профиль. В диалоге колонок доступны наборы «Проверка», «Контакты» и «Все поля»; ранее сохранённый выбор сохраняется.
+- «Запуски»: история обходов, ошибки и доступный индикатор прогресса.
 
-## Локальный запуск
+Все фильтры применяются кнопкой «Применить фильтры» с переходом на первую страницу. «Сбросить» очищает условия; при устаревшем номере страницы каталог возвращает первую. Подписи полей показывают выбранные условия, над таблицей указан диапазон результатов. Пояснения дат и статусов находятся под фильтрами.
 
-```bash
-python -m venv .venv
-.venv\Scripts\activate
-pip install -r requirements.txt
-uvicorn app.main:app --reload
-```
+Каталог поддерживает навигацию с клавиатуры и системный диалог колонок с Escape и возвратом фокуса. На узких экранах таблица прокручивается внутри страницы, колонка ФИО закреплена. Используется системный шрифт без внешних загрузок.
 
-Админка: `http://localhost:8000/admin`.
+Каталог формируется сервером: отдельная клиентская загрузка списка не нужна, переход показывает браузер. Пустой результат предлагает сбросить фильтры, пустая база — запустить парсинг. При ошибке базы каталог возвращает HTTP 503 и предлагает повторить загрузку с теми же фильтрами. При обновлении прогресса показывается состояние загрузки; при ошибке сохраняются последние значения с предупреждением об их актуальности и кнопкой «Повторить». Автоматические попытки продолжаются каждые 4 секунды, запрос ограничен 15 секундами; параллельные запросы не запускаются.
+
+## Docker Compose
+
+```bash
+docker compose up -d --build --remove-orphans
+```
+
+По умолчанию:
+
+- API и админка: `http://localhost:8000`
+- PostgreSQL: `postgres:5432` внутри сети Compose; порт на хост не опубликован.
 
-В админке доступны:
+Compose запускает `api`, `worker` и `postgres`. API привязан к localhost; для внешнего доступа нужен настроенный reverse proxy. REST API данных требует сессию администратора.
+
+Таблицы создаются приложением при старте. При обновлении существующей базы приложение также добавляет недостающие runtime-колонки, например `crawl_runs.skipped_count`. SQL-миграции для ручного применения лежат в `migrations/`.
+
+## Наполнение БД
+
+Основная карточка сотрудника хранится в `employees`: профиль, статус, даты обнаружения/увольнения, текущий JSON `current_data`, checksum и версия парсера. История успешных изменений сохраняется в `employee_snapshots` вместе с JSON-снимком и сжатым HTML профиля.
+
+Публикации теперь хранятся в двух видах:
+
+- краткий список остается внутри `employees.current_data.sections[].publications` для обратной совместимости;
+- детальные записи сохраняются в `employee_publications` и связываются с сотрудником через `employee_id`.
+
+`employee_publications` содержит `publication_id`, название, год, тип публикации, язык, статус, ссылку на карточку HSE Publications, DOI, внешние/document-ссылки, citation text, аннотацию, описание, авторов, raw JSON ответа `searchPubs` и `source_hash` для безопасного повторного upsert. Уникальность поддерживается по `(employee_id, publication_id)` и `(employee_id, source_hash)`, поэтому повторный crawl не должен создавать дубликаты.
 
-- `Dashboard`: общая статистика, последний добавленный сотрудник, прогресс текущего/последнего парсинга и ручной запуск.
-- `Directory`: настраиваемая таблица сотрудников с фильтрами, сортировкой, пагинацией и выбором колонок.
-- `Employees`: простая legacy-таблица сотрудников.
-- `Runs`: история запусков, ошибки и progress bar.
+Новости сотрудников также хранятся в двух видах:
+
+- краткий список остается внутри `employees.current_data.sections[].news_links`;
+- нормализованные карточки из вкладки «В новостях» сохраняются в `employee_news_links`.
+
+`employee_news_links` содержит название новости, ссылку, краткое описание, дату публикации, год публикации, raw JSON карточки и `source_hash`. Уникальность поддерживается по `(employee_id, url)` и `(employee_id, source_hash)`, поэтому повторный crawl не создает дубликаты.
 
-## Docker Compose
+## Кандидаты в диссертационный совет
 
-```bash
-docker compose up --build
-```
+Страница `/admin/council` доступна без авторизации. Она показывает сотрудников с докторской степенью в профиле и хотя бы одной публикацией в CSV или таблице `employee_publications`. Порог количества работ не применяется; сотрудники без степени, с кандидатской степенью или только PhD в список не включаются; уволенные сотрудники отмечены отдельно. Есть поиск по ФИО, сортировка по ФИО и каждому из трёх счётчиков публикаций в обоих направлениях и страницы по 25 сотрудников. Сортировка применяется ко всему списку и сохраняется при поиске и переходе между страницами. Кнопка «Скачать XLSX» выгружает всех найденных сотрудников с выбранными поиском и сортировкой, включая записи на других страницах. Файл содержит листы «Сотрудники», «Публикации» и «Параметры» с источниками, критериями и условиями отбора; экспорт доступен без авторизации по `/admin/council/export.xlsx`. У каждого сотрудника видны степень, число уникальных работ из CSV с Q1/Q2 за скользящие пять лет, число уникальных работ в БД и общий перечень без дублей с источниками, типами и подтверждёнными критериями.
 
-По умолчанию:
-
-- API и админка: `http://localhost:8000`
-- MCP: `http://localhost:8001/mcp`
-- Postgres: `localhost:5432`
-
-Таблицы создаются приложением при старте. SQL-миграция для ручного применения лежит в `migrations/001_init.sql`.
-
-## Парсинг
-
-Weekly worker запускается по `CRAWL_CRON`. Ручной запуск доступен в админке на `Dashboard` и странице `Runs` или через REST:
-
-```bash
-curl -X POST http://localhost:8000/api/crawl-runs --cookie "miem_admin_session=..."
-```
-
-Алгоритм обновления:
-
-- найденные сотрудники получают статус `active` и обновленный `last_seen_at`;
-- новые сотрудники добавляются в `employees`;
-- количество новых сотрудников за запуск сохраняется в `crawl_runs.new_count`;
-- активные сотрудники, исчезнувшие из текущего списка источника, получают статус `dismissed` и `dismissed_at`;
-- каждый успешный разбор сохраняет запись в `employee_snapshots`.
-
-Во время выполнения парсинга `found_count`, `parsed_count` и `error_count` обновляются в базе. Админка опрашивает `/api/crawl-runs/latest` и показывает прогресс как `parsed_count + error_count / found_count`.
-
-## MCP
-
-Endpoint: `POST /mcp`, авторизация `Authorization: Bearer <MCP_TOKEN>`.
-
-Поддерживаемые tools:
-
-- `search_employees(query, status?, limit?)`
-- `get_employee(profile_id_or_url)`
-- `list_employee_publications(profile_id_or_url)`
-- `list_employee_courses(profile_id_or_url)`
-- `get_crawl_status()`
-
-Пример:
-
-```bash
-curl http://localhost:8001/mcp \
-  -H "Authorization: Bearer change-me-mcp-token" \
-  -H "Content-Type: application/json" \
-  -d '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}'
-```
+CSV `app/data/council_articles.csv` сопоставляется по полному ФИО только при единственном совпадении. Публикации объединяются по DOI, затем по нормализованным названию и году, если нет конфликтующих DOI. Совпадение в двух источниках сохраняет оба источника, но учитывается один раз в общем количестве. Импорт CSV в БД не выдаётся за данные сайта. CSV не содержит типа материала, точной даты и подтверждения К1/RSCI: неизвестные сведения обозначены явно; счётчик CSV предварительный. Публичная страница не открывает контакты, служебные данные или операции админки.
 
+Список **предварительный**: в CSV указаны год и квартиль Scopus/WoS, но нет точной даты, типа материала, журнала, источника квартиля, подтверждения К1 и RSCI. На границах периода год не позволяет проверить точную дату. Статьи с Q1/Q2 учитываются с пометкой источника; отсутствие кандидата не доказывает недостаточную публикационную активность. Для подтверждённого отбора нужны проверка журналов и дат и дополнительные источники публикаций. Выгрузка поставляется вместе с релизом и обновляется отдельным изменением репозитория.
+
+## Парсинг
+
+Worker запускает обход по `CRAWL_CRON`. Ручной запуск также доступен в админке на `Dashboard` и странице `Runs` или через REST:
+
+```bash
+curl -X POST http://localhost:8000/api/crawl-runs --cookie "miem_admin_session=..."
+```
+
+Алгоритм обновления:
+
+- найденные сотрудники получают статус `active` и обновленный `last_seen_at`;
+- новые сотрудники добавляются в `employees`;
+- если профиль перенесен на другой URL, он сопоставляется с прежней записью по единственному точному совпадению ФИО;
+- старые URL сохраняются в истории `employee_profile_urls`;
+- количество новых сотрудников за запуск сохраняется в `crawl_runs.new_count`;
+- публикации из HSE Publications записываются в `employee_publications`, а краткий список остается в JSON профиля;
+- новости из блока «В новостях» записываются в `employee_news_links`, а краткий список остается в JSON профиля;
+- один `404` старого профиля переводит сотрудника в статус `verification_required`, а не в `dismissed`;
+- статус `dismissed` устанавливается только после нескольких последовательных проверок `404`/`410`;
+- сетевые ошибки и ответы `5xx` не считаются подтверждением увольнения;
+- если число кандидатов на увольнение превышает защитный лимит, автоматическое увольнение приостанавливается;
+- кнопка «Проверить уволенных» сверяет только их profile_key с текущим списком источника и возвращает найденных сотрудников в `active` без обновления содержимого профиля;
+- каждый успешный новый или измененный разбор сохраняет запись в `employee_snapshots`;
+- неизмененные профили учитываются в `crawl_runs.skipped_count` и не получают новый snapshot.
+
+Во время выполнения парсинга `found_count`, `parsed_count`, `skipped_count` и `error_count` обновляются в базе. Админка опрашивает `/api/crawl-runs/latest` и показывает прогресс как `(parsed_count + skipped_count + error_count) / found_count`.
+
 ## Обслуживание
-
-```bash
+
+```bash
 docker compose logs -f api
 docker compose logs -f worker
-docker compose exec postgres pg_dump -U miem miem_workers > backup.sql
+docker compose exec postgres pg_dump -U miem miem_workers > backup.sql
 docker compose down
 ```
 
-Версия сервиса: `0.2.1`. Админка всегда показывает версии backend и frontend в footer.
+Production deployment временно отключён до установки отдельного deployment-доступа Workers. Workflow `CI` запускает тесты на push в `main` и pull_request в `main`; шаги Upload release и Deploy and verify безусловно отключены для всех событий и значений input. Стандартные имена deployment secrets: `PROD_HOST`, `PROD_USER`, `PROD_SSH_KEY`, `PROD_KNOWN_HOSTS`; `.env` и ключи в репозиторий не добавляются. PostgreSQL backup выполняется на VPS раз в неделю user cron deploy-пользователя через `infra/postgres-backup.sh`: `pg_dumpall` сохраняет весь кластер в `~/backups/postgres/postgres-latest.sql`. Новый dump сначала создаётся во временный файл и заменяет старый только после успешного завершения; предыдущие dump-файлы `miem_workers` удаляются после успешного создания нового. Эта правка не меняет существующий backup cron.
+
+Ручной запуск с `diagnostic_only=true` (по умолчанию) выводит только SHA256 fingerprint ключа `PROD_SSH_KEY`, без подключения к VPS. Публичный ключ передаётся через pipe, не сохраняется в файл или artifacts; shell tracing отключён. При `diagnostic_only=false` диагностический job пропускается: это значение также не разрешает deployment. Возврат production deployment требует отдельной reviewed правки после установки нового доступа.
+
+## Проверки
+
+После установки зависимостей из `requirements.txt` выполните:
+
+```bash
+python -m pytest -q
+node --check app/static/admin.js
+```
+
+Тесты с данными используют временную SQLite; отдельные API smoke-тесты запускают приложение с его текущей конфигурацией. Браузерная проверка: `pip install playwright`, `python -m playwright install chromium`, затем `python tests/browser_admin.py`. Она проверяет реальный рендеринг страниц, клавиатуру, диалог, фильтры и восстановление прогресса после ошибки. Скриншоты сохраняются во временную папку; путь выводится в конце.
+
+Версия сервиса: `0.10.6`. Админка всегда показывает версии backend и frontend в footer.
+
+## Разовый импорт публикаций
+
+Инструкция, минимальные права временной роли и ограниченный SSH-запуск: [docs/publication-import.md](docs/publication-import.md). По умолчанию импортер выполняет dry-run; production-применение требует отдельного решения владельца.
+
+Интеграционная проверка импорта на отдельном PostgreSQL: [docs/postgresql-import-verification.md](docs/postgresql-import-verification.md).
+
+Отдельный тестовый стенд на VPS и read-only проверка ресурсов: [docs/postgresql-import-vps.md](docs/postgresql-import-vps.md). Стенд запускает только владелец после оценки ресурсов.

app/admin.py

@@ -1,171 +1,273 @@
-from fastapi import APIRouter, BackgroundTasks, Depends, Form, Request
-from fastapi.responses import HTMLResponse, RedirectResponse
-from fastapi.templating import Jinja2Templates
-from sqlalchemy import desc, func, or_, select
-from sqlalchemy.orm import Session
-
-from app.config import Settings, get_settings
-from app.db import SessionLocal, get_db
-from app.models import CrawlError, CrawlRun, Employee
-from app.security import SESSION_COOKIE, require_admin, sign_session, verify_admin
-from app.services.admin_data import list_employees_page, run_payload, stats_payload
+from fastapi import APIRouter, BackgroundTasks, Depends, Form, Request
+from fastapi.responses import HTMLResponse, RedirectResponse, Response
+from fastapi.templating import Jinja2Templates
+from sqlalchemy import desc, func, select
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.orm import Session
+
+from app.config import Settings, get_settings
+from app.db import SessionLocal, get_db
+from app.models import CrawlError, CrawlRun, Employee
+from app.security import SESSION_COOKIE, require_admin, sign_session, verify_admin
+from app.services.admin_data import (
+    employee_detail_payload,
+    format_admin_datetime,
+    list_employees_page,
+    run_detail_payload,
+    run_payload,
+    stats_payload,
+)
 from app.services.crawl_control import get_running_run, run_crawl_if_idle
-from app.version import BACKEND_VERSION, FRONTEND_VERSION
-
-router = APIRouter(prefix="/admin")
-templates = Jinja2Templates(directory="app/templates")
-
-
-@router.get("", response_class=HTMLResponse)
-def dashboard(request: Request, db: Session = Depends(get_db), settings: Settings = Depends(get_settings)):
-    require_admin(request, settings)
-    counts = stats_payload(db)
-    counts["runs"] = db.scalar(select(func.count()).select_from(CrawlRun)) or 0
-    counts["errors"] = db.scalar(select(func.count()).select_from(CrawlError)) or 0
-    runs = db.scalars(select(CrawlRun).order_by(desc(CrawlRun.started_at)).limit(10)).all()
-    return _render(request, "dashboard.html", {"counts": counts, "runs": runs, "latest_run": run_payload(runs[0]) if runs else None})
-
-
-@router.get("/login", response_class=HTMLResponse)
-def login_form(request: Request):
-    return _render(request, "login.html", {"error": None})
-
-
-@router.post("/login")
-def login(
-    request: Request,
-    username: str = Form(...),
-    password: str = Form(...),
-    settings: Settings = Depends(get_settings),
-):
-    if not verify_admin(username, password, settings):
-        return _render(request, "login.html", {"error": "Неверный логин или пароль"}, status_code=401)
-    redirect = RedirectResponse("/admin", status_code=303)
-    redirect.set_cookie(SESSION_COOKIE, sign_session(username, settings), httponly=True, samesite="lax")
-    return redirect
-
-
-@router.post("/logout")
-def logout():
-    redirect = RedirectResponse("/admin/login", status_code=303)
-    redirect.delete_cookie(SESSION_COOKIE)
-    return redirect
-
-
-@router.get("/employees", response_class=HTMLResponse)
-def employees(
-    request: Request,
-    status: str | None = None,
-    q: str | None = None,
-    db: Session = Depends(get_db),
-    settings: Settings = Depends(get_settings),
-):
-    require_admin(request, settings)
-    stmt = select(Employee)
-    if status:
-        stmt = stmt.where(Employee.status == status)
-    if q:
-        pattern = f"%{q}%"
-        stmt = stmt.where(or_(Employee.full_name.ilike(pattern), Employee.canonical_url.ilike(pattern)))
-    items = db.scalars(stmt.order_by(Employee.full_name).limit(200)).all()
-    return _render(request, "employees.html", {"employees": items, "status": status or "", "q": q or ""})
-
-
+from app.services.crawler import refresh_dismissed_status, refresh_employee
+from app.services.council import council_candidates
+from app.services.council_export import council_xlsx
+from app.version import BACKEND_VERSION, FRONTEND_VERSION
+
+router = APIRouter(prefix="/admin")
+templates = Jinja2Templates(directory="app/templates")
+
+
+@router.get("", response_class=HTMLResponse)
+def dashboard(request: Request, db: Session = Depends(get_db), settings: Settings = Depends(get_settings)):
+    require_admin(request, settings)
+    counts = stats_payload(db)
+    counts["runs"] = db.scalar(select(func.count()).select_from(CrawlRun)) or 0
+    counts["errors"] = db.scalar(select(func.count()).select_from(CrawlError)) or 0
+    run_models = db.scalars(select(CrawlRun).order_by(desc(CrawlRun.started_at)).limit(5)).all()
+    runs = [run_payload(run) for run in run_models]
+    return _render(request, "dashboard.html", {"counts": counts, "runs": runs, "latest_run": runs[0] if runs else None})
+
+
+@router.get("/login", response_class=HTMLResponse)
+def login_form(request: Request):
+    return _render(request, "login.html", {"error": None})
+
+
+@router.post("/login")
+def login(
+    request: Request,
+    username: str = Form(...),
+    password: str = Form(...),
+    settings: Settings = Depends(get_settings),
+):
+    if not verify_admin(username, password, settings):
+        return _render(request, "login.html", {"error": "Неверный логин или пароль"}, status_code=401)
+    redirect = RedirectResponse("/admin", status_code=303)
+    redirect.set_cookie(SESSION_COOKIE, sign_session(username, settings), httponly=True, samesite="lax")
+    return redirect
+
+
+@router.post("/logout")
+def logout():
+    redirect = RedirectResponse("/admin/login", status_code=303)
+    redirect.delete_cookie(SESSION_COOKIE)
+    return redirect
+
+
+@router.get("/employees", response_class=HTMLResponse)
+def employees(
+    request: Request,
+    status: str | None = None,
+    q: str | None = None,
+    settings: Settings = Depends(get_settings),
+):
+    require_admin(request, settings)
+    return RedirectResponse("/admin/directory", status_code=303)
+
+
 @router.get("/directory", response_class=HTMLResponse)
 def directory(
-    request: Request,
-    status: str | None = None,
-    q: str | None = None,
-    started_from: str | None = None,
+    request: Request,
+    status: str | None = None,
+    q: str | None = None,
+    started_from: str | None = None,
     started_to: str | None = None,
     has_email: str | None = None,
-    sort: str = "full_name",
-    direction: str = "asc",
-    limit: int = 50,
-    offset: int = 0,
-    db: Session = Depends(get_db),
-    settings: Settings = Depends(get_settings),
-):
-    require_admin(request, settings)
-    parsed_started_from = _parse_date(started_from)
-    parsed_started_to = _parse_date(started_to)
+    has_academic_degree: str | None = None,
+    sort: str = "full_name",
+    direction: str = "asc",
+    limit: int = 50,
+    offset: int = 0,
+    db: Session = Depends(get_db),
+    settings: Settings = Depends(get_settings),
+):
+    require_admin(request, settings)
+    parsed_started_from = _parse_date(started_from)
+    parsed_started_to = _parse_date(started_to)
     parsed_has_email = None if has_email in (None, "") else has_email == "true"
-    page = list_employees_page(
-        db,
-        status=status,
-        q=q,
-        started_from=parsed_started_from,
-        started_to=parsed_started_to,
-        has_email=parsed_has_email,
-        sort=sort,
-        direction=direction,
-        limit=limit,
-        offset=offset,
-    )
-    return _render(
-        request,
-        "directory.html",
-        {
-            "page": page,
-            "filters": {
-                "status": status or "",
-                "q": q or "",
-                "started_from": started_from or "",
-                "started_to": started_to or "",
+    parsed_has_academic_degree = None if has_academic_degree in (None, "") else has_academic_degree == "true"
+    try:
+        page = list_employees_page(
+            db,
+            status=status,
+            q=q,
+            started_from=parsed_started_from,
+            started_to=parsed_started_to,
+            has_email=parsed_has_email,
+            has_academic_degree=parsed_has_academic_degree,
+            sort=sort,
+            direction=direction,
+            limit=limit,
+            offset=offset,
+        )
+    except SQLAlchemyError:
+        db.rollback()
+        return _render(request, "directory_error.html", {}, status_code=503)
+    return _render(
+        request,
+        "directory.html",
+        {
+            "page": page,
+            "filters": {
+                "status": status or "",
+                "q": q or "",
+                "started_from": started_from or "",
+                "started_to": started_to or "",
                 "has_email": has_email or "",
-                "sort": sort,
-                "direction": direction,
-                "limit": limit,
-                "offset": offset,
-            },
-        },
+                "has_academic_degree": has_academic_degree or "",
+                "sort": sort,
+                "direction": direction,
+                "limit": page["limit"],
+                "offset": page["offset"],
+            },
+        },
     )
 
 
+@router.get("/council", response_class=HTMLResponse)
+def council(request: Request, q: str = "", offset: int = 0, sort: str = "csv_count", direction: str = "desc",
+            db: Session = Depends(get_db)):
+    try:
+        result = council_candidates(db, q=q, offset=offset, sort=sort, direction=direction)
+    except (SQLAlchemyError, OSError, ValueError, KeyError):
+        db.rollback()
+        return _render(request, "council.html", {"error": True, "public_page": True}, status_code=503)
+    return _render(request, "council.html", {"error": False, "public_page": True, **result})
+
+
+@router.get("/council/export.xlsx")
+def export_council(request: Request, q: str = "", sort: str = "csv_count", direction: str = "desc",
+                   db: Session = Depends(get_db)):
+    try:
+        result = council_candidates(db, q=q, sort=sort, direction=direction, paginate=False)
+        content = council_xlsx(result)
+    except (SQLAlchemyError, OSError, ValueError, KeyError):
+        db.rollback()
+        return _render(request, "council.html", {"error": True, "public_page": True, "export_error": True}, status_code=503)
+    return Response(content, media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+                    headers={"Content-Disposition": 'attachment; filename="council.xlsx"'})
+
+
 @router.get("/employees/{employee_id}", response_class=HTMLResponse)
-def employee_detail(
-    employee_id: int,
-    request: Request,
-    db: Session = Depends(get_db),
-    settings: Settings = Depends(get_settings),
-):
-    require_admin(request, settings)
-    employee = db.get(Employee, employee_id)
-    if not employee:
-        return RedirectResponse("/admin/employees", status_code=303)
-    snapshots = sorted(employee.snapshots, key=lambda item: item.captured_at, reverse=True)[:20]
-    return _render(request, "employee_detail.html", {"employee": employee, "snapshots": snapshots})
-
-
-@router.get("/runs", response_class=HTMLResponse)
-def runs(request: Request, db: Session = Depends(get_db), settings: Settings = Depends(get_settings)):
-    require_admin(request, settings)
-    items = db.scalars(select(CrawlRun).order_by(desc(CrawlRun.started_at)).limit(50)).all()
-    errors = db.scalars(select(CrawlError).order_by(desc(CrawlError.created_at)).limit(50)).all()
-    return _render(request, "runs.html", {"runs": items, "errors": errors})
-
-
-@router.post("/runs")
-def trigger_run(
-    request: Request,
-    background_tasks: BackgroundTasks,
-    db: Session = Depends(get_db),
-    settings: Settings = Depends(get_settings),
-):
-    require_admin(request, settings)
-    if get_running_run(db):
-        return RedirectResponse("/admin/runs", status_code=303)
-
-    def _crawl() -> None:
-        with SessionLocal() as db:
-            run_crawl_if_idle(db, settings)
-
-    background_tasks.add_task(_crawl)
-    return RedirectResponse("/admin/runs", status_code=303)
-
-
+def employee_detail(
+    employee_id: int,
+    request: Request,
+    db: Session = Depends(get_db),
+    settings: Settings = Depends(get_settings),
+):
+    require_admin(request, settings)
+    employee = db.get(Employee, employee_id)
+    if not employee:
+        return RedirectResponse("/admin/employees", status_code=303)
+    snapshots = [
+        {
+            "captured_display": format_admin_datetime(snapshot.captured_at),
+            "checksum": snapshot.checksum,
+            "parser_version": snapshot.parser_version,
+        }
+        for snapshot in sorted(employee.snapshots, key=lambda item: item.captured_at, reverse=True)[:20]
+    ]
+    return _render(
+        request,
+        "employee_detail.html",
+        {
+            "employee": employee,
+            "employee_view": employee_detail_payload(employee),
+            "snapshots": snapshots,
+            "refresh_status": request.query_params.get("refresh_status"),
+        },
+    )
+
+
+@router.post("/employees/{employee_id}/refresh")
+def refresh_employee_detail(
+    employee_id: int,
+    request: Request,
+    db: Session = Depends(get_db),
+    settings: Settings = Depends(get_settings),
+):
+    require_admin(request, settings)
+    employee = db.get(Employee, employee_id)
+    if not employee:
+        return RedirectResponse("/admin/directory", status_code=303)
+    run = refresh_employee(db, employee, settings)
+    status = "success" if run.status == "completed" else "error"
+    return RedirectResponse(f"/admin/employees/{employee_id}?refresh_status={status}", status_code=303)
+
+
+@router.get("/runs", response_class=HTMLResponse)
+def runs(request: Request, db: Session = Depends(get_db), settings: Settings = Depends(get_settings)):
+    require_admin(request, settings)
+    run_models = db.scalars(select(CrawlRun).order_by(desc(CrawlRun.started_at)).limit(50)).all()
+    items = [run_payload(run) for run in run_models]
+    errors = db.scalars(select(CrawlError).order_by(desc(CrawlError.created_at)).limit(50)).all()
+    return _render(request, "runs.html", {"runs": items, "errors": errors})
+
+
+@router.get("/runs/{run_id}", response_class=HTMLResponse)
+def run_detail(
+    run_id: int,
+    request: Request,
+    db: Session = Depends(get_db),
+    settings: Settings = Depends(get_settings),
+):
+    require_admin(request, settings)
+    run = db.get(CrawlRun, run_id)
+    if not run:
+        return RedirectResponse("/admin/runs", status_code=303)
+    return _render(request, "run_detail.html", {"run": run_detail_payload(db, run)})
+
+
+@router.post("/runs")
+def trigger_run(
+    request: Request,
+    background_tasks: BackgroundTasks,
+    db: Session = Depends(get_db),
+    settings: Settings = Depends(get_settings),
+):
+    require_admin(request, settings)
+    if get_running_run(db):
+        return RedirectResponse("/admin/runs", status_code=303)
+
+    def _crawl() -> None:
+        with SessionLocal() as db:
+            run_crawl_if_idle(db, settings)
+
+    background_tasks.add_task(_crawl)
+    return RedirectResponse("/admin/runs", status_code=303)
+
+
 @router.post("/crawl-now")
 def crawl_now(
+    request: Request,
+    background_tasks: BackgroundTasks,
+    db: Session = Depends(get_db),
+    settings: Settings = Depends(get_settings),
+):
+    require_admin(request, settings)
+    if get_running_run(db):
+        return RedirectResponse("/admin", status_code=303)
+
+    def _crawl() -> None:
+        with SessionLocal() as db:
+            run_crawl_if_idle(db, settings)
+
+    background_tasks.add_task(_crawl)
+    return RedirectResponse("/admin", status_code=303)
+
+
+@router.post("/dismissed/refresh")
+def refresh_dismissed(
     request: Request,
     background_tasks: BackgroundTasks,
     db: Session = Depends(get_db),
@@ -175,30 +277,30 @@ def crawl_now(
     if get_running_run(db):
         return RedirectResponse("/admin", status_code=303)
 
-    def _crawl() -> None:
+    def _refresh() -> None:
         with SessionLocal() as db:
-            run_crawl_if_idle(db, settings)
+            refresh_dismissed_status(db, settings)
 
-    background_tasks.add_task(_crawl)
+    background_tasks.add_task(_refresh)
     return RedirectResponse("/admin", status_code=303)
 
 
 def _render(request: Request, template: str, context: dict, status_code: int = 200) -> HTMLResponse:
-    payload = {
-        "request": request,
-        "backend_version": BACKEND_VERSION,
-        "frontend_version": FRONTEND_VERSION,
-        **context,
-    }
-    return templates.TemplateResponse(request, template, payload, status_code=status_code)
-
-
-def _parse_date(value: str | None):
-    if not value:
-        return None
-    try:
-        from datetime import date
-
-        return date.fromisoformat(value)
-    except ValueError:
-        return None
+    payload = {
+        "request": request,
+        "backend_version": BACKEND_VERSION,
+        "frontend_version": FRONTEND_VERSION,
+        **context,
+    }
+    return templates.TemplateResponse(request, template, payload, status_code=status_code)
+
+
+def _parse_date(value: str | None):
+    if not value:
+        return None
+    try:
+        from datetime import date
+
+        return date.fromisoformat(value)
+    except ValueError:
+        return None

app/api.py

@@ -8,7 +8,7 @@ from app.config import Settings, get_settings
 from app.db import SessionLocal, get_db
 from app.models import CrawlRun, Employee
 from app.security import require_admin
-from app.services.admin_data import employee_display_payload, list_employees_page, run_payload, stats_payload
+from app.services.admin_data import employee_display_payload, list_employees_page, run_detail_payload, run_payload, stats_payload
 from app.services.crawl_control import get_running_run, run_crawl_if_idle
 from app.version import BACKEND_VERSION, FRONTEND_VERSION
 
@@ -28,6 +28,7 @@ def list_employees(
     started_from: date | None = None,
     started_to: date | None = None,
     has_email: bool | None = None,
+    has_academic_degree: bool | None = None,
     sort: str = "full_name",
     direction: str = "asc",
     limit: int = 50,
@@ -43,6 +44,7 @@ def list_employees(
         started_from=started_from,
         started_to=started_to,
         has_email=has_email,
+        has_academic_degree=has_academic_degree,
         sort=sort,
         direction=direction,
         limit=limit,
@@ -88,6 +90,20 @@ def latest_crawl_run(
     return {"running": run_payload(running), "latest": run_payload(latest)}
 
 
+@router.get("/crawl-runs/{run_id}")
+def get_crawl_run(
+    run_id: int,
+    request: Request,
+    db: Session = Depends(get_db),
+    settings: Settings = Depends(get_settings),
+) -> dict:
+    require_admin(request, settings)
+    run = db.get(CrawlRun, run_id)
+    if not run:
+        return {"error": "not_found"}
+    return run_detail_payload(db, run) or {"error": "not_found"}
+
+
 @router.post("/crawl-runs")
 def trigger_crawl(
     request: Request,

app/config.py

@@ -1,5 +1,5 @@
 from functools import lru_cache
-from pydantic import Field
+from pydantic import Field, field_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 
@@ -13,12 +13,19 @@ class Settings(BaseSettings):
     request_timeout: int = 30
     request_delay_seconds: float = 1.0
     parser_use_playwright: bool = False
+    dismissal_confirmation_runs: int = Field(default=3, ge=1)
+    max_auto_dismissals_per_run: int = Field(default=25, ge=1)
 
     admin_username: str = "admin"
     admin_password: str = "admin"
     session_secret: str = Field(default="dev-session-secret", min_length=8)
-    mcp_token: str = "dev-mcp-token"
 
+    @field_validator("crawl_limit", mode="before")
+    @classmethod
+    def empty_crawl_limit_as_none(cls, value):
+        if value == "":
+            return None
+        return value
 
 @lru_cache
 def get_settings() -> Settings:

app/db.py

@@ -1,6 +1,6 @@
 from collections.abc import Generator
 
-from sqlalchemy import create_engine
+from sqlalchemy import create_engine, inspect, text
 from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
 
 from app.config import get_settings
@@ -25,6 +25,52 @@ def init_db() -> None:
     import app.models  # noqa: F401
 
     Base.metadata.create_all(bind=engine)
+    _ensure_runtime_schema()
+
+
+def _ensure_runtime_schema() -> None:
+    import app.models as models
+
+    inspector = inspect(engine)
+    table_names = set(inspector.get_table_names())
+    if "employees" in table_names and "employee_publications" not in table_names:
+        models.EmployeePublication.__table__.create(bind=engine, checkfirst=True)
+        inspector = inspect(engine)
+        table_names = set(inspector.get_table_names())
+    if "employees" in table_names and "employee_news_links" not in table_names:
+        models.EmployeeNewsLink.__table__.create(bind=engine, checkfirst=True)
+        inspector = inspect(engine)
+        table_names = set(inspector.get_table_names())
+    if "employees" in table_names and "employee_profile_urls" not in table_names:
+        models.EmployeeProfileUrl.__table__.create(bind=engine, checkfirst=True)
+        inspector = inspect(engine)
+    if "employees" in table_names:
+        employee_columns = {column["name"] for column in inspector.get_columns("employees")}
+        missing_columns = []
+        if "profile_unavailable_streak" not in employee_columns:
+            missing_columns.append("profile_unavailable_streak INTEGER NOT NULL DEFAULT 0")
+        if "last_profile_check_at" not in employee_columns:
+            missing_columns.append("last_profile_check_at TIMESTAMPTZ")
+        if "has_academic_degree" not in employee_columns:
+            missing_columns.append("has_academic_degree BOOLEAN NOT NULL DEFAULT FALSE")
+        if missing_columns:
+            with engine.begin() as connection:
+                for column in missing_columns:
+                    connection.execute(text(f"ALTER TABLE employees ADD COLUMN {column}"))
+                if engine.dialect.name == "postgresql":
+                    connection.execute(
+                        text(
+                            "UPDATE employees SET has_academic_degree = "
+                            "COALESCE(current_data::text ~* '(кандидат|доктор).{0,80}наук|ph\\.?[[:space:]]*d\\.?', FALSE)"
+                        )
+                    )
+                connection.execute(text("CREATE INDEX IF NOT EXISTS ix_employees_has_academic_degree ON employees (has_academic_degree)"))
+    if "crawl_runs" not in table_names:
+        return
+    crawl_run_columns = {column["name"] for column in inspector.get_columns("crawl_runs")}
+    if "skipped_count" not in crawl_run_columns:
+        with engine.begin() as connection:
+            connection.execute(text("ALTER TABLE crawl_runs ADD COLUMN skipped_count INTEGER NOT NULL DEFAULT 0"))
 
 
 def get_db() -> Generator[Session, None, None]:

app/main.py

@@ -4,14 +4,12 @@ from fastapi.staticfiles import StaticFiles
 from app.admin import router as admin_router
 from app.api import router as api_router
 from app.db import init_db
-from app.mcp import router as mcp_router
 from app.version import BACKEND_VERSION
 
 app = FastAPI(title="MIEM Employees", version=BACKEND_VERSION)
 app.mount("/static", StaticFiles(directory="app/static"), name="static")
 app.include_router(api_router)
 app.include_router(admin_router)
-app.include_router(mcp_router)
 
 
 @app.on_event("startup")

app/mcp.py

@@ -1,170 +0,0 @@
-import json
-
-from fastapi import APIRouter, Depends, Request
-from sqlalchemy import desc, or_, select
-from sqlalchemy.orm import Session
-
-from app.config import Settings, get_settings
-from app.db import get_db
-from app.models import CrawlRun, Employee
-from app.security import require_mcp_token
-
-router = APIRouter(prefix="/mcp")
-
-
-TOOLS = [
-    {
-        "name": "search_employees",
-        "description": "Search MIEM employees by name or profile URL.",
-        "inputSchema": {
-            "type": "object",
-            "properties": {
-                "query": {"type": "string"},
-                "status": {"type": "string", "enum": ["active", "dismissed"]},
-                "limit": {"type": "integer", "default": 20},
-            },
-            "required": ["query"],
-        },
-    },
-    {
-        "name": "get_employee",
-        "description": "Get one employee by profile id, profile key, or canonical URL.",
-        "inputSchema": {"type": "object", "properties": {"profile_id_or_url": {"type": "string"}}, "required": ["profile_id_or_url"]},
-    },
-    {
-        "name": "list_employee_publications",
-        "description": "List publications parsed from an employee profile.",
-        "inputSchema": {"type": "object", "properties": {"profile_id_or_url": {"type": "string"}}, "required": ["profile_id_or_url"]},
-    },
-    {
-        "name": "list_employee_courses",
-        "description": "List teaching courses parsed from an employee profile.",
-        "inputSchema": {"type": "object", "properties": {"profile_id_or_url": {"type": "string"}}, "required": ["profile_id_or_url"]},
-    },
-    {
-        "name": "get_crawl_status",
-        "description": "Return the latest crawl run status.",
-        "inputSchema": {"type": "object", "properties": {}},
-    },
-]
-
-
-@router.post("")
-async def mcp_http(
-    request: Request,
-    db: Session = Depends(get_db),
-    settings: Settings = Depends(get_settings),
-) -> dict:
-    require_mcp_token(request, settings)
-    payload = await request.json()
-    method = payload.get("method")
-    request_id = payload.get("id")
-    params = payload.get("params") or {}
-
-    try:
-        if method == "initialize":
-            result = {
-                "protocolVersion": "2024-11-05",
-                "serverInfo": {"name": "miem-employees", "version": "0.1.0"},
-                "capabilities": {"tools": {}},
-            }
-        elif method == "tools/list":
-            result = {"tools": TOOLS}
-        elif method == "tools/call":
-            result = _call_tool(db, params.get("name"), params.get("arguments") or {})
-        else:
-            return {"jsonrpc": "2.0", "id": request_id, "error": {"code": -32601, "message": "Method not found"}}
-        return {"jsonrpc": "2.0", "id": request_id, "result": result}
-    except Exception as exc:
-        return {"jsonrpc": "2.0", "id": request_id, "error": {"code": -32000, "message": str(exc)}}
-
-
-def _call_tool(db: Session, name: str, arguments: dict) -> dict:
-    if name == "search_employees":
-        return _tool_response(_search_employees(db, arguments))
-    if name == "get_employee":
-        employee = _find_employee(db, arguments["profile_id_or_url"])
-        return _tool_response(_employee_payload(employee) if employee else {"error": "not_found"})
-    if name == "list_employee_publications":
-        employee = _find_employee(db, arguments["profile_id_or_url"])
-        return _tool_response(_collect_section_items(employee, "publications"))
-    if name == "list_employee_courses":
-        employee = _find_employee(db, arguments["profile_id_or_url"])
-        return _tool_response(_collect_section_items(employee, "courses_by_year"))
-    if name == "get_crawl_status":
-        run = db.scalar(select(CrawlRun).order_by(desc(CrawlRun.started_at)).limit(1))
-        return _tool_response(_run_payload(run) if run else {"status": "never_run"})
-    raise ValueError(f"Unknown tool: {name}")
-
-
-def _search_employees(db: Session, arguments: dict) -> list[dict]:
-    query = arguments.get("query", "")
-    limit = min(int(arguments.get("limit") or 20), 100)
-    stmt = select(Employee)
-    if arguments.get("status"):
-        stmt = stmt.where(Employee.status == arguments["status"])
-    if query:
-        pattern = f"%{query}%"
-        stmt = stmt.where(or_(Employee.full_name.ilike(pattern), Employee.canonical_url.ilike(pattern)))
-    employees = db.scalars(stmt.order_by(Employee.full_name).limit(limit)).all()
-    return [_employee_payload(employee, include_data=False) for employee in employees]
-
-
-def _find_employee(db: Session, value: str) -> Employee | None:
-    pattern = value.strip()
-    stmt = select(Employee).where(
-        or_(
-            Employee.profile_key == pattern,
-            Employee.profile_id == pattern,
-            Employee.canonical_url == pattern,
-            Employee.canonical_url.ilike(f"%{pattern}%"),
-        )
-    )
-    return db.scalar(stmt.limit(1))
-
-
-def _collect_section_items(employee: Employee | None, section_type: str) -> dict:
-    if not employee or not employee.current_data:
-        return {"items": []}
-    items = []
-    for section in employee.current_data.get("sections") or []:
-        if section.get("type") != section_type:
-            continue
-        if section_type == "publications":
-            items.extend(section.get("publications") or [])
-        elif section_type == "courses_by_year":
-            items.extend(section.get("courses") or [])
-    return {"employee": _employee_payload(employee, include_data=False), "items": items}
-
-
-def _employee_payload(employee: Employee, include_data: bool = True) -> dict:
-    payload = {
-        "profile_key": employee.profile_key,
-        "profile_id": employee.profile_id,
-        "full_name": employee.full_name,
-        "status": employee.status,
-        "canonical_url": employee.canonical_url,
-        "last_seen_at": employee.last_seen_at.isoformat() if employee.last_seen_at else None,
-        "dismissed_at": employee.dismissed_at.isoformat() if employee.dismissed_at else None,
-    }
-    if include_data:
-        payload["data"] = employee.current_data
-    return payload
-
-
-def _run_payload(run: CrawlRun) -> dict:
-    return {
-        "id": run.id,
-        "status": run.status,
-        "source_url": run.source_url,
-        "started_at": run.started_at.isoformat() if run.started_at else None,
-        "finished_at": run.finished_at.isoformat() if run.finished_at else None,
-        "found_count": run.found_count,
-        "parsed_count": run.parsed_count,
-        "error_count": run.error_count,
-        "dismissed_count": run.dismissed_count,
-    }
-
-
-def _tool_response(data: object) -> dict:
-    return {"content": [{"type": "text", "text": json.dumps(data, ensure_ascii=False, default=str)}]}

app/models.py

@@ -1,6 +1,6 @@
 from datetime import datetime, timezone
 
-from sqlalchemy import DateTime, ForeignKey, Index, Integer, LargeBinary, String, Text, UniqueConstraint
+from sqlalchemy import Boolean, DateTime, ForeignKey, Index, Integer, LargeBinary, String, Text, UniqueConstraint
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 from sqlalchemy.types import JSON
@@ -21,6 +21,7 @@ class Employee(Base):
         UniqueConstraint("profile_key", name="uq_employees_profile_key"),
         Index("ix_employees_full_name", "full_name"),
         Index("ix_employees_status", "status"),
+        Index("ix_employees_has_academic_degree", "has_academic_degree"),
     )
 
     id: Mapped[int] = mapped_column(Integer, primary_key=True)
@@ -33,14 +34,37 @@ class Employee(Base):
     first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)
     last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)
     dismissed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    profile_unavailable_streak: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
+    last_profile_check_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     parser_version: Mapped[str | None] = mapped_column(String(32))
     current_data: Mapped[dict | None] = mapped_column(json_type)
+    has_academic_degree: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
     current_checksum: Mapped[str | None] = mapped_column(String(64))
     created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)
     updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow, nullable=False)
 
     snapshots: Mapped[list["EmployeeSnapshot"]] = relationship(back_populates="employee")
+    profile_urls: Mapped[list["EmployeeProfileUrl"]] = relationship(back_populates="employee", cascade="all, delete-orphan")
     tabs: Mapped[list["ProfileTab"]] = relationship(back_populates="employee", cascade="all, delete-orphan")
+    publications: Mapped[list["EmployeePublication"]] = relationship(back_populates="employee", cascade="all, delete-orphan")
+    news_links: Mapped[list["EmployeeNewsLink"]] = relationship(back_populates="employee", cascade="all, delete-orphan")
+    crawl_run_changes: Mapped[list["CrawlRunEmployeeChange"]] = relationship(back_populates="employee")
+
+
+class EmployeeProfileUrl(Base):
+    __tablename__ = "employee_profile_urls"
+    __table_args__ = (
+        UniqueConstraint("employee_id", "url", name="uq_employee_profile_urls_employee_url"),
+        Index("ix_employee_profile_urls_employee_id", "employee_id"),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    employee_id: Mapped[int] = mapped_column(ForeignKey("employees.id", ondelete="CASCADE"), nullable=False)
+    url: Mapped[str] = mapped_column(Text, nullable=False)
+    first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)
+    last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+    employee: Mapped[Employee] = relationship(back_populates="profile_urls")
 
 
 class EmployeeSnapshot(Base):
@@ -59,6 +83,68 @@ class EmployeeSnapshot(Base):
     employee: Mapped[Employee] = relationship(back_populates="snapshots")
 
 
+class EmployeePublication(Base):
+    __tablename__ = "employee_publications"
+    __table_args__ = (
+        UniqueConstraint("employee_id", "publication_id", name="uq_employee_publications_employee_publication"),
+        UniqueConstraint("employee_id", "source_hash", name="uq_employee_publications_employee_source_hash"),
+        Index("ix_employee_publications_employee_id", "employee_id"),
+        Index("ix_employee_publications_publication_id", "publication_id"),
+        Index("ix_employee_publications_doi_url", "doi_url"),
+        Index("ix_employee_publications_year", "year"),
+        Index("ix_employee_publications_publication_type", "publication_type"),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    employee_id: Mapped[int] = mapped_column(ForeignKey("employees.id", ondelete="CASCADE"), nullable=False)
+    publication_id: Mapped[str | None] = mapped_column(String(64))
+    title: Mapped[str] = mapped_column(Text, nullable=False)
+    year: Mapped[int | None] = mapped_column(Integer)
+    publication_type: Mapped[str | None] = mapped_column(String(64))
+    language: Mapped[str | None] = mapped_column(String(16))
+    status: Mapped[int | None] = mapped_column(Integer)
+    url: Mapped[str | None] = mapped_column(Text)
+    doi_url: Mapped[str | None] = mapped_column(Text)
+    other_url: Mapped[str | None] = mapped_column(Text)
+    document_url: Mapped[str | None] = mapped_column(Text)
+    citation_text: Mapped[str | None] = mapped_column(Text)
+    annotation: Mapped[dict | None] = mapped_column(json_type)
+    description: Mapped[dict | None] = mapped_column(json_type)
+    authors: Mapped[list | None] = mapped_column(json_type)
+    raw_data: Mapped[dict | None] = mapped_column(json_type)
+    source_hash: Mapped[str] = mapped_column(String(64), nullable=False)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow, nullable=False)
+
+    employee: Mapped[Employee] = relationship(back_populates="publications")
+
+
+class EmployeeNewsLink(Base):
+    __tablename__ = "employee_news_links"
+    __table_args__ = (
+        UniqueConstraint("employee_id", "url", name="uq_employee_news_links_employee_url"),
+        UniqueConstraint("employee_id", "source_hash", name="uq_employee_news_links_employee_source_hash"),
+        Index("ix_employee_news_links_employee_id", "employee_id"),
+        Index("ix_employee_news_links_url", "url"),
+        Index("ix_employee_news_links_published_at", "published_at"),
+        Index("ix_employee_news_links_published_year", "published_year"),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    employee_id: Mapped[int] = mapped_column(ForeignKey("employees.id", ondelete="CASCADE"), nullable=False)
+    title: Mapped[str] = mapped_column(Text, nullable=False)
+    url: Mapped[str | None] = mapped_column(Text)
+    summary: Mapped[str | None] = mapped_column(Text)
+    published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    published_year: Mapped[int | None] = mapped_column(Integer)
+    source_hash: Mapped[str] = mapped_column(String(64), nullable=False)
+    raw_data: Mapped[dict | None] = mapped_column(json_type)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow, nullable=False)
+
+    employee: Mapped[Employee] = relationship(back_populates="news_links")
+
+
 class CrawlRun(Base):
     __tablename__ = "crawl_runs"
 
@@ -69,11 +155,37 @@ class CrawlRun(Base):
     finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     found_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
     parsed_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
+    skipped_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
     new_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
     error_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
     dismissed_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
     message: Mapped[str | None] = mapped_column(Text)
 
+    employee_changes: Mapped[list["CrawlRunEmployeeChange"]] = relationship(back_populates="crawl_run")
+
+
+class CrawlRunEmployeeChange(Base):
+    __tablename__ = "crawl_run_employee_changes"
+    __table_args__ = (
+        Index("ix_crawl_run_employee_changes_run_id", "crawl_run_id"),
+        Index("ix_crawl_run_employee_changes_employee_id", "employee_id"),
+        Index("ix_crawl_run_employee_changes_change_type", "change_type"),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    crawl_run_id: Mapped[int] = mapped_column(ForeignKey("crawl_runs.id"), nullable=False)
+    employee_id: Mapped[int | None] = mapped_column(ForeignKey("employees.id"))
+    profile_key: Mapped[str] = mapped_column(String(255), nullable=False)
+    profile_url: Mapped[str] = mapped_column(Text, nullable=False)
+    full_name: Mapped[str | None] = mapped_column(Text)
+    change_type: Mapped[str] = mapped_column(String(32), nullable=False)
+    profile_available: Mapped[bool | None] = mapped_column()
+    message: Mapped[str | None] = mapped_column(Text)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+    crawl_run: Mapped[CrawlRun] = relationship(back_populates="employee_changes")
+    employee: Mapped[Employee | None] = relationship(back_populates="crawl_run_changes")
+
 
 class CrawlError(Base):
     __tablename__ = "crawl_errors"
@@ -108,3 +220,24 @@ class ParserSource(Base):
     source_url: Mapped[str] = mapped_column(Text, nullable=False)
     enabled: Mapped[bool] = mapped_column(default=True, nullable=False)
     created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+
+class ParseResourceCache(Base):
+    __tablename__ = "parse_resource_cache"
+    __table_args__ = (
+        UniqueConstraint("profile_key", "resource_key", "request_fingerprint", name="uq_parse_resource_cache_resource"),
+        Index("ix_parse_resource_cache_profile_key", "profile_key"),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    profile_key: Mapped[str] = mapped_column(String(255), nullable=False)
+    resource_key: Mapped[str] = mapped_column(String(255), nullable=False)
+    method: Mapped[str] = mapped_column(String(16), nullable=False)
+    url: Mapped[str] = mapped_column(Text, nullable=False)
+    request_fingerprint: Mapped[str] = mapped_column(String(64), nullable=False)
+    etag: Mapped[str | None] = mapped_column(Text)
+    last_modified: Mapped[str | None] = mapped_column(Text)
+    body_hash: Mapped[str] = mapped_column(String(64), nullable=False)
+    body_snapshot: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
+    parser_version: Mapped[str | None] = mapped_column(String(32))
+    fetched_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)

app/parser/profile.py

@@ -1,4 +1,7 @@
+import hashlib
+import json
 import re
+from datetime import datetime, timezone
 from urllib.parse import urljoin
 
 from bs4 import BeautifulSoup, NavigableString, Tag
@@ -99,6 +102,8 @@ def extract_person_header(soup: BeautifulSoup, source_url: str) -> dict:
 def extract_sections(soup: BeautifulSoup, source_url: str) -> list[dict]:
     sections = []
     for h2 in soup.select("h2"):
+        if h2.find_parent(class_="post") or h2.find_parent(attrs={"data-tab": "press_links_news"}):
+            continue
         title = normalize_ws(h2.get_text(" ", strip=True))
         if not title or "расписание занятий" in title.lower():
             continue
@@ -140,6 +145,21 @@ def extract_sections(soup: BeautifulSoup, source_url: str) -> list[dict]:
             if section_type in {"generic", "paragraphs"}:
                 section["type"] = "year_blocks"
         sections.append(section)
+    news_links = _parse_news_links(soup, source_url)
+    if news_links:
+        sections.append(
+            {
+                "title": "В новостях",
+                "slug": "v_novostyah",
+                "type": "news",
+                "raw_text": "",
+                "paragraphs": [],
+                "items": [item["title"] for item in news_links if item.get("title")],
+                "links": [{"text": item["title"], "url": item["url"]} for item in news_links if item.get("title") and item.get("url")],
+                "news_count": len(news_links),
+                "news_links": news_links,
+            }
+        )
     return sections
 
 
@@ -149,21 +169,42 @@ def parse_person_profile(
     headers: dict[str, str],
     timeout: int,
     use_playwright: bool = False,
+    resource_cache=None,
 ) -> dict | None:
     normalized_url = normalize_profile_url(source_url)
     if not normalized_url:
         return None
-    response = session.get(normalized_url, headers=headers, timeout=timeout)
-    response.raise_for_status()
-    html = response.text
+    profile_type, profile_id = parse_profile_identity(normalized_url)
+    cache_profile_key = f"{profile_type}:{profile_id}"
+    resource_manifest = []
+    html = _fetch_text(
+        session,
+        normalized_url,
+        headers,
+        timeout,
+        resource_cache=resource_cache,
+        profile_key=cache_profile_key,
+        resource_key="main-html",
+        resource_manifest=resource_manifest,
+    )
     if use_playwright:
         html = _render_with_playwright(normalized_url, html)
 
     soup = BeautifulSoup(html, "html.parser")
-    profile_type, profile_id = parse_profile_identity(normalized_url)
     header = extract_person_header(soup, normalized_url)
     tabs = extract_person_tabs(soup, normalized_url)
     sections = extract_sections(soup, normalized_url)
+    sections = enrich_sections_from_hse_widgets(
+        session,
+        soup,
+        normalized_url,
+        headers,
+        timeout,
+        sections,
+        resource_cache=resource_cache,
+        profile_key=cache_profile_key,
+        resource_manifest=resource_manifest,
+    )
     internal_links = [tab["href"] for tab in tabs if tab.get("href")]
 
     return {
@@ -180,9 +221,49 @@ def parse_person_profile(
         "employee_internal_links": internal_links,
         "parser_version": BACKEND_VERSION,
         "_html": html,
+        "_resource_manifest": resource_manifest,
     }
 
 
+def enrich_sections_from_hse_widgets(
+    session: Session,
+    soup: BeautifulSoup,
+    source_url: str,
+    headers: dict[str, str],
+    timeout: int,
+    sections: list[dict],
+    resource_cache=None,
+    profile_key: str | None = None,
+    resource_manifest: list[dict] | None = None,
+) -> list[dict]:
+    enriched = list(sections)
+    publications = _load_widget_publications(
+        session,
+        soup,
+        headers,
+        timeout,
+        resource_cache=resource_cache,
+        profile_key=profile_key,
+        resource_manifest=resource_manifest,
+    )
+    if publications:
+        enriched = _upsert_publications_section(enriched, publications)
+
+    theses = _load_widget_graduation_theses(
+        session,
+        soup,
+        source_url,
+        headers,
+        timeout,
+        resource_cache=resource_cache,
+        profile_key=profile_key,
+        resource_manifest=resource_manifest,
+    )
+    if theses:
+        enriched = _upsert_graduation_theses_section(enriched, theses)
+    return enriched
+
+
 def _render_with_playwright(source_url: str, fallback_html: str) -> str:
     try:
         from playwright.sync_api import sync_playwright
@@ -206,6 +287,161 @@ def _render_with_playwright(source_url: str, fallback_html: str) -> str:
         return fallback_html
 
 
+def _load_widget_publications(
+    session: Session,
+    soup: BeautifulSoup,
+    headers: dict[str, str],
+    timeout: int,
+    *,
+    resource_cache=None,
+    profile_key: str | None = None,
+    resource_manifest: list[dict] | None = None,
+) -> list[dict]:
+    script = soup.select_one('script[data-widget-name="AuthorSearch"][data-author]')
+    if not script:
+        return []
+    author_id = normalize_ws(script.get("data-author"))
+    if not author_id:
+        return []
+
+    publications = []
+    page_id = 1
+    per_page = 100
+    while page_id <= 20:
+        payload = {
+            "type": "ANY",
+            "filterParams": (
+                f'"acceptLanguage":"ru"|"fullTextPublicEnabled": 1|'
+                f'"pubsAuthor": {author_id}|"widgetName": "AuthorSearch"'
+            ),
+            "paginationParams": {
+                "publsSort": ["TITLE_ASC"],
+                "publsCount": per_page,
+                "pageId": page_id,
+            },
+        }
+        try:
+            if resource_cache and profile_key:
+                text = _fetch_text(
+                    session,
+                    "https://publications.hse.ru/api/searchPubs",
+                    headers,
+                    timeout,
+                    resource_cache=resource_cache,
+                    profile_key=profile_key,
+                    resource_key=f"publications-page-{page_id}",
+                    resource_manifest=resource_manifest,
+                    method="POST",
+                    json_payload=payload,
+                )
+                data = json.loads(text)
+            else:
+                response = session.post(
+                    "https://publications.hse.ru/api/searchPubs",
+                    json=payload,
+                    headers=headers,
+                    timeout=timeout,
+                )
+                response.raise_for_status()
+                data = response.json()
+        except Exception:
+            return publications
+
+        result = data.get("result") if isinstance(data, dict) else {}
+        items = _extract_publication_items(result)
+        if not items:
+            break
+        publications.extend(_normalize_publication_item(item, author_id) for item in items)
+
+        total = int(result.get("total") or 0)
+        if not result.get("more") and len(publications) >= total:
+            break
+        page_id += 1
+    return _dedupe_publications(publications)
+
+
+def _extract_publication_items(result: object) -> list[dict]:
+    if not isinstance(result, dict):
+        return []
+    return _flatten_publication_items(result.get("items"))
+
+
+def _flatten_publication_items(value: object) -> list[dict]:
+    if isinstance(value, list):
+        return [item for item in value if _is_publication_item(item)]
+    if not isinstance(value, dict):
+        return []
+
+    nested_items = value.get("items")
+    if isinstance(nested_items, list):
+        return [item for item in nested_items if _is_publication_item(item)]
+    if isinstance(nested_items, dict):
+        return _flatten_publication_items(nested_items)
+
+    publications = []
+    for child in value.values():
+        publications.extend(_flatten_publication_items(child))
+    return publications
+
+
+def _is_publication_item(value: object) -> bool:
+    return isinstance(value, dict) and ("id" in value or "title" in value)
+
+
+def _load_widget_graduation_theses(
+    session: Session,
+    soup: BeautifulSoup,
+    source_url: str,
+    headers: dict[str, str],
+    timeout: int,
+    *,
+    resource_cache=None,
+    profile_key: str | None = None,
+    resource_manifest: list[dict] | None = None,
+) -> list[dict]:
+    script = soup.select_one('script[src*="/n/stat/vkr/app.js"][data-person-id]')
+    if not script:
+        return []
+    person_id = normalize_ws(script.get("data-person-id"))
+    api_url = normalize_ws(script.get("data-api-url")) or "/n/vkr/api/"
+    if not person_id:
+        return []
+
+    request_headers = {**headers, "x-portal-language": "ru"}
+    try:
+        url = urljoin(source_url, api_url)
+        params = {"supervisorId": person_id}
+        if resource_cache and profile_key:
+            text = _fetch_text(
+                session,
+                url,
+                request_headers,
+                timeout,
+                resource_cache=resource_cache,
+                profile_key=profile_key,
+                resource_key="graduation-theses",
+                resource_manifest=resource_manifest,
+                params=params,
+            )
+            data = json.loads(text)
+        else:
+            response = session.get(
+                url,
+                params=params,
+                headers=request_headers,
+                timeout=timeout,
+            )
+            response.raise_for_status()
+            data = response.json()
+    except Exception:
+        return []
+
+    items = data.get("data") if isinstance(data, dict) else []
+    if not isinstance(items, list):
+        return []
+    return [_normalize_vkr_item(item, source_url) for item in items if isinstance(item, dict)]
+
+
 def _collect_between_h2(start_h2: Tag) -> list[Tag | NavigableString | str]:
     nodes = []
     for sibling in start_h2.next_siblings:
@@ -256,7 +492,7 @@ def _infer_section_type(title: str, nodes: list) -> str:
     lowered = title.lower()
     if _has_table(nodes):
         return "table"
-    if "публикац" in lowered:
+    if _is_publications_title(lowered):
         return "publications"
     if "учебные курсы" in lowered:
         return "courses_by_year"
@@ -267,6 +503,10 @@ def _infer_section_type(title: str, nodes: list) -> str:
     return "generic"
 
 
+def _is_publications_title(lowered_title: str) -> bool:
+    return lowered_title.startswith("публикац")
+
+
 def _has_table(nodes: list) -> bool:
     return any(isinstance(node, Tag) and (node.name == "table" or node.find("table")) for node in nodes)
 
@@ -353,6 +593,296 @@ def _parse_vkr_items(nodes: list) -> list[str]:
     return [item for item in dict.fromkeys(items) if item]
 
 
+def _parse_news_links(soup: BeautifulSoup, source_url: str) -> list[dict]:
+    news = []
+    for post in soup.select('[data-tab="press_links_news"] .post'):
+        if not isinstance(post, Tag):
+            continue
+        anchor = post.select_one(".post__content h2 a[href], h2 a[href], a[href]")
+        title = normalize_ws(anchor.get_text(" ", strip=True)) if anchor else ""
+        href = normalize_ws(anchor.get("href")) if anchor else ""
+        summary_node = post.select_one(".post__text")
+        summary = normalize_ws(summary_node.get_text(" ", strip=True)) if summary_node else ""
+        published_at = _parse_post_date(post)
+        if not title and not href:
+            continue
+        item = {
+            "title": title or href,
+            "url": urljoin(source_url, href) if href else None,
+            "summary": summary or None,
+            "published_at": published_at.isoformat() if published_at else None,
+            "published_year": published_at.year if published_at else _int_or_none(normalize_ws(_select_text(post, ".post-meta__year"))),
+            "raw_data": {
+                "title": title or href,
+                "url": href or None,
+                "summary": summary or None,
+                "date_text": normalize_ws(_select_text(post, ".post-meta__date")),
+            },
+        }
+        news.append(item)
+    return _dedupe_news_links(news)
+
+
+def _select_text(node: Tag, selector: str) -> str:
+    selected = node.select_one(selector)
+    return selected.get_text(" ", strip=True) if selected else ""
+
+
+def _parse_post_date(post: Tag) -> datetime | None:
+    day = _int_or_none(normalize_ws(_select_text(post, ".post-meta__day")))
+    month = _month_number(normalize_ws(_select_text(post, ".post-meta__month")))
+    year = _int_or_none(normalize_ws(_select_text(post, ".post-meta__year")))
+    if not day or not month or not year:
+        return None
+    try:
+        return datetime(year, month, day, tzinfo=timezone.utc)
+    except ValueError:
+        return None
+
+
+def _month_number(value: str) -> int | None:
+    lowered = value.lower().strip(".")
+    months = {
+        "янв": 1,
+        "январь": 1,
+        "января": 1,
+        "фев": 2,
+        "февр": 2,
+        "февраль": 2,
+        "февраля": 2,
+        "март": 3,
+        "мар": 3,
+        "марта": 3,
+        "апр": 4,
+        "апрель": 4,
+        "апреля": 4,
+        "май": 5,
+        "мая": 5,
+        "июнь": 6,
+        "июня": 6,
+        "июль": 7,
+        "июля": 7,
+        "авг": 8,
+        "август": 8,
+        "августа": 8,
+        "сент": 9,
+        "сен": 9,
+        "сентябрь": 9,
+        "сентября": 9,
+        "окт": 10,
+        "октябрь": 10,
+        "октября": 10,
+        "нояб": 11,
+        "ноябрь": 11,
+        "ноября": 11,
+        "дек": 12,
+        "декабрь": 12,
+        "декабря": 12,
+    }
+    return months.get(lowered)
+
+
+def _normalize_publication_item(item: dict, current_author_id: str | None = None) -> dict:
+    publication_id = str(item.get("id") or "").strip()
+    title = _html_to_text(item.get("title"))
+    year = _int_or_none(item.get("year"))
+    publication_type = str(item.get("type") or "").strip() or None
+    description = item.get("description") if isinstance(item.get("description"), dict) else {}
+    short_description = _localized_value(description.get("short")) or _localized_value(description.get("shortLeft"))
+    documents = item.get("documents") if isinstance(item.get("documents"), dict) else {}
+    language = item.get("language") if isinstance(item.get("language"), dict) else {}
+    annotation = _localized_text_map(item.get("annotation"))
+    authors = _normalize_publication_authors(item.get("authorsByType"), current_author_id)
+    citation_text = normalize_ws(str(description.get("main") or "")) or _build_publication_citation(title, authors, year)
+    text = normalize_ws(" ".join(part for part in [title, str(year or ""), short_description] if part))
+    return {
+        "id": publication_id or None,
+        "publication_id": publication_id or None,
+        "title": title or publication_id,
+        "year": year,
+        "type": publication_type,
+        "publication_type": publication_type,
+        "language": normalize_ws(language.get("name")) or None,
+        "status": _int_or_none(item.get("status")),
+        "url": f"https://publications.hse.ru/view/{publication_id}" if publication_id else None,
+        "doi_url": _document_href(documents, "DOI"),
+        "other_url": _document_href(documents, "OTHER_URL"),
+        "document_url": _document_href(documents, "DOCUMENT"),
+        "citation_text": citation_text or None,
+        "annotation": annotation,
+        "description": description or None,
+        "authors": authors,
+        "raw_data": item,
+        "text": text or title or publication_id,
+    }
+
+
+def _normalize_vkr_item(item: dict, source_url: str) -> dict:
+    thesis_id = item.get("id")
+    program = item.get("learnProgram") if isinstance(item.get("learnProgram"), dict) else {}
+    org_unit = item.get("orgUnit") if isinstance(item.get("orgUnit"), dict) else {}
+    supervisors = []
+    for supervisor in item.get("supervisors") or []:
+        if not isinstance(supervisor, dict):
+            continue
+        name = normalize_ws(supervisor.get("name"))
+        url = normalize_ws(supervisor.get("url"))
+        if name or url:
+            supervisors.append({"name": name or url, "url": url or None})
+    return {
+        "id": thesis_id,
+        "student": normalize_ws(item.get("student")),
+        "title": normalize_ws(item.get("title")),
+        "defense_year": item.get("year"),
+        "level": normalize_ws(item.get("level")),
+        "rating": item.get("rating"),
+        "project_url": urljoin(source_url, f"/edu/vkr/{thesis_id}") if thesis_id else None,
+        "program": normalize_ws(program.get("title")),
+        "program_url": urljoin(source_url, program.get("url")) if program.get("url") else None,
+        "org_unit": normalize_ws(org_unit.get("title")),
+        "org_unit_url": urljoin(source_url, org_unit.get("url")) if org_unit.get("url") else None,
+        "supervisors": supervisors,
+        "text": normalize_ws(" ".join(str(part) for part in [item.get("student"), item.get("title"), item.get("year")] if part)),
+    }
+
+
+def _upsert_publications_section(sections: list[dict], publications: list[dict]) -> list[dict]:
+    merged = []
+    inserted = False
+    for section in sections:
+        if section.get("type") != "publications":
+            merged.append(section)
+            continue
+        existing = section.get("publications") or []
+        section = {
+            **section,
+            "publications_count": max(section.get("publications_count") or 0, len(publications)),
+            "publications": _dedupe_publications([*existing, *publications]),
+        }
+        section["items"] = [item["text"] for item in section["publications"] if item.get("text")]
+        merged.append(section)
+        inserted = True
+    if not inserted:
+        merged.append(
+            {
+                "title": "Публикации и исследования",
+                "slug": "publikacii_i_issledovaniya",
+                "type": "publications",
+                "raw_text": "",
+                "paragraphs": [],
+                "items": [item["text"] for item in publications if item.get("text")],
+                "links": [],
+                "publications_count": len(publications),
+                "publications": publications,
+            }
+        )
+    return merged
+
+
+def _upsert_graduation_theses_section(sections: list[dict], theses: list[dict]) -> list[dict]:
+    section = {
+        "title": "Выпускные квалификационные работы студентов НИУ ВШЭ",
+        "slug": "vypusknye_kvalifikacionnye_raboty_studentov_niu_vshe",
+        "type": "graduation_theses",
+        "raw_text": "",
+        "paragraphs": [],
+        "items": [item["text"] for item in theses if item.get("text")],
+        "links": [{"text": item["title"], "url": item["project_url"]} for item in theses if item.get("title") and item.get("project_url")],
+        "theses_count": len(theses),
+        "theses": theses,
+    }
+    return [item for item in sections if item.get("type") != "graduation_theses"] + [section]
+
+
+def _dedupe_publications(items: list[dict]) -> list[dict]:
+    seen = set()
+    unique = []
+    for item in items:
+        key = item.get("id") or item.get("url") or item.get("title")
+        if key and key not in seen:
+            seen.add(key)
+            unique.append(item)
+    return unique
+
+
+def _dedupe_news_links(items: list[dict]) -> list[dict]:
+    seen = set()
+    unique = []
+    for item in items:
+        key = item.get("url") or item.get("title")
+        if key and key not in seen:
+            seen.add(key)
+            unique.append(item)
+    return unique
+
+
+def _html_to_text(value: object) -> str:
+    return normalize_ws(BeautifulSoup(str(value or ""), "html.parser").get_text(" ", strip=True))
+
+
+def _localized_text_map(value: object) -> dict[str, str]:
+    if not isinstance(value, dict):
+        return {}
+    localized = {}
+    for key in ("ru", "en", "publ"):
+        text = _html_to_text(value.get(key))
+        if text:
+            localized[key] = text
+    return localized
+
+
+def _localized_value(value: object) -> str:
+    if isinstance(value, dict):
+        return normalize_ws(value.get("ru") or value.get("publ") or value.get("en"))
+    return normalize_ws(str(value or ""))
+
+
+def _normalize_publication_authors(value: object, current_author_id: str | None) -> list[dict]:
+    if not isinstance(value, dict):
+        return []
+    authors = []
+    for author in value.get("author") or []:
+        if not isinstance(author, dict):
+            continue
+        title = author.get("title") if isinstance(author.get("title"), dict) else {}
+        reverse_title = author.get("reverseTitle") if isinstance(author.get("reverseTitle"), dict) else {}
+        author_id = normalize_ws(author.get("id"))
+        href = normalize_ws(author.get("href"))
+        authors.append(
+            {
+                "id": author_id or None,
+                "href": urljoin("https://www.hse.ru", href) if href else None,
+                "title_ru": _html_to_text(title.get("ru")),
+                "title_en": _html_to_text(title.get("en")),
+                "reverse_title_ru": _html_to_text(reverse_title.get("ru")),
+                "reverse_title_en": _html_to_text(reverse_title.get("en")),
+                "alt_name": normalize_ws(author.get("altName")) or None,
+                "other_name": normalize_ws(author.get("otherName")) or None,
+                "is_current_employee": bool(current_author_id and author_id == current_author_id),
+            }
+        )
+    return authors
+
+
+def _document_href(documents: dict, key: str) -> str | None:
+    document = documents.get(key)
+    if not isinstance(document, dict):
+        return None
+    return normalize_ws(document.get("href")) or None
+
+
+def _build_publication_citation(title: str, authors: list[dict], year: int | None) -> str:
+    author_names = [author.get("title_ru") or author.get("title_en") or author.get("alt_name") for author in authors]
+    return normalize_ws(". ".join(part for part in [", ".join(filter(None, author_names)), title, str(year or "")] if part))
+
+
+def _int_or_none(value: object) -> int | None:
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
 def _slugify(value: str) -> str:
     cleaned = re.sub(r"[^\w\s-]", "", value.lower(), flags=re.UNICODE)
     return re.sub(r"[-\s]+", "_", cleaned).strip("_") or "section"
@@ -378,3 +908,62 @@ def _dedupe_dicts(items: list[dict]) -> list[dict]:
             seen.add(key)
             unique.append(item)
     return unique
+
+
+def _fetch_text(
+    session: Session,
+    url: str,
+    headers: dict[str, str],
+    timeout: int,
+    *,
+    resource_cache=None,
+    profile_key: str | None = None,
+    resource_key: str,
+    resource_manifest: list[dict] | None,
+    method: str = "GET",
+    json_payload: object | None = None,
+    params: dict | None = None,
+) -> str:
+    if resource_cache and profile_key:
+        cached = resource_cache.fetch_text(
+            session,
+            profile_key=profile_key,
+            resource_key=resource_key,
+            method=method,
+            url=url,
+            headers=headers,
+            timeout=timeout,
+            json_payload=json_payload,
+            params=params,
+        )
+        if resource_manifest is not None:
+            resource_manifest.append(
+                {
+                    "resource_key": resource_key,
+                    "method": method,
+                    "url": url,
+                    "body_hash": cached.body_hash,
+                    "from_cache": cached.from_cache,
+                    "status_code": cached.status_code,
+                }
+            )
+        return cached.text
+
+    if method.upper() == "POST":
+        response = session.post(url, json=json_payload, headers=headers, timeout=timeout, params=params)
+    else:
+        response = session.get(url, headers=headers, timeout=timeout, params=params)
+    response.raise_for_status()
+    text = response.text
+    if resource_manifest is not None:
+        resource_manifest.append(
+            {
+                "resource_key": resource_key,
+                "method": method,
+                "url": url,
+                "body_hash": hashlib.sha256(text.encode("utf-8")).hexdigest(),
+                "from_cache": False,
+                "status_code": response.status_code,
+            }
+        )
+    return text

app/security.py

@@ -44,9 +44,3 @@ def require_admin(request: Request, settings: Settings) -> str:
     if not username:
         raise HTTPException(status_code=status.HTTP_303_SEE_OTHER, headers={"Location": "/admin/login"})
     return username
-
-
-def require_mcp_token(request: Request, settings: Settings) -> None:
-    auth = request.headers.get("authorization", "")
-    if not auth.startswith("Bearer ") or not hmac.compare_digest(auth.removeprefix("Bearer ").strip(), settings.mcp_token):
-        raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid MCP token")

app/services/academic_degrees.py

@@ -0,0 +1,33 @@
+import re
+from typing import Any
+
+
+_PATTERN = re.compile(r"\b(?:кандидат|доктор)(?:\s+[\w-]+){0,12}\s+наук\b|\bph\.?\s*d\.?(?!\w)", re.IGNORECASE)
+
+
+def academic_degrees(data: dict[str, Any] | None) -> list[str]:
+    degrees = []
+    seen = set()
+    for section in (data or {}).get("sections") or []:
+        if not isinstance(section, dict) or not re.search(
+            r"образован|степен|academic degree|education", str(section.get("title") or ""), re.IGNORECASE
+        ):
+            continue
+        values = [
+            *(entry.get("text") for entry in section.get("year_entries") or [] if isinstance(entry, dict)),
+            *(section.get("paragraphs") or []),
+            *(section.get("items") or []),
+            section.get("raw_text"),
+        ]
+        table = section.get("table") or {}
+        for row in table.get("rows") or []:
+            if isinstance(row, dict):
+                values.extend(row.get("cells") or [])
+        for value in values:
+            text = str(value or "").strip()
+            for match in _PATTERN.finditer(text):
+                degree = match.group(0).strip()
+                if degree.casefold() not in seen:
+                    seen.add(degree.casefold())
+                    degrees.append(degree)
+    return degrees

app/services/admin_data.py

@@ -3,11 +3,13 @@ from __future__ import annotations
 from datetime import date, datetime, time
 from math import ceil
 from typing import Any
+from zoneinfo import ZoneInfo
 
 from sqlalchemy import Select, Text, and_, desc, func, or_, select
 from sqlalchemy.orm import Session
 
-from app.models import CrawlRun, Employee
+from app.models import CrawlError, CrawlRun, CrawlRunEmployeeChange, Employee, EmployeeNewsLink
+from app.services.academic_degrees import academic_degrees
 
 EMPLOYEE_SORTS = {
     "full_name": Employee.full_name,
@@ -18,31 +20,59 @@ EMPLOYEE_SORTS = {
     "hse_start_year": Employee.current_data["hse_start_year"].as_integer(),
 }
 
-
 def employee_display_payload(employee: Employee) -> dict[str, Any]:
-    data = employee.current_data or {}
-    contacts = data.get("contacts") or {}
-    sections = data.get("sections") or []
-    emails = contacts.get("emails") or []
-    phones = contacts.get("phones") or []
+    data = _as_dict(employee.current_data)
+    contacts = _as_dict(data.get("contacts"))
+    sections = _as_list(data.get("sections"))
+    stored_news_links = _stored_news_links(employee)
+    positions = _clean_list(data.get("positions"))
+    emails = _clean_list(contacts.get("emails"))
+    phones = _clean_list(contacts.get("phones"))
+    degree_values = academic_degrees(data)
     return {
         "id": employee.id,
         "full_name": employee.full_name,
         "status": employee.status,
+        "status_display": _employee_status_display(employee.status),
         "canonical_url": employee.canonical_url,
-        "positions": data.get("positions") or [],
-        "positions_text": "; ".join(data.get("positions") or []),
+        "positions": positions,
+        "positions_text": "; ".join(positions),
         "hse_start_year": data.get("hse_start_year"),
         "emails": emails,
         "email_text": ", ".join(emails),
         "phones": phones,
         "phone_text": ", ".join(phones),
         "address": contacts.get("address"),
+        "academic_degree_text": "; ".join(degree_values),
         "publications_count": _count_section_items(sections, "publications"),
         "courses_count": _count_section_items(sections, "courses_by_year"),
+        "news_count": len(stored_news_links) or _count_section_items(sections, "news"),
         "first_seen_at": employee.first_seen_at.isoformat() if employee.first_seen_at else None,
         "last_seen_at": employee.last_seen_at.isoformat() if employee.last_seen_at else None,
         "dismissed_at": employee.dismissed_at.isoformat() if employee.dismissed_at else None,
+        "first_seen_display": format_admin_datetime(employee.first_seen_at),
+        "last_seen_display": format_admin_datetime(employee.last_seen_at),
+        "dismissed_display": format_admin_datetime(employee.dismissed_at),
+    }
+
+
+def employee_detail_payload(employee: Employee) -> dict[str, Any]:
+    data = _as_dict(employee.current_data)
+    contacts = _as_dict(data.get("contacts"))
+    return {
+        **employee_display_payload(employee),
+        "profile_type": employee.profile_type or data.get("profile_type"),
+        "profile_id": employee.profile_id or data.get("profile_id"),
+        "parser_version": employee.parser_version or data.get("parser_version"),
+        "contacts": {
+            "emails": _clean_list(contacts.get("emails")),
+            "phones": _clean_list(contacts.get("phones")),
+            "address": contacts.get("address"),
+            "contact_items": _normalize_contact_items(contacts.get("items")),
+        },
+        "external_ids": _normalize_external_ids(data.get("external_ids")),
+        "news_links": _detail_news_links(employee, data),
+        "sections": [_normalize_section(section) for section in _as_list(data.get("sections"))],
     }
 
 
@@ -53,6 +83,7 @@ def build_employee_query(
     started_from: date | None = None,
     started_to: date | None = None,
     has_email: bool | None = None,
+    has_academic_degree: bool | None = None,
 ) -> Select[tuple[Employee]]:
     stmt = select(Employee)
     filters = []
@@ -69,6 +100,8 @@ def build_employee_query(
         filters.append(Employee.current_data.cast(Text).ilike("%@%"))
     elif has_email is False:
         filters.append(or_(Employee.current_data.is_(None), ~Employee.current_data.cast(Text).ilike("%@%")))
+    if has_academic_degree is not None:
+        filters.append(Employee.has_academic_degree.is_(has_academic_degree))
     if filters:
         stmt = stmt.where(and_(*filters))
     return stmt
@@ -82,12 +115,13 @@ def list_employees_page(
     started_from: date | None = None,
     started_to: date | None = None,
     has_email: bool | None = None,
+    has_academic_degree: bool | None = None,
     sort: str = "full_name",
     direction: str = "asc",
     limit: int = 50,
     offset: int = 0,
 ) -> dict[str, Any]:
-    limit = max(1, min(limit, 200))
+    limit = limit if limit in {25, 50, 100} else 50
     offset = max(0, offset)
     base_stmt = build_employee_query(
         status=status,
@@ -95,13 +129,16 @@ def list_employees_page(
         started_from=started_from,
         started_to=started_to,
         has_email=has_email,
+        has_academic_degree=has_academic_degree,
     )
     total = db.scalar(select(func.count()).select_from(base_stmt.subquery())) or 0
+    if offset >= total:
+        offset = 0
     sort_column = EMPLOYEE_SORTS.get(sort, Employee.full_name)
     order = desc(sort_column) if direction == "desc" else sort_column
     employees = db.scalars(base_stmt.order_by(order).limit(limit).offset(offset)).all()
     return {
-        "items": [employee_display_payload(employee) for employee in employees],
+        "employees": [employee_display_payload(employee) for employee in employees],
         "total": total,
         "limit": limit,
         "offset": offset,
@@ -117,6 +154,10 @@ def stats_payload(db: Session) -> dict[str, Any]:
     return {
         "total": db.scalar(select(func.count()).select_from(Employee)) or 0,
         "active": db.scalar(select(func.count()).select_from(Employee).where(Employee.status == "active")) or 0,
+        "verification_required": db.scalar(
+            select(func.count()).select_from(Employee).where(Employee.status == "verification_required")
+        )
+        or 0,
         "dismissed": db.scalar(select(func.count()).select_from(Employee).where(Employee.status == "dismissed")) or 0,
         "new_in_last_run": latest_run.new_count if latest_run else 0,
         "latest_added": employee_display_payload(latest_added) if latest_added else None,
@@ -128,16 +169,20 @@ def stats_payload(db: Session) -> dict[str, Any]:
 def run_payload(run: CrawlRun | None) -> dict[str, Any] | None:
     if not run:
         return None
-    processed = run.parsed_count + run.error_count
+    processed = run.parsed_count + run.skipped_count + run.error_count
     percent = round((processed / run.found_count) * 100, 1) if run.found_count else 0
     return {
         "id": run.id,
         "source_url": run.source_url,
         "status": run.status,
+        "status_display": _run_status_display(run.status),
         "started_at": run.started_at.isoformat() if run.started_at else None,
         "finished_at": run.finished_at.isoformat() if run.finished_at else None,
+        "started_display": format_admin_datetime(run.started_at),
+        "finished_display": format_admin_datetime(run.finished_at),
         "found_count": run.found_count,
         "parsed_count": run.parsed_count,
+        "skipped_count": run.skipped_count,
         "new_count": run.new_count,
         "error_count": run.error_count,
         "dismissed_count": run.dismissed_count,
@@ -147,6 +192,97 @@ def run_payload(run: CrawlRun | None) -> dict[str, Any] | None:
     }
 
 
+def run_detail_payload(db: Session, run: CrawlRun | None) -> dict[str, Any] | None:
+    if not run:
+        return None
+    changes = db.scalars(
+        select(CrawlRunEmployeeChange)
+        .where(CrawlRunEmployeeChange.crawl_run_id == run.id)
+        .order_by(CrawlRunEmployeeChange.created_at, CrawlRunEmployeeChange.id)
+    ).all()
+    errors = db.scalars(select(CrawlError).where(CrawlError.crawl_run_id == run.id).order_by(CrawlError.created_at)).all()
+    grouped_changes = {"new": [], "missing_from_source": [], "verification_required": [], "dismissed": []}
+    for change in changes:
+        grouped_changes.setdefault(change.change_type, []).append(_change_payload(change))
+    return {
+        **(run_payload(run) or {}),
+        "changes_detail_available": bool(changes),
+        "changes": grouped_changes,
+        "errors": [_crawl_error_payload(error) for error in errors],
+    }
+
+
+def format_admin_datetime(value: Any) -> str:
+    if not value:
+        return "Не указано"
+    if isinstance(value, str):
+        try:
+            value = datetime.fromisoformat(value.replace("Z", "+00:00"))
+        except ValueError:
+            return value
+    if not isinstance(value, datetime):
+        return str(value)
+    if value.tzinfo:
+        value = value.astimezone(ZoneInfo("Europe/Moscow"))
+    return value.strftime("%d.%m.%Y %H:%M")
+
+
+def _employee_status_display(status: str | None) -> str:
+    labels = {"active": "Работает", "verification_required": "Требует проверки", "dismissed": "Уволен"}
+    return labels.get(status or "", status or "Не указано")
+
+
+def _run_status_display(status: str | None) -> str:
+    labels = {"running": "Выполняется", "completed": "Завершен", "failed": "Ошибка"}
+    return labels.get(status or "", status or "Не указано")
+
+
+def _change_payload(change: CrawlRunEmployeeChange) -> dict[str, Any]:
+    return {
+        "id": change.id,
+        "employee_id": change.employee_id,
+        "profile_key": change.profile_key,
+        "profile_url": change.profile_url,
+        "full_name": change.full_name,
+        "change_type": change.change_type,
+        "change_type_display": _change_type_display(change.change_type),
+        "profile_available": change.profile_available,
+        "profile_available_display": _profile_available_display(change.profile_available),
+        "message": change.message,
+        "created_at": change.created_at.isoformat() if change.created_at else None,
+        "created_display": format_admin_datetime(change.created_at),
+    }
+
+
+def _crawl_error_payload(error: CrawlError) -> dict[str, Any]:
+    return {
+        "id": error.id,
+        "crawl_run_id": error.crawl_run_id,
+        "profile_url": error.profile_url,
+        "error_type": error.error_type,
+        "message": error.message,
+        "created_at": error.created_at.isoformat() if error.created_at else None,
+        "created_display": format_admin_datetime(error.created_at),
+    }
+
+
+def _change_type_display(change_type: str | None) -> str:
+    labels = {
+        "new": "Новый",
+        "missing_from_source": "Потеряшка",
+        "dismissed": "Уволен",
+    }
+    return labels.get(change_type or "", change_type or "Не указано")
+
+
+def _profile_available_display(value: bool | None) -> str:
+    if value is True:
+        return "Профиль доступен"
+    if value is False:
+        return "Профиль недоступен"
+    return "Не проверялось"
+
+
 def _count_section_items(sections: list[dict[str, Any]], section_type: str) -> int:
     total = 0
     for section in sections:
@@ -156,4 +292,263 @@ def _count_section_items(sections: list[dict[str, Any]], section_type: str) -> i
             total += len(section.get("publications") or section.get("items") or [])
         elif section_type == "courses_by_year":
             total += len(section.get("courses") or [])
+        elif section_type == "news":
+            total += len(section.get("news_links") or section.get("items") or [])
     return total
+
+
+def _clean_list(values: Any) -> list[str]:
+    if values is None:
+        return []
+    if not isinstance(values, list):
+        values = [values]
+    return [str(value).strip() for value in values if str(value or "").strip()]
+
+
+def _as_dict(value: Any) -> dict[str, Any]:
+    return value if isinstance(value, dict) else {}
+
+
+def _as_list(value: Any) -> list[Any]:
+    if value is None:
+        return []
+    return value if isinstance(value, list) else [value]
+
+
+def _normalize_contact_items(items: Any) -> list[str]:
+    normalized = []
+    if not isinstance(items, list):
+        return normalized
+    for item in items:
+        if isinstance(item, dict):
+            value = item.get("raw") or item.get("value") or item.get("text")
+        else:
+            value = item
+        value = str(value or "").strip()
+        if value:
+            normalized.append(value)
+    return normalized
+
+
+def _normalize_external_ids(items: Any) -> list[dict[str, str | None]]:
+    normalized = []
+    if not isinstance(items, list):
+        return normalized
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        system = str(item.get("system") or "").strip()
+        value = str(item.get("value") or "").strip()
+        url = str(item.get("url") or "").strip()
+        if system or value or url:
+            normalized.append({"system": system or "ID", "value": value or url, "url": url or None})
+    return normalized
+
+
+def _normalize_section(section: Any) -> dict[str, Any]:
+    if not isinstance(section, dict):
+        return {"title": "Раздел", "type": "generic", "paragraphs": [str(section)], "items": [], "links": []}
+
+    section_type = section.get("type") or "generic"
+    paragraphs = _clean_list(section.get("paragraphs"))
+    items = _clean_list(section.get("items"))
+    raw_text = str(section.get("raw_text") or "").strip()
+    if not paragraphs and not items and raw_text:
+        paragraphs = [raw_text]
+
+    return {
+        "title": section.get("title") or "Раздел",
+        "type": section_type,
+        "raw_text": raw_text,
+        "paragraphs": paragraphs,
+        "list_items": items,
+        "links": _normalize_links(section.get("links")),
+        "year_entries": _normalize_year_entries(section.get("year_entries")),
+        "publications": _normalize_publications(section.get("publications")),
+        "publications_count": section.get("publications_count"),
+        "news_links": _normalize_news_links(section.get("news_links")),
+        "news_count": section.get("news_count"),
+        "theses": _normalize_theses(section.get("theses")),
+        "theses_count": section.get("theses_count"),
+        "academic_year": section.get("academic_year"),
+        "courses": _normalize_courses(section.get("courses")),
+        "table": _normalize_table(section.get("table")),
+    }
+
+
+def _normalize_links(items: Any) -> list[dict[str, str | None]]:
+    normalized = []
+    if not isinstance(items, list):
+        return normalized
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        text = str(item.get("text") or item.get("url") or "").strip()
+        url = str(item.get("url") or "").strip()
+        if text and url:
+            normalized.append({"text": text, "url": url})
+    return normalized
+
+
+def _stored_news_links(employee: Employee) -> list[dict[str, Any]]:
+    return [_stored_news_link_payload(item) for item in sorted(employee.news_links, key=_news_link_sort_key)]
+
+
+def _news_link_sort_key(item: EmployeeNewsLink) -> tuple:
+    timestamp = item.published_at.timestamp() if item.published_at else 0
+    return (-timestamp, item.title or "", item.id)
+
+
+def _stored_news_link_payload(item: EmployeeNewsLink) -> dict[str, Any]:
+    return {
+        "title": item.title,
+        "url": item.url,
+        "summary": item.summary,
+        "published_at": item.published_at.isoformat() if item.published_at else None,
+        "published_year": item.published_year,
+        "published_display": format_admin_date(item.published_at) if item.published_at else str(item.published_year or ""),
+    }
+
+
+def _detail_news_links(employee: Employee, data: dict[str, Any]) -> list[dict[str, Any]]:
+    stored = _stored_news_links(employee)
+    if stored:
+        return stored
+    for section in _as_list(data.get("sections")):
+        if isinstance(section, dict) and section.get("type") == "news":
+            return _normalize_news_links(section.get("news_links"))
+    return []
+
+
+def format_admin_date(value: Any) -> str:
+    if not value:
+        return ""
+    if isinstance(value, str):
+        try:
+            value = datetime.fromisoformat(value.replace("Z", "+00:00"))
+        except ValueError:
+            return value
+    if not isinstance(value, datetime):
+        return str(value)
+    if value.tzinfo:
+        value = value.astimezone(ZoneInfo("Europe/Moscow"))
+    return value.strftime("%d.%m.%Y")
+
+
+def _normalize_news_links(items: Any) -> list[dict[str, Any]]:
+    normalized = []
+    if not isinstance(items, list):
+        return normalized
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        title = str(item.get("title") or item.get("url") or "").strip()
+        url = str(item.get("url") or "").strip()
+        summary = str(item.get("summary") or "").strip()
+        published_at = str(item.get("published_at") or "").strip()
+        published_year = item.get("published_year")
+        if title or url:
+            normalized.append(
+                {
+                    "title": title or url,
+                    "url": url or None,
+                    "summary": summary or None,
+                    "published_at": published_at or None,
+                    "published_year": published_year,
+                    "published_display": format_admin_date(published_at) if published_at else str(published_year or ""),
+                }
+            )
+    return normalized
+
+
+def _normalize_year_entries(items: Any) -> list[dict[str, Any]]:
+    normalized = []
+    if not isinstance(items, list):
+        return normalized
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        text = str(item.get("text") or "").strip()
+        if text:
+            normalized.append({"year": item.get("year"), "text": text, "links": _normalize_links(item.get("links"))})
+    return normalized
+
+
+def _normalize_publications(items: Any) -> list[dict[str, str | None]]:
+    normalized = []
+    if not isinstance(items, list):
+        return normalized
+    for item in items:
+        if not isinstance(item, dict):
+            text = str(item or "").strip()
+            if text:
+                normalized.append({"title": text, "text": text, "url": None})
+            continue
+        title = str(item.get("title") or "").strip()
+        text = str(item.get("text") or title).strip()
+        url = str(item.get("url") or "").strip()
+        if title or text:
+            normalized.append({"title": title or text, "text": text or title, "url": url or None})
+    return normalized
+
+
+def _normalize_courses(items: Any) -> list[dict[str, str | None]]:
+    normalized = []
+    if not isinstance(items, list):
+        return normalized
+    for item in items:
+        if not isinstance(item, dict):
+            title = str(item or "").strip()
+            if title:
+                normalized.append({"title": title, "url": None})
+            continue
+        title = str(item.get("title") or "").strip()
+        url = str(item.get("url") or "").strip()
+        if title or url:
+            normalized.append({"title": title or url, "url": url or None})
+    return normalized
+
+
+def _normalize_theses(items: Any) -> list[dict[str, Any]]:
+    normalized = []
+    if not isinstance(items, list):
+        return normalized
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        title = str(item.get("title") or "").strip()
+        student = str(item.get("student") or "").strip()
+        if not title and not student:
+            continue
+        normalized.append(
+            {
+                "id": item.get("id"),
+                "student": student,
+                "title": title,
+                "defense_year": item.get("defense_year") or item.get("year"),
+                "level": str(item.get("level") or "").strip(),
+                "rating": item.get("rating"),
+                "project_url": str(item.get("project_url") or "").strip() or None,
+                "program": str(item.get("program") or "").strip(),
+                "program_url": str(item.get("program_url") or "").strip() or None,
+                "org_unit": str(item.get("org_unit") or "").strip(),
+                "org_unit_url": str(item.get("org_unit_url") or "").strip() or None,
+            }
+        )
+    return normalized
+
+
+def _normalize_table(table: Any) -> dict[str, Any] | None:
+    if not isinstance(table, dict):
+        return None
+    headers = _clean_list(table.get("headers"))
+    rows = []
+    for row in table.get("rows") or []:
+        if not isinstance(row, dict):
+            continue
+        cells = _clean_list(row.get("cells"))
+        if cells:
+            rows.append({"cells": cells, "link_url": row.get("link_url")})
+    if not headers and not rows:
+        return None
+    return {"headers": headers, "rows": rows}

app/services/council.py

@@ -0,0 +1,192 @@
+"""Public publication evidence from the council export and employee database."""
+
+import calendar
+import csv
+import re
+import unicodedata
+from collections import Counter, defaultdict
+from datetime import date, datetime
+from functools import lru_cache
+from pathlib import Path
+from urllib.parse import quote, urlsplit
+from zoneinfo import ZoneInfo
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session, selectinload
+
+from app.models import Employee
+from app.services.academic_degrees import academic_degrees
+
+
+EXPORT = Path(__file__).resolve().parents[1] / "data" / "council_articles.csv"
+DOCTOR = re.compile(r"^доктор\b.+\bнаук$", re.IGNORECASE)
+DOI = re.compile(r"10\.\d{4,9}/\S+", re.IGNORECASE)
+TYPES = {"ARTICLE": "Статья", "BOOK": "Книга", "CHAPTER": "Глава книги",
+         "BOOK_CHAPTER": "Глава книги", "REPORT": "Доклад", "PATENT": "Патент",
+         "PREPRINT": "Препринт", "CONFERENCE_PAPER": "Материал конференции"}
+
+
+def _name(value: str) -> str:
+    return " ".join(unicodedata.normalize("NFKC", value).casefold().replace("ё", "е").split())
+
+
+def _doi(value: str) -> str:
+    value = re.sub(r"^(?:https?://(?:dx\.)?doi\.org/|doi:\s*)", "", value.strip(), flags=re.I).lower()
+    return value if DOI.fullmatch(value) else ""
+
+
+def _safe_url(value: str | None) -> str | None:
+    try:
+        url = urlsplit(value or "")
+        if url.scheme in {"http", "https"} and url.hostname and not url.username and not url.password:
+            return value
+    except ValueError:
+        pass
+    return None
+
+
+def period(today: date) -> tuple[date, date]:
+    year = today.year - 5
+    return date(year, today.month, min(today.day, calendar.monthrange(year, today.month)[1])), today
+
+
+def _quartiles(row: dict) -> list[str]:
+    return [f"{label} {str(row.get(field) or '').strip().upper()}"
+            for label, field in (("Scopus", "Квартиль scopus"), ("Web of Science", "Квартиль wos"))
+            if str(row.get(field) or '').strip().upper() in {"Q1", "Q2"}]
+
+
+@lru_cache(maxsize=1)
+def _articles_by_author() -> dict[str, list[dict]]:
+    articles = defaultdict(list)
+    with EXPORT.open(encoding="utf-8-sig", newline="") as source:
+        reader = csv.DictReader(source, delimiter=";")
+        required = {"Название публикации", "Год", "DOI", "Авторы НИУ ВШЭ", "Квартиль scopus", "Квартиль wos"}
+        if not required <= set(reader.fieldnames or []):
+            raise ValueError("Missing council CSV headers")
+        for row_number, row in enumerate(reader, 2):
+            if None in row or any(value is None for value in row.values()):
+                raise ValueError("Malformed council CSV record")
+            title = row["Название публикации"].strip()
+            if not title:
+                continue
+            year = int(row["Год"]) if row["Год"].strip().isdigit() else None
+            article = {"title": title, "year": year, "doi": _doi(row["DOI"]),
+                       "types": [], "quartiles": _quartiles(row), "origins": ["Файл CSV"],
+                       "rows": [row_number], "in_csv": True, "in_db": False, "url": None}
+            for author in set(_name(value) for value in row["Авторы НИУ ВШЭ"].split(";")) - {""}:
+                articles[author].append(article)
+    return articles
+
+
+def _database_article(publication) -> dict:
+    raw = publication.raw_data if isinstance(publication.raw_data, dict) else {}
+    imported = raw.get("csv_enrichment")
+    from_site = bool(publication.publication_id or publication.url or any(key != "csv_enrichment" for key in raw))
+    origins = ["БД miem-workers · сайт ВШЭ"] if from_site else []
+    if imported:
+        origins.append("БД miem-workers · импорт CSV")
+    kind = publication.publication_type
+    return {"title": publication.title, "year": publication.year, "doi": _doi(publication.doi_url or ""),
+            "types": [TYPES.get(kind.upper(), f"Другой тип: {kind}")] if kind else [],
+            "quartiles": _quartiles(imported) if isinstance(imported, dict) else [],
+            "origins": origins or ["БД miem-workers"], "rows": [], "in_csv": False, "in_db": True,
+            "url": _safe_url(publication.url)}
+
+
+def _unique_articles(articles: list[dict], start: date, end: date) -> list[dict]:
+    title_years = defaultdict(set)
+    for article in articles:
+        if article["year"] is not None:
+            title_years[_name(article["title"])].add(article["year"])
+    def title_key(article):
+        title = _name(article["title"])
+        years = title_years[title]
+        year = article["year"]
+        return title, next(iter(years)) if year is None and len(years) == 1 else year
+
+    # A title/year fallback is safe only when it identifies at most one DOI.
+    title_dois = defaultdict(set)
+    for article in articles:
+        if article["doi"]:
+            title_dois[title_key(article)].add(article["doi"])
+    unique = {}
+    for article in articles:
+        fallback = title_key(article)
+        identifiers = title_dois[fallback]
+        identifier = article["doi"] or (next(iter(identifiers)) if len(identifiers) == 1 else "")
+        key = identifier or fallback
+        if key not in unique:
+            unique[key] = {**article, "doi": identifier,
+                           **{field: list(article[field]) for field in ("types", "quartiles", "origins", "rows")},
+                           "csv_qualifies": False}
+        merged = unique[key]
+        for field in ("types", "quartiles", "origins", "rows"):
+            merged[field] = list(dict.fromkeys([*merged[field], *article[field]]))
+        merged["in_csv"] |= article["in_csv"]
+        merged["in_db"] |= article["in_db"]
+        merged["url"] = merged["url"] or article["url"]
+        merged["year"] = merged["year"] or article["year"]
+        in_period = article["year"] is not None and start.year <= article["year"] <= end.year
+        merged["csv_qualifies"] |= bool(article["in_csv"] and article["quartiles"] and in_period)
+    for article in unique.values():
+        year = article["year"]
+        article["criteria"] = list(article["quartiles"])
+        if year is None:
+            article["criteria"].append("Год не указан — период не подтверждён")
+        elif start.year <= year <= end.year:
+            article["criteria"].append("В пределах пяти лет по году; точную дату нужно проверить" if year in {start.year, end.year}
+                                       else "В пределах пяти лет")
+        else:
+            article["criteria"].append("Вне пятилетнего периода")
+        if not article["quartiles"]:
+            article["criteria"].append("Q1/Q2, К1 и RSCI не подтверждены доступными данными")
+        article["doi_url"] = f"https://doi.org/{quote(article['doi'], safe='/')}" if article["doi"] else None
+        article["type_label"] = " / ".join(article["types"]) or "Тип не указан — статья не подтверждена"
+        if article["types"] and article["types"] != ["Статья"]:
+            article["csv_qualifies"] = False
+            article["criteria"].append("Тип материала не подтверждён как статья — не включён в счётчик CSV")
+    return sorted(unique.values(), key=lambda item: (not item["csv_qualifies"], -(item["year"] or 0), item["title"].casefold()))
+
+
+def council_candidates(db: Session, today: date | None = None, q: str = "", offset: int = 0,
+                       sort: str = "csv_count", direction: str = "desc", paginate: bool = True) -> dict:
+    today = today or datetime.now(ZoneInfo("Europe/Moscow")).date()
+    start, end = period(today)
+    employees = db.scalars(select(Employee).options(selectinload(Employee.publications))).all()
+    name_counts = Counter(_name(employee.full_name or "") for employee in employees)
+    by_author = _articles_by_author()
+    candidates = []
+    for employee in employees:
+        author = _name(employee.full_name or "")
+        ambiguous = bool(author and name_counts[author] > 1 and by_author.get(author))
+        csv_articles = by_author.get(author, []) if author and name_counts[author] == 1 else []
+        articles = _unique_articles([*csv_articles, *(_database_article(p) for p in employee.publications)], start, end)
+        if not articles:
+            continue
+        data = employee.current_data if isinstance(employee.current_data, dict) else {}
+        try:
+            degrees = academic_degrees(data)
+        except (AttributeError, TypeError):
+            degrees = []
+        if not any(DOCTOR.fullmatch(degree) for degree in degrees):
+            continue
+        candidates.append({"name": employee.full_name or "ФИО не указано", "degree": "; ".join(degrees) or "Учёная степень не указана",
+                           "count": len(articles), "csv_count": sum(a["csv_qualifies"] for a in articles),
+                           "db_count": sum(a["in_db"] for a in articles), "articles": articles,
+                           "profile_url": _safe_url(employee.canonical_url), "ambiguous": ambiguous,
+                           "dismissed": employee.status == "dismissed"})
+    sort = sort if sort in {"full_name", "csv_count", "db_count", "count"} else "csv_count"
+    direction = direction if direction in {"asc", "desc"} else "desc"
+    candidates.sort(key=lambda item: _name(item["name"]), reverse=sort == "full_name" and direction == "desc")
+    if sort != "full_name":
+        candidates.sort(key=lambda item: (item[sort], item["count"]), reverse=direction == "desc")
+    total = len(candidates)
+    query = _name(q)
+    candidates = [item for item in candidates if query in _name(item["name"])]
+    matched = len(candidates)
+    limit = 25
+    offset = min(max(0, offset), ((matched - 1) // limit) * limit) if matched else 0
+    return {"candidates": candidates[offset:offset + limit] if paginate else candidates,
+            "start": start, "end": end, "total": total,
+            "matched": matched, "q": q, "offset": offset, "limit": limit, "sort": sort, "direction": direction}

app/services/council_export.py

@@ -0,0 +1,70 @@
+"""XLSX report containing the same public evidence as the council page."""
+
+from io import BytesIO
+
+from openpyxl import Workbook
+from openpyxl.cell.cell import ILLEGAL_CHARACTERS_RE
+from openpyxl.styles import Alignment, Font, PatternFill
+from openpyxl.utils import get_column_letter
+
+
+def _append(sheet, values):
+    values = [ILLEGAL_CHARACTERS_RE.sub("", value) if isinstance(value, str) else value for value in values]
+    if any(isinstance(value, str) and len(value) > 32767 for value in values):
+        raise ValueError("Text exceeds Excel cell limit")
+    sheet.append(values)
+
+
+def council_xlsx(result: dict) -> bytes:
+    workbook = Workbook()
+    workbook.properties.creator = "MIEM Employees"
+    employees = workbook.active
+    employees.title = "Сотрудники"
+    publications = workbook.create_sheet("Публикации")
+    parameters = workbook.create_sheet("Параметры")
+    _append(employees, ["№", "ФИО", "Учёная степень", "CSV по критериям", "В БД", "Всего без дублей",
+                        "Статус", "Профиль ВШЭ", "Примечание"])
+    _append(publications, ["№ сотрудника", "ФИО", "Название публикации", "Год", "Тип материала", "Источники",
+                           "Критерии", "В счётчике CSV", "В БД", "DOI", "Ссылка на сайте", "Строки CSV"])
+    for number, candidate in enumerate(result["candidates"], 1):
+        note = "Совпадение ФИО: публикации CSV не присвоены автоматически" if candidate["ambiguous"] else ""
+        _append(employees, [number, candidate["name"], candidate["degree"], candidate["csv_count"],
+                            candidate["db_count"], candidate["count"], "Уволен" if candidate["dismissed"] else "Не отмечен как уволенный",
+                            candidate["profile_url"], note])
+        for article in candidate["articles"]:
+            _append(publications, [number, candidate["name"], article["title"], article["year"], article["type_label"],
+                                   "; ".join(article["origins"]), "; ".join(article["criteria"]),
+                                   "Да" if article["csv_qualifies"] else "Нет", "Да" if article["in_db"] else "Нет",
+                                   article["doi_url"], article["url"], ", ".join(map(str, article["rows"]))])
+    _append(parameters, ["Параметр", "Значение"])
+    for values in [
+        ["Начало периода", result["start"]], ["Конец периода", result["end"]], ["Поиск по ФИО", result["q"]],
+        ["Сортировать по", {"full_name": "ФИО", "csv_count": "CSV по критериям", "db_count": "В БД",
+                            "count": "Всего без дублей"}[result["sort"]]],
+        ["Направление", "По возрастанию" if result["direction"] == "asc" else "По убыванию"],
+        ["Сотрудников в выгрузке", result["matched"]], ["Всего сотрудников в списке", result["total"]],
+        ["Отбор по степени", "Доктор наук в профиле; достаточно хотя бы одной публикации из CSV или БД"],
+        ["Счётчик CSV", "Предварительный: Q1/Q2 Scopus или Web of Science в пятилетнем периоде по году. "
+         "Для пограничных лет нужно проверить дату, для неизвестного типа — подтвердить, что это статья. "
+         "Материалы с известным другим типом исключены. К1/RSCI не подтверждены доступными данными."],
+    ]:
+        _append(parameters, values)
+    for sheet, widths in [(employees, [7, 34, 40, 22, 12, 22, 30, 45, 55]),
+                          (publications, [16, 34, 65, 12, 40, 45, 65, 22, 12, 45, 45, 22]),
+                          (parameters, [32, 95])]:
+        sheet.freeze_panes = "A2"
+        sheet.auto_filter.ref = sheet.dimensions
+        for column, width in enumerate(widths, 1):
+            sheet.column_dimensions[get_column_letter(column)].width = width
+        for row in sheet:
+            for cell in row:
+                if isinstance(cell.value, str):
+                    # CSV/profile content is text, never an Excel formula or error code.
+                    cell.data_type = "s"
+                cell.alignment = Alignment(vertical="top", wrap_text=True)
+        for cell in sheet[1]:
+            cell.font = Font(bold=True, color="FFFFFF")
+            cell.fill = PatternFill("solid", fgColor="263640")
+    output = BytesIO()
+    workbook.save(output)
+    return output.getvalue()

app/services/crawler.py

@@ -1,78 +1,162 @@
-import gzip
-import hashlib
-import json
+import gzip
+import hashlib
+import json
+import re
 import time
-from datetime import datetime, timezone
-
-import requests
-from sqlalchemy import select
-from sqlalchemy.orm import Session
-
-from app.config import Settings
-from app.models import CrawlError, CrawlRun, Employee, EmployeeSnapshot, ParserSource, ProfileTab
-from app.parser.collector import collect_profile_links
-from app.parser.profile import parse_person_profile
+import unicodedata
+from datetime import datetime, timezone
+
+import requests
+from sqlalchemy import inspect, select
+from sqlalchemy.orm import Session
+
+from app.config import Settings
+from app.models import (
+    CrawlError,
+    CrawlRun,
+    CrawlRunEmployeeChange,
+    Employee,
+    EmployeeNewsLink,
+    EmployeePublication,
+    EmployeeProfileUrl,
+    EmployeeSnapshot,
+    ParserSource,
+    ProfileTab,
+)
+from app.parser.collector import collect_profile_links
+from app.parser.profile import parse_person_profile
 from app.parser.profile_url import profile_key
-
-HEADERS = {
-    "User-Agent": "Mozilla/5.0 (compatible; MIEMEmployeesBot/0.1.0; +https://miem.hse.ru/)"
-}
-
-
+from app.services.academic_degrees import academic_degrees
+from app.services.resource_cache import ResourceCache
+
+HEADERS = {
+    "User-Agent": "Mozilla/5.0 (compatible; MIEMEmployeesBot/0.1.0; +https://miem.hse.ru/)"
+}
+
+
 def run_crawl(db: Session, settings: Settings) -> CrawlRun:
+    source = _ensure_source(db, settings.source_url)
+    run = CrawlRun(source_url=source.source_url, status="running")
+    db.add(run)
+    db.commit()
+    db.refresh(run)
+
+    found_keys: set[str] = set()
+    parsed_count = 0
+    skipped_count = 0
+    try:
+        with requests.Session() as session:
+            resource_cache = ResourceCache(db)
+            urls = collect_profile_links(session, source.source_url, HEADERS, settings.request_timeout)
+            if settings.crawl_limit:
+                urls = urls[: settings.crawl_limit]
+            run.found_count = len(urls)
+            db.commit()
+
+            for url in urls:
+                key = profile_key(url)
+                if key:
+                    found_keys.add(key)
+                try:
+                    parsed = parse_person_profile(
+                        session,
+                        url,
+                        HEADERS,
+                        settings.request_timeout,
+                        settings.parser_use_playwright,
+                        resource_cache=resource_cache,
+                    )
+                    if not parsed:
+                        continue
+                    employee, changed = _upsert_employee(db, run, parsed)
+                    if employee.profile_key:
+                        found_keys.add(employee.profile_key)
+                    if changed:
+                        parsed_count += 1
+                    else:
+                        skipped_count += 1
+                    run.parsed_count = parsed_count
+                    run.skipped_count = skipped_count
+                    db.commit()
+                except Exception as exc:
+                    run.error_count += 1
+                    db.add(
+                        CrawlError(
+                            crawl_run_id=run.id,
+                            profile_url=url,
+                            error_type=type(exc).__name__,
+                            message=str(exc),
+                        )
+                    )
+                    db.commit()
+                finally:
+                    time.sleep(settings.request_delay_seconds)
+
+            run.dismissed_count = _mark_dismissed(
+                db,
+                run,
+                found_keys,
+                session,
+                settings.request_timeout,
+                confirmation_runs=settings.dismissal_confirmation_runs,
+                max_auto_dismissals=settings.max_auto_dismissals_per_run,
+            )
+        run.status = "completed"
+    except Exception as exc:
+        run.status = "failed"
+        run.message = str(exc)
+    finally:
+        run.finished_at = datetime.now(timezone.utc)
+        db.commit()
+        db.refresh(run)
+    return run
+
+
+def refresh_dismissed_status(db: Session, settings: Settings) -> CrawlRun:
     source = _ensure_source(db, settings.source_url)
     run = CrawlRun(source_url=source.source_url, status="running")
     db.add(run)
     db.commit()
     db.refresh(run)
 
-    found_keys: set[str] = set()
-    parsed_count = 0
     try:
+        employees = db.scalars(select(Employee).where(Employee.status == "dismissed")).all()
+        run.found_count = len(employees)
         with requests.Session() as session:
             urls = collect_profile_links(session, source.source_url, HEADERS, settings.request_timeout)
-            if settings.crawl_limit:
-                urls = urls[: settings.crawl_limit]
-            run.found_count = len(urls)
-            db.commit()
-
-            for url in urls:
-                key = profile_key(url)
-                if key:
-                    found_keys.add(key)
-                try:
-                    parsed = parse_person_profile(
-                        session,
-                        url,
-                        HEADERS,
-                        settings.request_timeout,
-                        settings.parser_use_playwright,
-                    )
-                    if not parsed:
-                        continue
-                    _upsert_employee(db, run, parsed)
-                    parsed_count += 1
-                    run.parsed_count = parsed_count
-                    db.commit()
-                except Exception as exc:
-                    run.error_count += 1
-                    db.add(
-                        CrawlError(
-                            crawl_run_id=run.id,
-                            profile_url=url,
-                            error_type=type(exc).__name__,
-                            message=str(exc),
-                        )
-                    )
-                    db.commit()
-                finally:
-                    time.sleep(settings.request_delay_seconds)
-
-        run.dismissed_count = _mark_dismissed(db, found_keys)
+        source_keys = {key for url in urls if (key := profile_key(url))}
+        now = datetime.now(timezone.utc)
+        for employee in employees:
+            if employee.profile_key not in source_keys:
+                run.skipped_count += 1
+                continue
+            employee.status = "active"
+            employee.dismissed_at = None
+            employee.last_seen_at = now
+            employee.profile_unavailable_streak = 0
+            employee.last_profile_check_at = now
+            _record_employee_change(
+                db,
+                run,
+                employee,
+                "reactivated",
+                profile_available=True,
+                message="Сотрудник снова найден в исходном списке.",
+            )
+            run.parsed_count += 1
         run.status = "completed"
     except Exception as exc:
         run.status = "failed"
+        run.error_count = 1
         run.message = str(exc)
+        db.add(
+            CrawlError(
+                crawl_run_id=run.id,
+                profile_url=source.source_url,
+                error_type=type(exc).__name__,
+                message=str(exc),
+            )
+        )
     finally:
         run.finished_at = datetime.now(timezone.utc)
         db.commit()
@@ -80,81 +164,635 @@ def run_crawl(db: Session, settings: Settings) -> CrawlRun:
     return run
 
 
-def _ensure_source(db: Session, source_url: str) -> ParserSource:
-    source = db.scalar(select(ParserSource).where(ParserSource.source_url == source_url))
-    if source:
-        return source
-    source = ParserSource(source_url=source_url, enabled=True)
-    db.add(source)
-    db.commit()
-    db.refresh(source)
-    return source
+def refresh_employee(db: Session, employee: Employee, settings: Settings) -> CrawlRun:
+    run = CrawlRun(source_url=employee.canonical_url, status="running", found_count=1)
+    db.add(run)
+    db.commit()
+    db.refresh(run)
+
+    try:
+        with requests.Session() as session:
+            resource_cache = ResourceCache(db)
+            parsed = parse_person_profile(
+                session,
+                employee.canonical_url,
+                HEADERS,
+                settings.request_timeout,
+                settings.parser_use_playwright,
+                resource_cache=resource_cache,
+            )
+        if not parsed:
+            raise ValueError("Профиль не удалось распарсить.")
+        if _parsed_profile_key(parsed) != employee.profile_key:
+            raise ValueError("Распарсенный профиль не совпадает с обновляемым сотрудником.")
+
+        _, changed = _upsert_employee(db, run, parsed)
+        if changed:
+            run.parsed_count = 1
+        else:
+            run.skipped_count = 1
+        run.status = "completed"
+    except Exception as exc:
+        run.status = "failed"
+        run.error_count = 1
+        run.message = str(exc)
+        db.add(
+            CrawlError(
+                crawl_run_id=run.id,
+                profile_url=employee.canonical_url,
+                error_type=type(exc).__name__,
+                message=str(exc),
+            )
+        )
+    finally:
+        run.finished_at = datetime.now(timezone.utc)
+        db.commit()
+        db.refresh(run)
+    return run
+
+
+def _ensure_source(db: Session, source_url: str) -> ParserSource:
+    source = db.scalar(select(ParserSource).where(ParserSource.source_url == source_url))
+    if source:
+        return source
+    source = ParserSource(source_url=source_url, enabled=True)
+    db.add(source)
+    db.commit()
+    db.refresh(source)
+    return source
+
+
+def _parsed_profile_key(parsed: dict) -> str:
+    return f"{parsed.get('profile_type')}:{parsed.get('profile_id')}"
+
+
+def _upsert_employee(db: Session, run: CrawlRun, parsed: dict) -> tuple[Employee, bool]:
+    html = parsed.pop("_html", None)
+    parsed.pop("_resource_manifest", None)
+    checksum = _checksum(parsed)
+    key = _parsed_profile_key(parsed)
+    employee = db.scalar(select(Employee).where(Employee.profile_key == key))
+    if not employee:
+        employee = _find_employee_with_moved_profile(db, parsed)
+    now = datetime.now(timezone.utc)
+    if not employee:
+        employee = Employee(
+            profile_key=key,
+            profile_type=parsed.get("profile_type"),
+            profile_id=parsed.get("profile_id"),
+            canonical_url=parsed["source_url"],
+            first_seen_at=now,
+        )
+        db.add(employee)
+        run.new_count += 1
+        is_new = True
+    else:
+        is_new = False
+
+    parser_version = parsed.get("parser_version")
+    changed = is_new or employee.current_checksum != checksum or employee.parser_version != parser_version
+    previous_url = employee.canonical_url if employee.canonical_url != parsed["source_url"] else None
+    employee.profile_key = key
+    employee.profile_type = parsed.get("profile_type")
+    employee.profile_id = parsed.get("profile_id")
+    employee.canonical_url = parsed["source_url"]
+    employee.full_name = parsed.get("full_name")
+    employee.status = "active"
+    employee.last_seen_at = now
+    employee.dismissed_at = None
+    employee.profile_unavailable_streak = 0
+    employee.last_profile_check_at = now
+    employee.parser_version = parser_version
+    employee.has_academic_degree = bool(academic_degrees(parsed))
+    if changed:
+        employee.current_data = parsed
+    employee.current_checksum = checksum
+    db.flush()
+    _sync_profile_url_history(db, employee, previous_url, employee.canonical_url, now)
+
+    if is_new:
+        _record_employee_change(
+            db,
+            run,
+            employee,
+            "new",
+            profile_available=True,
+            message="Сотрудник впервые найден в источнике.",
+        )
+
+    if changed:
+        db.query(ProfileTab).filter(ProfileTab.employee_id == employee.id).delete()
+        for tab in parsed.get("tabs") or []:
+            db.add(
+                ProfileTab(
+                    employee_id=employee.id,
+                    title=tab.get("title") or "",
+                    href=tab.get("href") or "",
+                    data_index=tab.get("data_index"),
+                )
+            )
+
+        db.add(
+            EmployeeSnapshot(
+                employee_id=employee.id,
+                crawl_run_id=run.id,
+                parsed_data=parsed,
+                html_snapshot=gzip.compress(html.encode("utf-8")) if html else None,
+                checksum=checksum,
+                parser_version=parser_version,
+            )
+        )
+    db.flush()
+    _try_sync_employee_publications(db, run, employee, parsed)
+    _try_sync_employee_news_links(db, run, employee, parsed)
+    return employee, changed
+
+
+def _find_employee_with_moved_profile(db: Session, parsed: dict) -> Employee | None:
+    full_name = parsed.get("full_name")
+    if not full_name:
+        return None
+    candidates = db.scalars(select(Employee).where(Employee.full_name == full_name)).all()
+    if len(candidates) == 1:
+        return candidates[0]
+
+    parsed_identity = _profile_identity_values(parsed)
+    if not parsed_identity:
+        return None
+    matches = [candidate for candidate in candidates if parsed_identity & _employee_identity_values(candidate)]
+    return matches[0] if len(matches) == 1 else None
+
+
+def _profile_identity_values(profile: dict) -> set[str]:
+    if not isinstance(profile, dict):
+        return set()
+    values = set()
+    contacts = profile.get("contacts") or {}
+    if not isinstance(contacts, dict):
+        contacts = {}
+    for email in contacts.get("emails") or []:
+        normalized = str(email).strip().lower()
+        if normalized:
+            values.add(f"email:{normalized}")
+    for item in profile.get("external_ids") or []:
+        if not isinstance(item, dict):
+            continue
+        system = str(item.get("system") or "").strip().lower()
+        value = str(item.get("value") or "").strip().lower()
+        if system and value:
+            values.add(f"external:{system}:{value}")
+    return values
+
+
+def _employee_identity_values(employee: Employee) -> set[str]:
+    return _profile_identity_values(employee.current_data or {})
+
+
+def _sync_profile_url_history(
+    db: Session,
+    employee: Employee,
+    previous_url: str | None,
+    current_url: str,
+    seen_at: datetime,
+) -> None:
+    urls = {url for url in (previous_url, current_url) if url}
+    for url in urls:
+        history = db.scalar(
+            select(EmployeeProfileUrl).where(
+                EmployeeProfileUrl.employee_id == employee.id,
+                EmployeeProfileUrl.url == url,
+            )
+        )
+        if history:
+            history.last_seen_at = seen_at
+        else:
+            db.add(
+                EmployeeProfileUrl(
+                    employee_id=employee.id,
+                    url=url,
+                    first_seen_at=seen_at,
+                    last_seen_at=seen_at,
+                )
+            )
+
+
+def _try_sync_employee_publications(db: Session, run: CrawlRun, employee: Employee, parsed: dict) -> None:
+    try:
+        if not _publication_payloads(parsed):
+            return
+        if not _employee_publications_table_exists(db):
+            return
+        with db.begin_nested():
+            _sync_employee_publications(db, employee, parsed)
+    except Exception as exc:
+        db.add(
+            CrawlError(
+                crawl_run_id=run.id,
+                profile_url=employee.canonical_url,
+                error_type=type(exc).__name__,
+                message=f"Не удалось сохранить публикации сотрудника: {exc}",
+            )
+        )
+
+
+def _employee_publications_table_exists(db: Session) -> bool:
+    return inspect(db.connection()).has_table(EmployeePublication.__tablename__)
+
+
+def _sync_employee_publications(db: Session, employee: Employee, parsed: dict) -> None:
+    publications = _publication_payloads(parsed)
+    imported = [item for item in db.scalars(select(EmployeePublication).where(
+        EmployeePublication.employee_id == employee.id
+    )).all() if _csv_enrichment(item) is not None]
+    seen_hashes = set()
+    for publication in publications:
+        source_hash = _publication_hash(publication)
+        seen_hashes.add(source_hash)
+        publication_id = _clean_optional(publication.get("publication_id") or publication.get("id"))
+        existing = None
+        if publication_id:
+            existing = db.scalar(
+                select(EmployeePublication).where(
+                    EmployeePublication.employee_id == employee.id,
+                    EmployeePublication.publication_id == publication_id,
+                )
+            )
+        if not existing:
+            existing = db.scalar(
+                select(EmployeePublication).where(
+                    EmployeePublication.employee_id == employee.id,
+                    EmployeePublication.source_hash == source_hash,
+                )
+            )
+        if not existing:
+            existing = _match_imported_publication(imported, publication)
+        if not existing:
+            existing = EmployeePublication(employee_id=employee.id, source_hash=source_hash, title=_publication_title(publication))
+            db.add(existing)
+        _apply_publication(existing, publication, source_hash)
+
+    if seen_hashes:
+        stale = db.scalars(
+            select(EmployeePublication).where(
+                EmployeePublication.employee_id == employee.id,
+                EmployeePublication.source_hash.not_in(seen_hashes),
+            )
+        ).all()
+        for item in stale:
+            if _csv_enrichment(item) is None:
+                db.delete(item)
 
 
-def _upsert_employee(db: Session, run: CrawlRun, parsed: dict) -> Employee:
-    html = parsed.pop("_html", None)
-    checksum = _checksum(parsed)
-    key = f"{parsed.get('profile_type')}:{parsed.get('profile_id')}"
-    employee = db.scalar(select(Employee).where(Employee.profile_key == key))
-    now = datetime.now(timezone.utc)
-    if not employee:
-        employee = Employee(
-            profile_key=key,
-            profile_type=parsed.get("profile_type"),
-            profile_id=parsed.get("profile_id"),
-            canonical_url=parsed["source_url"],
-            first_seen_at=now,
+def _csv_enrichment(publication: EmployeePublication) -> dict | None:
+    raw = publication.raw_data
+    metadata = raw.get("csv_enrichment") if isinstance(raw, dict) else None
+    return metadata if isinstance(metadata, dict) else None
+
+
+def _match_imported_publication(imported: list[EmployeePublication], publication: dict) -> EmployeePublication | None:
+    def doi(value):
+        return re.sub(r"^(?:https?://(?:dx\.)?doi\.org/|doi:\s*)", "", (value or "").strip(), flags=re.I).lower()
+
+    def title(value):
+        return " ".join(unicodedata.normalize("NFKC", value or "").casefold().replace("ё", "е").split())
+
+    identifier = doi(publication.get("doi_url"))
+    year = _int_or_none(publication.get("year"))
+    name = title(publication.get("title") or publication.get("text"))
+    matches = [item for item in imported if
+               (identifier and doi(item.doi_url) == identifier) or
+               (name and title(item.title) == name and item.year in (None, year))]
+    if len(matches) > 1:
+        raise ValueError("Ambiguous imported publication; synchronization rolled back")
+    if not matches:
+        return None
+    match = matches[0]
+    if ((identifier and doi(match.doi_url) and identifier != doi(match.doi_url)) or
+            (year and match.year and year != match.year)):
+        raise ValueError("Conflicting imported publication; synchronization rolled back")
+    return match
+
+
+def _publication_payloads(parsed: dict) -> list[dict]:
+    publications = []
+    for section in parsed.get("sections") or []:
+        if not isinstance(section, dict) or section.get("type") != "publications":
+            continue
+        for publication in section.get("publications") or []:
+            if isinstance(publication, dict):
+                publications.append(publication)
+    return publications
+
+
+def _apply_publication(target: EmployeePublication, publication: dict, source_hash: str) -> None:
+    enrichment = _csv_enrichment(target)
+    publication_id = _clean_optional(publication.get("publication_id") or publication.get("id"))
+    if enrichment is not None and target.publication_id and publication_id and target.publication_id != publication_id:
+        raise ValueError(
+            f"Ambiguous HSE IDs for CSV publication {target.id}: "
+            f"{target.publication_id} != {publication_id}; publication synchronization rolled back"
         )
-        db.add(employee)
-        run.new_count += 1
-
-    employee.full_name = parsed.get("full_name")
-    employee.status = "active"
-    employee.last_seen_at = now
-    employee.dismissed_at = None
-    employee.parser_version = parsed.get("parser_version")
-    employee.current_data = parsed
-    employee.current_checksum = checksum
-    db.flush()
-
-    db.query(ProfileTab).filter(ProfileTab.employee_id == employee.id).delete()
-    for tab in parsed.get("tabs") or []:
-        db.add(
-            ProfileTab(
-                employee_id=employee.id,
-                title=tab.get("title") or "",
-                href=tab.get("href") or "",
-                data_index=tab.get("data_index"),
-            )
-        )
-
-    db.add(
-        EmployeeSnapshot(
-            employee_id=employee.id,
-            crawl_run_id=run.id,
-            parsed_data=parsed,
-            html_snapshot=gzip.compress(html.encode("utf-8")) if html else None,
-            checksum=checksum,
-            parser_version=parsed.get("parser_version"),
-        )
-    )
-    return employee
-
-
-def _mark_dismissed(db: Session, found_keys: set[str]) -> int:
-    dismissed = 0
-    active = db.scalars(select(Employee).where(Employee.status == "active")).all()
-    now = datetime.now(timezone.utc)
-    for employee in active:
-        if employee.profile_key in found_keys:
-            continue
-        employee.status = "dismissed"
-        employee.dismissed_at = now
-        dismissed += 1
-    db.commit()
-    return dismissed
-
-
-def _checksum(data: dict) -> str:
-    payload = json.dumps(data, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
-    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+    preserved = {field: getattr(target, field) for field in ("title", "year", "doi_url")}
+    target.publication_id = (publication_id or target.publication_id) if enrichment is not None else publication_id
+    target.title = _publication_title(publication)
+    target.year = _int_or_none(publication.get("year"))
+    target.publication_type = _clean_optional(publication.get("publication_type") or publication.get("type"))
+    target.language = _clean_optional(publication.get("language"))
+    target.status = _int_or_none(publication.get("status"))
+    target.url = _clean_optional(publication.get("url"))
+    target.doi_url = _clean_optional(publication.get("doi_url"))
+    target.other_url = _clean_optional(publication.get("other_url"))
+    target.document_url = _clean_optional(publication.get("document_url"))
+    target.citation_text = _clean_optional(publication.get("citation_text") or publication.get("text"))
+    target.annotation = publication.get("annotation") if isinstance(publication.get("annotation"), dict) else None
+    target.description = publication.get("description") if isinstance(publication.get("description"), dict) else None
+    target.authors = publication.get("authors") if isinstance(publication.get("authors"), list) else None
+    target.raw_data = publication.get("raw_data") if isinstance(publication.get("raw_data"), dict) else publication
+    if enrichment is not None:
+        target.raw_data = {**target.raw_data, "csv_enrichment": enrichment}
+        for field in ("title", "year", "doi_url"):
+            if not getattr(target, field) or (field == "title" and not (publication.get("title") or publication.get("text"))):
+                setattr(target, field, preserved[field])
+    target.source_hash = source_hash
+
+
+def _publication_hash(publication: dict) -> str:
+    return _payload_hash(publication.get("raw_data") if isinstance(publication.get("raw_data"), dict) else publication)
+
+
+def _payload_hash(value: object) -> str:
+    payload = json.dumps(_stable_checksum_payload(value), ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str)
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+def _publication_title(publication: dict) -> str:
+    return _clean_optional(publication.get("title") or publication.get("text") or publication.get("id")) or "Untitled publication"
+
+
+def _clean_optional(value: object) -> str | None:
+    text = str(value or "").strip()
+    return text or None
+
+
+def _int_or_none(value: object) -> int | None:
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _try_sync_employee_news_links(db: Session, run: CrawlRun, employee: Employee, parsed: dict) -> None:
+    try:
+        if not _news_link_payloads(parsed):
+            return
+        if not _employee_news_links_table_exists(db):
+            return
+        with db.begin_nested():
+            _sync_employee_news_links(db, employee, parsed)
+    except Exception as exc:
+        db.add(
+            CrawlError(
+                crawl_run_id=run.id,
+                profile_url=employee.canonical_url,
+                error_type=type(exc).__name__,
+                message=f"Не удалось сохранить новости сотрудника: {exc}",
+            )
+        )
+
+
+def _employee_news_links_table_exists(db: Session) -> bool:
+    return inspect(db.connection()).has_table(EmployeeNewsLink.__tablename__)
+
+
+def _sync_employee_news_links(db: Session, employee: Employee, parsed: dict) -> None:
+    news_links = _news_link_payloads(parsed)
+    seen_hashes = set()
+    for news_link in news_links:
+        source_hash = _news_link_hash(news_link)
+        seen_hashes.add(source_hash)
+        url = _clean_optional(news_link.get("url"))
+        existing = None
+        if url:
+            existing = db.scalar(
+                select(EmployeeNewsLink).where(
+                    EmployeeNewsLink.employee_id == employee.id,
+                    EmployeeNewsLink.url == url,
+                )
+            )
+        if not existing:
+            existing = db.scalar(
+                select(EmployeeNewsLink).where(
+                    EmployeeNewsLink.employee_id == employee.id,
+                    EmployeeNewsLink.source_hash == source_hash,
+                )
+            )
+        if not existing:
+            existing = EmployeeNewsLink(employee_id=employee.id, source_hash=source_hash, title=_news_link_title(news_link))
+            db.add(existing)
+        _apply_news_link(existing, news_link, source_hash)
+
+    if seen_hashes:
+        stale = db.scalars(
+            select(EmployeeNewsLink).where(
+                EmployeeNewsLink.employee_id == employee.id,
+                EmployeeNewsLink.source_hash.not_in(seen_hashes),
+            )
+        ).all()
+        for item in stale:
+            db.delete(item)
+
+
+def _news_link_payloads(parsed: dict) -> list[dict]:
+    news_links = []
+    for section in parsed.get("sections") or []:
+        if not isinstance(section, dict) or section.get("type") != "news":
+            continue
+        for item in section.get("news_links") or []:
+            if isinstance(item, dict):
+                news_links.append(item)
+    return news_links
+
+
+def _apply_news_link(target: EmployeeNewsLink, news_link: dict, source_hash: str) -> None:
+    target.title = _news_link_title(news_link)
+    target.url = _clean_optional(news_link.get("url"))
+    target.summary = _clean_optional(news_link.get("summary"))
+    target.published_at = _datetime_or_none(news_link.get("published_at"))
+    target.published_year = _int_or_none(news_link.get("published_year"))
+    target.raw_data = news_link.get("raw_data") if isinstance(news_link.get("raw_data"), dict) else news_link
+    target.source_hash = source_hash
+
+
+def _news_link_hash(news_link: dict) -> str:
+    return _payload_hash(news_link.get("raw_data") if isinstance(news_link.get("raw_data"), dict) else news_link)
+
+
+def _news_link_title(news_link: dict) -> str:
+    return _clean_optional(news_link.get("title") or news_link.get("url")) or "Untitled news"
+
+
+def _datetime_or_none(value: object) -> datetime | None:
+    if isinstance(value, datetime):
+        return value
+    if not value:
+        return None
+    try:
+        parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+    except ValueError:
+        return None
+    return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)
+
+
+def _mark_dismissed(
+    db: Session,
+    run: CrawlRun,
+    found_keys: set[str],
+    session: requests.Session,
+    timeout: int,
+    *,
+    confirmation_runs: int = 3,
+    max_auto_dismissals: int | None = 25,
+) -> int:
+    dismissed = 0
+    candidates = db.scalars(
+        select(Employee).where(Employee.status.in_(("active", "verification_required")))
+    ).all()
+    now = datetime.now(timezone.utc)
+    unavailable = []
+    for employee in candidates:
+        if employee.profile_key in found_keys:
+            continue
+        profile_available = _profile_check(session, employee.canonical_url, timeout)
+        employee.last_profile_check_at = now
+        if profile_available is None:
+            db.add(
+                CrawlError(
+                    crawl_run_id=run.id,
+                    profile_url=employee.canonical_url,
+                    error_type="ProfileAvailabilityCheckError",
+                    message="Не удалось надёжно проверить доступность профиля; статус сотрудника не изменён.",
+                )
+            )
+            continue
+        if profile_available:
+            employee.profile_unavailable_streak = 0
+            if employee.status == "verification_required":
+                employee.status = "active"
+            _record_employee_change(
+                db,
+                run,
+                employee,
+                "missing_from_source",
+                profile_available=True,
+                message="Профиль доступен, но ссылка отсутствует в исходном списке.",
+            )
+            continue
+        next_streak = employee.profile_unavailable_streak + 1
+        unavailable.append((employee, next_streak))
+
+    dismissal_blocked = bool(
+        max_auto_dismissals is not None and len(unavailable) > max_auto_dismissals
+    )
+    if dismissal_blocked:
+        run.message = (
+            f"Автоматическое увольнение приостановлено: {len(unavailable)} профилей "
+            f"одновременно не подтвердились (лимит {max_auto_dismissals})."
+        )
+
+    for employee, next_streak in unavailable:
+        employee.profile_unavailable_streak = next_streak
+        if next_streak < confirmation_runs or dismissal_blocked:
+            employee.status = "verification_required"
+            _record_employee_change(
+                db,
+                run,
+                employee,
+                "verification_required",
+                profile_available=False,
+                message=(
+                    "Профиль не подтвердился. Автоматическое увольнение отложено до "
+                    f"{confirmation_runs} последовательных проверок."
+                    if not dismissal_blocked
+                    else "Автоматическое увольнение отложено из-за массовой ошибки проверки профилей."
+                ),
+            )
+            continue
+
+        employee.status = "dismissed"
+        employee.dismissed_at = now
+        _record_employee_change(
+            db,
+            run,
+            employee,
+            "dismissed",
+            profile_available=False,
+            message=(
+                "Сотрудник отсутствует в исходном списке, профиль не подтвердился "
+                f"{confirmation_runs} раза подряд."
+            ),
+        )
+        dismissed += 1
+    db.commit()
+    return dismissed
+
+
+def _profile_is_available(session: requests.Session, url: str, timeout: int) -> bool:
+    return _profile_check(session, url, timeout) is True
+
+
+def _profile_check(session: requests.Session, url: str, timeout: int) -> bool | None:
+    try:
+        response = session.get(url, headers=HEADERS, timeout=timeout, allow_redirects=True)
+        if response.status_code < 400:
+            return True
+        if response.status_code in {404, 410}:
+            return False
+        return None
+    except requests.RequestException:
+        return None
+
+
+def _record_employee_change(
+    db: Session,
+    run: CrawlRun,
+    employee: Employee,
+    change_type: str,
+    *,
+    profile_available: bool | None,
+    message: str,
+) -> None:
+    db.add(
+        CrawlRunEmployeeChange(
+            crawl_run_id=run.id,
+            employee_id=employee.id,
+            profile_key=employee.profile_key,
+            profile_url=employee.canonical_url,
+            full_name=employee.full_name,
+            change_type=change_type,
+            profile_available=profile_available,
+            message=message,
+        )
+    )
+
+
+def _checksum(data: dict) -> str:
+    payload = json.dumps(_stable_checksum_payload(data), ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+def _stable_checksum_payload(value):
+    if isinstance(value, dict):
+        return {key: _stable_checksum_payload(item) for key, item in value.items()}
+    if isinstance(value, list):
+        return [_stable_checksum_payload(item) for item in value]
+    if isinstance(value, str):
+        return _normalize_date_dependent_experience(value)
+    return value
+
+
+def _normalize_date_dependent_experience(value: str) -> str:
+    return re.sub(
+        r"(?i)(стаж(?:\s+работы)?(?:\s+в\s+ниу\s+вшэ|\s+в\s+вшэ)?\s*:?\s*)\d+\s*(?:год(?:а|ов)?|лет)",
+        r"\1<experience-years>",
+        value,
+    )

app/services/resource_cache.py

@@ -0,0 +1,147 @@
+from __future__ import annotations
+
+import gzip
+import hashlib
+import json
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Any
+
+import requests
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from app.models import ParseResourceCache
+from app.version import BACKEND_VERSION
+
+
+@dataclass(frozen=True)
+class CachedResource:
+    text: str
+    body_hash: str
+    from_cache: bool
+    status_code: int
+
+
+class ResourceCache:
+    def __init__(self, db: Session):
+        self.db = db
+
+    def fetch_text(
+        self,
+        session: requests.Session,
+        *,
+        profile_key: str,
+        resource_key: str,
+        method: str,
+        url: str,
+        headers: dict[str, str],
+        timeout: int,
+        json_payload: Any | None = None,
+        params: dict[str, Any] | None = None,
+    ) -> CachedResource:
+        method = method.upper()
+        fingerprint = _request_fingerprint(method=method, url=url, json_payload=json_payload, params=params)
+        cached = self.db.scalar(
+            select(ParseResourceCache).where(
+                ParseResourceCache.profile_key == profile_key,
+                ParseResourceCache.resource_key == resource_key,
+                ParseResourceCache.request_fingerprint == fingerprint,
+            )
+        )
+
+        request_headers = dict(headers)
+        if cached:
+            if cached.etag:
+                request_headers["If-None-Match"] = cached.etag
+            if cached.last_modified:
+                request_headers["If-Modified-Since"] = cached.last_modified
+
+        response = _send(
+            session,
+            method=method,
+            url=url,
+            headers=request_headers,
+            timeout=timeout,
+            json_payload=json_payload,
+            params=params,
+        )
+        if response.status_code == 304 and cached:
+            cached.fetched_at = datetime.now(timezone.utc)
+            self.db.flush()
+            return CachedResource(
+                text=gzip.decompress(cached.body_snapshot).decode("utf-8"),
+                body_hash=cached.body_hash,
+                from_cache=True,
+                status_code=response.status_code,
+            )
+
+        response.raise_for_status()
+        text = response.text
+        body_hash = _body_hash(text)
+        etag = response.headers.get("ETag") if hasattr(response, "headers") else None
+        last_modified = response.headers.get("Last-Modified") if hasattr(response, "headers") else None
+
+        if cached:
+            cached.method = method
+            cached.url = url
+            cached.etag = etag
+            cached.last_modified = last_modified
+            cached.body_hash = body_hash
+            cached.body_snapshot = gzip.compress(text.encode("utf-8"))
+            cached.parser_version = BACKEND_VERSION
+            cached.fetched_at = datetime.now(timezone.utc)
+        else:
+            self.db.add(
+                ParseResourceCache(
+                    profile_key=profile_key,
+                    resource_key=resource_key,
+                    method=method,
+                    url=url,
+                    request_fingerprint=fingerprint,
+                    etag=etag,
+                    last_modified=last_modified,
+                    body_hash=body_hash,
+                    body_snapshot=gzip.compress(text.encode("utf-8")),
+                    parser_version=BACKEND_VERSION,
+                    fetched_at=datetime.now(timezone.utc),
+                )
+            )
+        self.db.flush()
+        return CachedResource(text=text, body_hash=body_hash, from_cache=False, status_code=response.status_code)
+
+
+def _send(
+    session: requests.Session,
+    *,
+    method: str,
+    url: str,
+    headers: dict[str, str],
+    timeout: int,
+    json_payload: Any | None,
+    params: dict[str, Any] | None,
+) -> requests.Response:
+    if method == "POST":
+        return session.post(url, json=json_payload, headers=headers, timeout=timeout, params=params)
+    return session.get(url, headers=headers, timeout=timeout, params=params)
+
+
+def _request_fingerprint(
+    *,
+    method: str,
+    url: str,
+    json_payload: Any | None,
+    params: dict[str, Any] | None,
+) -> str:
+    payload = {
+        "method": method,
+        "url": url,
+        "json": json_payload,
+        "params": params,
+    }
+    encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+    return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
+
+
+def _body_hash(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()

app/static/admin.js

@@ -3,13 +3,15 @@
     "full_name",
     "status",
     "positions",
-    "hse_start_year",
-    "email",
     "last_seen_at",
-    "dismissed_at",
     "profile",
   ];
   const storageKey = "miem.directory.columns";
+  const columnPresets = {
+    review: ["full_name", "status", "academic_degree", "last_seen_at", "profile"],
+    contacts: ["full_name", "status", "email", "phone", "address", "profile"],
+    full: ["full_name", "status", "positions", "hse_start_year", "email", "phone", "address", "academic_degree", "publications_count", "courses_count", "news_count", "first_seen_at", "last_seen_at", "dismissed_at", "profile"],
+  };
 
   function readColumns() {
     try {
@@ -39,18 +41,33 @@
     if (!document.querySelector("[data-directory-table]")) return;
     let columns = readColumns();
     const modal = document.querySelector("[data-columns-modal]");
+    const trigger = document.querySelector("[data-columns-open]");
     applyColumns(columns);
 
-    document.querySelectorAll("[data-columns-open]").forEach((button) => {
-      button.addEventListener("click", () => {
-        modal.hidden = false;
-      });
+    trigger.addEventListener("click", () => {
+      modal.showModal();
+      trigger.setAttribute("aria-expanded", "true");
+      modal.querySelector("input")?.focus();
     });
     document.querySelectorAll("[data-columns-close]").forEach((button) => {
       button.addEventListener("click", () => {
-        modal.hidden = true;
+        modal.close();
       });
     });
+    document.querySelectorAll("[data-columns-preset]").forEach((button) => {
+      button.addEventListener("click", () => {
+        columns = columnPresets[button.dataset.columnsPreset];
+        writeColumns(columns);
+        applyColumns(columns);
+      });
+    });
+    modal.addEventListener("click", (event) => {
+      if (event.target === modal) modal.close();
+    });
+    modal.addEventListener("close", () => {
+      trigger.setAttribute("aria-expanded", "false");
+      trigger.focus();
+    });
     document.querySelectorAll("[data-column-toggle]").forEach((checkbox) => {
       checkbox.addEventListener("change", () => {
         columns = Array.from(document.querySelectorAll("[data-column-toggle]:checked")).map((item) => item.value);
@@ -59,47 +76,72 @@
         applyColumns(columns);
       });
     });
-    document.querySelectorAll("[data-row-href]").forEach((row) => {
-      row.addEventListener("click", (event) => {
-        if (event.target.closest("a, button, input, select, label")) return;
-        window.location.href = row.dataset.rowHref;
-      });
-    });
   }
 
   function setupProgress() {
     const panel = document.querySelector("[data-progress-panel]");
     if (!panel) return;
+    const state = document.createElement("p");
+    state.className = "progress-panel__empty";
+    state.setAttribute("role", "status");
+    panel.append(state);
+    let pending = false;
 
     const update = (run) => {
       if (!run) return;
       const status = document.querySelector("[data-progress-status]");
       const processed = document.querySelector("[data-progress-processed]");
       const found = document.querySelector("[data-progress-found]");
+      const skipped = document.querySelector("[data-progress-skipped]");
       const errors = document.querySelector("[data-progress-errors]");
       const fill = document.querySelector("[data-progress-fill]");
       const percent = document.querySelector("[data-progress-percent]");
-      if (status) status.textContent = run.status;
+      const error = document.querySelector("[data-progress-error]");
+      if (status) status.textContent = run.status_display || run.status;
       if (processed) processed.textContent = run.processed_count;
       if (found) found.textContent = run.found_count;
+      if (skipped) skipped.textContent = run.skipped_count;
       if (errors) errors.textContent = run.error_count;
       if (fill) fill.style.width = `${run.progress_percent}%`;
       if (percent) percent.textContent = run.progress_percent;
+      if (fill) {
+        fill.parentElement.setAttribute("aria-valuenow", run.progress_percent);
+        fill.parentElement.setAttribute("aria-valuetext", `${run.progress_percent}%`);
+      }
+      if (error) error.hidden = true;
     };
 
     const poll = async () => {
+      if (pending) return true;
+      pending = true;
+      state.textContent = "Обновляем прогресс…";
+      panel.setAttribute("aria-busy", "true");
       try {
-        const response = await fetch("/api/crawl-runs/latest", { credentials: "same-origin" });
-        if (!response.ok) return false;
+        const response = await fetch("/api/crawl-runs/latest", {
+          credentials: "same-origin",
+          signal: AbortSignal.timeout(15000),
+        });
+        if (!response.ok) throw new Error("progress request failed");
         const data = await response.json();
         const run = data.running || data.latest;
         update(run);
+        state.textContent = run ? "Прогресс обновлён" : "Запусков пока нет. Запустите парсинг.";
+        const error = document.querySelector("[data-progress-error]");
+        if (error) error.hidden = true;
         return Boolean(data.running);
       } catch (_error) {
-        return false;
+        const error = document.querySelector("[data-progress-error]");
+        if (error) error.hidden = false;
+        state.textContent = "Показаны последние полученные данные. Прогресс может быть устаревшим.";
+        return true;
+      } finally {
+        pending = false;
+        panel.setAttribute("aria-busy", "false");
       }
     };
 
+    document.querySelectorAll("[data-progress-retry]").forEach((button) => button.addEventListener("click", poll));
+
     const interval = window.setInterval(async () => {
       const keepGoing = await poll();
       if (!keepGoing) window.clearInterval(interval);

app/templates/base.html

@@ -4,22 +4,28 @@
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <title>{% block title %}MIEM Employees{% endblock %}</title>
-    <link rel="stylesheet" href="/static/admin.css">
+    <link rel="stylesheet" href="/static/admin.css?v={{ frontend_version }}">
   </head>
   <body class="admin">
+    <a class="admin__skip-link" href="#main-content">Перейти к содержимому</a>
     <header class="admin__header">
-      <h1 class="admin__brand">MIEM Employees</h1>
+      <h1 class="admin__brand"><a class="admin__brand-link" href="{{ '/admin/council' if public_page else '/admin' }}">MIEM Employees</a></h1>
       <nav class="admin__nav">
-        <a class="admin__link" href="/admin">Dashboard</a>
-        <a class="admin__link" href="/admin/directory">Directory</a>
-        <a class="admin__link" href="/admin/employees">Employees</a>
-        <a class="admin__link" href="/admin/runs">Runs</a>
+        {% if public_page %}
+        <a class="admin__link" href="/admin/council" aria-current="page">Диссертационный совет</a>
+        <a class="admin__link" href="/admin/login">Вход в админку</a>
+        {% else %}
+        <a class="admin__link" href="/admin"{% if request.url.path == "/admin" %} aria-current="page"{% endif %}>Обзор</a>
+        <a class="admin__link" href="/admin/directory"{% if request.url.path == "/admin/directory" %} aria-current="page"{% endif %}>Сотрудники</a>
+        <a class="admin__link" href="/admin/council"{% if request.url.path == "/admin/council" %} aria-current="page"{% endif %}>Диссертационный совет</a>
+        <a class="admin__link" href="/admin/runs"{% if request.url.path.startswith("/admin/runs") %} aria-current="page"{% endif %}>Запуски</a>
         <form method="post" action="/admin/logout">
-          <button class="button button--ghost" type="submit">Logout</button>
+          <button class="button button--ghost" type="submit">Выйти</button>
         </form>
+        {% endif %}
       </nav>
     </header>
-    <main class="admin__main">
+    <main class="admin__main" id="main-content">
       {% block content %}{% endblock %}
     </main>
     <footer class="admin__footer">

app/templates/council.html

@@ -0,0 +1,107 @@
+{% extends "base.html" %}
+{% block title %}Диссертационный совет · MIEM Employees{% endblock %}
+{% block content %}
+<section class="council" aria-labelledby="council-title">
+  <header class="council__hero">
+    <div class="council__hero-copy">
+      <h2 class="council__title" id="council-title">Диссертационный совет</h2>
+      <p class="council__intro">Публикации сотрудников для проверки состава совета. В списке сотрудники с докторской степенью в профиле и хотя бы одной публикацией в файле CSV или базе miem-workers. Минимальный порог количества работ не применяется.</p>
+    </div>
+  </header>
+
+  {% if error %}
+  <div class="council__message" role="alert">
+    <h3>{% if export_error %}Не удалось создать XLSX{% else %}Не удалось загрузить публикации{% endif %}</h3>
+    <p>{% if export_error %}Не удалось выгрузить данные. Вернитесь к списку и попробуйте скачать файл позже.{% else %}Данные временно недоступны. Попробуйте обновить страницу позже.{% endif %}</p>
+    <a class="admin__link" href="/admin/council">Повторить загрузку</a>
+  </div>
+  {% else %}
+  <details class="council__notice">
+    <summary><strong>Как читать счётчики и критерии</strong></summary>
+    <p>«CSV по критериям» — уникальные публикации с отметкой Scopus или Web of Science Q1/Q2 за {{ start.strftime('%d.%m.%Y') }} — {{ end.strftime('%d.%m.%Y') }}. «В БД» — все уникальные публикации сотрудника из miem-workers, включая импортированные из CSV. Общий перечень объединяет оба источника; совпадения не увеличивают итоговое количество.</p>
+    <p>В CSV нет типа материала, точной даты и подтверждения К1 или RSCI. Счётчик CSV предварительный: для пограничных лет нужно проверить дату, для материалов без типа — подтвердить, что это статьи. Материалы, для которых в БД указан другой тип, в этот счётчик не входят. Отсутствие отметки означает недостаток данных. Наличие публикаций не подтверждает соответствие всем требованиям совета.</p>
+  </details>
+
+  <form class="council__search" method="get" action="/admin/council" role="search">
+    <label class="directory__field" for="council-search">Поиск по ФИО
+      <input class="directory__input" id="council-search" name="q" type="search" value="{{ q }}" placeholder="Фамилия или полное имя">
+    </label>
+    <label class="directory__field">Сортировать по
+      <select class="directory__input" name="sort">
+        {% for value, label in [('full_name', 'ФИО'), ('csv_count', 'CSV по критериям'), ('db_count', 'Публикации в БД'), ('count', 'Всего без дублей')] %}
+        <option value="{{ value }}"{% if sort == value %} selected{% endif %}>{{ label }}</option>
+        {% endfor %}
+      </select>
+    </label>
+    <label class="directory__field">Направление
+      <select class="directory__input" name="direction">
+        <option value="asc"{% if direction == 'asc' %} selected{% endif %}>По возрастанию</option>
+        <option value="desc"{% if direction == 'desc' %} selected{% endif %}>По убыванию</option>
+      </select>
+    </label>
+    <button class="button" type="submit">Применить</button>
+    <button class="button button--secondary" type="submit" formaction="/admin/council/export.xlsx">Скачать XLSX</button>
+    {% if q %}<a class="admin__link" href="?sort={{ sort }}&amp;direction={{ direction }}">Сбросить поиск</a>{% endif %}
+  </form>
+  <div class="council__list-heading">
+    <h3>Сотрудники</h3>
+    <p role="status">{% if matched %}Показаны {{ offset + 1 }}–{{ [offset + limit, matched]|min }} из {{ matched }}{% else %}Найдено: 0{% endif %}{% if q %} · Всего в списке: {{ total }}{% endif %}</p>
+  </div>
+  {% if candidates %}
+  <p class="council__hint">Откройте сотрудника, чтобы увидеть публикации, источники и основания. XLSX содержит всех найденных сотрудников и их публикации, включая другие страницы списка.</p>
+  <div class="council__list">
+    {% for candidate in candidates %}
+    <details class="council__candidate">
+      <summary class="council__candidate-summary">
+        <span class="council__candidate-identity"><strong>{{ candidate.name }}</strong><small>{{ candidate.degree }}{% if candidate.dismissed %} · Уволен{% endif %}</small></span>
+        <span class="council__counts">
+          <span class="council__candidate-count"><strong>{{ candidate.csv_count }}</strong><small>CSV по критериям</small></span>
+          <span class="council__candidate-count"><strong>{{ candidate.db_count }}</strong><small>В БД</small></span>
+          <span class="council__candidate-count"><strong>{{ candidate.count }}</strong><small>Всего без дублей</small></span>
+        </span>
+        <span class="council__candidate-cue" aria-hidden="true"></span>
+      </summary>
+      <div class="council__candidate-body">
+        {% if candidate.profile_url %}<a class="admin__link" href="{{ candidate.profile_url }}" target="_blank" rel="noopener noreferrer">Профиль на сайте ВШЭ</a>{% endif %}
+        {% if candidate.ambiguous %}<p class="council__hint">В базе есть сотрудники с одинаковым ФИО. Публикации из файла не присвоены автоматически; показаны записи, связанные с этим сотрудником в БД.</p>{% endif %}
+        <ol class="council__articles">
+          {% for article in candidate.articles %}
+          <li class="council__article">
+            <div class="council__article-heading">
+              <strong>{{ article.title }}</strong>
+              <span>{{ article.year or 'Год не указан' }}</span>
+            </div>
+            <div class="council__article-meta">
+              {% for origin in article.origins %}<span class="council__source">{{ origin }}</span>{% endfor %}
+              <span>{{ article.type_label }}</span>
+              {% if article.doi_url %}<a class="admin__link" href="{{ article.doi_url }}" target="_blank" rel="noopener noreferrer">DOI</a>{% endif %}
+              {% if article.url %}<a class="admin__link" href="{{ article.url }}" target="_blank" rel="noopener noreferrer">Публикация на сайте</a>{% endif %}
+              {% if article.rows %}<span>Строки CSV: {{ article.rows|join(', ') }}</span>{% endif %}
+            </div>
+            <ul class="council__evidence" aria-label="Критерии и подтверждения">
+              {% for criterion in article.criteria %}<li>{{ criterion }}</li>{% endfor %}
+            </ul>
+          </li>
+          {% endfor %}
+        </ol>
+      </div>
+    </details>
+    {% endfor %}
+  </div>
+  {% if matched > limit %}
+  <nav class="directory__pagination" aria-label="Страницы сотрудников">
+    {% if offset > 0 %}<a class="admin__link" href="?q={{ q|urlencode }}&amp;sort={{ sort }}&amp;direction={{ direction }}&amp;offset={{ offset - limit }}">Назад</a>{% endif %}
+    <span class="directory__page">{{ offset // limit + 1 }} / {{ (matched + limit - 1) // limit }}</span>
+    {% if offset + limit < matched %}<a class="admin__link" href="?q={{ q|urlencode }}&amp;sort={{ sort }}&amp;direction={{ direction }}&amp;offset={{ offset + limit }}">Далее</a>{% endif %}
+  </nav>
+  {% endif %}
+  {% else %}
+  <div class="council__message">
+    <h4>{% if q %}Сотрудники не найдены{% else %}Сотрудники с докторской степенью и публикациями пока не найдены{% endif %}</h4>
+    <p>{% if q %}Попробуйте другую фамилию или сбросьте поиск.{% else %}Для включения в список нужны докторская степень в профиле и хотя бы одна публикация, однозначно связанная с сотрудником.{% endif %}</p>
+    {% if q %}<a class="admin__link" href="?sort={{ sort }}&amp;direction={{ direction }}">Сбросить поиск</a>{% endif %}
+  </div>
+  {% endif %}
+  {% endif %}
+</section>
+{% endblock %}

app/templates/dashboard.html

@@ -1,62 +1,70 @@
-{% extends "base.html" %}
-{% block title %}Dashboard · MIEM Employees{% endblock %}
-{% block content %}
-<section class="admin__grid">
-  <div class="metric"><div class="metric__label">Total</div><div class="metric__value">{{ counts.total }}</div></div>
-  <div class="metric"><div class="metric__label">Active</div><div class="metric__value">{{ counts.active }}</div></div>
-  <div class="metric"><div class="metric__label">New in last run</div><div class="metric__value">{{ counts.new_in_last_run }}</div></div>
-  <div class="metric"><div class="metric__label">Dismissed</div><div class="metric__value">{{ counts.dismissed }}</div></div>
-</section>
-<section class="stats-strip">
-  <div class="stats-strip__item">
-    <span class="stats-strip__label">Latest added</span>
-    {% if counts.latest_added %}
-    <a class="stats-strip__value" href="/admin/employees/{{ counts.latest_added.id }}">{{ counts.latest_added.full_name or counts.latest_added.canonical_url }}</a>
-    {% else %}
-    <span class="stats-strip__value">No employees yet</span>
-    {% endif %}
-  </div>
-  <div class="stats-strip__item">
-    <span class="stats-strip__label">Runs</span>
-    <span class="stats-strip__value">{{ counts.runs }}</span>
-  </div>
-  <div class="stats-strip__item">
-    <span class="stats-strip__label">Errors</span>
-    <span class="stats-strip__value">{{ counts.errors }}</span>
-  </div>
-</section>
-<section class="panel progress-panel" data-progress-panel>
-  <div class="progress-panel__header">
-    <h2 class="panel__title">Parsing progress</h2>
-    <form method="post" action="/admin/crawl-now">
-      <button class="button" type="submit">Start crawl now</button>
-    </form>
-  </div>
-  {% set run = counts.current_running_run or latest_run %}
-  <div class="progress-panel__body" data-progress-body>
-    <div class="progress-panel__meta">
-      <span data-progress-status>{{ run.status if run else "idle" }}</span>
-      <span><span data-progress-processed>{{ run.processed_count if run else 0 }}</span> / <span data-progress-found>{{ run.found_count if run else 0 }}</span> processed</span>
-      <span><span data-progress-errors>{{ run.error_count if run else 0 }}</span> errors</span>
-    </div>
-    <div class="progress-bar" aria-label="Parsing progress">
-      <div class="progress-bar__fill" data-progress-fill style="width: {{ run.progress_percent if run else 0 }}%"></div>
-    </div>
-    <div class="progress-panel__percent"><span data-progress-percent>{{ run.progress_percent if run else 0 }}</span>%</div>
-  </div>
-</section>
-<section class="panel">
-  <h2 class="panel__title">Latest runs</h2>
-  <table class="table">
-    <thead><tr><th class="table__head">ID</th><th class="table__head">Status</th><th class="table__head">Parsed</th><th class="table__head">Errors</th><th class="table__head">Started</th></tr></thead>
-    <tbody>
-      {% for run in runs %}
-      <tr><td class="table__cell">{{ run.id }}</td><td class="table__cell">{{ run.status }}</td><td class="table__cell">{{ run.parsed_count }}</td><td class="table__cell">{{ run.error_count }}</td><td class="table__cell">{{ run.started_at }}</td></tr>
-      {% endfor %}
-    </tbody>
-  </table>
-</section>
-{% endblock %}
-{% block scripts %}
-<script src="/static/admin.js"></script>
-{% endblock %}
+{% extends "base.html" %}
+{% block title %}Обзор · MIEM Employees{% endblock %}
+{% block content %}
+<section class="admin__grid">
+  <a class="metric metric--link" href="/admin/directory"><span class="metric__label">Всего в базе</span><span class="metric__value">{{ counts.total }}</span></a>
+  <a class="metric metric--link" href="/admin/directory?status=active"><span class="metric__label">Работают</span><span class="metric__value">{{ counts.active }}</span></a>
+  <a class="metric metric--link" href="/admin/directory?status=verification_required"><span class="metric__label">Требуют проверки</span><span class="metric__value">{{ counts.verification_required }}</span></a>
+  <a class="metric metric--link" href="{% if latest_run %}/admin/runs/{{ latest_run.id }}#new-employees{% else %}/admin/runs{% endif %}"><span class="metric__label">Новые за запуск</span><span class="metric__value">{{ counts.new_in_last_run }}</span></a>
+  <a class="metric metric--link" href="/admin/directory?status=dismissed"><span class="metric__label">Уволены</span><span class="metric__value">{{ counts.dismissed }}</span></a>
+</section>
+<section class="stats-strip">
+  <div class="stats-strip__item">
+    <span class="stats-strip__label">Последний добавленный</span>
+    {% if counts.latest_added %}
+    <a class="stats-strip__value" href="/admin/employees/{{ counts.latest_added.id }}">{{ counts.latest_added.full_name or counts.latest_added.canonical_url }}</a>
+    {% else %}
+    <span class="stats-strip__value">Сотрудников пока нет</span>
+    {% endif %}
+  </div>
+  <a class="stats-strip__item stats-strip__item--link" href="/admin/runs">
+    <span class="stats-strip__label">Запуски</span>
+    <span class="stats-strip__value">{{ counts.runs }}</span>
+  </a>
+  <div class="stats-strip__item">
+    <span class="stats-strip__label">Ошибки</span>
+    <span class="stats-strip__value">{{ counts.errors }}</span>
+  </div>
+</section>
+<section class="panel progress-panel" data-progress-panel>
+  <div class="progress-panel__header">
+    <h2 class="panel__title">Прогресс парсинга</h2>
+    <div class="progress-panel__actions">
+      <form method="post" action="/admin/crawl-now">
+        <button class="button" type="submit">Запустить парсинг</button>
+      </form>
+      <form method="post" action="/admin/dismissed/refresh">
+        <button class="button button--secondary" type="submit">Проверить уволенных</button>
+      </form>
+    </div>
+  </div>
+  {% set run = counts.current_running_run or latest_run %}
+  <div class="progress-panel__body" data-progress-body>
+    <div class="progress-panel__meta">
+      <span data-progress-status>{{ run.status_display if run else "Ожидание" }}</span>
+      <span>обработано: <span data-progress-processed>{{ run.processed_count if run else 0 }}</span> / <span data-progress-found>{{ run.found_count if run else 0 }}</span></span>
+      <span>без изменений: <span data-progress-skipped>{{ run.skipped_count if run else 0 }}</span></span>
+      <span>ошибок: <span data-progress-errors>{{ run.error_count if run else 0 }}</span></span>
+    </div>
+    <div class="progress-bar" role="progressbar" aria-label="Прогресс парсинга" aria-valuemin="0" aria-valuemax="100" aria-valuenow="{{ run.progress_percent if run else 0 }}" aria-valuetext="{{ run.progress_percent if run else 0 }}%">
+      <div class="progress-bar__fill" data-progress-fill style="width: {{ run.progress_percent if run else 0 }}%"></div>
+    </div>
+    <div class="progress-panel__percent"><span data-progress-percent>{{ run.progress_percent if run else 0 }}</span>%</div>
+    <p class="progress-panel__error" data-progress-error role="status" hidden>Не удалось обновить прогресс. <button class="button button--ghost button--compact" type="button" data-progress-retry>Повторить</button></p>
+  </div>
+</section>
+<section class="panel">
+  <h2 class="panel__title">Последние запуски</h2>
+  <table class="table">
+    <thead><tr><th class="table__head">ID</th><th class="table__head">Статус</th><th class="table__head">Обработано</th><th class="table__head">Без изменений</th><th class="table__head">Ошибки</th><th class="table__head">Старт</th></tr></thead>
+    <tbody>
+      {% for run in runs %}
+      <tr class="table__row"><td class="table__cell"><a class="admin__link" href="/admin/runs/{{ run.id }}">{{ run.id }}</a></td><td class="table__cell">{{ run.status_display }}</td><td class="table__cell">{{ run.parsed_count }}</td><td class="table__cell">{{ run.skipped_count }}</td><td class="table__cell">{{ run.error_count }}</td><td class="table__cell">{{ run.started_display }}</td></tr>
+      {% endfor %}
+    </tbody>
+  </table>
+</section>
+{% endblock %}
+{% block scripts %}
+<script src="/static/admin.js"></script>
+{% endblock %}

app/templates/directory.html

@@ -1,79 +1,124 @@
 {% extends "base.html" %}
-{% block title %}Directory · MIEM Employees{% endblock %}
+{% block title %}Сотрудники · MIEM Employees{% endblock %}
 {% block content %}
 <section class="directory">
   <div class="directory__header">
     <div>
-      <h2 class="directory__title">Directory</h2>
-      <p class="directory__summary">{{ page.total }} employees found</p>
+      <h2 class="directory__title">Сотрудники</h2>
+      {% set range_start = page.offset + 1 if page.total else 0 %}
+      {% set range_end = [page.offset + page.employees|length, page.total]|min %}
+      <p class="directory__summary" role="status">Показаны {{ range_start }}–{{ range_end }} из {{ page.total }}</p>
     </div>
-    <button class="button" type="button" data-columns-open>Columns</button>
+    <button class="button" type="button" data-columns-open aria-controls="columns-dialog" aria-expanded="false">Настроить колонки</button>
   </div>
 
   <form class="directory__filters" method="get" action="/admin/directory">
-    <input class="directory__input" name="q" value="{{ filters.q }}" placeholder="Name or URL">
-    <select class="directory__input" name="status">
-      <option value="" {% if not filters.status %}selected{% endif %}>All statuses</option>
-      <option value="active" {% if filters.status == "active" %}selected{% endif %}>Active</option>
-      <option value="dismissed" {% if filters.status == "dismissed" %}selected{% endif %}>Dismissed</option>
-    </select>
-    <select class="directory__input" name="has_email">
-      <option value="" {% if not filters.has_email %}selected{% endif %}>Any email</option>
-      <option value="true" {% if filters.has_email == "true" %}selected{% endif %}>Has email</option>
-      <option value="false" {% if filters.has_email == "false" %}selected{% endif %}>No email</option>
-    </select>
-    <input class="directory__input" type="date" name="started_from" value="{{ filters.started_from }}" aria-label="First seen from">
-    <input class="directory__input" type="date" name="started_to" value="{{ filters.started_to }}" aria-label="First seen to">
-    <select class="directory__input" name="sort">
-      {% for value, label in [("full_name", "Name"), ("status", "Status"), ("hse_start_year", "HSE start"), ("first_seen_at", "First seen"), ("last_seen_at", "Last seen"), ("dismissed_at", "Dismissed")] %}
-      <option value="{{ value }}" {% if filters.sort == value %}selected{% endif %}>Sort: {{ label }}</option>
+    <fieldset class="directory__filter-group directory__filter-group--search">
+      <legend class="directory__filter-legend">Поиск</legend>
+      <label class="directory__field">ФИО или ссылка<input class="directory__input" name="q" value="{{ filters.q }}" placeholder="Например, Иванов или hse.ru"></label>
+    </fieldset>
+    <fieldset class="directory__filter-group">
+      <legend class="directory__filter-legend">Статус и данные</legend>
+      <label class="directory__field">Статус<select class="directory__input" name="status">
+      <option value="" {% if not filters.status %}selected{% endif %}>Все статусы</option>
+      <option value="active" {% if filters.status == "active" %}selected{% endif %}>Работает</option>
+      <option value="verification_required" {% if filters.status == "verification_required" %}selected{% endif %}>Требует проверки</option>
+      <option value="dismissed" {% if filters.status == "dismissed" %}selected{% endif %}>Уволен</option>
+      </select></label>
+      <label class="directory__field">Email<select class="directory__input" name="has_email">
+      <option value="" {% if not filters.has_email %}selected{% endif %}>Любой email</option>
+      <option value="true" {% if filters.has_email == "true" %}selected{% endif %}>Есть email</option>
+      <option value="false" {% if filters.has_email == "false" %}selected{% endif %}>Нет email</option>
+      </select></label>
+      <label class="directory__field">Учёная степень<select class="directory__input" name="has_academic_degree">
+      <option value="" {% if not filters.has_academic_degree %}selected{% endif %}>Любая учёная степень</option>
+      <option value="true" {% if filters.has_academic_degree == "true" %}selected{% endif %}>Есть учёная степень</option>
+      <option value="false" {% if filters.has_academic_degree == "false" %}selected{% endif %}>Нет учёной степени</option>
+      </select></label>
+    </fieldset>
+    <fieldset class="directory__filter-group">
+      <legend class="directory__filter-legend">Период обнаружения</legend>
+      <label class="directory__field">От<input class="directory__input" type="date" name="started_from" value="{{ filters.started_from }}"></label>
+      <label class="directory__field">До<input class="directory__input" type="date" name="started_to" value="{{ filters.started_to }}"></label>
+    </fieldset>
+    <fieldset class="directory__filter-group">
+      <legend class="directory__filter-legend">Сортировка и страница</legend>
+      <label class="directory__field">Сортировать по<select class="directory__input" name="sort">
+      {% for value, label in [("full_name", "ФИО"), ("status", "Статус"), ("hse_start_year", "Год начала"), ("first_seen_at", "Впервые найден"), ("last_seen_at", "Последний раз найден"), ("dismissed_at", "Дата увольнения")] %}
+      <option value="{{ value }}" {% if filters.sort == value %}selected{% endif %}>{{ label }}</option>
       {% endfor %}
-    </select>
-    <select class="directory__input" name="direction">
-      <option value="asc" {% if filters.direction == "asc" %}selected{% endif %}>Ascending</option>
-      <option value="desc" {% if filters.direction == "desc" %}selected{% endif %}>Descending</option>
-    </select>
-    <button class="button" type="submit">Apply</button>
+      </select></label>
+      <label class="directory__field">Направление<select class="directory__input" name="direction">
+      <option value="asc" {% if filters.direction == "asc" %}selected{% endif %}>По возрастанию</option>
+      <option value="desc" {% if filters.direction == "desc" %}selected{% endif %}>По убыванию</option>
+      </select></label>
+      <label class="directory__field">Сотрудников на странице<select class="directory__input" name="limit">
+      {% for value in [25, 50, 100] %}
+      <option value="{{ value }}" {% if filters.limit == value %}selected{% endif %}>{{ value }}</option>
+      {% endfor %}
+      </select></label>
+    </fieldset>
+    <div class="directory__filter-actions">
+      <button class="button" type="submit">Применить фильтры</button>
+      <a class="button button--ghost" href="/admin/directory">Сбросить</a>
+    </div>
   </form>
 
+  <details class="directory__help">
+    <summary>Что означают даты и статусы</summary>
+    <p>«Год начала» — начало работы в ВШЭ. «Впервые найден» и «Последний раз найден» — даты обнаружения сотрудника в списке источника, а не даты приёма и увольнения. «Требует проверки» означает, что профиль недоступен и увольнение ещё не подтверждено.</p>
+  </details>
+
   <div class="directory__table-wrap">
     <table class="directory-table" data-directory-table>
       <thead>
         <tr>
-          <th class="directory-table__head" data-column="full_name">Name</th>
-          <th class="directory-table__head" data-column="status">Status</th>
-          <th class="directory-table__head" data-column="positions">Positions</th>
-          <th class="directory-table__head" data-column="hse_start_year">HSE start</th>
+          <th class="directory-table__head" data-column="full_name">ФИО</th>
+          <th class="directory-table__head" data-column="status">Статус</th>
+          <th class="directory-table__head" data-column="positions">Должности</th>
+          <th class="directory-table__head" data-column="hse_start_year">Год начала</th>
           <th class="directory-table__head" data-column="email">Email</th>
-          <th class="directory-table__head" data-column="phone">Phone</th>
-          <th class="directory-table__head" data-column="address">Address</th>
-          <th class="directory-table__head" data-column="publications_count">Publications</th>
-          <th class="directory-table__head" data-column="courses_count">Courses</th>
-          <th class="directory-table__head" data-column="first_seen_at">First seen</th>
-          <th class="directory-table__head" data-column="last_seen_at">Last seen</th>
-          <th class="directory-table__head" data-column="dismissed_at">Dismissed</th>
-          <th class="directory-table__head" data-column="profile">Profile</th>
+          <th class="directory-table__head" data-column="phone">Телефон</th>
+          <th class="directory-table__head" data-column="address">Адрес</th>
+          <th class="directory-table__head" data-column="academic_degree">Учёная степень</th>
+          <th class="directory-table__head" data-column="publications_count">Публикации</th>
+          <th class="directory-table__head" data-column="courses_count">Курсы</th>
+          <th class="directory-table__head" data-column="news_count">Новости</th>
+          <th class="directory-table__head" data-column="first_seen_at">Впервые найден</th>
+          <th class="directory-table__head" data-column="last_seen_at">Последний раз найден</th>
+          <th class="directory-table__head" data-column="dismissed_at">Дата увольнения</th>
+          <th class="directory-table__head" data-column="profile">Профиль</th>
         </tr>
       </thead>
       <tbody>
-        {% for employee in page.items %}
-        <tr class="directory-table__row" data-row-href="/admin/employees/{{ employee.id }}">
-          <td class="directory-table__cell" data-column="full_name">{{ employee.full_name or "No name" }}</td>
-          <td class="directory-table__cell" data-column="status"><span class="badge {% if employee.status == "dismissed" %}badge--dismissed{% endif %}">{{ employee.status }}</span></td>
+        {% for employee in page.employees %}
+        <tr class="directory-table__row">
+          <td class="directory-table__cell" data-column="full_name"><a class="admin__link" href="/admin/employees/{{ employee.id }}">{{ employee.full_name or "Без имени" }}</a></td>
+          <td class="directory-table__cell" data-column="status"><span class="badge {% if employee.status == "dismissed" %}badge--dismissed{% elif employee.status == "verification_required" %}badge--verification{% endif %}">{{ employee.status_display }}</span></td>
           <td class="directory-table__cell" data-column="positions">{{ employee.positions_text }}</td>
           <td class="directory-table__cell" data-column="hse_start_year">{{ employee.hse_start_year or "" }}</td>
           <td class="directory-table__cell" data-column="email">{{ employee.email_text }}</td>
           <td class="directory-table__cell" data-column="phone">{{ employee.phone_text }}</td>
           <td class="directory-table__cell" data-column="address">{{ employee.address or "" }}</td>
+          <td class="directory-table__cell" data-column="academic_degree">{{ employee.academic_degree_text }}</td>
           <td class="directory-table__cell" data-column="publications_count">{{ employee.publications_count }}</td>
           <td class="directory-table__cell" data-column="courses_count">{{ employee.courses_count }}</td>
-          <td class="directory-table__cell" data-column="first_seen_at">{{ employee.first_seen_at or "" }}</td>
-          <td class="directory-table__cell" data-column="last_seen_at">{{ employee.last_seen_at or "" }}</td>
-          <td class="directory-table__cell" data-column="dismissed_at">{{ employee.dismissed_at or "" }}</td>
-          <td class="directory-table__cell" data-column="profile"><a class="admin__link" href="{{ employee.canonical_url }}">Open</a></td>
+          <td class="directory-table__cell" data-column="news_count">{{ employee.news_count }}</td>
+          <td class="directory-table__cell" data-column="first_seen_at">{{ employee.first_seen_display }}</td>
+          <td class="directory-table__cell" data-column="last_seen_at">{{ employee.last_seen_display }}</td>
+          <td class="directory-table__cell" data-column="dismissed_at">{{ employee.dismissed_display }}</td>
+          <td class="directory-table__cell" data-column="profile"><a class="admin__link" href="{{ employee.canonical_url }}" target="_blank" rel="noopener">Внешний профиль</a></td>
         </tr>
         {% else %}
-        <tr><td class="directory-table__empty" colspan="13">No employees match these filters.</td></tr>
+        <tr><td class="directory-table__empty" colspan="15">
+          <strong>Сотрудники не найдены</strong>
+          {% if filters.q or filters.status or filters.has_email or filters.has_academic_degree or filters.started_from or filters.started_to %}
+          <span>Попробуйте изменить запрос или сбросить фильтры.</span>
+          <a class="admin__link" href="/admin/directory">Сбросить фильтры</a>
+          {% else %}
+          <span>Запустите парсинг, чтобы загрузить сотрудников.</span>
+          {% endif %}
+        </td></tr>
         {% endfor %}
       </tbody>
     </table>
@@ -83,29 +128,34 @@
     {% set prev_offset = filters.offset - filters.limit %}
     {% set next_offset = filters.offset + filters.limit %}
     {% if filters.offset > 0 %}
-    <a class="admin__link" href="{{ request.url.include_query_params(offset=prev_offset) }}">Previous</a>
+    <a class="admin__link" href="{{ request.url.include_query_params(offset=prev_offset) }}">Назад</a>
     {% endif %}
-    <span class="directory__page">Page {{ page.page }}{% if page.pages %} of {{ page.pages }}{% endif %}</span>
+    <span class="directory__page">Страница {{ page.page }}{% if page.pages %} из {{ page.pages }}{% endif %}</span>
     {% if next_offset < page.total %}
-    <a class="admin__link" href="{{ request.url.include_query_params(offset=next_offset) }}">Next</a>
+    <a class="admin__link" href="{{ request.url.include_query_params(offset=next_offset) }}">Вперед</a>
     {% endif %}
   </div>
 </section>
 
-<div class="columns-modal" data-columns-modal hidden>
-  <div class="columns-modal__backdrop" data-columns-close></div>
-  <section class="columns-modal__panel" aria-label="Column settings">
+<dialog class="columns-modal" id="columns-dialog" data-columns-modal aria-labelledby="columns-dialog-title">
+  <section class="columns-modal__panel">
     <div class="columns-modal__header">
-      <h3 class="columns-modal__title">Visible columns</h3>
-      <button class="button button--ghost" type="button" data-columns-close>Close</button>
+      <h3 class="columns-modal__title" id="columns-dialog-title">Отображаемые колонки</h3>
+      <button class="button button--ghost" type="button" data-columns-close>Закрыть</button>
     </div>
     <div class="columns-modal__grid">
-      {% for key, label in [("full_name", "Name"), ("status", "Status"), ("positions", "Positions"), ("hse_start_year", "HSE start"), ("email", "Email"), ("phone", "Phone"), ("address", "Address"), ("publications_count", "Publications"), ("courses_count", "Courses"), ("first_seen_at", "First seen"), ("last_seen_at", "Last seen"), ("dismissed_at", "Dismissed"), ("profile", "Profile")] %}
+      {% for key, label in [("full_name", "ФИО"), ("status", "Статус"), ("positions", "Должности"), ("hse_start_year", "Год начала"), ("email", "Email"), ("phone", "Телефон"), ("address", "Адрес"), ("academic_degree", "Учёная степень"), ("publications_count", "Публикации"), ("courses_count", "Курсы"), ("news_count", "Новости"), ("first_seen_at", "Впервые найден"), ("last_seen_at", "Последний раз найден"), ("dismissed_at", "Дата увольнения"), ("profile", "Профиль")] %}
       <label class="columns-modal__option"><input class="columns-modal__checkbox" type="checkbox" value="{{ key }}" data-column-toggle> {{ label }}</label>
       {% endfor %}
     </div>
+    <fieldset class="columns-modal__presets">
+      <legend class="columns-modal__preset-label">Быстрый набор</legend>
+      <button class="button button--ghost button--compact" type="button" data-columns-preset="review">Проверка</button>
+      <button class="button button--ghost button--compact" type="button" data-columns-preset="contacts">Контакты</button>
+      <button class="button button--ghost button--compact" type="button" data-columns-preset="full">Все поля</button>
+    </fieldset>
   </section>
-</div>
+</dialog>
 {% endblock %}
 {% block scripts %}
 <script src="/static/admin.js"></script>

app/templates/directory_error.html

@@ -0,0 +1,9 @@
+{% extends "base.html" %}
+{% block title %}Каталог недоступен · MIEM Employees{% endblock %}
+{% block content %}
+<section class="panel">
+  <h2 class="panel__title">Не удалось загрузить сотрудников</h2>
+  <p>База данных временно недоступна. Повторите запрос; выбранные фильтры сохранятся.</p>
+  <a class="button" href="{{ request.url.path }}{% if request.url.query %}?{{ request.url.query }}{% endif %}">Повторить загрузку</a>
+</section>
+{% endblock %}

app/templates/employee_detail.html

@@ -1,26 +1,255 @@
 {% extends "base.html" %}
-{% block title %}{{ employee.full_name }} · MIEM Employees{% endblock %}
+{% block title %}{{ employee_view.full_name }} · MIEM Employees{% endblock %}
 {% block content %}
-<section class="panel">
-  <h2 class="panel__title">{{ employee.full_name or employee.profile_key }}</h2>
-  <p><span class="badge {% if employee.status == "dismissed" %}badge--dismissed{% endif %}">{{ employee.status }}</span></p>
-  <p><a class="admin__link" href="{{ employee.canonical_url }}">{{ employee.canonical_url }}</a></p>
-  <h3>Tabs</h3>
-  <ul>
-    {% for tab in employee.tabs %}
-    <li><a class="admin__link" href="{{ tab.href }}">{{ tab.title }}</a></li>
-    {% endfor %}
-  </ul>
-  <h3>Current data</h3>
-  <pre class="code">{{ employee.current_data | tojson(indent=2) }}</pre>
+<section class="employee-card">
+  <div class="employee-card__header">
+    <div class="employee-card__identity">
+      <h2 class="employee-card__title">{{ employee_view.full_name or employee.profile_key }}</h2>
+      <span class="badge {% if employee_view.status == "dismissed" %}badge--dismissed{% endif %}">{{ employee_view.status_display }}</span>
+    </div>
+    <div class="employee-card__actions">
+      <form method="post" action="/admin/employees/{{ employee.id }}/refresh">
+        <button class="button button--compact" type="submit">Обновить данные</button>
+      </form>
+      <a class="admin__link" href="{{ employee_view.canonical_url }}">{{ employee_view.canonical_url }}</a>
+    </div>
+  </div>
+  {% if refresh_status == "success" %}
+  <p class="employee-card__notice employee-card__notice--success">Данные сотрудника обновлены.</p>
+  {% elif refresh_status == "error" %}
+  <p class="employee-card__notice employee-card__notice--error">Не удалось обновить данные сотрудника.</p>
+  {% endif %}
+
+  <section class="employee-card__section">
+    <h3 class="employee-section__title">Основная информация</h3>
+    <dl class="employee-card__meta">
+      <div class="employee-card__meta-item">
+        <dt class="employee-card__meta-label">Должности</dt>
+        <dd class="employee-card__meta-value">
+          {% if employee_view.positions %}
+          <ul class="employee-card__list">
+            {% for position in employee_view.positions %}
+            <li class="employee-card__list-item">{{ position }}</li>
+            {% endfor %}
+          </ul>
+          {% else %}
+          Не указано
+          {% endif %}
+        </dd>
+      </div>
+      <div class="employee-card__meta-item"><dt class="employee-card__meta-label">Год начала работы в ВШЭ</dt><dd class="employee-card__meta-value">{{ employee_view.hse_start_year or "Не указано" }}</dd></div>
+      <div class="employee-card__meta-item"><dt class="employee-card__meta-label">Тип профиля</dt><dd class="employee-card__meta-value">{{ employee_view.profile_type or "Не указано" }}</dd></div>
+      <div class="employee-card__meta-item"><dt class="employee-card__meta-label">ID профиля</dt><dd class="employee-card__meta-value">{{ employee_view.profile_id or "Не указано" }}</dd></div>
+      <div class="employee-card__meta-item"><dt class="employee-card__meta-label">Впервые найден</dt><dd class="employee-card__meta-value">{{ employee_view.first_seen_display }}</dd></div>
+      <div class="employee-card__meta-item"><dt class="employee-card__meta-label">Последний раз найден</dt><dd class="employee-card__meta-value">{{ employee_view.last_seen_display }}</dd></div>
+      <div class="employee-card__meta-item"><dt class="employee-card__meta-label">Дата увольнения</dt><dd class="employee-card__meta-value">{{ employee_view.dismissed_display }}</dd></div>
+    </dl>
+  </section>
+
+  <section class="employee-card__section">
+    <h3 class="employee-section__title">Контакты</h3>
+    <dl class="employee-card__meta">
+      <div class="employee-card__meta-item">
+        <dt class="employee-card__meta-label">Email</dt>
+        <dd class="employee-card__meta-value">
+          {% if employee_view.contacts.emails %}
+          <ul class="employee-card__list">
+            {% for email in employee_view.contacts.emails %}
+            <li class="employee-card__list-item"><a class="admin__link" href="mailto:{{ email }}">{{ email }}</a></li>
+            {% endfor %}
+          </ul>
+          {% else %}
+          Не указано
+          {% endif %}
+        </dd>
+      </div>
+      <div class="employee-card__meta-item">
+        <dt class="employee-card__meta-label">Телефоны</dt>
+        <dd class="employee-card__meta-value">{{ employee_view.contacts.phones | join(", ") if employee_view.contacts.phones else "Не указано" }}</dd>
+      </div>
+      <div class="employee-card__meta-item">
+        <dt class="employee-card__meta-label">Адрес</dt>
+        <dd class="employee-card__meta-value">{{ employee_view.contacts.address or "Не указано" }}</dd>
+      </div>
+      {% if employee_view.contacts.contact_items %}
+      <div class="employee-card__meta-item employee-card__meta-item--wide">
+        <dt class="employee-card__meta-label">Прочее</dt>
+        <dd class="employee-card__meta-value">
+          <ul class="employee-card__list">
+            {% for item in employee_view.contacts.contact_items %}
+            <li class="employee-card__list-item">{{ item }}</li>
+            {% endfor %}
+          </ul>
+        </dd>
+      </div>
+      {% endif %}
+    </dl>
+  </section>
+
+  {% if employee_view.external_ids %}
+  <section class="employee-card__section">
+    <h3 class="employee-section__title">Внешние идентификаторы</h3>
+    <ul class="employee-card__list">
+      {% for external_id in employee_view.external_ids %}
+      <li class="employee-card__list-item">
+        <strong>{{ external_id.system }}:</strong>
+        {% if external_id.url %}
+        <a class="admin__link" href="{{ external_id.url }}">{{ external_id.value }}</a>
+        {% else %}
+        {{ external_id.value }}
+        {% endif %}
+      </li>
+      {% endfor %}
+    </ul>
+  </section>
+  {% endif %}
+
+  {% if employee_view.news_links %}
+  <section class="employee-card__section">
+    <h3 class="employee-section__title">В новостях</h3>
+    <ul class="employee-card__list">
+      {% for news in employee_view.news_links %}
+      <li class="employee-card__list-item">
+        {% if news.published_display %}<div class="employee-section__meta"><span class="employee-section__meta-item">{{ news.published_display }}</span></div>{% endif %}
+        {% if news.url %}
+        <a class="admin__link" href="{{ news.url }}">{{ news.title }}</a>
+        {% else %}
+        {{ news.title }}
+        {% endif %}
+        {% if news.summary %}<div class="employee-section__text">{{ news.summary }}</div>{% endif %}
+      </li>
+      {% endfor %}
+    </ul>
+  </section>
+  {% endif %}
+
+  <section class="employee-card__section">
+    <h3 class="employee-section__title">Разделы профиля</h3>
+    {% if employee_view.sections %}
+    <div class="employee-card__sections">
+      {% for section in employee_view.sections %}
+      <article class="employee-section">
+        <div class="employee-section__header">
+          <h4 class="employee-section__title">{{ section.title }}</h4>
+          <span class="employee-section__type">{{ section.type }}</span>
+        </div>
+
+        {% if section.type == "year_blocks" and section.year_entries %}
+        <ul class="employee-card__list">
+          {% for entry in section.year_entries %}
+          <li class="employee-card__list-item">{% if entry.year %}<strong>{{ entry.year }}:</strong> {% endif %}{{ entry.text }}</li>
+          {% endfor %}
+        </ul>
+        {% elif section.type == "publications" and section.publications %}
+        {% if section.publications_count %}<p class="employee-section__note">Всего: {{ section.publications_count }}</p>{% endif %}
+        <ul class="employee-card__list">
+          {% for publication in section.publications %}
+          <li class="employee-card__list-item">
+            {% if publication.url %}
+            <a class="admin__link" href="{{ publication.url }}">{{ publication.title }}</a>
+            {% else %}
+            {{ publication.title }}
+            {% endif %}
+            {% if publication.text and publication.text != publication.title %}<div class="employee-section__text">{{ publication.text }}</div>{% endif %}
+          </li>
+          {% endfor %}
+        </ul>
+        {% elif section.type == "courses_by_year" and section.courses %}
+        {% if section.academic_year %}<p class="employee-section__note">Учебный год: {{ section.academic_year }}</p>{% endif %}
+        <ul class="employee-card__list">
+          {% for course in section.courses %}
+          <li class="employee-card__list-item">
+            {% if course.url %}
+            <a class="admin__link" href="{{ course.url }}">{{ course.title }}</a>
+            {% else %}
+            {{ course.title }}
+            {% endif %}
+          </li>
+          {% endfor %}
+        </ul>
+        {% elif section.type == "graduation_theses" and section.theses %}
+        {% if section.theses_count %}<p class="employee-section__note">Всего: {{ section.theses_count }}</p>{% endif %}
+        <ul class="employee-card__list">
+          {% for thesis in section.theses %}
+          <li class="employee-card__list-item">
+            {% if thesis.student %}<strong>{{ thesis.student }}</strong>{% endif %}
+            {% if thesis.title %}
+            <div class="employee-section__text">
+              {% if thesis.project_url %}
+              <a class="admin__link" href="{{ thesis.project_url }}">{{ thesis.title }}</a>
+              {% else %}
+              {{ thesis.title }}
+              {% endif %}
+            </div>
+            {% endif %}
+            <div class="employee-section__meta">
+              {% if thesis.defense_year %}<span class="employee-section__meta-item">Год защиты: {{ thesis.defense_year }}</span>{% endif %}
+              {% if thesis.level %}<span class="employee-section__meta-item">{{ thesis.level }}</span>{% endif %}
+              {% if thesis.rating is not none %}<span class="employee-section__meta-item">Оценка: {{ thesis.rating }}</span>{% endif %}
+              {% if thesis.program %}
+              <span class="employee-section__meta-item">
+                {% if thesis.program_url %}<a class="admin__link" href="{{ thesis.program_url }}">{{ thesis.program }}</a>{% else %}{{ thesis.program }}{% endif %}
+              </span>
+              {% endif %}
+            </div>
+          </li>
+          {% endfor %}
+        </ul>
+        {% elif section.type == "table" and section.table %}
+        <div class="employee-section__table-wrap">
+          <table class="employee-section__table">
+            {% if section.table.headers %}
+            <thead><tr>{% for header in section.table.headers %}<th class="employee-section__head">{{ header }}</th>{% endfor %}</tr></thead>
+            {% endif %}
+            <tbody>
+              {% for row in section.table.rows %}
+              <tr>
+                {% for cell in row.cells %}
+                <td class="employee-section__cell">{{ cell }}</td>
+                {% endfor %}
+              </tr>
+              {% endfor %}
+            </tbody>
+          </table>
+        </div>
+        {% else %}
+        {% if section.paragraphs %}
+        {% for paragraph in section.paragraphs %}
+        <p class="employee-section__text">{{ paragraph }}</p>
+        {% endfor %}
+        {% endif %}
+        {% if section.list_items %}
+        <ul class="employee-card__list">
+          {% for item in section.list_items %}
+          <li class="employee-card__list-item">{{ item }}</li>
+          {% endfor %}
+        </ul>
+        {% endif %}
+        {% endif %}
+
+        {% if section.links and section.type not in ["courses_by_year", "graduation_theses"] %}
+        <div class="employee-section__links">
+          {% for link in section.links %}
+          <a class="employee-section__link" href="{{ link.url }}">{{ link.text }}</a>
+          {% endfor %}
+        </div>
+        {% endif %}
+      </article>
+      {% endfor %}
+    </div>
+    {% else %}
+    <p class="employee-section__text">Разделы профиля не найдены.</p>
+    {% endif %}
+  </section>
 </section>
+
 <section class="panel">
-  <h2 class="panel__title">Snapshots</h2>
+  <h2 class="panel__title">Снапшоты</h2>
   <table class="table">
-    <thead><tr><th class="table__head">Captured</th><th class="table__head">Checksum</th><th class="table__head">Parser</th></tr></thead>
+    <thead><tr><th class="table__head">Дата</th><th class="table__head">Checksum</th><th class="table__head">Парсер</th></tr></thead>
     <tbody>
       {% for snapshot in snapshots %}
-      <tr><td class="table__cell">{{ snapshot.captured_at }}</td><td class="table__cell">{{ snapshot.checksum }}</td><td class="table__cell">{{ snapshot.parser_version }}</td></tr>
+      <tr><td class="table__cell">{{ snapshot.captured_display }}</td><td class="table__cell">{{ snapshot.checksum }}</td><td class="table__cell">{{ snapshot.parser_version }}</td></tr>
       {% endfor %}
     </tbody>
   </table>

app/templates/employees.html

@@ -1,29 +0,0 @@
-{% extends "base.html" %}
-{% block title %}Employees · MIEM Employees{% endblock %}
-{% block content %}
-<section class="panel">
-  <h2 class="panel__title">Employees</h2>
-  <form class="form" method="get" action="/admin/employees">
-    <input class="form__input" name="q" value="{{ q }}" placeholder="Name or URL">
-    <select class="form__select" name="status">
-      <option value="" {% if not status %}selected{% endif %}>All</option>
-      <option value="active" {% if status == "active" %}selected{% endif %}>Active</option>
-      <option value="dismissed" {% if status == "dismissed" %}selected{% endif %}>Dismissed</option>
-    </select>
-    <button class="button" type="submit">Search</button>
-  </form>
-  <table class="table">
-    <thead><tr><th class="table__head">Name</th><th class="table__head">Status</th><th class="table__head">Last seen</th><th class="table__head">Profile</th></tr></thead>
-    <tbody>
-      {% for employee in employees %}
-      <tr>
-        <td class="table__cell"><a class="admin__link" href="/admin/employees/{{ employee.id }}">{{ employee.full_name or employee.profile_key }}</a></td>
-        <td class="table__cell"><span class="badge {% if employee.status == "dismissed" %}badge--dismissed{% endif %}">{{ employee.status }}</span></td>
-        <td class="table__cell">{{ employee.last_seen_at }}</td>
-        <td class="table__cell"><a class="admin__link" href="{{ employee.canonical_url }}">{{ employee.canonical_url }}</a></td>
-      </tr>
-      {% endfor %}
-    </tbody>
-  </table>
-</section>
-{% endblock %}

app/templates/login.html

@@ -3,18 +3,18 @@
   <head>
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1">
-    <title>Login · MIEM Employees</title>
-    <link rel="stylesheet" href="/static/admin.css">
+    <title>Вход · MIEM Employees</title>
+    <link rel="stylesheet" href="/static/admin.css?v={{ frontend_version }}">
   </head>
   <body class="admin">
     <main class="admin__main">
       <section class="panel">
-        <h1 class="panel__title">Admin login</h1>
+        <h1 class="panel__title">Вход в админку</h1>
         {% if error %}<p>{{ error }}</p>{% endif %}
         <form class="form" method="post" action="/admin/login">
-          <label class="form__label">Login <input class="form__input" name="username" autocomplete="username"></label>
-          <label class="form__label">Password <input class="form__input" name="password" type="password" autocomplete="current-password"></label>
-          <button class="button" type="submit">Sign in</button>
+          <label class="form__label">Логин <input class="form__input" name="username" autocomplete="username"></label>
+          <label class="form__label">Пароль <input class="form__input" name="password" type="password" autocomplete="current-password"></label>
+          <button class="button" type="submit">Войти</button>
         </form>
       </section>
     </main>

app/templates/run_detail.html

@@ -0,0 +1,65 @@
+{% extends "base.html" %}
+{% block title %}Запуск {{ run.id }} · MIEM Employees{% endblock %}
+{% block content %}
+<section class="panel">
+  <div class="progress-panel__header">
+    <div>
+      <h2 class="panel__title">Запуск {{ run.id }}</h2>
+      <p class="progress-panel__empty">{{ run.started_display }} · {{ run.status_display }}</p>
+    </div>
+    <a class="admin__link" href="/admin/runs">Все запуски</a>
+  </div>
+  <div class="stats-strip">
+    <div class="stats-strip__item"><span class="stats-strip__label">Найдено</span><span class="stats-strip__value">{{ run.found_count }}</span></div>
+    <div class="stats-strip__item"><span class="stats-strip__label">Обработано</span><span class="stats-strip__value">{{ run.parsed_count }}</span></div>
+    <div class="stats-strip__item"><span class="stats-strip__label">Без изменений</span><span class="stats-strip__value">{{ run.skipped_count }}</span></div>
+    <div class="stats-strip__item"><span class="stats-strip__label">Новые</span><span class="stats-strip__value">{{ run.new_count }}</span></div>
+    <div class="stats-strip__item"><span class="stats-strip__label">Потеряшки</span><span class="stats-strip__value">{{ run.changes.missing_from_source | length }}</span></div>
+    <div class="stats-strip__item"><span class="stats-strip__label">Уволены</span><span class="stats-strip__value">{{ run.dismissed_count }}</span></div>
+    <div class="stats-strip__item"><span class="stats-strip__label">Ошибки</span><span class="stats-strip__value">{{ run.error_count }}</span></div>
+  </div>
+  {% if not run.changes_detail_available %}
+  <p class="progress-panel__empty">Детализация сотрудников для этого запуска недоступна. Она сохраняется только для новых запусков после обновления.</p>
+  {% endif %}
+</section>
+
+{% for group, title in [("new", "Новые сотрудники"), ("missing_from_source", "Потеряшки"), ("verification_required", "Требуют проверки"), ("dismissed", "Уволенные")] %}
+<section class="panel"{% if group == "new" %} id="new-employees"{% endif %}>
+  <h2 class="panel__title">{{ title }}</h2>
+  {% set items = run.changes[group] %}
+  {% if items %}
+  <table class="table">
+    <thead><tr><th class="table__head">ФИО</th><th class="table__head">Профиль</th><th class="table__head">Проверка</th><th class="table__head">Комментарий</th></tr></thead>
+    <tbody>
+      {% for item in items %}
+      <tr>
+        <td class="table__cell">{% if item.employee_id %}<a class="admin__link" href="/admin/employees/{{ item.employee_id }}">{{ item.full_name or item.profile_key }}</a>{% else %}{{ item.full_name or item.profile_key }}{% endif %}</td>
+        <td class="table__cell"><a class="admin__link" href="{{ item.profile_url }}">{{ item.profile_url }}</a></td>
+        <td class="table__cell">{{ item.profile_available_display }}</td>
+        <td class="table__cell">{{ item.message or "" }}</td>
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+  {% else %}
+  <p class="progress-panel__empty">Нет записей.</p>
+  {% endif %}
+</section>
+{% endfor %}
+
+<section class="panel">
+  <h2 class="panel__title">Ошибки запуска</h2>
+  {% if run.errors %}
+  <table class="table">
+    <thead><tr><th class="table__head">Профиль</th><th class="table__head">Ошибка</th><th class="table__head">Время</th></tr></thead>
+    <tbody>
+      {% for error in run.errors %}
+      <tr><td class="table__cell">{{ error.profile_url or "" }}</td><td class="table__cell">{{ error.error_type }}: {{ error.message }}</td><td class="table__cell">{{ error.created_display }}</td></tr>
+      {% endfor %}
+    </tbody>
+  </table>
+  {% else %}
+  <p class="progress-panel__empty">Ошибок нет.</p>
+  {% endif %}
+</section>
+{% endblock %}

app/templates/runs.html

@@ -1,52 +1,56 @@
 {% extends "base.html" %}
-{% block title %}Runs · MIEM Employees{% endblock %}
+{% block title %}Запуски · MIEM Employees{% endblock %}
 {% block content %}
 <section class="panel">
   <div class="progress-panel__header">
-    <h2 class="panel__title">Crawl runs</h2>
-    <form method="post" action="/admin/runs"><button class="button" type="submit">Start crawl now</button></form>
+    <h2 class="panel__title">Запуски парсинга</h2>
+    <form method="post" action="/admin/runs"><button class="button" type="submit">Запустить парсинг</button></form>
   </div>
   {% set run = runs[0] if runs else none %}
   {% if run %}
-  {% set processed = run.parsed_count + run.error_count %}
+  {% set processed = run.parsed_count + run.skipped_count + run.error_count %}
   {% set percent = ((processed / run.found_count) * 100) | round(1) if run.found_count else 0 %}
   <div class="progress-panel" data-progress-panel>
     <div class="progress-panel__meta">
-      <span data-progress-status>{{ run.status }}</span>
-      <span><span data-progress-processed>{{ processed }}</span> / <span data-progress-found>{{ run.found_count }}</span> processed</span>
-      <span><span data-progress-errors>{{ run.error_count }}</span> errors</span>
+      <span data-progress-status>{{ run.status_display }}</span>
+      <span>обработано: <span data-progress-processed>{{ processed }}</span> / <span data-progress-found>{{ run.found_count }}</span></span>
+      <span>без изменений: <span data-progress-skipped>{{ run.skipped_count }}</span></span>
+      <span>ошибок: <span data-progress-errors>{{ run.error_count }}</span></span>
     </div>
-    <div class="progress-bar" aria-label="Parsing progress">
+    <div class="progress-bar" role="progressbar" aria-label="Прогресс парсинга" aria-valuemin="0" aria-valuemax="100" aria-valuenow="{{ percent }}" aria-valuetext="{{ percent }}%">
       <div class="progress-bar__fill" data-progress-fill style="width: {{ percent }}%"></div>
     </div>
   <div class="progress-panel__percent"><span data-progress-percent>{{ percent }}</span>%</div>
+  <p class="progress-panel__error" data-progress-error role="status" hidden>Не удалось обновить прогресс. <button class="button button--ghost button--compact" type="button" data-progress-retry>Повторить</button></p>
   </div>
   {% else %}
   <div class="progress-panel" data-progress-panel>
     <div class="progress-panel__meta">
-      <span data-progress-status>idle</span>
-      <span><span data-progress-processed>0</span> / <span data-progress-found>0</span> processed</span>
-      <span><span data-progress-errors>0</span> errors</span>
+      <span data-progress-status>Ожидание</span>
+      <span>обработано: <span data-progress-processed>0</span> / <span data-progress-found>0</span></span>
+      <span>без изменений: <span data-progress-skipped>0</span></span>
+      <span>ошибок: <span data-progress-errors>0</span></span>
     </div>
-    <div class="progress-bar" aria-label="Parsing progress">
+    <div class="progress-bar" role="progressbar" aria-label="Прогресс парсинга" aria-valuemin="0" aria-valuemax="100" aria-valuenow="0" aria-valuetext="0%">
       <div class="progress-bar__fill" data-progress-fill style="width: 0%"></div>
     </div>
     <div class="progress-panel__percent"><span data-progress-percent>0</span>%</div>
+    <p class="progress-panel__error" data-progress-error role="status" hidden>Не удалось обновить прогресс. <button class="button button--ghost button--compact" type="button" data-progress-retry>Повторить</button></p>
   </div>
   {% endif %}
   <table class="table">
-    <thead><tr><th class="table__head">ID</th><th class="table__head">Status</th><th class="table__head">Found</th><th class="table__head">Parsed</th><th class="table__head">New</th><th class="table__head">Errors</th><th class="table__head">Dismissed</th></tr></thead>
+    <thead><tr><th class="table__head">ID</th><th class="table__head">Статус</th><th class="table__head">Найдено</th><th class="table__head">Обработано</th><th class="table__head">Без изменений</th><th class="table__head">Новые</th><th class="table__head">Ошибки</th><th class="table__head">Уволены</th><th class="table__head">Старт</th></tr></thead>
     <tbody>
       {% for run in runs %}
-      <tr><td class="table__cell">{{ run.id }}</td><td class="table__cell">{{ run.status }}</td><td class="table__cell">{{ run.found_count }}</td><td class="table__cell">{{ run.parsed_count }}</td><td class="table__cell">{{ run.new_count }}</td><td class="table__cell">{{ run.error_count }}</td><td class="table__cell">{{ run.dismissed_count }}</td></tr>
+      <tr class="table__row"><td class="table__cell"><a class="admin__link" href="/admin/runs/{{ run.id }}">{{ run.id }}</a></td><td class="table__cell">{{ run.status_display }}</td><td class="table__cell">{{ run.found_count }}</td><td class="table__cell">{{ run.parsed_count }}</td><td class="table__cell">{{ run.skipped_count }}</td><td class="table__cell">{{ run.new_count }}</td><td class="table__cell">{{ run.error_count }}</td><td class="table__cell">{{ run.dismissed_count }}</td><td class="table__cell">{{ run.started_display }}</td></tr>
       {% endfor %}
     </tbody>
   </table>
 </section>
 <section class="panel">
-  <h2 class="panel__title">Recent errors</h2>
+  <h2 class="panel__title">Последние ошибки</h2>
   <table class="table">
-    <thead><tr><th class="table__head">Run</th><th class="table__head">Profile</th><th class="table__head">Error</th></tr></thead>
+    <thead><tr><th class="table__head">Запуск</th><th class="table__head">Профиль</th><th class="table__head">Ошибка</th></tr></thead>
     <tbody>
       {% for error in errors %}
       <tr><td class="table__cell">{{ error.crawl_run_id }}</td><td class="table__cell">{{ error.profile_url }}</td><td class="table__cell">{{ error.error_type }}: {{ error.message }}</td></tr>

app/version.py

@@ -1,3 +1,3 @@
-APP_VERSION = "0.2.1"
-FRONTEND_VERSION = "0.2.1"
-BACKEND_VERSION = "0.2.1"
+APP_VERSION = "0.13.1"
+FRONTEND_VERSION = "0.13.1"
+BACKEND_VERSION = "0.13.1"

app/worker.py

@@ -17,7 +17,14 @@ def crawl_once() -> None:
     settings = get_settings()
     with SessionLocal() as db:
         run = run_crawl(db, settings)
-        logger.info("crawl finished: id=%s status=%s parsed=%s errors=%s", run.id, run.status, run.parsed_count, run.error_count)
+        logger.info(
+            "crawl finished: id=%s status=%s parsed=%s skipped=%s errors=%s",
+            run.id,
+            run.status,
+            run.parsed_count,
+            run.skipped_count,
+            run.error_count,
+        )
 
 
 def main() -> None:

docker-compose.yml

@@ -1,6 +1,7 @@
 services:
   postgres:
     image: postgres:16-alpine
+    restart: always
     environment:
       POSTGRES_DB: ${POSTGRES_DB:-miem_workers}
       POSTGRES_USER: ${POSTGRES_USER:-miem}
@@ -15,18 +16,20 @@ services:
 
   api:
     build: .
+    restart: always
     command: uvicorn app.main:app --host 0.0.0.0 --port 8000
     env_file: .env
     environment:
       DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-miem}:${POSTGRES_PASSWORD:-miem_password}@postgres:5432/${POSTGRES_DB:-miem_workers}
     ports:
-      - "127.0.0.1:8000:8000"
+      - "127.0.0.1:${API_PORT:-8000}:8000"
     depends_on:
       postgres:
         condition: service_healthy
 
   worker:
     build: .
+    restart: always
     command: python -m app.worker
     env_file: .env
     environment:
@@ -35,17 +38,5 @@ services:
       postgres:
         condition: service_healthy
 
-  mcp:
-    build: .
-    command: uvicorn app.main:app --host 0.0.0.0 --port 8000
-    env_file: .env
-    environment:
-      DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-miem}:${POSTGRES_PASSWORD:-miem_password}@postgres:5432/${POSTGRES_DB:-miem_workers}
-    ports:
-      - "127.0.0.1:8001:8000"
-    depends_on:
-      postgres:
-        condition: service_healthy
-
 volumes:
   postgres_data:

docs/postgresql-import-verification.md

@@ -0,0 +1,99 @@
+# Проверка импорта на изолированном PostgreSQL
+
+Запускается средствами проекта: pytest, SQLAlchemy, psycopg и отдельный Docker
+Compose. Не использует production Compose, `.env`, SSH или данные сотрудников.
+PostgreSQL 16 и runner работают в отдельной `internal`-сети без опубликованных
+портов. Данные PostgreSQL находятся в tmpfs, без постоянных volumes. Базы:
+`miem_import_test` и `miem_crawl_test`. Все сотрудники, статьи и credentials
+синтетические; тестовые пароли непригодны для реальных систем.
+
+Dockerfile копирует только нужный код и requirements. Его отдельный dockerignore
+исключает `.git`, `.env`, файлы ключей/credentials, CSV владельца, данные
+`app/data` и остальные файлы workspace. Runtime не получает переменные хоста.
+CLI subprocess дополнительно получает только явно заданные тестовые libpq
+параметры; `.pgpass`/service-файлы отключены. Не изменяйте адрес `postgres-test`
+или имена тестовых БД для подключения к существующему серверу.
+
+Синтетическая роль `import_test` создаётся только init-скриптом одноразового
+тестового кластера. Fixture выдаёт ей документированные column grants и USAGE
+sequence. Проверяется отсутствие CREATE схемы, INSERT сотрудников, DELETE
+публикаций и sequence UPDATE. Это не создание временного доступа к рабочей БД.
+Таблицы создаются моделью приложения и очищаются только в тестовых БД.
+
+HTTP заблокирован test fixture; входные профили полного обхода подставляются
+локально. Это проверяет настоящий SQL, транзакции, savepoint и сериализацию JSON,
+но не парсинг сайта ВШЭ. Fixtures SQLite и подмена psycopg для интеграционных
+тестов не используются.
+
+## Что проверяется
+
+1. Dry-run: все строки всех таблиц и состояния sequences совпадают до/после.
+2. Apply: одна вставка, одно обновление, пропуск отсутствующего сотрудника.
+3. Повторный импорт: ноль INSERT/UPDATE, неизменные таблицы/sequences.
+4. CHECK constraint тестовой БД вызывает ошибку третьего действия после UPDATE
+   и успешного INSERT; все строки возвращаются в исходное состояние. Sequence
+   может продвинуться при rollback — это ожидаемое поведение PostgreSQL.
+5. Обновление только года без raw_data сохраняет существующий JSONB.
+6. Поздние строки CSV дополняют метаданные; первое непустое значение сохраняется
+   при конфликте, отчёт содержит названия конфликтующих полей. Повтор идемпотентен.
+7. Два последовательных crawl сохраняют CSV-статью, отсутствующую у ВШЭ;
+   появившаяся статья получает HSE ID без второй записи и потери метаданных.
+8. Полный `run_crawl` обрабатывает двух сотрудников. Два HSE ID на одну CSV-запись
+   откатывают savepoint первого, сохраняя JSONB, авторов, ID, хеш и updated_at;
+   публикация второго сохраняется. Проверяются обе очередности ID и уже
+   установленная привязка; неоднозначность отражается в CrawlError.
+
+Всего 10 PostgreSQL-тестов (восьмой сценарий параметризован). В обычном
+`python -m pytest -q` они пропускаются. Opt-in `MIEM_POSTGRES_INTEGRATION=1`
+разрешён только с точным тестовым DATABASE_URL Compose-проекта.
+
+## Команды владельцу
+
+Выполняйте на локальной машине с Docker Engine/Compose v2 из корня репозитория
+после независимого ревью. Команды явно выбирают локальный Unix socket и очищают
+наследуемое окружение. Для Docker Desktop/Windows используйте Linux/WSL с
+доступным локальным `/var/run/docker.sock`. Для отдельного стенда на VPS используйте
+[отдельную инструкцию](postgresql-import-vps.md) и `compose.vps.yml`: этот локальный
+Compose не предназначен для запуска на сервере с работающими сервисами.
+Первый шаг:
+
+```sh
+env -i PATH="$PATH" docker --host unix:///var/run/docker.sock version
+```
+
+Ожидается exit code 0 и секции `Client` и `Server`. `command not found`,
+`permission denied` или только Client с ошибкой подключения означает, что
+до запуска этого варианта нужен локальный Docker. При его отсутствии сначала
+выполните только read-only диагностику из инструкции VPS.
+
+Дальнейшие точные команды сохранены здесь для воспроизводимого запуска:
+
+```sh
+env -i PATH="$PATH" docker --host unix:///var/run/docker.sock compose --env-file /dev/null --project-name miem-publication-integration -f tests/postgres/compose.yml config --quiet
+env -i PATH="$PATH" docker --host unix:///var/run/docker.sock compose --env-file /dev/null --project-name miem-publication-integration -f tests/postgres/compose.yml build tests
+env -i PATH="$PATH" docker --host unix:///var/run/docker.sock compose --env-file /dev/null --project-name miem-publication-integration -f tests/postgres/compose.yml up --abort-on-container-exit --exit-code-from tests
+```
+
+Первые две команды проверяют Compose и собирают runner с уже используемыми
+зависимостями. Сборка/скачивание образов требуют registry/PyPI, но тестовая
+runtime-сеть не имеет внешнего доступа и запросов к ВШЭ нет. При успехе третьей
+команды ожидаются `10 passed` и exit code 0. Фактическую совместимость образа,
+column grants и PostgreSQL можно считать подтверждённой **только после такого
+выполнения**, не по unit-тестам или статическому чтению файлов.
+
+После сохранения вывода тестов удалите именно этот тестовый проект:
+
+```sh
+env -i PATH="$PATH" docker --host unix:///var/run/docker.sock compose --env-file /dev/null --project-name miem-publication-integration -f tests/postgres/compose.yml down
+```
+
+Это уничтожит только синтетические данные в tmpfs тестового PostgreSQL. Не
+используйте `docker system prune`, production Compose или удаление рабочих
+volumes. Перед повторным прогоном выполните этот же `down`, чтобы следующий
+кластер начинался с чистого init.sql; не запускайте два прогона одновременно.
+
+## Статус подготовки
+
+В среде подготовки Docker отсутствует; настоящие PostgreSQL-тесты не запускались.
+Локально доступны только unit-тесты, проверка Python и статические проверки.
+Production/VPS и доступ к рабочей БД не изменялись.

docs/postgresql-import-vps.md

@@ -0,0 +1,153 @@
+# PostgreSQL-проверка на отдельном стенде VPS
+
+Команды выполняет владелец под root. Подготовка не подключается к VPS и не
+запускает стенд. Не используйте production Compose/`.env`, Docker context,
+volumes, сети, mounts или production credentials. Никаких production-ролей БД,
+SSH-ключей и правил firewall эти команды не создают и не изменяют.
+
+## Сначала только read-only проверка
+
+Следующий блок не создаёт контейнеров, не скачивает образов и не меняет данные.
+Docker CLI использует локальный Unix socket и очищенное окружение; конфигурация
+CLI указывает на отдельный каталог, который здесь не создаётся.
+
+```sh
+free -h
+nproc
+uptime
+df -h / /var/lib/docker
+env -i PATH="$PATH" docker --config /root/miem-publication-integration/docker-config --host unix:///var/run/docker.sock info --format 'CPUs={{.NCPU}} RAM_bytes={{.MemTotal}} cgroup={{.CgroupVersion}} DockerRootDir={{.DockerRootDir}} warnings={{json .Warnings}}'
+env -i PATH="$PATH" docker --config /root/miem-publication-integration/docker-config --host unix:///var/run/docker.sock compose version
+env -i PATH="$PATH" docker --config /root/miem-publication-integration/docker-config --host unix:///var/run/docker.sock stats --no-stream --format 'table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.PIDs}}'
+env -i PATH="$PATH" docker --config /root/miem-publication-integration/docker-config --host unix:///var/run/docker.sock image inspect postgres:16-alpine python:3.12-slim miem-publication-integration-postgres:pr55 miem-publication-integration-runner:pr55 --format '{{.RepoTags}} id={{.Id}} revision={{index .Config.Labels "org.opencontainers.image.revision"}}'
+```
+
+Как читать результат:
+
+- `free`: нужна колонка **available**, а не только free. Предварительный запас —
+  не менее 2 GiB: стенд ограничен суммарно 1 GiB, ещё минимум 1 GiB остаётся
+  системе/сервисам. Это ориентир, решение зависит от их нагрузки и пиков.
+- `nproc` и `uptime`: сравните число CPU и load average. Постоянный load на уровне
+  числа CPU или выше — основание отложить тест, а не останавливать сервисы.
+- `df`: желательно не менее 4 GiB свободно для загрузки образов и запаса работы
+  Docker. Если DockerRootDir отличается от `/var/lib/docker`, отдельно выполните
+  `df -h` для показанного каталога. Не используйте prune для освобождения места.
+- `stats`: показывает потребление уже работающих контейнеров, но не их пиковые
+  требования. Docker warnings о недоступных memory/swap/cgroup limits нужно
+  устранить до запуска; без действующих лимитов этот вариант не запускать.
+- `image inspect`: существующие образы показывают tags, ID и revision.
+  `No such image` означает отсутствие образа, **не** разрешение на pull/build.
+  Для запуска нужны два образа `miem-publication-integration-*:pr55` с revision
+  проверенного commit PR #55. Base-образы нужны только для сборки, не для самого
+  запуска уже подготовленного стенда.
+
+Пришлите только этот диагностический вывод. Не присылайте `docker inspect`
+production-контейнеров целиком, env, конфигурации подключения или секреты.
+До оценки ресурсов и завершения независимого review остальные действия ниже
+не выполнять.
+
+## Изоляция и лимиты подготовленного стенда
+
+Используется только `tests/postgres/compose.vps.yml`, не override production или
+локального Compose. Project name фиксирован:
+`miem-publication-integration-vps`.
+
+| Ресурс | Лимит / область |
+|---|---|
+| PostgreSQL | Отдельный контейнер `postgres-test`, 0.5 CPU, 512 MiB RAM, без дополнительного swap, 128 процессов |
+| PostgreSQL настройки | shared_buffers 64 MiB, max_connections 12 |
+| Test runner | 0.5 CPU, 512 MiB RAM, без дополнительного swap, 128 процессов; UID 65534, read-only rootfs, no-new-privileges, без capabilities |
+| Данные PostgreSQL | Только named volume `miem-publication-integration-vps_pgdata`; local driver tmpfs до 256 MiB |
+| Временные файлы runner | tmpfs `/tmp` до 128 MiB; входят в memory limit контейнера |
+| Сеть | Только `miem-publication-integration-vps_isolated`, internal; опубликованных портов нет |
+| Логи | json-file, до 5 MiB на контейнер, один файл |
+
+Volume с tmpfs выбран намеренно: отдельное имя для адресной очистки и предел
+размера синтетических данных. Данные не сохраняются после остановки/размонтирования.
+Образы содержат только код и синтетический init.sql. Host bind mounts,
+Docker socket, production networks и env_file отсутствуют. Docker socket
+доступен только CLI владельца на хосте, ни одному контейнеру он не передаётся.
+Нет restart policy. Pull запрещён; ресурсы не создаются автоматически при
+отсутствии заранее подготовленного образа.
+
+Лимиты процесса сборки/загрузки образов отличаются от runtime: Compose CPU/RAM
+limits **не ограничивают Docker build**. Поэтому на VPS с рабочими сервисами
+не выполняйте незапланированную сборку. Если готовых образов нет, отдельно
+согласуйте место сборки и окно загрузки после read-only диагностики.
+
+## Подготовка образов после review (не запуск стенда)
+
+На отдельной согласованной машине сборки с Docker, из проверенной копии PR #55,
+укажите полный SHA проверенного commit в переменной `TEST_COMMIT` (не секрет).
+Сначала `git status --short` должен быть пустым, `git rev-parse HEAD` — совпадать
+с TEST_COMMIT. Это не команды для немедленного выполнения на VPS.
+
+```sh
+git status --short
+git rev-parse HEAD
+env -i PATH="$PATH" docker --host unix:///var/run/docker.sock build --label org.opencontainers.image.revision="$TEST_COMMIT" -t miem-publication-integration-runner:pr55 -f tests/postgres/Dockerfile .
+env -i PATH="$PATH" docker --host unix:///var/run/docker.sock build --label org.opencontainers.image.revision="$TEST_COMMIT" -t miem-publication-integration-postgres:pr55 -f tests/postgres/Dockerfile.postgres tests/postgres
+env -i PATH="$PATH" docker --host unix:///var/run/docker.sock image save --output miem-publication-integration-images.tar miem-publication-integration-postgres:pr55 miem-publication-integration-runner:pr55
+sha256sum miem-publication-integration-images.tar tests/postgres/compose.vps.yml
+```
+
+Dockerfile-specific ignore-файлы исключают секреты, реальные CSV и остальные
+файлы workspace из обоих контекстов сборки. Передайте владельцу только архив
+этих двух образов и `compose.vps.yml` с проверенными SHA256, без полного workspace
+и `.env`. Транспорт выбирает владелец; инструкция не получает SSH/production
+credentials. Base-образы могут скачиваться только на согласованной машине сборки.
+
+На VPS после оценки ресурсов/отдельного разрешения владельца: файлы размещаются
+в `/root/miem-publication-integration`, Compose переименован в `compose.vps.yml`,
+архив — в `images.tar`. Проверка checksum должна совпасть с источником.
+
+```sh
+install -d -m 700 /root/miem-publication-integration /root/miem-publication-integration/docker-config
+sha256sum /root/miem-publication-integration/images.tar /root/miem-publication-integration/compose.vps.yml
+env -i PATH="$PATH" docker --config /root/miem-publication-integration/docker-config --host unix:///var/run/docker.sock image load --input /root/miem-publication-integration/images.tar
+env -i PATH="$PATH" docker --config /root/miem-publication-integration/docker-config --host unix:///var/run/docker.sock image inspect miem-publication-integration-postgres:pr55 miem-publication-integration-runner:pr55 --format '{{.RepoTags}} id={{.Id}} revision={{index .Config.Labels "org.opencontainers.image.revision"}}'
+env -i PATH="$PATH" docker --config /root/miem-publication-integration/docker-config --host unix:///var/run/docker.sock compose --env-file /dev/null --project-name miem-publication-integration-vps -f /root/miem-publication-integration/compose.vps.yml config --quiet
+```
+
+Проверьте отсутствие уже существующих ресурсов с этим project name. Если они
+есть, выясните их назначение; не запускайте второй стенд и не удаляйте чужие
+ресурсы. Следующие команды только читают labels Docker Compose:
+
+```sh
+env -i PATH="$PATH" docker --config /root/miem-publication-integration/docker-config --host unix:///var/run/docker.sock ps -a --filter label=com.docker.compose.project=miem-publication-integration-vps
+env -i PATH="$PATH" docker --config /root/miem-publication-integration/docker-config --host unix:///var/run/docker.sock network ls --filter label=com.docker.compose.project=miem-publication-integration-vps
+env -i PATH="$PATH" docker --config /root/miem-publication-integration/docker-config --host unix:///var/run/docker.sock volume ls --filter label=com.docker.compose.project=miem-publication-integration-vps
+```
+
+## Запуск и адресная очистка (пока не выполнять)
+
+Только после согласования владельцем ресурсов и проверенного состава стенда:
+
+```sh
+env -i PATH="$PATH" docker --config /root/miem-publication-integration/docker-config --host unix:///var/run/docker.sock compose --env-file /dev/null --project-name miem-publication-integration-vps -f /root/miem-publication-integration/compose.vps.yml up --no-build --pull never --abort-on-container-exit --exit-code-from tests
+```
+
+Ожидаемый успех — `10 passed`, exit code 0. Сохраните вывод/код выхода и логи:
+
+```sh
+env -i PATH="$PATH" docker --config /root/miem-publication-integration/docker-config --host unix:///var/run/docker.sock compose --env-file /dev/null --project-name miem-publication-integration-vps -f /root/miem-publication-integration/compose.vps.yml logs --no-color > /root/miem-publication-integration/test.log
+```
+
+**Следующая команда удаляет контейнеры, внутреннюю сеть и named volume именно
+этого теста; все его синтетические данные будут потеряны.** Перед ней сохраните
+отчёт. Она не затрагивает ресурсы других Compose-проектов:
+
+```sh
+env -i PATH="$PATH" docker --config /root/miem-publication-integration/docker-config --host unix:///var/run/docker.sock compose --env-file /dev/null --project-name miem-publication-integration-vps -f /root/miem-publication-integration/compose.vps.yml down --volumes
+```
+
+Не используйте `--remove-orphans`, `docker system prune`, общие rm/rmi/volume prune
+или down рабочего проекта. Base-образы и образы теста оставьте: они могут быть
+общими для других проверок. Архив и журнал также не удаляются этой командой.
+Для подтверждения очистки повторите три read-only команды с project label:
+в выводе должны остаться только заголовки.
+
+## Статус
+
+VPS-стенд не запускался, Docker/Compose-проверка не выполнялась в среде
+подготовки из-за отсутствия Docker. PostgreSQL-интеграция ещё не подтверждена.

docs/publication-import.md

@@ -0,0 +1,312 @@
+# Разовый импорт публикаций
+
+Инструмент: `scripts/import_publications.py`. Он не импортирует приложение, не читает
+`.env`, не вызывает `init_db`, не создаёт таблиц и не удаляет записи. Подключение
+берётся только из libpq environment/service, настроенных владельцем. Production в
+рамках подготовки не проверялся и не изменялся.
+
+Локальные проверки: синтетические сотрудники, повторный запуск, дубликаты,
+неоднозначности, конфликты DOI/года, сохранение заполненных значений и
+транзакционный путь на тестовом соединении (read-only / rollback при второй
+ошибочной вставке / успешный commit). Прогон на настоящем PostgreSQL и проверка
+grants не выполнялись; первый owner-run dry-run с read-only ролью обязателен.
+
+## Вход и сопоставление
+
+Файл владельца `Статьи.csv`: UTF-8 с возможным BOM, разделитель `;`, 1797 записей,
+SHA256 `1f236be56d99677b1403df29a4c1d3a7e6d10cb90bd355d678201a027921b0ce`.
+Обязательные колонки: `Название публикации`, `Год`, `DOI`, `Авторы НИУ ВШЭ`.
+Номер строки в отчёте — номер CSV-записи с заголовком, а не физической строки.
+
+Сотрудник сопоставляется по полному ФИО: Unicode NFKC, регистр, пробелы и ё/е.
+Учитываются все существующие статусы; новые сотрудники не создаются. Ноль
+совпадений → `employee_not_found`, несколько → `ambiguous_employee`. Инициалы,
+переводы, перестановки слов и приблизительные совпадения не используются.
+
+Публикация сопоставляется **внутри конкретного сотрудника** по нормализованному DOI
+или полному названию и году. Отсутствующий год БД допускает заполнение при
+совпадении названия. Если эти способы указывают на разные записи, импорт
+пропускается (`ambiguous_publication`). Разные заполненные DOI или годы —
+`doi_conflict` / `year_conflict`. Существующие дубликаты не объединяются и не
+удаляются. Одна статья у разных сотрудников — предусмотренные моделью авторские
+связи, а не дубли внутри карточки.
+
+Повтор CSV внутри сотрудника не создаёт записи. Каждая поздняя строка дополняет
+ещё пустые метаданные; первый непустой источник имеет приоритет, в том числе
+ранее сохранённое значение БД. Повтор без изменений — `csv_duplicate`, дополнение
+запланированной вставки — `merged_csv_duplicate`. При конфликте непустых
+метаданных первое значение сохраняется; остальные пустые поля всё равно
+дополняются. Отчёт содержит `metadata_conflict_first_value_kept`, список
+`conflict_fields` (только названия полей, без значений) и сводный счётчик
+`metadata_conflicts`. Конфликт DOI/года идентификации по-прежнему пропускает
+всю связь. Если строка без DOI была запланирована первой, следующая строка с
+DOI может дополнить ту же вставку. При повторном запуске уже заполненные записи дают
+`already_present`. Хеш новой записи детерминирован по сотруднику и DOI либо
+названию/году. `source_hash` существующих записей не меняется.
+
+Пустые ячейки ничего не стирают. Пустые название/авторы, некорректный год
+(допустимо 1000–9999) или DOI пропускаются как `invalid_fields`. Повреждённая
+структура CSV прерывает весь запуск до подключения. Несовместимый JSON БД
+пропускается (`unsupported_raw_data`, `unsupported_csv_metadata`).
+
+## Таблицы, столбцы и права
+
+| Объект | Операция / столбцы |
+|---|---|
+| `public.employees` | Только SELECT: `id`, `full_name` |
+| `public.employee_publications` | SELECT: `id`, `employee_id`, `title`, `year`, `doi_url`, `raw_data` |
+| Новая публикация | INSERT: `employee_id`, `title`, `year`, `doi_url` при наличии, `raw_data`, `source_hash`, `created_at`, `updated_at` |
+| Существующая публикация | UPDATE: только пустые `title`, `year`, `doi_url`; `raw_data`; служебный `updated_at` |
+| Sequence идентификатора публикации | USAGE для `nextval`; имя определяется `pg_get_serial_sequence`, обычно `public.employee_publications_id_seq` |
+
+Остальные таблицы и sequences не затрагиваются. В `raw_data` сохраняются прежние
+ключи; добавляется/дополняется `csv_enrichment` только отсутствующими или пустыми
+значениями. Уже заполненные CSV-метаданные не заменяются. Поля `authors`,
+`publication_id`, тип материала, язык, аннотация, ссылки HSE и статус не меняются.
+CSV содержит **только авторов НИУ ВШЭ**, поэтому он не считается полным составом
+авторов и не заменяет `authors`. Идентификаторы Scopus/WoS, квартиль и остальные
+непустые исходные ячейки сохраняются в `raw_data.csv_enrichment` без интерпретации.
+Карточки `employees.current_data` и snapshots не изменяются; отчёт и данные
+таблицы не означают обновления всех представлений интерфейса.
+
+Минимальные собственные grants роли: CONNECT к БД, USAGE схемы public,
+перечисленные SELECT/INSERT/UPDATE по столбцам, USAGE найденной sequence.
+Не нужны DELETE, TRUNCATE, CREATE, ALTER, ownership, superuser, доступ к прочим
+таблицам и sequence UPDATE/setval. Роль новая, без членства в других ролях.
+Права PUBLIC всё равно наследуются: владелец должен отдельно проверить их;
+этот набор команд не меняет общие права PUBLIC и не гарантирует изоляцию при
+уже существующих широких общих grants. В частности TEMP может быть выдан PUBLIC.
+
+## Применение и ограничения
+
+Без `--apply` используется PostgreSQL `SERIALIZABLE READ ONLY`, без INSERT,
+UPDATE и nextval, с rollback. С `--apply` весь план применяется одной транзакцией
+SERIALIZABLE. Advisory lock предотвращает одновременное применение этим
+импортером. Таймаут запроса — 60 секунд, ожидания блокировок — 5 секунд.
+Ошибка записи/конфликт сериализации откатывает все изменения таблиц; автоматических
+повторов нет. Sequence может получить пропуски при откате — это нормальное
+поведение PostgreSQL. Изменение входного файла во время запуска прерывает импорт.
+
+JSON-отчёт содержит количества insert/update/skip, причины и пары
+`номер CSV-записи — employee_id`, без ФИО, названий статей и секретов. Счётчики
+относятся к авторским связям, а не уникальным статьям. Консоль выводит только
+сводку. Отчёт сначала пишется в `.pending` до commit, после успеха переименовывается.
+Если остался `.pending`, следующий запуск блокируется: сначала проверьте исход
+транзакции. Отчёт `.pending` не доказывает commit; ошибка переименования после
+commit также возможна. Безопасный повторный dry-run позволяет проверить состояние.
+При необходимости владелец перемещает `.pending` в архив перед повтором.
+
+**До применения нужен свежий backup БД.** Во время импорта владелец должен
+исключить конкурентные записи публикаций на время транзакции, согласовав окно
+работы; эта подготовка не отключает crawler. Не выполняйте production apply до
+независимого ревью и развёртывания исправления crawler из этого PR.
+
+До исправления `_upsert_employee` запускал синхронизацию публикаций даже для
+неизменённой карточки. При пустом списке публикаций или отсутствии таблицы
+синхронизация пропускалась. При непустом списке `_sync_employee_publications`
+удалял все записи сотрудника, чей `source_hash` отсутствовал среди хешей ответа
+ВШЭ, включая CSV-вставки. Найденные по `publication_id`/хешу записи полностью
+получали новый `raw_data`, поэтому теряли CSV-метаданные даже без удаления.
+
+Исправление сохраняет записи с dict `raw_data.csv_enrichment` при очистке
+устаревших публикаций. При обновлении сохраняется этот ключ и заполненные
+title/year/doi_url, если ВШЭ вернул пустые значения. Непустые значения ВШЭ
+продолжают обновлять канонические поля, а исходные CSV-значения остаются в
+метаданных. Если статья впервые появилась у ВШЭ, crawler сопоставляет её с
+импортированной записью по DOI или названию/году до новой вставки, сохраняет id
+и добавляет HSE publication_id. Конфликт/неоднозначность откатывает весь savepoint
+синхронизации публикаций и регистрирует CrawlError; карточка сотрудника продолжает
+обрабатываться. Удаление обычных устаревших HSE-записей остаётся прежним. Здесь
+не приведены команды рестарта или отключения сервисов.
+
+Уже установленный HSE publication_id CSV-записи нельзя заменить другим HSE ID,
+даже когда DOI/название/год совпадают. Это проверяется перед изменением полей
+при любом пути сопоставления (ID, source_hash, CSV-поля). Поэтому две HSE-статьи
+не могут по очереди перезаписать одну CSV-запись: весь savepoint откатывается
+независимо от порядка статей. Отсутствующий ID во входе не очищает существующую
+CSV-привязку. Отчёт обхода (`errors` в деталях запуска) содержит CrawlError с
+идентификатором записи БД и конфликтующими HSE ID, без ФИО и названий статей.
+
+## Команды владельцу (не выполнялись)
+
+Ниже — подготовка разового доступа после ревью. Выполняет владелец на сервере.
+Рабочая директория — проверенная копия этого PR; файлы программы и CSV должны
+быть защищены от записи временным SSH-пользователем. Пароли/ключи не включать
+в репозиторий или чат. Имена контейнеров соответствуют текущему Compose-проекту;
+владелец проверяет их локально перед выполнением.
+
+Сначала резервная копия в приватный каталог владельца:
+
+```sh
+install -d -m 700 /var/backups/miem-publication-import
+docker exec miem_workers-postgres-1 pg_dump -U miem -d miem_workers -Fc -f /tmp/publications-before.dump
+docker cp miem_workers-postgres-1:/tmp/publications-before.dump /var/backups/miem-publication-import/before.dump
+chmod 600 /var/backups/miem-publication-import/before.dump
+```
+
+При повторной операции используйте новый путь backup; не перезаписывайте нужную
+предыдущую копию. Проверьте exit codes и архив `pg_restore --list` до выдачи write.
+
+Откройте psql владельцем БД:
+
+```sh
+docker exec -it miem_workers-postgres-1 psql -U miem -d miem_workers
+```
+
+Выполните SQL (без пароля в SQL/истории):
+
+```sql
+\set ON_ERROR_STOP on
+SELECT format('CREATE ROLE miem_publication_import LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS CONNECTION LIMIT 1 VALID UNTIL %L', CURRENT_TIMESTAMP + interval '24 hours') \gexec
+\password miem_publication_import
+GRANT CONNECT ON DATABASE miem_workers TO miem_publication_import;
+GRANT USAGE ON SCHEMA public TO miem_publication_import;
+GRANT SELECT (id, full_name) ON public.employees TO miem_publication_import;
+GRANT SELECT (id, employee_id, title, year, doi_url, raw_data) ON public.employee_publications TO miem_publication_import;
+-- Сначала выдаются только read-права для dry-run.
+SELECT pg_get_serial_sequence('public.employee_publications', 'id');
+SELECT has_schema_privilege('miem_publication_import', 'public', 'CREATE') AS unexpected_create,
+       has_table_privilege('miem_publication_import', 'public.employees', 'INSERT') AS unexpected_employee_insert,
+       has_table_privilege('miem_publication_import', 'public.employee_publications', 'DELETE') AS unexpected_delete;
+```
+
+Три проверки должны вернуть false. Не выдавайте write-права при неожиданном
+доступе. Если sequence не найдена, выясните фактический механизм id перед импортом.
+Срок роли ограничивает парольную авторизацию, но не заменяет отзыв доступа и
+завершение открытых сессий.
+
+После принятого dry-run, непосредственно перед разрешённым apply:
+
+```sql
+GRANT INSERT (employee_id, title, year, doi_url, raw_data, source_hash, created_at, updated_at) ON public.employee_publications TO miem_publication_import;
+GRANT UPDATE (title, year, doi_url, raw_data, updated_at) ON public.employee_publications TO miem_publication_import;
+SELECT format('GRANT USAGE ON SEQUENCE %s TO miem_publication_import', pg_get_serial_sequence('public.employee_publications', 'id')) \gexec
+```
+
+Подготовьте внутри API-контейнера защищённые файлы libpq `pg_service.conf` и
+`pgpass` из окружения/секрет-хранилища владельца: service
+`miem_publication_import`, БД `miem_workers`, пользователь
+`miem_publication_import`. Используйте внутренний PostgreSQL endpoint Compose,
+порт 5432; не публикуйте порт БД. Поле password хранится только в `pgpass`,
+доступном UID 65534 с mode 600. Не копируйте `.env` приложения. Этот шаг
+зависит от секрет-хранилища владельца; универсальной команды получения секрета нет.
+
+Установите файлы (команды root; каталог контейнера не должен быть volume с
+непроверенными файлами). `Статьи.csv` расположен рядом с каталогом репозитория:
+
+```sh
+docker exec miem_workers-api-1 mkdir -p /opt/miem-import /tmp/miem-import
+docker cp scripts/import_publications.py miem_workers-api-1:/opt/miem-import/import_publications.py
+docker cp ../Статьи.csv miem_workers-api-1:/opt/miem-import/articles.csv
+docker exec miem_workers-api-1 chown -R root:root /opt/miem-import
+docker exec miem_workers-api-1 chmod 755 /opt/miem-import
+docker exec miem_workers-api-1 chmod 644 /opt/miem-import/import_publications.py /opt/miem-import/articles.csv
+docker exec miem_workers-api-1 chown 65534:65534 /opt/miem-import/pg_service.conf /opt/miem-import/pgpass /tmp/miem-import
+docker exec miem_workers-api-1 chmod 600 /opt/miem-import/pg_service.conf /opt/miem-import/pgpass
+docker exec miem_workers-api-1 chmod 700 /tmp/miem-import
+docker exec miem_workers-api-1 sha256sum /opt/miem-import/articles.csv
+install -m 755 -o root -g root scripts/import-publications-run /usr/local/sbin/miem-publication-import
+install -m 755 -o root -g root scripts/import-publications-ssh /usr/local/sbin/miem-publication-import-ssh
+useradd --create-home --shell /bin/sh miem-pub-import
+install -d -m 700 -o miem-pub-import -g miem-pub-import /home/miem-pub-import/.ssh
+```
+
+Проверьте SHA256 CSV по указанному выше значению и script по проверенной копии PR.
+Секретные файлы должны существовать перед командами chown/chmod. Root-владелец
+каталога не позволяет UID 65534 заменить программу/CSV; этот UID может писать
+только отчёт и свои файлы конфигурации. Не предоставляйте SSH-пользователю группу
+docker или общий sudo. Каталоги и executables выше должны быть root-owned и
+недоступны для записи этому пользователю. Убедитесь, что абсолютный путь Docker
+на сервере — `/usr/bin/docker`.
+
+Создайте `/etc/sudoers.d/miem-publication-import` с mode 440:
+
+```sudoers
+miem-pub-import ALL=(root) NOPASSWD: /usr/local/sbin/miem-publication-import dry-run, /usr/local/sbin/miem-publication-import apply
+```
+
+```sh
+visudo -cf /etc/sudoers.d/miem-publication-import
+```
+
+В `/home/miem-pub-import/.ssh/authorized_keys` добавьте **только публичный** ключ
+владельца с ограничениями (замените `<PUBLIC_KEY>` самим публичным ключом):
+
+```text
+restrict,command="/usr/local/sbin/miem-publication-import-ssh" <PUBLIC_KEY>
+```
+
+```sh
+chown root:root /home/miem-pub-import/.ssh/authorized_keys
+chmod 644 /home/miem-pub-import/.ssh/authorized_keys
+chown root:root /home/miem-pub-import /home/miem-pub-import/.ssh
+chmod 755 /home/miem-pub-import
+chmod 755 /home/miem-pub-import/.ssh
+```
+
+Root-владение home/.ssh предотвращает замену ограниченного authorized_keys.
+Убедитесь, что SSH не разрешает этому пользователю другие ключи, пароль,
+keyboard-interactive, пользовательские environment/rc или общий shell.
+При необходимости задайте в sshd `Match User miem-pub-import` с
+`AuthenticationMethods publickey`, `PasswordAuthentication no`,
+`KbdInteractiveAuthentication no`, `DisableForwarding yes`, `PermitTTY no`,
+`PermitUserRC no` и указанным ForceCommand. Проверка/reload SSH зависит от
+конфигурации сервера; перед reload сохраните действующую сессию владельца.
+
+С клиентской машины параметры хоста/ключа берутся из окружения владельца:
+
+```sh
+ssh -i "$IMPORT_SSH_KEY" -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=yes "miem-pub-import@$PROD_HOST" 'import-publications dry-run'
+```
+
+После проверки сводки, выдачи write-grants и отдельного решения владельца:
+
+```sh
+ssh -i "$IMPORT_SSH_KEY" -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=yes "miem-pub-import@$PROD_HOST" 'import-publications apply'
+ssh -i "$IMPORT_SSH_KEY" -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=yes "miem-pub-import@$PROD_HOST" 'import-publications dry-run'
+```
+
+Повторный dry-run должен показать ноль insert/update. При concurrent changes
+сводки могут различаться; без production-проверки фактические количества неизвестны.
+Владелец забирает отчёты без предоставления SCP/general shell временному ключу:
+
+```sh
+docker cp miem_workers-api-1:/tmp/miem-import/apply.json ./publication-import-apply.json
+docker cp miem_workers-api-1:/tmp/miem-import/dry-run.json ./publication-import-dry-run.json
+```
+
+## Отзыв доступа
+
+Владелец прекращает новые SSH-входы: перемещает временный authorized_keys в свой
+закрытый архив и удаляет только этот sudoers-файл. Не удаляйте данные БД или volumes.
+
+```sh
+mv /home/miem-pub-import/.ssh/authorized_keys /var/backups/miem-publication-import/authorized_keys.disabled
+rm /etc/sudoers.d/miem-publication-import
+```
+
+В psql владельцем:
+
+```sql
+ALTER ROLE miem_publication_import NOLOGIN;
+SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE usename = 'miem_publication_import' AND pid <> pg_backend_pid();
+REVOKE ALL PRIVILEGES ON DATABASE miem_workers FROM miem_publication_import;
+REVOKE ALL PRIVILEGES ON SCHEMA public FROM miem_publication_import;
+REVOKE SELECT (id, full_name) ON public.employees FROM miem_publication_import;
+REVOKE SELECT (id, employee_id, title, year, doi_url, raw_data),
+       INSERT (employee_id, title, year, doi_url, raw_data, source_hash, created_at, updated_at),
+       UPDATE (title, year, doi_url, raw_data, updated_at)
+       ON public.employee_publications FROM miem_publication_import;
+SELECT format('REVOKE ALL PRIVILEGES ON SEQUENCE %s FROM miem_publication_import', pg_get_serial_sequence('public.employee_publications', 'id')) \gexec
+DROP ROLE miem_publication_import;
+```
+
+После сохранения отчётов и backup удалите только временные секреты подключения:
+
+```sh
+docker exec miem_workers-api-1 rm /opt/miem-import/pgpass /opt/miem-import/pg_service.conf
+```
+
+Эти действия не используют `DROP OWNED` и не удаляют публикации. SQL роли не
+является миграцией схемы приложения. Если DROP ROLE не проходит, проверьте
+оставшиеся grants; не применяйте CASCADE/удаление данных.

infra/install-postgres-backup-cron.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+set -eu
+
+temporary=$(mktemp)
+app_dir=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)
+trap 'rm -f -- "$temporary"' EXIT HUP INT TERM
+
+crontab -l 2>/dev/null | grep -v 'postgres-backup.sh' > "$temporary" || true
+printf '%s\n' "30 3 * * 0 flock -w 3600 /tmp/miem-postgres-backup.lock $app_dir/infra/postgres-backup.sh || logger -t miem-postgres-backup -- 'backup skipped or failed'" >> "$temporary"
+crontab "$temporary"

infra/postgres-backup.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+set -eu
+umask 077
+
+app_dir=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)
+backup_dir="$HOME/backups/postgres"
+backup="$backup_dir/postgres-latest.sql"
+temporary="$backup.tmp.$$"
+
+log() { logger -t miem-postgres-backup -- "$*" 2>/dev/null || true; }
+
+mkdir -p "$backup_dir"
+trap 'status=$?; rm -f -- "$temporary"; [ "$status" -eq 0 ] || log "backup failed"; exit "$status"' EXIT HUP INT TERM
+cd "$app_dir"
+
+docker compose exec -T postgres sh -c 'pg_dumpall -U "$POSTGRES_USER"' > "$temporary"
+test -s "$temporary"
+mv "$temporary" "$backup"
+find "$HOME/backups/miem_workers" -maxdepth 1 -type f -name 'miem_workers-*.sql' -delete 2>/dev/null || true
+log "created $backup"

migrations/001_init.sql

@@ -13,6 +13,7 @@ CREATE TABLE IF NOT EXISTS crawl_runs (
   finished_at TIMESTAMPTZ,
   found_count INTEGER NOT NULL DEFAULT 0,
   parsed_count INTEGER NOT NULL DEFAULT 0,
+  skipped_count INTEGER NOT NULL DEFAULT 0,
   new_count INTEGER NOT NULL DEFAULT 0,
   error_count INTEGER NOT NULL DEFAULT 0,
   dismissed_count INTEGER NOT NULL DEFAULT 0,
@@ -32,6 +33,7 @@ CREATE TABLE IF NOT EXISTS employees (
   dismissed_at TIMESTAMPTZ,
   parser_version VARCHAR(32),
   current_data JSONB,
+  has_academic_degree BOOLEAN NOT NULL DEFAULT FALSE,
   current_checksum VARCHAR(64),
   created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
   updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
@@ -39,6 +41,7 @@ CREATE TABLE IF NOT EXISTS employees (
 
 CREATE INDEX IF NOT EXISTS ix_employees_full_name ON employees (full_name);
 CREATE INDEX IF NOT EXISTS ix_employees_status ON employees (status);
+CREATE INDEX IF NOT EXISTS ix_employees_has_academic_degree ON employees (has_academic_degree);
 
 CREATE TABLE IF NOT EXISTS employee_snapshots (
   id SERIAL PRIMARY KEY,
@@ -73,3 +76,22 @@ CREATE TABLE IF NOT EXISTS profile_tabs (
 );
 
 CREATE INDEX IF NOT EXISTS ix_profile_tabs_employee_id ON profile_tabs (employee_id);
+
+CREATE TABLE IF NOT EXISTS parse_resource_cache (
+  id SERIAL PRIMARY KEY,
+  profile_key VARCHAR(255) NOT NULL,
+  resource_key VARCHAR(255) NOT NULL,
+  method VARCHAR(16) NOT NULL,
+  url TEXT NOT NULL,
+  request_fingerprint VARCHAR(64) NOT NULL,
+  etag TEXT,
+  last_modified TEXT,
+  body_hash VARCHAR(64) NOT NULL,
+  body_snapshot BYTEA NOT NULL,
+  parser_version VARCHAR(32),
+  fetched_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  CONSTRAINT uq_parse_resource_cache_resource UNIQUE (profile_key, resource_key, request_fingerprint)
+);
+
+CREATE INDEX IF NOT EXISTS ix_parse_resource_cache_profile_key
+  ON parse_resource_cache (profile_key);

migrations/003_crawl_run_employee_changes.sql

@@ -0,0 +1,21 @@
+CREATE TABLE IF NOT EXISTS crawl_run_employee_changes (
+  id SERIAL PRIMARY KEY,
+  crawl_run_id INTEGER NOT NULL REFERENCES crawl_runs(id),
+  employee_id INTEGER REFERENCES employees(id),
+  profile_key VARCHAR(255) NOT NULL,
+  profile_url TEXT NOT NULL,
+  full_name TEXT,
+  change_type VARCHAR(32) NOT NULL,
+  profile_available BOOLEAN,
+  message TEXT,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE INDEX IF NOT EXISTS ix_crawl_run_employee_changes_run_id
+  ON crawl_run_employee_changes (crawl_run_id);
+
+CREATE INDEX IF NOT EXISTS ix_crawl_run_employee_changes_employee_id
+  ON crawl_run_employee_changes (employee_id);
+
+CREATE INDEX IF NOT EXISTS ix_crawl_run_employee_changes_change_type
+  ON crawl_run_employee_changes (change_type);

migrations/004_dataset_versions.sql

@@ -0,0 +1,29 @@
+CREATE TABLE IF NOT EXISTS dataset_versions (
+  id SERIAL PRIMARY KEY,
+  hash VARCHAR(64) NOT NULL UNIQUE,
+  previous_hash VARCHAR(64),
+  crawl_run_id INTEGER REFERENCES crawl_runs(id),
+  employee_count INTEGER NOT NULL DEFAULT 0,
+  active_count INTEGER NOT NULL DEFAULT 0,
+  dismissed_count INTEGER NOT NULL DEFAULT 0,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE INDEX IF NOT EXISTS ix_dataset_versions_created_at
+  ON dataset_versions (created_at);
+
+CREATE TABLE IF NOT EXISTS dataset_version_items (
+  id SERIAL PRIMARY KEY,
+  dataset_version_id INTEGER NOT NULL REFERENCES dataset_versions(id),
+  profile_key VARCHAR(255) NOT NULL,
+  employee_id INTEGER REFERENCES employees(id),
+  status VARCHAR(32) NOT NULL,
+  checksum VARCHAR(64) NOT NULL,
+  CONSTRAINT uq_dataset_version_items_version_profile UNIQUE (dataset_version_id, profile_key)
+);
+
+CREATE INDEX IF NOT EXISTS ix_dataset_version_items_hash
+  ON dataset_version_items (dataset_version_id);
+
+CREATE INDEX IF NOT EXISTS ix_dataset_version_items_profile_key
+  ON dataset_version_items (profile_key);

migrations/005_parse_resource_cache.sql

@@ -0,0 +1,21 @@
+ALTER TABLE crawl_runs
+ADD COLUMN IF NOT EXISTS skipped_count INTEGER NOT NULL DEFAULT 0;
+
+CREATE TABLE IF NOT EXISTS parse_resource_cache (
+  id SERIAL PRIMARY KEY,
+  profile_key VARCHAR(255) NOT NULL,
+  resource_key VARCHAR(255) NOT NULL,
+  method VARCHAR(16) NOT NULL,
+  url TEXT NOT NULL,
+  request_fingerprint VARCHAR(64) NOT NULL,
+  etag TEXT,
+  last_modified TEXT,
+  body_hash VARCHAR(64) NOT NULL,
+  body_snapshot BYTEA NOT NULL,
+  parser_version VARCHAR(32),
+  fetched_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  CONSTRAINT uq_parse_resource_cache_resource UNIQUE (profile_key, resource_key, request_fingerprint)
+);
+
+CREATE INDEX IF NOT EXISTS ix_parse_resource_cache_profile_key
+  ON parse_resource_cache (profile_key);

migrations/006_employee_publications.sql

@@ -0,0 +1,39 @@
+CREATE TABLE IF NOT EXISTS employee_publications (
+  id SERIAL PRIMARY KEY,
+  employee_id INTEGER NOT NULL REFERENCES employees(id) ON DELETE CASCADE,
+  publication_id VARCHAR(64),
+  title TEXT NOT NULL,
+  year INTEGER,
+  publication_type VARCHAR(64),
+  language VARCHAR(16),
+  status INTEGER,
+  url TEXT,
+  doi_url TEXT,
+  other_url TEXT,
+  document_url TEXT,
+  citation_text TEXT,
+  annotation JSONB,
+  description JSONB,
+  authors JSONB,
+  raw_data JSONB,
+  source_hash VARCHAR(64) NOT NULL,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  CONSTRAINT uq_employee_publications_employee_publication UNIQUE (employee_id, publication_id),
+  CONSTRAINT uq_employee_publications_employee_source_hash UNIQUE (employee_id, source_hash)
+);
+
+CREATE INDEX IF NOT EXISTS ix_employee_publications_employee_id
+  ON employee_publications (employee_id);
+
+CREATE INDEX IF NOT EXISTS ix_employee_publications_publication_id
+  ON employee_publications (publication_id);
+
+CREATE INDEX IF NOT EXISTS ix_employee_publications_doi_url
+  ON employee_publications (doi_url);
+
+CREATE INDEX IF NOT EXISTS ix_employee_publications_year
+  ON employee_publications (year);
+
+CREATE INDEX IF NOT EXISTS ix_employee_publications_publication_type
+  ON employee_publications (publication_type);

migrations/007_employee_news_links.sql

@@ -0,0 +1,27 @@
+CREATE TABLE IF NOT EXISTS employee_news_links (
+  id SERIAL PRIMARY KEY,
+  employee_id INTEGER NOT NULL REFERENCES employees(id) ON DELETE CASCADE,
+  title TEXT NOT NULL,
+  url TEXT,
+  summary TEXT,
+  published_at TIMESTAMPTZ,
+  published_year INTEGER,
+  source_hash VARCHAR(64) NOT NULL,
+  raw_data JSONB,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  CONSTRAINT uq_employee_news_links_employee_url UNIQUE (employee_id, url),
+  CONSTRAINT uq_employee_news_links_employee_source_hash UNIQUE (employee_id, source_hash)
+);
+
+CREATE INDEX IF NOT EXISTS ix_employee_news_links_employee_id
+  ON employee_news_links (employee_id);
+
+CREATE INDEX IF NOT EXISTS ix_employee_news_links_url
+  ON employee_news_links (url);
+
+CREATE INDEX IF NOT EXISTS ix_employee_news_links_published_at
+  ON employee_news_links (published_at);
+
+CREATE INDEX IF NOT EXISTS ix_employee_news_links_published_year
+  ON employee_news_links (published_year);

migrations/008_profile_verification.sql

@@ -0,0 +1,17 @@
+ALTER TABLE employees
+ADD COLUMN IF NOT EXISTS profile_unavailable_streak INTEGER NOT NULL DEFAULT 0;
+
+ALTER TABLE employees
+ADD COLUMN IF NOT EXISTS last_profile_check_at TIMESTAMPTZ;
+
+CREATE TABLE IF NOT EXISTS employee_profile_urls (
+  id SERIAL PRIMARY KEY,
+  employee_id INTEGER NOT NULL REFERENCES employees(id) ON DELETE CASCADE,
+  url TEXT NOT NULL,
+  first_seen_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  last_seen_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  CONSTRAINT uq_employee_profile_urls_employee_url UNIQUE (employee_id, url)
+);
+
+CREATE INDEX IF NOT EXISTS ix_employee_profile_urls_employee_id
+  ON employee_profile_urls (employee_id);

migrations/009_academic_degree_filter.sql

@@ -0,0 +1,11 @@
+ALTER TABLE employees
+ADD COLUMN IF NOT EXISTS has_academic_degree BOOLEAN NOT NULL DEFAULT FALSE;
+
+UPDATE employees
+SET has_academic_degree = COALESCE(
+  current_data::text ~* '(кандидат|доктор).{0,80}наук|ph\.?[[:space:]]*d\.?',
+  FALSE
+);
+
+CREATE INDEX IF NOT EXISTS ix_employees_has_academic_degree
+  ON employees (has_academic_degree);

pyproject.toml

@@ -1,28 +1,29 @@
-[project]
-name = "miem-workers"
-version = "0.1.0"
-description = "MIEM employees parser, admin API, and MCP server"
-requires-python = ">=3.11"
-dependencies = [
-  "apscheduler>=3.10.4",
-  "beautifulsoup4>=4.12.3",
-  "fastapi>=0.115.0",
-  "httpx>=0.27.0",
-  "jinja2>=3.1.4",
-  "lxml>=5.2.0",
-  "psycopg[binary]>=3.2.0",
-  "pydantic-settings>=2.4.0",
-  "python-multipart>=0.0.9",
-  "requests>=2.32.0",
-  "sqlalchemy>=2.0.32",
-  "uvicorn[standard]>=0.30.0",
-]
-
-[project.optional-dependencies]
-dev = [
-  "pytest>=8.3.0",
-]
-
-[tool.pytest.ini_options]
-testpaths = ["tests"]
-pythonpath = ["."]
+[project]
+name = "miem-workers"
+version = "0.13.1"
+description = "MIEM employees parser, admin API, and web admin"
+requires-python = ">=3.11"
+dependencies = [
+  "apscheduler>=3.10.4",
+  "beautifulsoup4>=4.12.3",
+  "fastapi>=0.115.0",
+  "httpx>=0.27.0",
+  "jinja2>=3.1.4",
+  "lxml>=5.2.0",
+  "psycopg[binary]>=3.2.0",
+  "pydantic-settings>=2.4.0",
+  "openpyxl>=3.1.5,<4",
+  "python-multipart>=0.0.9",
+  "requests>=2.32.0",
+  "sqlalchemy>=2.0.32",
+  "uvicorn[standard]>=0.30.0",
+]
+
+[project.optional-dependencies]
+dev = [
+  "pytest>=8.3.0",
+]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["."]

requirements.txt

@@ -6,6 +6,7 @@ jinja2>=3.1.4
 lxml>=5.2.0
 psycopg[binary]>=3.2.0
 pydantic-settings>=2.4.0
+openpyxl>=3.1.5,<4
 python-multipart>=0.0.9
 requests>=2.32.0
 sqlalchemy>=2.0.32

scripts/import-publications-run

@@ -0,0 +1,25 @@
+#!/bin/sh
+# Install root-owned; caller cannot choose container, executable, input or env.
+set -eu
+case "${1:-}" in
+    dry-run) mode=dry-run ;;
+    apply) mode=apply ;;
+    *) exit 1 ;;
+esac
+[ "$#" -eq 1 ] || exit 1
+if [ "$mode" = apply ]; then
+    exec /usr/bin/docker exec --user 65534:65534 \
+        -e PGSERVICEFILE=/opt/miem-import/pg_service.conf \
+        -e PGSERVICE=miem_publication_import \
+        -e PGPASSFILE=/opt/miem-import/pgpass \
+        -e PYTHONNOUSERSITE=1 \
+        miem_workers-api-1 python -I /opt/miem-import/import_publications.py \
+        /opt/miem-import/articles.csv --apply --report /tmp/miem-import/apply.json
+fi
+exec /usr/bin/docker exec --user 65534:65534 \
+    -e PGSERVICEFILE=/opt/miem-import/pg_service.conf \
+    -e PGSERVICE=miem_publication_import \
+    -e PGPASSFILE=/opt/miem-import/pgpass \
+    -e PYTHONNOUSERSITE=1 \
+    miem_workers-api-1 python -I /opt/miem-import/import_publications.py \
+    /opt/miem-import/articles.csv --report /tmp/miem-import/dry-run.json

scripts/import-publications-ssh

@@ -0,0 +1,8 @@
+#!/bin/sh
+# Forced SSH command: no arguments, paths, forwarding or shell from the client.
+set -eu
+case "${SSH_ORIGINAL_COMMAND:-}" in
+    'import-publications dry-run') exec sudo -n /usr/local/sbin/miem-publication-import dry-run ;;
+    'import-publications apply') exec sudo -n /usr/local/sbin/miem-publication-import apply ;;
+    *) printf '%s\n' 'Allowed: import-publications dry-run | import-publications apply' >&2; exit 1 ;;
+esac

scripts/import_publications.py

@@ -0,0 +1,192 @@
+"""Owner-run CSV enrichment; no app startup, .env loading, DDL or deletes."""
+
+import argparse
+import csv
+import hashlib
+import json
+import re
+import unicodedata
+from collections import Counter, defaultdict
+from pathlib import Path
+
+
+def normalized(value):
+    return " ".join(unicodedata.normalize("NFKC", value or "").casefold().replace("ё", "е").split())
+
+
+def doi(value):
+    return re.sub(r"^(?:https?://(?:dx\.)?doi\.org/|doi:\s*)", "", (value or "").strip(), flags=re.I).lower()
+
+
+def read_rows(path):
+    with path.open(encoding="utf-8-sig", newline="") as source:
+        reader = csv.DictReader(source, delimiter=";")
+        required = {"Название публикации", "Год", "DOI", "Авторы НИУ ВШЭ"}
+        if not required <= set(reader.fieldnames or []):
+            raise ValueError("Missing required CSV headers")
+        rows = []
+        for number, row in enumerate(reader, 2):
+            if None in row or any(v is None for v in row.values()):
+                raise ValueError(f"Malformed CSV record {number}")
+            rows.append((number, {k.strip(): v.strip() for k, v in row.items()}))
+        return rows
+
+
+def plan(rows, employees, publications):
+    names = defaultdict(list)
+    by_employee = defaultdict(list)
+    for employee in employees:
+        if normalized(employee["full_name"]):
+            names[normalized(employee["full_name"])].append(employee["id"])
+    for publication in publications:
+        by_employee[publication["employee_id"]].append(dict(publication))
+    actions, audit, seen = [], [], set()
+    for number, row in rows:
+        title, identifier = row["Название публикации"], doi(row["DOI"])
+        author_names = sorted(set(normalized(a) for a in row["Авторы НИУ ВШЭ"].split(";")) - {""})
+        year = int(row["Год"]) if row["Год"].isdigit() else None
+        if not title or not year or not 1000 <= year <= 9999 or not author_names or (identifier and not re.fullmatch(r"10\.\d{4,9}/\S+", identifier)):
+            audit.append({"row": number, "employee_id": None, "result": "skip", "reason": "invalid_fields"})
+            continue
+        for author in author_names:
+            ids = names.get(author, [])
+            entry = {"row": number, "employee_id": ids[0] if len(ids) == 1 else None}
+            def skip(reason):
+                audit.append({**entry, "result": "skip", "reason": reason})
+            if len(ids) != 1:
+                skip("employee_not_found" if not ids else "ambiguous_employee")
+                continue
+            employee_id = ids[0]
+            candidates = by_employee[employee_id]
+            exact = [p for p in candidates if identifier and doi(p.get("doi_url")) == identifier]
+            title_matches = [p for p in candidates if normalized(p["title"]) == normalized(title) and p.get("year") in (None, year)]
+            matches = {p["id"]: p for p in exact + title_matches}
+            if len(matches) > 1:
+                skip("ambiguous_publication")
+                continue
+            existing = next(iter(matches.values()), None)
+            if existing and identifier and doi(existing.get("doi_url")) and doi(existing["doi_url"]) != identifier:
+                skip("doi_conflict")
+                continue
+            if existing and existing.get("year") and existing["year"] != year:
+                skip("year_conflict")
+                continue
+            key = (employee_id, identifier or (normalized(title), year))
+            duplicate = key in seen
+            seen.add(key)
+            metadata = {k: v for k, v in row.items() if v}
+            raw = existing.get("raw_data") if existing else None
+            if raw is not None and not isinstance(raw, dict):
+                skip("unsupported_raw_data")
+                continue
+            raw = dict(raw or {})
+            old_metadata = raw.get("csv_enrichment", {})
+            if not isinstance(old_metadata, dict):
+                skip("unsupported_csv_metadata")
+                continue
+            conflicts = sorted(k for k, v in metadata.items() if old_metadata.get(k) and old_metadata[k] != v)
+            if conflicts:
+                entry["conflict_fields"] = conflicts
+            merged = dict(old_metadata)
+            for k, v in metadata.items():
+                if not merged.get(k):
+                    merged[k] = v
+            raw["csv_enrichment"] = merged
+            values = {"raw_data": raw}
+            if identifier and (not existing or not existing.get("doi_url")):
+                values["doi_url"] = "https://doi.org/" + identifier
+            if not existing or not existing.get("year"):
+                values["year"] = year
+            if not existing or not existing.get("title"):
+                values["title"] = title
+            if existing:
+                values = {k: v for k, v in values.items() if existing.get(k) != v}
+                if not values:
+                    skip("metadata_conflict_first_value_kept" if conflicts else "csv_duplicate" if duplicate else "already_present")
+                    continue
+                if isinstance(existing["id"], str):
+                    actions[int(existing["id"].split("-")[1])]["values"].update(values)
+                    existing.update(values)
+                    skip("metadata_conflict_first_value_kept" if conflicts else "merged_csv_duplicate")
+                    continue
+                action = {"result": "update", "id": existing["id"], "values": values}
+                existing.update(values)
+            else:
+                source_hash = hashlib.sha256(json.dumps(key, ensure_ascii=False, sort_keys=True).encode()).hexdigest()
+                values.update(employee_id=employee_id, source_hash=source_hash)
+                action = {"result": "insert", "values": values}
+                by_employee[employee_id].append({**values, "id": f"planned-{len(actions)}"})
+            actions.append(action)
+            audit.append({**entry, "result": action["result"], "reason": "metadata_conflict_first_value_kept" if conflicts else ""})
+    return actions, audit
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("file", type=Path)
+    parser.add_argument("--apply", action="store_true", help="Commit; default is read-only dry-run")
+    parser.add_argument("--report", type=Path, required=True)
+    args = parser.parse_args()
+    pending_report = args.report.with_suffix(args.report.suffix + ".pending")
+    if args.file.resolve() in (args.report.resolve(), pending_report.resolve()):
+        raise ValueError("Report must not overwrite input")
+    source_digest = hashlib.sha256(args.file.read_bytes()).hexdigest()
+    rows = read_rows(args.file)
+    # libpq environment/service only; never load application settings or .env.
+    import psycopg
+    from psycopg.rows import dict_row
+    from psycopg.types.json import Jsonb
+    from psycopg import sql
+
+    # Check report access before any possible commit; pending file is not a success report.
+    with pending_report.open("x", encoding="utf-8"):
+        pass
+    with psycopg.connect("", row_factory=dict_row) as connection:
+        connection.execute("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE" + ("" if args.apply else " READ ONLY"))
+        connection.execute("SET LOCAL search_path = pg_catalog, public")
+        connection.execute("SET LOCAL statement_timeout = '60s'")
+        connection.execute("SET LOCAL lock_timeout = '5s'")
+        if args.apply:
+            locked = connection.execute("SELECT pg_try_advisory_xact_lock(716203841)").fetchone()
+            if not next(iter(locked.values())):
+                raise RuntimeError("Another import is running")
+        employees = connection.execute("SELECT id, full_name FROM public.employees").fetchall()
+        publications = connection.execute("SELECT id, employee_id, title, year, doi_url, raw_data FROM public.employee_publications").fetchall()
+        actions, audit = plan(rows, employees, publications)
+        if args.apply:
+            for action in actions:
+                values = dict(action["values"])
+                if "raw_data" in values:
+                    values["raw_data"] = Jsonb(values["raw_data"])
+                columns = list(values)
+                if action["result"] == "insert":
+                    query = sql.SQL("INSERT INTO public.employee_publications ({}, created_at, updated_at) VALUES ({}, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)").format(
+                        sql.SQL(", ").join(map(sql.Identifier, columns)), sql.SQL(", ").join(sql.Placeholder() for _ in columns))
+                    connection.execute(query, list(values.values()))
+                else:
+                    query = sql.SQL("UPDATE public.employee_publications SET {}, updated_at = CURRENT_TIMESTAMP WHERE id = %s").format(
+                        sql.SQL(", ").join(sql.SQL("{} = %s").format(sql.Identifier(k)) for k in columns))
+                    result = connection.execute(query, [*values.values(), action["id"]])
+                    if result.rowcount != 1:
+                        raise RuntimeError("Publication changed concurrently")
+        if hashlib.sha256(args.file.read_bytes()).hexdigest() != source_digest:
+            raise RuntimeError("Input changed during import")
+        summary = {"mode": "apply" if args.apply else "dry-run", "csv_rows": len(rows),
+                   "sha256": source_digest,
+                   "counts": {key: sum(item["result"] == key for item in audit) for key in ("insert", "update", "skip")},
+                   "metadata_conflicts": sum(bool(item.get("conflict_fields")) for item in audit),
+                   "skip_reasons": dict(Counter(item["reason"] for item in audit if item["result"] == "skip"))}
+        pending_report.write_text(json.dumps({"summary": summary, "records": audit}, ensure_ascii=False, indent=2), encoding="utf-8")
+        if not args.apply:
+            connection.rollback()
+    pending_report.replace(args.report)
+    print(json.dumps(summary, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as error:
+        # Driver errors may contain connection credentials or personal data.
+        print(f"Import failed ({type(error).__name__}); transaction not partially applied. Check configuration and permissions.")
+        raise SystemExit(1)

tests/browser_admin.py

@@ -0,0 +1,143 @@
+"""Run from the repository root: python tests/browser_admin.py.
+
+Requires Playwright and Chromium. Uses in-memory data and intercepted requests;
+never contacts the production service. Optional BROWSER_EXECUTABLE selects an
+already installed Chromium instead of the Playwright download.
+"""
+import os
+from pathlib import Path
+import sys
+from tempfile import mkdtemp
+from urllib.parse import urlsplit
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from fastapi.testclient import TestClient
+from playwright.sync_api import expect, sync_playwright
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.pool import StaticPool
+
+from app.config import Settings, get_settings
+from app.db import Base, get_db
+from app.main import app
+from app.models import Employee
+from app.security import SESSION_COOKIE, sign_session
+
+
+def main():
+    engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
+    Base.metadata.create_all(engine)
+    factory = sessionmaker(bind=engine)
+    with factory() as db:
+        db.add_all(Employee(profile_key=f"staff:{i}", canonical_url=f"https://www.hse.ru/staff/{i}",
+                            full_name=f"Сотрудник {i:02d}", current_data={"positions": ["Преподаватель"]})
+                   for i in range(30))
+        db.commit()
+
+    def database():
+        with factory() as db:
+            yield db
+
+    settings = Settings(_env_file=None, session_secret="browser-test-secret")
+    app.dependency_overrides[get_db] = database
+    app.dependency_overrides[get_settings] = lambda: settings
+    client = TestClient(app)
+    client.cookies.set(SESSION_COOKIE, sign_session(settings.admin_username, settings))
+    output = Path(mkdtemp(prefix="miem-admin-browser-"))
+    pending = []
+
+    def serve(route):
+        url = urlsplit(route.request.url)
+        if url.path == "/api/crawl-runs/latest":
+            pending.append(route)
+            return
+        response = client.get(url.path + ("?" + url.query if url.query else ""))
+        route.fulfill(status=response.status_code, body=response.content,
+                      content_type=response.headers.get("content-type", "text/plain"))
+
+    try:
+        with sync_playwright() as playwright:
+            browser = playwright.chromium.launch(executable_path=os.getenv("BROWSER_EXECUTABLE"))
+            page = browser.new_page(viewport={"width": 1440, "height": 1000})
+            page.route("**/*", serve)
+            errors = []
+            page.on("pageerror", lambda error: errors.append(str(error)))
+            page.goto("http://miem.test/admin/directory?limit=25&offset=25")
+            expect(page.get_by_role("status")).to_have_text("Показаны 26–30 из 30")
+            assert page.locator("th:visible").count() == 5
+            assert page.locator(".directory__table-wrap").evaluate("e => e.scrollWidth <= e.clientWidth")
+            skip_link = page.get_by_role("link", name="Перейти к содержимому")
+            assert skip_link.evaluate("e => getComputedStyle(e).left === '-999px'")
+            page.keyboard.press("Tab")
+            expect(skip_link).to_be_focused()
+            assert skip_link.evaluate("e => getComputedStyle(e).left === '8px'")
+            page.keyboard.press("Tab")
+            assert skip_link.evaluate("e => getComputedStyle(e).left === '-999px'")
+
+            for width in (1440, 1024, 921, 920, 768, 621, 620, 390, 320):
+                page.set_viewport_size({"width": width, "height": 844})
+                assert page.locator(".directory__filters").evaluate("e => e.scrollWidth <= e.clientWidth")
+                assert page.locator(".directory__input").evaluate_all("""controls => controls.every(control => {
+                  const field = control.closest('.directory__field').getBoundingClientRect();
+                  const rect = control.getBoundingClientRect();
+                  return rect.left >= field.left - 1 && rect.right <= field.right + 1;
+                })""")
+            page.set_viewport_size({"width": 1440, "height": 1000})
+            page.get_by_label("ФИО или ссылка").fill("Сотрудник 00")
+            page.get_by_role("button", name="Применить фильтры").click()
+            expect(page.get_by_role("status")).to_have_text("Показаны 1–1 из 1")
+            assert "offset=" not in page.url
+            trigger = page.get_by_role("button", name="Настроить колонки")
+            trigger.focus()
+            page.keyboard.press("Enter")
+            dialog = page.get_by_role("dialog", name="Отображаемые колонки")
+            expect(dialog).to_be_visible()
+            for _ in range(22):
+                page.keyboard.press("Tab")
+                # Native dialogs may let Tab reach browser chrome, never background controls.
+                assert dialog.evaluate("e => document.activeElement === document.body || e.contains(document.activeElement)")
+            page.keyboard.press("Escape")
+            expect(trigger).to_be_focused()
+            trigger.click()
+            page.get_by_role("button", name="Контакты", exact=True).click()
+            page.keyboard.press("Escape")
+            page.reload()
+            expect(page.locator('th[data-column="email"]')).to_be_visible()
+            page.evaluate("localStorage.clear()")
+            page.goto("http://miem.test/admin/directory")
+            page.screenshot(path=str(output / "desktop.png"), full_page=True)
+            page.set_viewport_size({"width": 390, "height": 844})
+            assert page.evaluate("document.documentElement.scrollWidth <= innerWidth")
+            assert page.locator('th[data-column="full_name"]').evaluate("e => getComputedStyle(e).position") == "sticky"
+            page.screenshot(path=str(output / "mobile.png"), full_page=True)
+            trigger.click()
+            expect(dialog).to_be_visible()
+            assert dialog.evaluate("e => e.getBoundingClientRect().right <= innerWidth")
+            page.screenshot(path=str(output / "mobile-dialog.png"), full_page=True)
+            page.keyboard.press("Escape")
+            page.get_by_label("ФИО или ссылка").fill("Нет совпадений")
+            page.get_by_role("button", name="Применить фильтры").click()
+            expect(page.get_by_text("Сотрудники не найдены", exact=True)).to_be_visible()
+            expect(page.get_by_role("link", name="Сбросить фильтры", exact=True)).to_be_visible()
+            page.goto("http://miem.test/admin")
+            page.wait_for_function("document.querySelector('[data-progress-panel]').getAttribute('aria-busy') === 'true'")
+            pending.pop(0).fulfill(status=503, body="unavailable")
+            expect(page.locator("[data-progress-error]")).to_be_visible()
+            expect(page.get_by_text("Показаны последние полученные данные.", exact=False)).to_be_visible()
+            page.get_by_role("button", name="Повторить").click()
+            page.wait_for_timeout(100)
+            pending.pop(0).fulfill(json={"running": None, "latest": None})
+            expect(page.locator("[data-progress-error]")).to_be_hidden()
+            expect(page.get_by_text("Запусков пока нет. Запустите парсинг.", exact=True)).to_be_visible()
+            assert not errors, errors
+            browser.close()
+    finally:
+        app.dependency_overrides.clear()
+        client.close()
+        engine.dispose()
+    print(f"Browser checks passed; screenshots: {output}")
+
+
+if __name__ == "__main__":
+    main()

tests/browser_council.py

@@ -0,0 +1,154 @@
+"""Offline public council UX check: python tests/browser_council.py.
+
+Requires Playwright/Chromium; BROWSER_EXECUTABLE may select an installed browser.
+Synthetic employees and intercepted requests; no production access.
+"""
+import os
+from io import BytesIO
+from pathlib import Path
+import sys
+from urllib.parse import urlsplit
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from fastapi.testclient import TestClient
+from openpyxl import load_workbook
+from playwright.sync_api import expect, sync_playwright
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.pool import StaticPool
+
+from app.db import Base, get_db
+from app.main import app
+from app.models import Employee, EmployeePublication
+from app.services import council
+
+
+def main():
+    engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
+    Base.metadata.create_all(engine)
+    factory = sessionmaker(bind=engine)
+    with factory() as db:
+        for number in range(28):
+            person = Employee(profile_key=f"test:{number}", full_name=f"Сотрудник {number:02d}",
+                              canonical_url="https://www.hse.ru/staff/test",
+                              current_data={"sections": [{"title": "Учёные степени",
+                                            "items": ["Доктор технических наук"]}]})
+            db.add(person)
+            db.flush()
+            db.add(EmployeePublication(employee_id=person.id, title="Исследование методов передачи данных в фотонных интегральных схемах",
+                                       year=2024, publication_type="ARTICLE", publication_id=f"{number}",
+                                       url=f"https://publications.hse.ru/view/{number}", source_hash=f"test:{number}"))
+            if number == 27:
+                db.add(EmployeePublication(employee_id=person.id, title="Дополнительная статья", year=2024,
+                                           publication_type="ARTICLE", source_hash="test:extra"))
+        for name, degree in [("Без степени", None), ("Кандидат наук", "Кандидат технических наук")]:
+            person = Employee(profile_key=name, full_name=name, canonical_url="https://www.hse.ru/staff/test",
+                              current_data={"sections": [{"title": "Учёные степени", "items": [degree] if degree else []}]})
+            db.add(person)
+            db.flush()
+            db.add(EmployeePublication(employee_id=person.id, title="Не включать в совет", year=2024, source_hash=name))
+        db.commit()
+
+    def database():
+        with factory() as db:
+            yield db
+
+    original = council._articles_by_author
+    council._articles_by_author = lambda: {"сотрудник 26": [
+        {"title": f"Публикация из CSV {number}", "year": 2024, "doi": f"10.1234/test-{number}",
+         "types": [], "quartiles": ["Scopus Q1"], "origins": ["Файл CSV"], "rows": [number + 2],
+         "in_csv": True, "in_db": False, "url": None} for number in range(2)]}
+    app.dependency_overrides[get_db] = database
+    client = TestClient(app)
+    output = Path(".impeccable/review")
+    output.mkdir(parents=True, exist_ok=True)
+
+    def serve(route):
+        url = urlsplit(route.request.url)
+        response = client.get(url.path + ("?" + url.query if url.query else ""))
+        route.fulfill(status=response.status_code, body=response.content,
+                      headers={key: value for key, value in response.headers.items()
+                               if key in {"content-type", "content-disposition"}})
+
+    try:
+        with sync_playwright() as playwright:
+            browser = playwright.chromium.launch(executable_path=os.getenv("BROWSER_EXECUTABLE"))
+            page = browser.new_page(viewport={"width": 1440, "height": 1000})
+            page.route("**/*", serve)
+            errors = []
+            page.on("pageerror", lambda error: errors.append(str(error)))
+            page.goto("http://miem.test/admin/council")
+            expect(page.get_by_role("status")).to_have_text("Показаны 1–25 из 28")
+            assert "Без степени" not in page.content() and "Кандидат наук" not in page.content()
+            expect(page.get_by_role("link", name="Вход в админку")).to_be_visible()
+            page.get_by_role("link", name="Далее").click()
+            expect(page.get_by_role("status")).to_have_text("Показаны 26–28 из 28")
+            page.get_by_role("link", name="Назад").click()
+            summary = page.locator(".council__candidate-summary").first
+            summary.focus()
+            page.keyboard.press("Enter")
+            expect(page.get_by_text("БД miem-workers · сайт ВШЭ").first).to_be_visible()
+            expect(page.get_by_text("Q1/Q2, К1 и RSCI не подтверждены доступными данными").first).to_be_visible()
+            assert page.locator(".council__candidate").first.evaluate("e => e.open")
+            page.screenshot(path=str(output / "desktop.png"), full_page=True)
+            page.screenshot(path=str(output / "sorting-desktop.png"), full_page=False)
+            for width in (1024, 768, 700, 390, 320):
+                page.set_viewport_size({"width": width, "height": 844})
+                assert page.evaluate("document.documentElement.scrollWidth <= innerWidth"), width
+                assert summary.evaluate("e => e.scrollWidth <= e.clientWidth"), width
+            page.set_viewport_size({"width": 390, "height": 844})
+            page.screenshot(path=str(output / "mobile.png"), full_page=True)
+            page.screenshot(path=str(output / "sorting-mobile.png"), full_page=False)
+            for sort, direction, first in [("full_name", "asc", "Сотрудник 00"),
+                                            ("full_name", "desc", "Сотрудник 27"),
+                                            ("db_count", "desc", "Сотрудник 27"),
+                                            ("count", "desc", "Сотрудник 26"),
+                                            ("csv_count", "desc", "Сотрудник 26")]:
+                page.get_by_label("Сортировать по").select_option(sort)
+                page.get_by_label("Направление").select_option(direction)
+                page.get_by_role("button", name="Применить", exact=True).click()
+                expect(page.locator(".council__candidate-identity strong").first).to_have_text(first)
+            page.get_by_label("Сортировать по").select_option("full_name")
+            page.get_by_label("Направление").select_option("desc")
+            page.get_by_role("button", name="Применить", exact=True).click()
+            page.get_by_role("link", name="Далее").click()
+            expect(page.get_by_label("Сортировать по")).to_have_value("full_name")
+            expect(page.get_by_label("Направление")).to_have_value("desc")
+            expect(page.locator(".council__candidate-identity strong").first).to_have_text("Сотрудник 02")
+            with page.expect_download() as pending_download:
+                page.get_by_role("button", name="Скачать XLSX", exact=True).click()
+            download = pending_download.value
+            assert download.suggested_filename == "council.xlsx"
+            download.save_as(str(output / "council.xlsx"))
+            workbook = load_workbook(output / "council.xlsx")
+            assert workbook["Сотрудники"].max_row == 29
+            assert workbook["Сотрудники"]["B2"].value == "Сотрудник 27"
+            page.get_by_label("Поиск по ФИО").fill("Сотрудник 27")
+            page.get_by_role("button", name="Применить", exact=True).click()
+            expect(page.get_by_role("status")).to_have_text("Показаны 1–1 из 1 · Всего в списке: 28")
+            expect(page.get_by_label("Сортировать по")).to_have_value("full_name")
+            expect(page.get_by_label("Направление")).to_have_value("desc")
+            with page.expect_download() as pending_download:
+                page.get_by_role("button", name="Скачать XLSX", exact=True).click()
+            filtered = load_workbook(BytesIO(Path(pending_download.value.path()).read_bytes()))
+            assert filtered["Сотрудники"].max_row == 2
+            page.get_by_label("Поиск по ФИО").fill("Нет совпадений")
+            page.get_by_role("button", name="Применить", exact=True).click()
+            expect(page.get_by_text("Сотрудники не найдены", exact=True)).to_be_visible()
+            page.get_by_role("link", name="Сбросить поиск").first.click()
+            expect(page.get_by_role("status")).to_have_text("Показаны 1–25 из 28")
+            expect(page.get_by_label("Сортировать по")).to_have_value("full_name")
+            expect(page.get_by_label("Направление")).to_have_value("desc")
+            assert not errors, errors
+            browser.close()
+    finally:
+        council._articles_by_author = original
+        app.dependency_overrides.clear()
+        client.close()
+        engine.dispose()
+    print(f"Council browser checks passed; screenshots: {output}")
+
+
+if __name__ == "__main__":
+    main()

tests/postgres/Dockerfile

@@ -0,0 +1,7 @@
+FROM python:3.12-slim
+WORKDIR /app
+COPY requirements.txt .
+RUN python -m pip install --no-cache-dir -r requirements.txt
+COPY app ./app
+COPY scripts/import_publications.py ./scripts/import_publications.py
+COPY tests/conftest.py tests/test_publication_import_postgres.py ./tests/

tests/postgres/Dockerfile.dockerignore

@@ -0,0 +1,18 @@
+**
+!requirements.txt
+!app/
+!app/**
+!scripts/
+!scripts/import_publications.py
+!tests/
+!tests/conftest.py
+!tests/test_publication_import_postgres.py
+!tests/postgres/
+!tests/postgres/Dockerfile
+!tests/postgres/Dockerfile.dockerignore
+app/data/
+**/__pycache__/
+**/.env*
+**/*credentials*
+**/*.key
+**/*.pem

tests/postgres/Dockerfile.postgres

@@ -0,0 +1,2 @@
+FROM postgres:16-alpine
+COPY init.sql /docker-entrypoint-initdb.d/10-test.sql

tests/postgres/Dockerfile.postgres.dockerignore

@@ -0,0 +1,4 @@
+**
+!init.sql
+!Dockerfile.postgres
+!Dockerfile.postgres.dockerignore

tests/postgres/compose.vps.yml

@@ -0,0 +1,69 @@
+name: miem-publication-integration-vps
+
+services:
+  postgres-test:
+    image: miem-publication-integration-postgres:pr55
+    pull_policy: never
+    environment:
+      POSTGRES_USER: test_admin
+      POSTGRES_PASSWORD: isolated_test_admin
+      POSTGRES_DB: miem_import_test
+    command: ["postgres", "-c", "shared_buffers=64MB", "-c", "max_connections=12"]
+    cpus: 0.5
+    mem_limit: 512m
+    memswap_limit: 512m
+    pids_limit: 128
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "test_admin", "-d", "miem_import_test"]
+      interval: 2s
+      timeout: 5s
+      retries: 30
+    logging:
+      driver: json-file
+      options:
+        max-size: "5m"
+        max-file: "1"
+    networks: [isolated]
+
+  tests:
+    image: miem-publication-integration-runner:pr55
+    pull_policy: never
+    user: "65534:65534"
+    read_only: true
+    cap_drop: [ALL]
+    security_opt: ["no-new-privileges:true"]
+    cpus: 0.5
+    mem_limit: 512m
+    memswap_limit: 512m
+    pids_limit: 128
+    tmpfs:
+      - /tmp:size=128m,mode=1777
+    environment:
+      DATABASE_URL: postgresql+psycopg://test_admin:isolated_test_admin@postgres-test:5432/miem_import_test
+      MIEM_POSTGRES_INTEGRATION: "1"
+      PYTHONDONTWRITEBYTECODE: "1"
+      PYTEST_ADDOPTS: "-p no:cacheprovider"
+    command: ["python", "-m", "pytest", "-q", "tests/test_publication_import_postgres.py"]
+    depends_on:
+      postgres-test:
+        condition: service_healthy
+    logging:
+      driver: json-file
+      options:
+        max-size: "5m"
+        max-file: "1"
+    networks: [isolated]
+
+networks:
+  isolated:
+    internal: true
+
+volumes:
+  pgdata:
+    driver: local
+    driver_opts:
+      type: tmpfs
+      device: tmpfs
+      o: size=256m

tests/postgres/compose.yml

@@ -0,0 +1,36 @@
+name: miem-publication-integration
+
+services:
+  postgres-test:
+    image: postgres:16-alpine
+    environment:
+      POSTGRES_USER: test_admin
+      POSTGRES_PASSWORD: isolated_test_admin
+      POSTGRES_DB: miem_import_test
+    tmpfs:
+      - /var/lib/postgresql/data
+    volumes:
+      - ./init.sql:/docker-entrypoint-initdb.d/10-test.sql:ro
+    healthcheck:
+      test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "test_admin", "-d", "miem_import_test"]
+      interval: 2s
+      timeout: 5s
+      retries: 30
+    networks: [isolated]
+
+  tests:
+    build:
+      context: ../..
+      dockerfile: tests/postgres/Dockerfile
+    environment:
+      DATABASE_URL: postgresql+psycopg://test_admin:isolated_test_admin@postgres-test:5432/miem_import_test
+      MIEM_POSTGRES_INTEGRATION: "1"
+    command: ["python", "-m", "pytest", "-q", "tests/test_publication_import_postgres.py"]
+    depends_on:
+      postgres-test:
+        condition: service_healthy
+    networks: [isolated]
+
+networks:
+  isolated:
+    internal: true

tests/postgres/init.sql

@@ -0,0 +1,6 @@
+-- Synthetic credentials for this ephemeral, private test cluster only.
+CREATE DATABASE miem_crawl_test OWNER test_admin;
+CREATE ROLE import_test LOGIN PASSWORD 'isolated_test_import'
+    NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS
+    CONNECTION LIMIT 1;
+GRANT CONNECT ON DATABASE miem_import_test, miem_crawl_test TO import_test;

tests/test_academic_degrees.py

@@ -0,0 +1,26 @@
+from app.services.academic_degrees import academic_degrees
+
+
+def test_academic_degrees_extracts_all_degree_names_without_surrounding_text():
+    data = {
+        "sections": [
+            {
+                "title": "Образование",
+                "year_entries": [
+                    {"text": "2008 — кандидат технических наук, доцент"},
+                    {"text": "2018 — Доктор физико-математических наук"},
+                ],
+            },
+            {
+                "title": "Учёные степени",
+                "table": {"rows": [{"cells": ["Ph.D.", "Доктор физико-математических наук"]}]},
+            },
+            {"title": "Публикации", "items": ["Доктор медицинских наук выступил автором статьи"]},
+        ]
+    }
+
+    assert academic_degrees(data) == [
+        "кандидат технических наук",
+        "Доктор физико-математических наук",
+        "Ph.D.",
+    ]

tests/test_admin_data.py

@@ -1,7 +1,23 @@
 from datetime import datetime, timezone
 
-from app.models import CrawlRun, Employee
-from app.services.admin_data import employee_display_payload, list_employees_page, run_payload, stats_payload
+from app.models import CrawlError, CrawlRun, CrawlRunEmployeeChange, Employee, EmployeeNewsLink
+from app.services.admin_data import (
+    employee_detail_payload,
+    employee_display_payload,
+    format_admin_datetime,
+    list_employees_page,
+    run_detail_payload,
+    run_payload,
+    stats_payload,
+)
+
+
+def test_format_admin_datetime_handles_datetime_string_and_none():
+    value = datetime(2026, 4, 28, 17, 13, 34, tzinfo=timezone.utc)
+
+    assert format_admin_datetime(value) == "28.04.2026 20:13"
+    assert format_admin_datetime("2026-04-28T17:13:34.448605+00:00") == "28.04.2026 20:13"
+    assert format_admin_datetime(None) == "Не указано"
 
 
 def test_employee_display_payload_extracts_common_fields(db_session):
@@ -19,6 +35,7 @@ def test_employee_display_payload_extracts_common_fields(db_session):
             "sections": [
                 {"type": "publications", "publications": [{"title": "Paper"}]},
                 {"type": "courses_by_year", "courses": [{"title": "Course"}]},
+                {"type": "news", "news_links": [{"title": "News", "url": "https://example.test/news"}]},
             ],
         },
     )
@@ -26,9 +43,197 @@ def test_employee_display_payload_extracts_common_fields(db_session):
     payload = employee_display_payload(employee)
 
     assert payload["positions_text"] == "Professor"
+    assert payload["status_display"] == "Работает"
     assert payload["email_text"] == "person@hse.ru"
     assert payload["publications_count"] == 1
     assert payload["courses_count"] == 1
+    assert payload["news_count"] == 1
+    assert payload["first_seen_display"] != "Не указано"
+
+
+def test_list_employees_page_filters_and_displays_academic_degrees(db_session):
+    db_session.add_all(
+        [
+            Employee(
+                profile_key="staff:degree",
+                canonical_url="https://www.hse.ru/staff/degree",
+                full_name="Doctor",
+                status="active",
+                first_seen_at=datetime.now(timezone.utc),
+                last_seen_at=datetime.now(timezone.utc),
+                has_academic_degree=True,
+                current_data={
+                    "sections": [
+                        {
+                            "title": "Образование и учёные степени",
+                            "year_entries": [{"year": 2020, "text": "Доктор технических наук"}],
+                        }
+                    ]
+                },
+            ),
+            Employee(
+                profile_key="staff:no-degree",
+                canonical_url="https://www.hse.ru/staff/no-degree",
+                full_name="Master",
+                status="active",
+                first_seen_at=datetime.now(timezone.utc),
+                last_seen_at=datetime.now(timezone.utc),
+                has_academic_degree=False,
+                current_data={"sections": [{"title": "Образование", "items": ["Магистратура"]}]},
+            ),
+        ]
+    )
+    db_session.commit()
+
+    page = list_employees_page(db_session, has_academic_degree=True)
+
+    assert page["total"] == 1
+    assert page["employees"][0]["full_name"] == "Doctor"
+    assert page["employees"][0]["academic_degree_text"] == "Доктор технических наук"
+
+    page = list_employees_page(db_session, has_academic_degree=False)
+
+    assert page["employees"][0]["full_name"] == "Master"
+
+
+def test_employee_detail_payload_normalizes_human_readable_sections(db_session):
+    employee = Employee(
+        profile_key="staff:person",
+        profile_type="staff",
+        profile_id="person",
+        canonical_url="https://www.hse.ru/staff/person",
+        full_name="Person Name",
+        status="active",
+        first_seen_at=datetime.now(timezone.utc),
+        last_seen_at=datetime.now(timezone.utc),
+        current_data={
+            "positions": ["Professor"],
+            "hse_start_year": 2024,
+            "contacts": {
+                "emails": ["person@hse.ru"],
+                "phones": ["+79990000000"],
+                "address": "Moscow",
+                "items": [{"raw": "consultation hours"}],
+            },
+            "external_ids": [{"system": "ORCID", "value": "0000", "url": "https://orcid.org/0000"}],
+            "sections": [
+                {
+                    "title": "Education",
+                    "type": "year_blocks",
+                    "year_entries": [{"year": 2020, "text": "Master degree"}],
+                },
+                {
+                    "title": "Publications",
+                    "type": "publications",
+                    "publications": [{"title": "Paper", "text": "Paper details", "url": "https://example.test/paper"}],
+                },
+                {
+                    "title": "Courses",
+                    "type": "courses_by_year",
+                    "academic_year": "2025/2026",
+                    "courses": [{"title": "Course", "url": "https://example.test/course"}],
+                },
+                {
+                    "title": "ВКР",
+                    "type": "graduation_theses",
+                    "theses_count": 1,
+                    "theses": [
+                        {
+                            "student": "Student Name",
+                            "title": "Thesis title",
+                            "defense_year": 2025,
+                            "project_url": "https://www.hse.ru/edu/vkr/1",
+                        }
+                    ],
+                },
+                {
+                    "title": "Fallback",
+                    "type": "generic",
+                    "raw_text": "Fallback text",
+                },
+                {
+                    "title": "В новостях",
+                    "type": "news",
+                    "news_links": [
+                        {
+                            "title": "News title",
+                            "url": "https://example.test/news",
+                            "summary": "News summary",
+                            "published_at": "2026-04-28T00:00:00+00:00",
+                            "published_year": 2026,
+                        }
+                    ],
+                },
+            ],
+        },
+    )
+
+    payload = employee_detail_payload(employee)
+
+    assert payload["contacts"]["emails"] == ["person@hse.ru"]
+    assert payload["contacts"]["contact_items"] == ["consultation hours"]
+    assert payload["external_ids"][0]["system"] == "ORCID"
+    assert payload["sections"][0]["year_entries"][0]["text"] == "Master degree"
+    assert payload["sections"][1]["publications"][0]["title"] == "Paper"
+    assert payload["sections"][2]["courses"][0]["title"] == "Course"
+    assert payload["sections"][3]["theses"][0]["student"] == "Student Name"
+    assert payload["sections"][4]["paragraphs"] == ["Fallback text"]
+    assert payload["sections"][5]["news_links"][0]["title"] == "News title"
+    assert payload["news_links"][0]["published_display"] == "28.04.2026"
+
+
+def test_employee_payload_prefers_stored_news_links(db_session):
+    employee = Employee(
+        profile_key="staff:news",
+        canonical_url="https://www.hse.ru/staff/news",
+        full_name="News Person",
+        status="active",
+        first_seen_at=datetime.now(timezone.utc),
+        last_seen_at=datetime.now(timezone.utc),
+        current_data={"sections": [{"type": "news", "news_links": [{"title": "Old news"}]}]},
+    )
+    db_session.add(employee)
+    db_session.commit()
+    db_session.add(
+        EmployeeNewsLink(
+            employee_id=employee.id,
+            title="Stored news",
+            url="https://example.test/stored",
+            summary="Stored summary",
+            published_at=datetime(2026, 4, 28, tzinfo=timezone.utc),
+            published_year=2026,
+            source_hash="b" * 64,
+        )
+    )
+    db_session.commit()
+
+    display = employee_display_payload(employee)
+    detail = employee_detail_payload(employee)
+
+    assert display["news_count"] == 1
+    assert detail["news_links"][0]["title"] == "Stored news"
+    assert detail["news_links"][0]["published_display"] == "28.04.2026"
+
+
+def test_employee_payloads_tolerate_malformed_current_data(db_session):
+    employee = Employee(
+        profile_key="staff:broken",
+        canonical_url="https://www.hse.ru/staff/broken",
+        full_name="Broken Data",
+        status="active",
+        first_seen_at=datetime.now(timezone.utc),
+        last_seen_at=datetime.now(timezone.utc),
+        current_data="not-a-dict",
+    )
+
+    display = employee_display_payload(employee)
+    detail = employee_detail_payload(employee)
+
+    assert display["positions"] == []
+    assert display["email_text"] == ""
+    assert detail["contacts"]["emails"] == []
+    assert detail["contacts"]["contact_items"] == []
+    assert detail["sections"] == []
 
 
 def test_list_employees_page_filters_sorts_and_paginates(db_session):
@@ -59,7 +264,20 @@ def test_list_employees_page_filters_sorts_and_paginates(db_session):
     page = list_employees_page(db_session, status="active", sort="full_name", direction="asc", limit=10)
 
     assert page["total"] == 1
-    assert page["items"][0]["full_name"] == "Alpha"
+    assert page["employees"][0]["full_name"] == "Alpha"
+    assert page["limit"] == 50
+
+
+def test_catalog_recovers_from_out_of_range_offset(db_session):
+    db_session.add(Employee(profile_key="staff:page", canonical_url="https://www.hse.ru/staff/page", full_name="Page Person"))
+    db_session.commit()
+
+    page = list_employees_page(db_session, q="Page", offset=100, limit=25)
+    assert (page["offset"], page["page"], page["total"]) == (0, 1, 1)
+    assert len(page["employees"]) == 1
+    empty = list_employees_page(db_session, q="missing", offset=100)
+    assert empty["offset"] == 0
+    assert empty["employees"] == []
 
 
 def test_stats_payload_uses_latest_run_new_count(db_session):
@@ -89,10 +307,52 @@ def test_run_payload_calculates_progress():
         status="running",
         found_count=10,
         parsed_count=4,
+        skipped_count=2,
         error_count=1,
     )
 
     payload = run_payload(run)
 
-    assert payload["processed_count"] == 5
-    assert payload["progress_percent"] == 50.0
+    assert payload["processed_count"] == 7
+    assert payload["progress_percent"] == 70.0
+    assert payload["status_display"] == "Выполняется"
+
+
+def test_run_detail_payload_groups_changes_and_handles_old_runs(db_session):
+    old_run = CrawlRun(source_url="https://miem.hse.ru/persons", status="completed")
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="completed", new_count=1)
+    employee = Employee(
+        profile_key="staff:new",
+        canonical_url="https://www.hse.ru/staff/new",
+        full_name="New Person",
+        status="active",
+        first_seen_at=datetime.now(timezone.utc),
+        last_seen_at=datetime.now(timezone.utc),
+    )
+    db_session.add_all([old_run, run, employee])
+    db_session.commit()
+    db_session.add(
+        CrawlRunEmployeeChange(
+            crawl_run_id=run.id,
+            employee_id=employee.id,
+            profile_key=employee.profile_key,
+            profile_url=employee.canonical_url,
+            full_name=employee.full_name,
+            change_type="new",
+            profile_available=True,
+            message="added",
+        )
+    )
+    db_session.add(
+        CrawlError(crawl_run_id=run.id, profile_url=employee.canonical_url, error_type="ValueError", message="bad")
+    )
+    db_session.commit()
+
+    payload = run_detail_payload(db_session, run)
+    old_payload = run_detail_payload(db_session, old_run)
+
+    assert payload["changes_detail_available"] is True
+    assert payload["changes"]["new"][0]["full_name"] == "New Person"
+    assert payload["errors"][0]["error_type"] == "ValueError"
+    assert old_payload["changes_detail_available"] is False
+    assert old_payload["changes"]["new"] == []

tests/test_admin_templates.py

@@ -0,0 +1,133 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+from jinja2 import Environment, FileSystemLoader
+
+
+def test_base_navigation_is_russian_and_has_no_legacy_employees_link():
+    template = Environment(loader=FileSystemLoader("app/templates")).get_template("base.html").render(
+        request=SimpleNamespace(url=SimpleNamespace(path="/admin"))
+    )
+
+    assert "Обзор" in template
+    assert "Сотрудники" in template
+    assert "Запуски" in template
+    assert "Выйти" in template
+    assert '<a class="admin__brand-link" href="/admin">MIEM Employees</a>' in template
+    assert ">Employees<" not in template
+    assert "/admin/employees" not in template
+
+
+def test_directory_template_is_russian_and_uses_display_dates():
+    template = Path("app/templates/directory.html").read_text(encoding="utf-8")
+
+    assert "Сотрудники" in template
+    assert "колонки" in template
+    assert "Применить фильтры" in template
+    assert "Сотрудников на странице" in template
+    assert "{% for value in [25, 50, 100] %}" in template
+    assert "Показаны" in template
+    assert "Новости" in template
+    assert "Есть учёная степень" in template
+    assert 'data-column="academic_degree"' in template
+    assert "employee.news_count" in template
+    assert "employee.first_seen_display" in template
+    assert "employee.last_seen_display" in template
+    assert "employee.dismissed_display" in template
+    assert "verification_required" in template
+    assert '<label class="directory__field">Статус' in template
+    assert 'id="columns-dialog"' in template
+    assert 'aria-controls="columns-dialog"' in template
+    assert 'aria-labelledby="columns-dialog-title"' in template
+    assert "Сбросить фильтры" in template
+    assert 'data-columns-preset="review"' in template
+    assert '<dialog class="columns-modal"' in template
+    assert 'id="columns-dialog"' in template
+    assert 'aria-controls="columns-dialog"' in template
+    assert "Сбросить фильтры" in template
+    assert "Directory" not in template
+    assert "employees found" not in template
+
+
+def test_admin_employees_route_redirects_to_directory():
+    source = Path("app/admin.py").read_text(encoding="utf-8")
+
+    assert 'RedirectResponse("/admin/directory", status_code=303)' in source
+
+
+def test_dashboard_limits_latest_runs_to_five():
+    source = Path("app/admin.py").read_text(encoding="utf-8")
+
+    assert "order_by(desc(CrawlRun.started_at)).limit(5)" in source
+    assert "order_by(desc(CrawlRun.started_at)).limit(10)" not in source
+
+
+def test_runs_template_links_to_run_detail():
+    template = Path("app/templates/runs.html").read_text(encoding="utf-8")
+
+    assert '<a class="admin__link" href="/admin/runs/{{ run.id }}">' in template
+    assert "onclick=" not in template
+    assert 'role="link"' not in template
+
+
+def test_run_detail_template_extends_base_and_shows_change_groups():
+    template = Path("app/templates/run_detail.html").read_text(encoding="utf-8")
+
+    assert '{% extends "base.html" %}' in template
+    assert 'id="new-employees"' in template
+    assert "Новые сотрудники" in template
+    assert "Потеряшки" in template
+    assert "Требуют проверки" in template
+    assert "Уволенные" in template
+    assert "Детализация сотрудников для этого запуска недоступна" in template
+
+
+def test_dashboard_metric_cards_link_to_admin_targets():
+    template = Path("app/templates/dashboard.html").read_text(encoding="utf-8")
+
+    assert 'href="/admin/directory"' in template
+    assert 'href="/admin/directory?status=active"' in template
+    assert 'href="/admin/directory?status=verification_required"' in template
+    assert '/admin/runs/{{ latest_run.id }}#new-employees' in template
+    assert 'href="/admin/directory?status=dismissed"' in template
+    assert 'href="/admin/runs"' in template
+
+
+def test_dashboard_has_dismissed_status_refresh_action():
+    template = Path("app/templates/dashboard.html").read_text(encoding="utf-8")
+
+    assert 'action="/admin/dismissed/refresh"' in template
+    assert "Проверить уволенных" in template
+
+
+def test_progress_template_has_accessible_progressbar():
+    for name in ("dashboard.html", "runs.html"):
+        template = Path(f"app/templates/{name}").read_text(encoding="utf-8")
+        assert 'role="progressbar"' in template
+        assert 'aria-valuemin="0"' in template
+        assert 'aria-valuemax="100"' in template
+        assert "Прогресс парсинга" in template
+
+
+def test_dashboard_latest_run_rows_link_to_run_detail():
+    template = Path("app/templates/dashboard.html").read_text(encoding="utf-8")
+
+    assert '<a class="admin__link" href="/admin/runs/{{ run.id }}">' in template
+    assert "onclick=" not in template
+    assert 'role="link"' not in template
+
+
+def test_admin_js_uses_native_dialog_for_column_settings():
+    source = Path("app/static/admin.js").read_text(encoding="utf-8")
+
+    assert "showModal()" in source
+    assert "modal.close()" in source
+    assert 'setAttribute("aria-expanded", "true")' in source
+    assert "data-progress-retry" in source
+
+
+def test_base_template_has_skip_link_and_current_navigation_state():
+    template = Path("app/templates/base.html").read_text(encoding="utf-8")
+
+    assert 'href="#main-content"' in template
+    assert 'aria-current="page"' in template

tests/test_api.py

@@ -0,0 +1,154 @@
+from datetime import datetime, timezone
+from types import SimpleNamespace
+
+from fastapi.testclient import TestClient
+from sqlalchemy import create_engine, select
+from sqlalchemy.exc import OperationalError
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.pool import StaticPool
+
+from app.config import Settings, get_settings
+from app.db import Base, get_db
+from app.main import app
+from app.models import CrawlRun, CrawlRunEmployeeChange, Employee
+from app.security import SESSION_COOKIE, sign_session
+
+
+def test_health_returns_versions():
+    response = TestClient(app).get("/api/health")
+
+    assert response.status_code == 200
+    assert response.json()["backend_version"] == "0.13.1"
+
+
+def test_api_employees_and_stats_require_admin_session():
+    engine = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False}, poolclass=StaticPool)
+    Base.metadata.create_all(engine)
+    session_factory = sessionmaker(bind=engine)
+    db = session_factory()
+    employee = Employee(
+        profile_key="staff:alpha",
+        profile_type="staff",
+        profile_id="alpha",
+        canonical_url="https://www.hse.ru/staff/alpha",
+        full_name="Alpha Person",
+        status="active",
+        first_seen_at=datetime.now(timezone.utc),
+        last_seen_at=datetime.now(timezone.utc),
+        current_data={"contacts": {"emails": ["alpha@hse.ru"]}, "sections": []},
+    )
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="completed", new_count=1)
+    db.add_all([employee, run])
+    db.commit()
+    db.add(CrawlRunEmployeeChange(
+        crawl_run_id=run.id,
+        employee_id=employee.id,
+        profile_key=employee.profile_key,
+        profile_url=employee.canonical_url,
+        full_name=employee.full_name,
+        change_type="new",
+        profile_available=True,
+        message="added",
+    ))
+    db.commit()
+    run_id = run.id
+    db.close()
+
+    settings = Settings(admin_username="admin", admin_password="password", session_secret="session-secret")
+
+    def override_db():
+        session = session_factory()
+        try:
+            yield session
+        finally:
+            session.close()
+
+    app.dependency_overrides[get_db] = override_db
+    app.dependency_overrides[get_settings] = lambda: settings
+    client = TestClient(app)
+    client.cookies.set(SESSION_COOKIE, sign_session("admin", settings))
+
+    employees = client.get("/api/employees", params={"q": "Alpha", "has_email": True})
+    stats = client.get("/api/stats")
+    run_details = client.get(f"/api/crawl-runs/{run_id}")
+
+    assert employees.status_code == 200
+    assert employees.json()["total"] == 1
+    assert stats.status_code == 200
+    assert stats.json()["new_in_last_run"] == 1
+    assert run_details.status_code == 200
+    assert run_details.json()["changes"]["new"][0]["full_name"] == "Alpha Person"
+    directory = client.get("/admin/directory?q=Alpha&offset=100")
+    assert directory.status_code == 200
+    assert "Показаны 1–1 из 1" in directory.text
+    assert 'name="offset"' not in directory.text
+    assert "Alpha Person" in directory.text
+    empty = client.get("/admin/directory?q=missing&offset=100")
+    assert "Показаны 0–0 из 0" in empty.text
+    assert "Сбросить фильтры" in empty.text
+    app.dependency_overrides.clear()
+
+
+def test_admin_refresh_employee_route_updates_only_requested_employee(monkeypatch):
+    engine = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False}, poolclass=StaticPool)
+    Base.metadata.create_all(engine)
+    session_factory = sessionmaker(bind=engine)
+    db = session_factory()
+    db.add(Employee(
+        profile_key="org_person:133709486",
+        profile_type="org_person",
+        profile_id="133709486",
+        canonical_url="https://www.hse.ru/org/persons/133709486",
+        full_name="Будков Юрий Алексеевич",
+        status="active",
+    ))
+    db.commit()
+    employee_id = db.scalar(select(Employee.id))
+    db.close()
+
+    settings = Settings(admin_username="admin", admin_password="password", session_secret="session-secret")
+
+    def override_db():
+        session = session_factory()
+        try:
+            yield session
+        finally:
+            session.close()
+
+    calls = []
+
+    def fake_refresh_employee(db, refreshed_employee, route_settings):
+        calls.append((refreshed_employee.id, route_settings))
+        return SimpleNamespace(status="completed")
+
+    app.dependency_overrides[get_db] = override_db
+    app.dependency_overrides[get_settings] = lambda: settings
+    monkeypatch.setattr("app.admin.refresh_employee", fake_refresh_employee)
+    client = TestClient(app)
+    client.cookies.set(SESSION_COOKIE, sign_session("admin", settings))
+
+    response = client.post(f"/admin/employees/{employee_id}/refresh", follow_redirects=False)
+
+    assert response.status_code == 303
+    assert response.headers["location"] == f"/admin/employees/{employee_id}?refresh_status=success"
+    assert calls == [(employee_id, settings)]
+    app.dependency_overrides.clear()
+
+
+def test_directory_database_error_has_retry_and_preserves_filters(db_session, monkeypatch):
+    settings = Settings(_env_file=None, session_secret="test-session-secret")
+    app.dependency_overrides[get_db] = lambda: db_session
+    app.dependency_overrides[get_settings] = lambda: settings
+    def unavailable(*args, **kwargs):
+        raise OperationalError("SELECT", {}, Exception("private database details"))
+    monkeypatch.setattr("app.admin.list_employees_page", unavailable)
+    try:
+        client = TestClient(app)
+        client.cookies.set(SESSION_COOKIE, sign_session(settings.admin_username, settings))
+        response = client.get("/admin/directory?q=Alice")
+        assert response.status_code == 503
+        assert 'href="/admin/directory?q=Alice"' in response.text
+        assert "Повторить загрузку" in response.text
+        assert "private database details" not in response.text
+    finally:
+        app.dependency_overrides.clear()

tests/test_api_mcp.py

@@ -1,159 +0,0 @@
-from datetime import datetime, timezone
-
-from fastapi.testclient import TestClient
-from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.pool import StaticPool
-
-from app.config import Settings, get_settings
-from app.db import Base, get_db
-from app.main import app
-from app.models import CrawlRun, Employee
-from app.security import SESSION_COOKIE, sign_session
-
-
-def test_health_returns_versions():
-    client = TestClient(app)
-
-    response = client.get("/api/health")
-
-    assert response.status_code == 200
-    assert response.json()["backend_version"] == "0.2.1"
-
-
-def test_mcp_requires_token_and_lists_tools():
-    engine = create_engine(
-        "sqlite:///:memory:",
-        connect_args={"check_same_thread": False},
-        poolclass=StaticPool,
-    )
-    Base.metadata.create_all(engine)
-    Session = sessionmaker(bind=engine)
-
-    def override_db():
-        session = Session()
-        try:
-            yield session
-        finally:
-            session.close()
-
-    app.dependency_overrides[get_db] = override_db
-    app.dependency_overrides[get_settings] = lambda: Settings(mcp_token="secret", session_secret="session-secret")
-    client = TestClient(app)
-
-    unauthorized = client.post("/mcp", json={"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}})
-    authorized = client.post(
-        "/mcp",
-        headers={"Authorization": "Bearer secret"},
-        json={"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}},
-    )
-
-    assert unauthorized.status_code == 401
-    assert authorized.status_code == 200
-    assert authorized.json()["result"]["tools"][0]["name"] == "search_employees"
-
-    app.dependency_overrides.clear()
-
-
-def test_mcp_search_employees_returns_matching_employee():
-    engine = create_engine(
-        "sqlite:///:memory:",
-        connect_args={"check_same_thread": False},
-        poolclass=StaticPool,
-    )
-    Base.metadata.create_all(engine)
-    Session = sessionmaker(bind=engine)
-    session = Session()
-    session.add(
-        Employee(
-            profile_key="staff:avsergeev",
-            profile_type="staff",
-            profile_id="avsergeev",
-            canonical_url="https://www.hse.ru/staff/avsergeev",
-            full_name="Сергеев Алексей Викторович",
-            status="active",
-            first_seen_at=datetime.now(timezone.utc),
-            last_seen_at=datetime.now(timezone.utc),
-            current_data={"sections": []},
-        )
-    )
-    session.commit()
-    session.close()
-
-    def override_db():
-        db = Session()
-        try:
-            yield db
-        finally:
-            db.close()
-
-    app.dependency_overrides[get_db] = override_db
-    app.dependency_overrides[get_settings] = lambda: Settings(mcp_token="secret", session_secret="session-secret")
-    client = TestClient(app)
-
-    response = client.post(
-        "/mcp",
-        headers={"Authorization": "Bearer secret"},
-        json={
-            "jsonrpc": "2.0",
-            "id": 1,
-            "method": "tools/call",
-            "params": {"name": "search_employees", "arguments": {"query": "Сергеев"}},
-        },
-    )
-
-    assert response.status_code == 200
-    assert "Сергеев Алексей Викторович" in response.json()["result"]["content"][0]["text"]
-
-    app.dependency_overrides.clear()
-
-
-def test_api_employees_and_stats_require_admin_session():
-    engine = create_engine(
-        "sqlite:///:memory:",
-        connect_args={"check_same_thread": False},
-        poolclass=StaticPool,
-    )
-    Base.metadata.create_all(engine)
-    Session = sessionmaker(bind=engine)
-    db = Session()
-    db.add(
-        Employee(
-            profile_key="staff:alpha",
-            profile_type="staff",
-            profile_id="alpha",
-            canonical_url="https://www.hse.ru/staff/alpha",
-            full_name="Alpha Person",
-            status="active",
-            first_seen_at=datetime.now(timezone.utc),
-            last_seen_at=datetime.now(timezone.utc),
-            current_data={"contacts": {"emails": ["alpha@hse.ru"]}, "sections": []},
-        )
-    )
-    db.add(CrawlRun(source_url="https://miem.hse.ru/persons", status="completed", new_count=1))
-    db.commit()
-    db.close()
-
-    settings = Settings(admin_username="admin", admin_password="password", session_secret="session-secret")
-
-    def override_db():
-        session = Session()
-        try:
-            yield session
-        finally:
-            session.close()
-
-    app.dependency_overrides[get_db] = override_db
-    app.dependency_overrides[get_settings] = lambda: settings
-    client = TestClient(app)
-    client.cookies.set(SESSION_COOKIE, sign_session("admin", settings))
-
-    employees = client.get("/api/employees", params={"q": "Alpha", "has_email": True})
-    stats = client.get("/api/stats")
-
-    assert employees.status_code == 200
-    assert employees.json()["total"] == 1
-    assert stats.status_code == 200
-    assert stats.json()["new_in_last_run"] == 1
-
-    app.dependency_overrides.clear()

tests/test_config.py

@@ -0,0 +1,22 @@
+from app.config import Settings
+
+
+def test_empty_crawl_limit_is_treated_as_none():
+    settings = Settings(crawl_limit="")
+
+    assert settings.crawl_limit is None
+
+
+def test_numeric_crawl_limit_is_parsed():
+    settings = Settings(crawl_limit="25")
+
+    assert settings.crawl_limit == 25
+
+
+def test_dismissal_safety_settings_have_conservative_defaults(monkeypatch):
+    monkeypatch.delenv("DISMISSAL_CONFIRMATION_RUNS", raising=False)
+    monkeypatch.delenv("MAX_AUTO_DISMISSALS_PER_RUN", raising=False)
+    settings = Settings(_env_file=None)
+
+    assert settings.dismissal_confirmation_runs == 3
+    assert settings.max_auto_dismissals_per_run == 25

tests/test_council.py

@@ -0,0 +1,310 @@
+import csv
+from datetime import date
+from io import BytesIO
+from zipfile import ZipFile
+
+import pytest
+from fastapi.testclient import TestClient
+from openpyxl import Workbook, load_workbook
+from openpyxl.worksheet.worksheet import Worksheet
+from sqlalchemy.exc import OperationalError
+
+from app.db import get_db
+from app.main import app
+from app.models import Employee, EmployeePublication
+from app.services import council
+from app.services.council_export import _append
+
+
+@pytest.fixture()
+def export(monkeypatch, tmp_path):
+    path = tmp_path / "articles.csv"
+    def write(rows):
+        fields = ["Название публикации", "Год", "DOI", "EID (Scopus)", "UT (WoS)",
+                  "Авторы НИУ ВШЭ", "Квартиль scopus", "Квартиль wos"]
+        with path.open("w", encoding="utf-8-sig", newline="") as output:
+            writer = csv.DictWriter(output, fieldnames=fields, delimiter=";")
+            writer.writeheader()
+            writer.writerows(rows)
+        council._articles_by_author.cache_clear()
+    monkeypatch.setattr(council, "EXPORT", path)
+    write([])
+    yield write
+    council._articles_by_author.cache_clear()
+
+
+def employee(db, name="Иванов Иван Иванович", degree="Доктор технических наук", status="active", key="doctor"):
+    person = Employee(profile_key=key, canonical_url="https://www.hse.ru/staff/test", full_name=name,
+                      status=status, current_data={"contacts": {"emails": ["private@example.org"]},
+                      "sections": [{"title": "Учёные степени", "items": [degree] if degree else []}]})
+    db.add(person)
+    db.flush()
+    return person
+
+
+def publication(db, person, title="Shared work", doi=None, year=2024, kind=None, **extra):
+    item = EmployeePublication(employee_id=person.id, title=title, year=year, doi_url=doi,
+                               publication_type=kind, source_hash=f"hash-{len(db.new)}-{title}-{doi}", **extra)
+    db.add(item)
+    return item
+
+
+def row(title="Shared work", doi="10.1234/shared", year="2024", author="Иванов Иван Иванович", **extra):
+    return {"Название публикации": title, "Год": year, "DOI": doi, "Авторы НИУ ВШЭ": author,
+            "Квартиль scopus": "Q1", **extra}
+
+
+def test_council_union_counts_provenance_and_types(db_session, export):
+    export([row(), row(**{"Квартиль scopus": "", "Квартиль wos": "Q2"}),
+            row(title="CSV only", doi="", year="2022"),
+            row(title="Not eligible", doi="10.1234/old", year="2019")])
+    person = employee(db_session, degree="Доктор технических наук")
+    publication(db_session, person, doi="https://dx.doi.org/10.1234/SHARED", kind="ARTICLE",
+                publication_id="100", url="https://publications.hse.ru/view/100")
+    publication(db_session, person, title=" shared   WORK ", kind="ARTICLE")
+    publication(db_session, person, title="Book only", kind="BOOK", year=2020)
+    publication(db_session, person, title="Imported", raw_data={"csv_enrichment": {"Квартиль wos": "Q2"}})
+    db_session.commit()
+    result = council.council_candidates(db_session, date(2026, 9, 30))
+    assert result["total"] == 1
+    candidate = result["candidates"][0]
+    assert (candidate["csv_count"], candidate["db_count"], candidate["count"]) == (2, 3, 5)
+    shared = next(a for a in candidate["articles"] if a["doi"] == "10.1234/shared")
+    assert shared["type_label"] == "Статья"
+    assert shared["quartiles"] == ["Scopus Q1", "Web of Science Q2"]
+    assert shared["rows"] == [2, 3]
+    assert "Файл CSV" in shared["origins"] and "БД miem-workers · сайт ВШЭ" in shared["origins"]
+    imported = next(a for a in candidate["articles"] if a["title"] == "Imported")
+    assert imported["origins"] == ["БД miem-workers · импорт CSV"]
+    assert not imported["csv_qualifies"]
+    assert "Тип не указан" in imported["type_label"]
+    assert council.period(date(2024, 2, 29))[0] == date(2019, 2, 28)
+
+
+def test_only_doctors_with_a_publication_no_count_threshold(db_session, export):
+    export([row(author="Доктор", title="Single qualifying"),
+            row(author="Без степени", title="No degree"),
+            row(author="Кандидат", title="Candidate work"),
+            row(author="PhD", title="PhD work"),
+            row(author="Две степени", title="Mixed degrees", **{"Квартиль scopus": "Q3"}),
+            row(author="Уволен", title="Old", year="2010")])
+    employee(db_session, "Доктор", key="csv-only")
+    for name, degree in [("Без степени", None), ("Кандидат", "Кандидат технических наук"), ("PhD", "Ph.D.")]:
+        person = employee(db_session, name, degree, key=name)
+        publication(db_session, person)
+    employee(db_session, "Две степени", "Кандидат технических наук; Доктор физико-математических наук", key="mixed")
+    employee(db_session, "Уволен", status="dismissed", key="dismissed")
+    only_db = employee(db_session, "Только БД", key="database")
+    publication(db_session, only_db, kind="PATENT")
+    employee(db_session, "Без публикаций", key="empty")
+    db_session.commit()
+    items = council.council_candidates(db_session, date(2026, 9, 30))["candidates"]
+    assert {c["name"] for c in items} == {"Доктор", "Две степени", "Уволен", "Только БД"}
+    assert next(c for c in items if c["name"] == "Доктор")["csv_count"] == 1
+    assert next(c for c in items if c["name"] == "Только БД")["db_count"] == 1
+    assert next(c for c in items if c["name"] == "Две степени")["csv_count"] == 0
+    assert next(c for c in items if c["name"] == "Уволен")["dismissed"]
+
+
+def test_no_false_merge_when_title_has_conflicting_dois(db_session, export):
+    export([row(doi="10.1234/one"), row(doi="10.1234/two"), row(doi="")])
+    person = employee(db_session)
+    publication(db_session, person, title="Renamed work", doi="https://doi.org/10.1234/one")
+    db_session.commit()
+    item = council.council_candidates(db_session, date(2026, 9, 30))["candidates"][0]
+    assert item["count"] == 3
+    assert item["csv_count"] == 3 and item["db_count"] == 1
+
+
+def test_csv_same_name_does_not_attribute_to_wrong_employee(db_session, export):
+    export([row()])
+    for key in ["one", "two"]:
+        person = employee(db_session, key=key)
+        publication(db_session, person, title=key)
+    db_session.commit()
+    items = council.council_candidates(db_session, date(2026, 9, 30))["candidates"]
+    assert len(items) == 2
+    assert all(c["ambiguous"] and c["csv_count"] == 0 and c["count"] == 1 for c in items)
+
+
+def test_missing_database_year_merges_only_unambiguous_title_and_non_article_excluded(db_session, export):
+    export([row(title="Known book"), row(title="Article", doi="10.1234/article")])
+    person = employee(db_session)
+    publication(db_session, person, title="Known book", year=None, kind="BOOK")
+    publication(db_session, person, title="Article", year=None, kind="ARTICLE")
+    db_session.commit()
+    candidate = council.council_candidates(db_session, date(2026, 9, 30))["candidates"][0]
+    assert (candidate["count"], candidate["csv_count"], candidate["db_count"]) == (2, 1, 2)
+    book = next(a for a in candidate["articles"] if a["type_label"] == "Книга")
+    assert not book["csv_qualifies"]
+    assert "не включён в счётчик CSV" in book["criteria"][-1]
+
+
+def test_search_and_pagination_include_all_workers(db_session, export):
+    for number in range(28):
+        person = employee(db_session, f"Петров {number:02d}", key=f"person-{number}")
+        publication(db_session, person, title=f"Work {number}")
+    db_session.commit()
+    first = council.council_candidates(db_session)
+    last = council.council_candidates(db_session, offset=25)
+    assert first["total"] == 28 and len(first["candidates"]) == 25
+    assert len(last["candidates"]) == 3
+    assert {c["name"] for c in first["candidates"]}.isdisjoint(c["name"] for c in last["candidates"])
+    assert council.council_candidates(db_session, q="петров 27")["matched"] == 1
+    assert council.council_candidates(db_session, offset=-1)["offset"] == 0
+    assert council.council_candidates(db_session, offset=999)["offset"] == 25
+    descending = council.council_candidates(db_session, sort="full_name", direction="desc")
+    last_descending = council.council_candidates(db_session, sort="full_name", direction="desc", offset=25)
+    assert descending["candidates"][0]["name"] == "Петров 27"
+    assert [c["name"] for c in last_descending["candidates"]] == ["Петров 02", "Петров 01", "Петров 00"]
+
+
+def test_sort_by_name_and_each_unique_publication_count(db_session, export):
+    export([row(author="Анна", title=f"CSV {number}", doi=f"10.1234/csv-{number}") for number in range(3)]
+           + [row(author="Борис", title="CSV Boris", doi="10.1234/boris")])
+    anna = employee(db_session, "Анна", key="anna")
+    publication(db_session, anna, title="CSV 0", doi="https://doi.org/10.1234/csv-0")
+    for name, count in [("Борис", 2), ("Вера", 4)]:
+        person = employee(db_session, name, key=name)
+        for number in range(count):
+            publication(db_session, person, title=f"DB {name} {number}")
+    db_session.commit()
+    expected = {
+        "full_name": {"asc": ["Анна", "Борис", "Вера"], "desc": ["Вера", "Борис", "Анна"]},
+        "csv_count": {"asc": ["Вера", "Борис", "Анна"], "desc": ["Анна", "Борис", "Вера"]},
+        "db_count": {"asc": ["Анна", "Борис", "Вера"], "desc": ["Вера", "Борис", "Анна"]},
+        "count": {"asc": ["Анна", "Борис", "Вера"], "desc": ["Вера", "Анна", "Борис"]},
+    }
+    for sort, directions in expected.items():
+        for direction, names in directions.items():
+            result = council.council_candidates(db_session, sort=sort, direction=direction)
+            assert [c["name"] for c in result["candidates"]] == names
+    invalid = council.council_candidates(db_session, sort="unknown", direction="unknown")
+    assert (invalid["sort"], invalid["direction"]) == ("csv_count", "desc")
+    assert [c["name"] for c in invalid["candidates"]] == expected["csv_count"]["desc"]
+    app.dependency_overrides[get_db] = lambda: db_session
+    try:
+        response = TestClient(app).get("/admin/council?sort=full_name&direction=desc")
+        assert response.status_code == 200
+        assert response.text.index("Вера") < response.text.index("Борис") < response.text.index("Анна")
+        assert '<option value="full_name" selected>' in response.text
+        assert '<option value="desc" selected>' in response.text
+    finally:
+        app.dependency_overrides.clear()
+
+
+def test_public_page_data_minimization_and_safe_links(db_session, export):
+    export([row(title='<script>alert("csv")</script>', doi="javascript:bad")])
+    person = employee(db_session)
+    person.canonical_url = "javascript:bad"
+    publication(db_session, person, url="javascript:alert(1)", doi="javascript:bad", kind="BOOK")
+    db_session.commit()
+    app.dependency_overrides[get_db] = lambda: db_session
+    try:
+        client = TestClient(app)
+        response = client.get("/admin/council", follow_redirects=False)
+        assert response.status_code == 200
+        assert "Иванов Иван Иванович" in response.text and "Доктор технических наук" in response.text
+        assert "Минимальный порог количества работ не применяется" in response.text
+        assert "CSV по критериям" in response.text and "В БД" in response.text and "Всего без дублей" in response.text
+        assert "Книга" in response.text and "Тип не указан" in response.text
+        assert "&lt;script&gt;" in response.text and "<script>" not in response.text
+        assert "javascript:" not in response.text and "private@example.org" not in response.text
+        assert "/admin/employees/" not in response.text and "/admin/logout" not in response.text
+        assert "Backend 0.13.1 · Frontend 0.13.1" in response.text
+        assert "Вход в админку" in response.text
+        assert client.get("/admin/directory", follow_redirects=False).status_code in (302, 303, 401, 403)
+        assert "Сотрудники не найдены" in client.get("/admin/council?q=missing").text
+    finally:
+        app.dependency_overrides.clear()
+
+
+def test_public_error_state_missing_export_and_database_failure(db_session, export, monkeypatch):
+    app.dependency_overrides[get_db] = lambda: db_session
+    try:
+        council.EXPORT.unlink()
+        response = TestClient(app).get("/admin/council")
+        assert response.status_code == 503 and "Повторить загрузку" in response.text
+        def fail(*args, **kwargs):
+            raise OperationalError("query", {}, Exception("private error"))
+        monkeypatch.setattr("app.admin.council_candidates", fail)
+        response = TestClient(app).get("/admin/council")
+        assert response.status_code == 503 and "private error" not in response.text
+    finally:
+        app.dependency_overrides.clear()
+
+
+def test_xlsx_exports_all_filtered_workers_in_order_and_deduplicated_evidence(db_session, export):
+    export([row(author="Петров 27"), row(author="Петров 27", **{"Квартиль wos": "Q2"})])
+    for number in range(28):
+        person = employee(db_session, f"Петров {number:02d}", key=f"xlsx-{number}")
+        publication(db_session, person, title="Shared work" if number == 27 else f"Work {number}",
+                    doi="https://doi.org/10.1234/shared" if number == 27 else None,
+                    kind="ARTICLE", publication_id=str(number), url=f"https://publications.hse.ru/view/{number}")
+    excluded = employee(db_session, "Без степени", degree=None, key="excluded")
+    publication(db_session, excluded)
+    db_session.commit()
+    app.dependency_overrides[get_db] = lambda: db_session
+    try:
+        client = TestClient(app)
+        response = client.get("/admin/council/export.xlsx?q=Петров&sort=full_name&direction=desc&offset=25")
+        assert response.status_code == 200
+        assert response.headers["content-type"] == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+        assert response.headers["content-disposition"] == 'attachment; filename="council.xlsx"'
+        book = load_workbook(BytesIO(response.content))
+        assert book.sheetnames == ["Сотрудники", "Публикации", "Параметры"]
+        people, works = book["Сотрудники"], book["Публикации"]
+        assert people.max_row == 29 and works.max_row == 29
+        assert people["B2"].value == "Петров 27" and people["B29"].value == "Петров 00"
+        assert [people.cell(2, n).value for n in [4, 5, 6]] == [1, 1, 1]
+        assert works["C2"].value == "Shared work" and works["E2"].value == "Статья"
+        assert "Файл CSV" in works["F2"].value and "сайт ВШЭ" in works["F2"].value
+        assert "Scopus Q1" in works["G2"].value and "Web of Science Q2" in works["G2"].value
+        assert works["J2"].value == "https://doi.org/10.1234/shared" and works["L2"].value == "2, 3"
+        assert people["D2"].data_type == "n" and people.freeze_panes == "A2"
+        assert works.auto_filter.ref == works.dimensions
+        text = " ".join(str(cell.value) for sheet in book for row in sheet for cell in row)
+        assert "private@example.org" not in text and "Без степени" not in text
+        assert load_workbook(BytesIO(client.get("/admin/council/export.xlsx?q=Петров%2027").content))["Сотрудники"].max_row == 2
+        empty = load_workbook(BytesIO(client.get("/admin/council/export.xlsx?q=missing").content))
+        assert empty["Сотрудники"].max_row == 1 and empty["Публикации"].max_row == 1
+    finally:
+        app.dependency_overrides.clear()
+
+
+def test_xlsx_treats_profile_text_as_text_and_rejects_silent_truncation(db_session, export):
+    person = employee(db_session, '=HYPERLINK("https://example.org","name")')
+    publication(db_session, person, title="=1+1\x01")
+    publication(db_session, person, title="#N/A")
+    db_session.commit()
+    app.dependency_overrides[get_db] = lambda: db_session
+    try:
+        client = TestClient(app)
+        response = client.get("/admin/council/export.xlsx")
+        assert response.status_code == 200
+        book = load_workbook(BytesIO(response.content))
+        assert book["Сотрудники"]["B2"].value == person.full_name
+        assert book["Сотрудники"]["B2"].data_type == "s"
+        titles = {book["Публикации"].cell(n, 3).value: book["Публикации"].cell(n, 3).data_type for n in [2, 3]}
+        assert titles == {"=1+1": "s", "#N/A": "s"}
+        with ZipFile(BytesIO(response.content)) as archive:
+            assert all(b"<f>" not in archive.read(n) for n in archive.namelist() if n.startswith("xl/worksheets/"))
+        person.full_name = "Long " + "x" * 32768
+        db_session.commit()
+        failure = client.get("/admin/council/export.xlsx")
+        assert failure.status_code == 503 and "Не удалось создать XLSX" in failure.text
+        assert "content-disposition" not in failure.headers
+    finally:
+        app.dependency_overrides.clear()
+
+
+def test_xlsx_appending_rows_does_not_scan_growing_sheet(monkeypatch):
+    def fail_on_scan(sheet):
+        pytest.fail("Appending rows must not recompute worksheet bounds by scanning existing cells")
+    monkeypatch.setattr(Worksheet, "max_row", property(fail_on_scan))
+    monkeypatch.setattr(Worksheet, "max_column", property(fail_on_scan))
+    sheet = Workbook().active
+    for number in range(1000):
+        _append(sheet, [number, "text"])
+    assert sheet.cell(1000, 1).value == 999
+    assert sheet.cell(1000, 2).value == "text"

tests/test_crawler.py

@@ -1,58 +1,690 @@
-from datetime import datetime, timezone
-
-from app.models import CrawlRun, Employee
-from app.services.crawler import _mark_dismissed, _upsert_employee
+import gzip
+import pytest
+from datetime import datetime, timezone
+
+from app.models import (
+    CrawlError,
+    CrawlRun,
+    CrawlRunEmployeeChange,
+    Employee,
+    EmployeeNewsLink,
+    EmployeePublication,
+    EmployeeSnapshot,
+    ParseResourceCache,
+)
+from app.config import Settings
+from app.services.crawler import _checksum, _mark_dismissed, _upsert_employee, refresh_dismissed_status
+from app.services.resource_cache import ResourceCache
+
+
+class FakeResponse:
+    def __init__(self, status_code):
+        self.status_code = status_code
+
+
+class FakeSession:
+    def __init__(self, statuses):
+        self.statuses = statuses
+
+    def get(self, url, **_kwargs):
+        return FakeResponse(self.statuses[url])
+
+
+class ConditionalResponse:
+    def __init__(self, status_code, text="", headers=None):
+        self.status_code = status_code
+        self._text = text
+        self.headers = headers or {}
+        self.text_read = False
+
+    @property
+    def text(self):
+        self.text_read = True
+        return self._text
+
+    def raise_for_status(self):
+        return None
+
+
+class ConditionalSession:
+    def __init__(self):
+        self.requests = []
+        self.not_modified_response = ConditionalResponse(304)
+
+    def get(self, url, **kwargs):
+        self.requests.append((url, kwargs))
+        if kwargs["headers"].get("If-None-Match") == '"cached"':
+            return self.not_modified_response
+        return ConditionalResponse(200, "fresh", {"ETag": '"fresh"'})
 
 
-def test_mark_dismissed_only_marks_missing_active_employees(db_session):
-    db_session.add(
-        Employee(
-            profile_key="staff:kept",
-            canonical_url="https://www.hse.ru/staff/kept",
-            status="active",
-            first_seen_at=datetime.now(timezone.utc),
-            last_seen_at=datetime.now(timezone.utc),
-        )
+def test_refresh_dismissed_status_reactivates_only_profiles_in_source(monkeypatch, db_session):
+    now = datetime.now(timezone.utc)
+    found = Employee(
+        profile_key="staff:returned",
+        canonical_url="https://www.hse.ru/staff/returned",
+        status="dismissed",
+        dismissed_at=now,
+        first_seen_at=now,
+        last_seen_at=now,
     )
-    db_session.add(
-        Employee(
-            profile_key="staff:gone",
-            canonical_url="https://www.hse.ru/staff/gone",
-            status="active",
-            first_seen_at=datetime.now(timezone.utc),
-            last_seen_at=datetime.now(timezone.utc),
-        )
+    still_dismissed = Employee(
+        profile_key="staff:gone",
+        canonical_url="https://www.hse.ru/staff/gone",
+        status="dismissed",
+        dismissed_at=now,
+        first_seen_at=now,
+        last_seen_at=now,
     )
+    db_session.add_all([found, still_dismissed])
     db_session.commit()
+    monkeypatch.setattr(
+        "app.services.crawler.collect_profile_links",
+        lambda *_args, **_kwargs: ["https://www.hse.ru/staff/returned"],
+    )
 
-    dismissed = _mark_dismissed(db_session, {"staff:kept"})
+    run = refresh_dismissed_status(db_session, Settings())
 
-    assert dismissed == 1
-    assert db_session.query(Employee).filter_by(profile_key="staff:kept").one().status == "active"
-    gone = db_session.query(Employee).filter_by(profile_key="staff:gone").one()
-    assert gone.status == "dismissed"
-    assert gone.dismissed_at is not None
+    assert run.status == "completed"
+    assert run.parsed_count == 1
+    assert run.skipped_count == 1
+    assert found.status == "active"
+    assert found.dismissed_at is None
+    assert still_dismissed.status == "dismissed"
+
+
+def test_mark_dismissed_records_missing_source_when_profile_is_available(db_session):
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    db_session.add(run)
+    db_session.add(
+        Employee(
+            profile_key="staff:kept",
+            canonical_url="https://www.hse.ru/staff/kept",
+            status="active",
+            first_seen_at=datetime.now(timezone.utc),
+            last_seen_at=datetime.now(timezone.utc),
+        )
+    )
+    db_session.add(
+        Employee(
+            profile_key="staff:missing",
+            canonical_url="https://www.hse.ru/staff/missing",
+            status="active",
+            first_seen_at=datetime.now(timezone.utc),
+            last_seen_at=datetime.now(timezone.utc),
+        )
+    )
+    db_session.commit()
+
+    dismissed = _mark_dismissed(
+        db_session,
+        run,
+        {"staff:kept"},
+        FakeSession({"https://www.hse.ru/staff/missing": 200}),
+        30,
+    )
+
+    assert dismissed == 0
+    assert db_session.query(Employee).filter_by(profile_key="staff:kept").one().status == "active"
+    missing = db_session.query(Employee).filter_by(profile_key="staff:missing").one()
+    assert missing.status == "active"
+    assert missing.dismissed_at is None
+    change = db_session.query(CrawlRunEmployeeChange).one()
+    assert change.change_type == "missing_from_source"
+    assert change.profile_available is True
+
+
+def test_mark_dismissed_requires_consecutive_unavailable_checks(db_session):
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    employee = Employee(
+        profile_key="staff:gone",
+        canonical_url="https://www.hse.ru/staff/gone",
+        status="active",
+        first_seen_at=datetime.now(timezone.utc),
+        last_seen_at=datetime.now(timezone.utc),
+    )
+    db_session.add_all([run, employee])
+    db_session.commit()
+
+    first_check = _mark_dismissed(
+        db_session,
+        run,
+        set(),
+        FakeSession({"https://www.hse.ru/staff/gone": 404}),
+        30,
+        confirmation_runs=2,
+    )
+
+    assert first_check == 0
+    assert employee.status == "verification_required"
+    assert employee.dismissed_at is None
+    assert employee.profile_unavailable_streak == 1
+    assert db_session.query(CrawlRunEmployeeChange).one().change_type == "verification_required"
+
+    second_check = _mark_dismissed(
+        db_session,
+        run,
+        set(),
+        FakeSession({"https://www.hse.ru/staff/gone": 404}),
+        30,
+        confirmation_runs=2,
+    )
+
+    assert second_check == 1
+    assert employee.status == "dismissed"
+    assert employee.dismissed_at is not None
+    assert employee.profile_unavailable_streak == 2
+    change = db_session.query(CrawlRunEmployeeChange).order_by(CrawlRunEmployeeChange.id).all()[-1]
+    assert change.change_type == "dismissed"
+    assert change.profile_available is False
+
+
+def test_mark_dismissed_does_not_dismiss_on_server_error(db_session):
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    employee = Employee(
+        profile_key="staff:temporary-error",
+        canonical_url="https://www.hse.ru/staff/temporary-error",
+        status="active",
+        first_seen_at=datetime.now(timezone.utc),
+        last_seen_at=datetime.now(timezone.utc),
+    )
+    db_session.add_all([run, employee])
+    db_session.commit()
+
+    dismissed = _mark_dismissed(
+        db_session,
+        run,
+        set(),
+        FakeSession({"https://www.hse.ru/staff/temporary-error": 503}),
+        30,
+        confirmation_runs=1,
+    )
+
+    assert dismissed == 0
+    assert employee.status == "active"
+    assert employee.profile_unavailable_streak == 0
+    assert db_session.query(CrawlError).one().error_type == "ProfileAvailabilityCheckError"
+
+
+def test_available_profile_resets_verification_streak(db_session):
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    employee = Employee(
+        profile_key="staff:restored",
+        canonical_url="https://www.hse.ru/staff/restored",
+        status="active",
+        first_seen_at=datetime.now(timezone.utc),
+        last_seen_at=datetime.now(timezone.utc),
+    )
+    db_session.add_all([run, employee])
+    db_session.commit()
+
+    _mark_dismissed(
+        db_session,
+        run,
+        set(),
+        FakeSession({"https://www.hse.ru/staff/restored": 404}),
+        30,
+        confirmation_runs=3,
+    )
+    _mark_dismissed(
+        db_session,
+        run,
+        set(),
+        FakeSession({"https://www.hse.ru/staff/restored": 200}),
+        30,
+        confirmation_runs=3,
+    )
+
+    assert employee.status == "active"
+    assert employee.profile_unavailable_streak == 0
+
+
+def test_mark_dismissed_blocks_mass_auto_dismissals(db_session):
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    employees = [
+        Employee(
+            profile_key=f"staff:gone-{index}",
+            canonical_url=f"https://www.hse.ru/staff/gone-{index}",
+            status="active",
+            first_seen_at=datetime.now(timezone.utc),
+            last_seen_at=datetime.now(timezone.utc),
+        )
+        for index in range(2)
+    ]
+    db_session.add_all([run, *employees])
+    db_session.commit()
+
+    dismissed = _mark_dismissed(
+        db_session,
+        run,
+        set(),
+        FakeSession({employee.canonical_url: 404 for employee in employees}),
+        30,
+        confirmation_runs=1,
+        max_auto_dismissals=1,
+    )
+
+    assert dismissed == 0
+    assert {employee.status for employee in employees} == {"verification_required"}
+    assert "приостановлено" in run.message
+
+
+def test_upsert_employee_increments_new_count_and_records_change_for_new_employee(db_session):
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    db_session.add(run)
+    db_session.commit()
+
+    employee, _ = _upsert_employee(
+        db_session,
+        run,
+        {
+            "source_url": "https://www.hse.ru/staff/newperson",
+            "profile_type": "staff",
+            "profile_id": "newperson",
+            "full_name": "New Person",
+            "tabs": [],
+            "sections": [
+                {
+                    "title": "Образование и учёные степени",
+                    "year_entries": [{"text": "Кандидат технических наук"}],
+                }
+            ],
+            "parser_version": "0.2.0",
+            "_html": "<html></html>",
+        },
+    )
+    db_session.commit()
+
+    assert run.new_count == 1
+    assert employee.has_academic_degree is True
+    change = db_session.query(CrawlRunEmployeeChange).one()
+    assert change.change_type == "new"
+    assert change.full_name == "New Person"
+
+
+def test_upsert_employee_reconciles_profile_moved_to_new_url(db_session):
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    employee = Employee(
+        profile_key="staff:abelov",
+        canonical_url="https://www.hse.ru/staff/abelov",
+        full_name="Белов Александр Владимирович",
+        status="active",
+        first_seen_at=datetime.now(timezone.utc),
+        last_seen_at=datetime.now(timezone.utc),
+    )
+    db_session.add_all([run, employee])
+    db_session.commit()
+    employee_id = employee.id
+
+    updated, changed = _upsert_employee(
+        db_session,
+        run,
+        {
+            "source_url": "https://www.hse.ru/org/persons/47634735",
+            "profile_type": "org_person",
+            "profile_id": "47634735",
+            "full_name": "Белов Александр Владимирович",
+            "tabs": [],
+            "sections": [],
+            "parser_version": "0.7.0",
+            "_html": "<html></html>",
+        },
+    )
+    db_session.commit()
+
+    assert changed is True
+    assert updated.id == employee_id
+    assert updated.profile_key == "org_person:47634735"
+    assert updated.canonical_url == "https://www.hse.ru/org/persons/47634735"
+    assert updated.status == "active"
+    assert run.new_count == 0
+    assert db_session.query(Employee).count() == 1
+    assert {item.url for item in updated.profile_urls} == {
+        "https://www.hse.ru/staff/abelov",
+        "https://www.hse.ru/org/persons/47634735",
+    }
+
+
+def test_resource_cache_uses_etag_and_reuses_cached_body_on_304(db_session):
+    db_session.add(
+        ParseResourceCache(
+            profile_key="staff:cached",
+            resource_key="main-html",
+            method="GET",
+            url="https://www.hse.ru/staff/cached",
+            request_fingerprint="020d59db7b358d9023d0f185bcbf5a9c085d3cf2bf91d92d48eee9147e8d0f01",
+            etag='"cached"',
+            body_hash="cached-hash",
+            body_snapshot=gzip.compress("cached body".encode("utf-8")),
+            parser_version="0.6.0",
+        )
+    )
+    db_session.commit()
+    session = ConditionalSession()
+
+    result = ResourceCache(db_session).fetch_text(
+        session,
+        profile_key="staff:cached",
+        resource_key="main-html",
+        method="GET",
+        url="https://www.hse.ru/staff/cached",
+        headers={"User-Agent": "test"},
+        timeout=10,
+    )
+
+    assert session.requests[0][1]["headers"]["If-None-Match"] == '"cached"'
+    assert result.text == "cached body"
+    assert result.from_cache is True
+    assert session.not_modified_response.text_read is False
+
+
+def test_upsert_employee_skips_snapshot_when_checksum_is_unchanged(db_session):
+    first_run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    second_run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    db_session.add_all([first_run, second_run])
+    db_session.commit()
+
+    _, first_changed = _upsert_employee(db_session, first_run, _parsed_employee("same"))
+    _, second_changed = _upsert_employee(db_session, second_run, _parsed_employee("same"))
+    db_session.commit()
+
+    assert first_changed is True
+    assert second_changed is False
+    assert db_session.query(EmployeeSnapshot).count() == 1
+
+
+def test_upsert_employee_saves_publications_and_reuses_existing_rows(db_session):
+    first_run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    second_run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    db_session.add_all([first_run, second_run])
+    db_session.commit()
+
+    parsed = _parsed_employee("published")
+    parsed["sections"] = [
+        {
+            "type": "publications",
+            "publications": [
+                {
+                    "id": "888959076",
+                    "publication_id": "888959076",
+                    "title": "Detailed Publication",
+                    "year": 2023,
+                    "publication_type": "ARTICLE",
+                    "language": "ru",
+                    "status": 1,
+                    "url": "https://publications.hse.ru/view/888959076",
+                    "doi_url": "https://doi.org/10.1/test",
+                    "citation_text": "Detailed citation",
+                    "annotation": {"ru": "Аннотация"},
+                    "description": {"main": "Detailed citation"},
+                    "authors": [{"id": "1", "title_ru": "Автор"}],
+                    "raw_data": {"id": "888959076", "title": "Detailed Publication"},
+                }
+            ],
+        }
+    ]
+
+    employee, _ = _upsert_employee(db_session, first_run, parsed)
+    db_session.commit()
+    _upsert_employee(db_session, second_run, _parsed_employee_with_publication("published"))
+    db_session.commit()
+
+    publications = db_session.query(EmployeePublication).filter_by(employee_id=employee.id).all()
+    assert len(publications) == 1
+    assert publications[0].doi_url == "https://doi.org/10.1/test"
+    assert publications[0].authors == [{"id": "1", "title_ru": "Автор"}]
+
+
+def test_upsert_employee_records_publication_errors_without_failing_employee(monkeypatch, db_session):
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    db_session.add(run)
+    db_session.commit()
+
+    def broken_sync(*_args, **_kwargs):
+        raise RuntimeError("boom")
+
+    monkeypatch.setattr("app.services.crawler._sync_employee_publications", broken_sync)
+
+    employee, changed = _upsert_employee(db_session, run, _parsed_employee_with_publication("error-safe"))
+    db_session.commit()
+
+    assert changed is True
+    assert employee.full_name == "Same Person"
+    assert db_session.query(Employee).filter_by(profile_key="staff:error-safe").one()
+    error = db_session.query(CrawlError).one()
+    assert "публикации" in error.message.lower()
+
+
+def test_upsert_employee_saves_news_links_and_reuses_existing_rows(db_session):
+    first_run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    second_run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    db_session.add_all([first_run, second_run])
+    db_session.commit()
+
+    employee, _ = _upsert_employee(db_session, first_run, _parsed_employee_with_news("news-person"))
+    db_session.commit()
+    _upsert_employee(db_session, second_run, _parsed_employee_with_news("news-person"))
+    db_session.commit()
+
+    news_links = db_session.query(EmployeeNewsLink).filter_by(employee_id=employee.id).all()
+    assert len(news_links) == 1
+    assert news_links[0].title == "News Title"
+    assert news_links[0].url == "https://www.hse.ru/news/1.html"
+    assert news_links[0].published_year == 2026
+
+
+def test_upsert_employee_records_news_errors_without_failing_employee(monkeypatch, db_session):
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    db_session.add(run)
+    db_session.commit()
+
+    def broken_sync(*_args, **_kwargs):
+        raise RuntimeError("boom")
+
+    monkeypatch.setattr("app.services.crawler._sync_employee_news_links", broken_sync)
+
+    employee, changed = _upsert_employee(db_session, run, _parsed_employee_with_news("news-error-safe"))
+    db_session.commit()
+
+    assert changed is True
+    assert employee.full_name == "Same Person"
+    assert db_session.query(Employee).filter_by(profile_key="staff:news-error-safe").one()
+    error = db_session.query(CrawlError).one()
+    assert "новости" in error.message.lower()
+
+
+def test_checksum_changes_when_widget_data_changes():
+    base = _parsed_employee("widgets")
+    changed = _parsed_employee("widgets")
+    changed["sections"] = [
+        {
+            "type": "publications",
+            "publications": [{"id": "1", "title": "New publication"}],
+        }
+    ]
+
+    assert _checksum(base) != _checksum(changed)
+
+
+def test_checksum_ignores_date_dependent_experience_text():
+    first = _parsed_employee("experience")
+    second = _parsed_employee("experience")
+    first["sections"] = [{"raw_text": "Стаж работы в НИУ ВШЭ: 5 лет"}]
+    second["sections"] = [{"raw_text": "Стаж работы в НИУ ВШЭ: 6 лет"}]
+
+    assert _checksum(first) == _checksum(second)
+
+
+def _parsed_employee(profile_id: str) -> dict:
+    return {
+        "source_url": f"https://www.hse.ru/staff/{profile_id}",
+        "profile_type": "staff",
+        "profile_id": profile_id,
+        "full_name": "Same Person",
+        "tabs": [],
+        "sections": [],
+        "parser_version": "0.6.0",
+        "_html": "<html></html>",
+    }
+
+
+def _parsed_employee_with_publication(profile_id: str) -> dict:
+    parsed = _parsed_employee(profile_id)
+    parsed["sections"] = [
+        {
+            "type": "publications",
+            "publications": [
+                {
+                    "id": "888959076",
+                    "publication_id": "888959076",
+                    "title": "Detailed Publication",
+                    "year": 2023,
+                    "publication_type": "ARTICLE",
+                    "language": "ru",
+                    "status": 1,
+                    "url": "https://publications.hse.ru/view/888959076",
+                    "doi_url": "https://doi.org/10.1/test",
+                    "citation_text": "Detailed citation",
+                    "annotation": {"ru": "Аннотация"},
+                    "description": {"main": "Detailed citation"},
+                    "authors": [{"id": "1", "title_ru": "Автор"}],
+                    "raw_data": {"id": "888959076", "title": "Detailed Publication"},
+                }
+            ],
+        }
+    ]
+    return parsed
+
+
+def _parsed_employee_with_news(profile_id: str) -> dict:
+    parsed = _parsed_employee(profile_id)
+    parsed["sections"] = [
+        {
+            "type": "news",
+            "news_links": [
+                {
+                    "title": "News Title",
+                    "url": "https://www.hse.ru/news/1.html",
+                    "summary": "News summary",
+                    "published_at": "2026-04-28T00:00:00+00:00",
+                    "published_year": 2026,
+                    "raw_data": {"title": "News Title", "url": "https://www.hse.ru/news/1.html"},
+                }
+            ],
+        }
+    ]
+    return parsed
 
 
-def test_upsert_employee_increments_new_count_for_new_employee(db_session):
+def test_csv_enrichment_survives_repeated_crawl_without_duplicate(db_session):
+    from scripts.import_publications import plan
+    from app.services.crawler import _sync_employee_publications
+
     run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
     db_session.add(run)
     db_session.commit()
-
-    _upsert_employee(
-        db_session,
-        run,
-        {
-            "source_url": "https://www.hse.ru/staff/newperson",
-            "profile_type": "staff",
-            "profile_id": "newperson",
-            "full_name": "New Person",
-            "tabs": [],
-            "sections": [],
-            "parser_version": "0.2.0",
-            "_html": "<html></html>",
-        },
-    )
+    employee, _ = _upsert_employee(db_session, run, _parsed_employee_with_publication("csv-preserved"))
     db_session.commit()
+    existing = db_session.query(EmployeePublication).filter_by(employee_id=employee.id).one()
+    rows = [(number, {"Название публикации": title, "Год": str(year), "DOI": identifier,
+                      "Авторы НИУ ВШЭ": employee.full_name, "Квартиль scopus": "Q1"})
+            for number, title, year, identifier in (
+                (2, existing.title, existing.year, ""),
+                (3, "CSV only", 2024, "10.1234/csv-only"),
+                (4, "Later HSE", 2024, "10.1234/later"))]
+    actions, _ = plan(rows, [{"id": employee.id, "full_name": employee.full_name}],
+                      [{field: getattr(existing, field) for field in
+                        ("id", "employee_id", "title", "year", "doi_url", "raw_data")}])
+    for action in actions:
+        if action["result"] == "insert":
+            db_session.add(EmployeePublication(**action["values"]))
+        else:
+            for field, value in action["values"].items():
+                setattr(existing, field, value)
+    db_session.add(EmployeePublication(employee_id=employee.id, title="Obsolete HSE",
+                                       source_hash="obsolete", raw_data={"id": "obsolete"}))
+    db_session.commit()
+    original_id = existing.id
+    imported_ids = {p.title: p.id for p in db_session.query(EmployeePublication).all()}
 
-    assert run.new_count == 1
+    parsed = _parsed_employee_with_publication("csv-preserved")
+    publication = parsed["sections"][0]["publications"][0]
+    publication.pop("year")
+    publication.pop("doi_url")
+    publication["raw_data"]["fresh"] = True
+    for _ in range(2):
+        _sync_employee_publications(db_session, employee, parsed)
+        db_session.commit()
+    remaining = db_session.query(EmployeePublication).filter_by(employee_id=employee.id).all()
+    assert len(remaining) == 3
+    assert all(p.raw_data["csv_enrichment"]["Квартиль scopus"] == "Q1" for p in remaining)
+    refreshed = db_session.get(EmployeePublication, original_id)
+    assert refreshed.raw_data["fresh"] is True
+    assert refreshed.year == 2023 and refreshed.doi_url == "https://doi.org/10.1/test"
+    assert db_session.get(EmployeePublication, imported_ids["CSV only"]) is not None
+
+    parsed["sections"][0]["publications"].append({
+        "publication_id": "later-id", "title": "Later HSE", "year": 2024,
+        "doi_url": "http://dx.doi.org/10.1234/LATER", "raw_data": {"id": "later-id"}})
+    for _ in range(2):
+        _sync_employee_publications(db_session, employee, parsed)
+        db_session.commit()
+    assert db_session.query(EmployeePublication).filter_by(employee_id=employee.id).count() == 3
+    adopted = db_session.get(EmployeePublication, imported_ids["Later HSE"])
+    assert adopted.publication_id == "later-id"
+    assert adopted.raw_data["csv_enrichment"]["Квартиль scopus"] == "Q1"
+
+
+def test_ambiguous_csv_match_rolls_back_publication_sync(db_session):
+    from app.services.crawler import _try_sync_employee_publications
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    db_session.add(run)
+    db_session.commit()
+    employee, _ = _upsert_employee(db_session, run, _parsed_employee("csv-ambiguous"))
+    for index in range(2):
+        db_session.add(EmployeePublication(employee_id=employee.id, title="Same", year=2024,
+            doi_url="https://doi.org/10.1234/same", source_hash=f"csv-{index}",
+            raw_data={"csv_enrichment": {"Год": "2024"}}))
+    db_session.commit()
+    parsed = {"sections": [{"type": "publications", "publications": [
+        {"publication_id": "new", "title": "New"},
+        {"publication_id": "same", "title": "Same", "doi_url": "https://doi.org/10.1234/same", "year": 2024}]}]}
+    _try_sync_employee_publications(db_session, run, employee, parsed)
+    db_session.commit()
+    assert db_session.query(EmployeePublication).filter_by(employee_id=employee.id).count() == 2
+    assert db_session.query(CrawlError).filter_by(crawl_run_id=run.id, error_type="ValueError").count() == 1
+
+
+@pytest.mark.parametrize("bound_id", [None, "hse-A"])
+@pytest.mark.parametrize("incoming_ids", [("hse-A", "hse-B"), ("hse-B", "hse-A")])
+def test_two_hse_ids_cannot_overwrite_one_csv_record(db_session, bound_id, incoming_ids):
+    from app.services.crawler import _try_sync_employee_publications
+    from app.services.admin_data import run_detail_payload
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    db_session.add(run)
+    db_session.commit()
+    employee, _ = _upsert_employee(db_session, run, _parsed_employee("csv-hse-id-conflict"))
+    original_raw = {"csv_enrichment": {"Год": "2024"}, "original": "keep"}
+    original_authors = [{"title_ru": "Synthetic original author"}]
+    record = EmployeePublication(employee_id=employee.id, title="Same", year=2024,
+        publication_id=bound_id, source_hash="csv-original", raw_data=original_raw,
+        authors=original_authors)
+    db_session.add(record)
+    db_session.commit()
+    original_id = record.id
+    parsed = {"sections": [{"type": "publications", "publications": [
+        {"publication_id": identifier, "title": "Same", "year": 2024,
+         "authors": [{"id": identifier}], "raw_data": {"id": identifier}}
+        for identifier in incoming_ids]}]}
+    _try_sync_employee_publications(db_session, run, employee, parsed)
+    db_session.commit()
+    db_session.expire_all()
+    kept = db_session.get(EmployeePublication, original_id)
+    assert db_session.query(EmployeePublication).filter_by(employee_id=employee.id).count() == 1
+    assert kept.publication_id == bound_id and kept.source_hash == "csv-original"
+    assert kept.raw_data == original_raw and kept.authors == original_authors
+    errors = run_detail_payload(db_session, run)["errors"]
+    assert len(errors) == 1
+    assert errors[0]["error_type"] == "ValueError"
+    assert "Ambiguous HSE IDs" in errors[0]["message"]
+    assert "hse-A" in errors[0]["message"] and "hse-B" in errors[0]["message"]

tests/test_db_schema.py

@@ -0,0 +1,144 @@
+from sqlalchemy import create_engine, inspect, text
+
+from app.db import _ensure_runtime_schema
+
+
+def test_runtime_schema_adds_skipped_count_to_existing_crawl_runs_table(monkeypatch):
+    engine = create_engine("sqlite:///:memory:")
+    with engine.begin() as connection:
+        connection.execute(
+            text(
+                """
+                CREATE TABLE crawl_runs (
+                  id INTEGER PRIMARY KEY,
+                  source_url TEXT NOT NULL,
+                  status VARCHAR(32) NOT NULL DEFAULT 'running',
+                  found_count INTEGER NOT NULL DEFAULT 0,
+                  parsed_count INTEGER NOT NULL DEFAULT 0
+                )
+                """
+            )
+        )
+    monkeypatch.setattr("app.db.engine", engine)
+
+    _ensure_runtime_schema()
+
+    columns = {column["name"] for column in inspect(engine).get_columns("crawl_runs")}
+    assert "skipped_count" in columns
+
+
+def test_runtime_schema_creates_employee_publications_table_when_employees_exist(monkeypatch):
+    engine = create_engine("sqlite:///:memory:")
+    with engine.begin() as connection:
+        connection.execute(
+            text(
+                """
+                CREATE TABLE employees (
+                  id INTEGER PRIMARY KEY,
+                  profile_key VARCHAR(255) NOT NULL UNIQUE,
+                  canonical_url TEXT NOT NULL,
+                  status VARCHAR(32) NOT NULL DEFAULT 'active',
+                  first_seen_at DATETIME NOT NULL,
+                  last_seen_at DATETIME NOT NULL,
+                  created_at DATETIME NOT NULL,
+                  updated_at DATETIME NOT NULL
+                )
+                """
+            )
+        )
+        connection.execute(
+            text(
+                """
+                CREATE TABLE crawl_runs (
+                  id INTEGER PRIMARY KEY,
+                  source_url TEXT NOT NULL,
+                  status VARCHAR(32) NOT NULL DEFAULT 'running',
+                  found_count INTEGER NOT NULL DEFAULT 0,
+                  parsed_count INTEGER NOT NULL DEFAULT 0,
+                  skipped_count INTEGER NOT NULL DEFAULT 0
+                )
+                """
+            )
+        )
+    monkeypatch.setattr("app.db.engine", engine)
+
+    _ensure_runtime_schema()
+    _ensure_runtime_schema()
+
+    inspector = inspect(engine)
+    assert "employee_publications" in inspector.get_table_names()
+    columns = {column["name"] for column in inspector.get_columns("employee_publications")}
+    assert {"employee_id", "publication_id", "doi_url", "authors", "raw_data", "source_hash"}.issubset(columns)
+
+
+def test_runtime_schema_creates_employee_news_links_table_when_employees_exist(monkeypatch):
+    engine = create_engine("sqlite:///:memory:")
+    with engine.begin() as connection:
+        connection.execute(
+            text(
+                """
+                CREATE TABLE employees (
+                  id INTEGER PRIMARY KEY,
+                  profile_key VARCHAR(255) NOT NULL UNIQUE,
+                  canonical_url TEXT NOT NULL,
+                  status VARCHAR(32) NOT NULL DEFAULT 'active',
+                  first_seen_at DATETIME NOT NULL,
+                  last_seen_at DATETIME NOT NULL,
+                  created_at DATETIME NOT NULL,
+                  updated_at DATETIME NOT NULL
+                )
+                """
+            )
+        )
+        connection.execute(
+            text(
+                """
+                CREATE TABLE crawl_runs (
+                  id INTEGER PRIMARY KEY,
+                  source_url TEXT NOT NULL,
+                  status VARCHAR(32) NOT NULL DEFAULT 'running',
+                  found_count INTEGER NOT NULL DEFAULT 0,
+                  parsed_count INTEGER NOT NULL DEFAULT 0,
+                  skipped_count INTEGER NOT NULL DEFAULT 0
+                )
+                """
+            )
+        )
+    monkeypatch.setattr("app.db.engine", engine)
+
+    _ensure_runtime_schema()
+    _ensure_runtime_schema()
+
+    inspector = inspect(engine)
+    assert "employee_news_links" in inspector.get_table_names()
+    columns = {column["name"] for column in inspector.get_columns("employee_news_links")}
+    assert {"employee_id", "title", "url", "summary", "published_at", "published_year", "source_hash", "raw_data"}.issubset(columns)
+
+
+def test_runtime_schema_adds_profile_verification_fields(monkeypatch):
+    engine = create_engine("sqlite:///:memory:")
+    with engine.begin() as connection:
+        connection.execute(
+            text(
+                """
+                CREATE TABLE employees (
+                  id INTEGER PRIMARY KEY,
+                  profile_key VARCHAR(255) NOT NULL UNIQUE,
+                  canonical_url TEXT NOT NULL,
+                  status VARCHAR(32) NOT NULL DEFAULT 'active',
+                  first_seen_at DATETIME NOT NULL,
+                  last_seen_at DATETIME NOT NULL,
+                  created_at DATETIME NOT NULL,
+                  updated_at DATETIME NOT NULL
+                )
+                """
+            )
+        )
+    monkeypatch.setattr("app.db.engine", engine)
+
+    _ensure_runtime_schema()
+
+    inspector = inspect(engine)
+    columns = {column["name"] for column in inspector.get_columns("employees")}
+    assert {"profile_unavailable_streak", "last_profile_check_at", "has_academic_degree"}.issubset(columns)
+    assert "employee_profile_urls" in inspector.get_table_names()

tests/test_deploy_diagnostic.py

@@ -0,0 +1,70 @@
+import os
+from pathlib import Path
+import re
+import subprocess
+import textwrap
+
+import pytest
+
+
+WORKFLOW = (Path(__file__).parents[1] / ".gitea/workflows/ci.yml").read_text()
+
+
+@pytest.mark.parametrize("keygen_exit", [0, 1])
+def test_fingerprint_output_and_cleanup(tmp_path, keygen_exit):
+    step = WORKFLOW.split("      - name: Deployment key fingerprint\n", 1)[1]
+    step = step.split("      - name:", 1)[0]
+    script = textwrap.dedent(step.split("        run: |\n", 1)[1])
+    keygen = tmp_path / "ssh-keygen"
+    keygen.write_text(
+        "#!/bin/sh\n"
+        'case "$1" in\n'
+        '-y) printf "PUBLIC_MATERIAL\\n"; exit "$KEYGEN_EXIT" ;;\n'
+        '-lf) cat >/dev/null; printf "256 SHA256:test comment (ED25519)\\n" ;;\n'
+        '*) exit 99 ;;\n'
+        "esac\n"
+    )
+    keygen.chmod(0o700)
+    (tmp_path / "deploy_key").write_text("PRIVATE_MATERIAL")
+    (tmp_path / "known_hosts").write_text("HOST_MATERIAL")
+    result = subprocess.run(
+        ["bash", "-x", "-c", script], capture_output=True, text=True,
+        env={**os.environ, "RUNNER_TEMP": str(tmp_path),
+             "PATH": f"{tmp_path}:{os.environ['PATH']}",
+             "KEYGEN_EXIT": str(keygen_exit)},
+    )
+    assert (result.returncode == 0) == (keygen_exit == 0)
+    if keygen_exit == 0:
+        assert result.stdout == "SHA256:test\n"
+    assert "MATERIAL" not in result.stdout + result.stderr
+    assert "comment" not in result.stdout + result.stderr
+    assert not (tmp_path / "deploy_key").exists()
+    assert not (tmp_path / "known_hosts").exists()
+
+
+@pytest.mark.parametrize("event", ["push", "pull_request", "workflow_dispatch"])
+@pytest.mark.parametrize("diagnostic_only", ["true", "false", ""])
+def test_launch_conditions(event, diagnostic_only):
+    job = WORKFLOW.split("\n  deploy:\n", 1)[1]
+    condition = re.search(r"^    if: (.+)$", job, re.MULTILINE).group(1)
+    terms = re.findall(r"(github\.[a-z_.]+) == '([^']*)'", condition)
+    assert condition == " && ".join(f"{name} == '{value}'" for name, value in terms)
+    context = {"github.event_name": event,
+               "github.event.inputs.diagnostic_only": diagnostic_only}
+    runs = all(context[name] == value for name, value in terms)
+    assert runs == (event == "workflow_dispatch" and diagnostic_only == "true")
+
+
+def test_both_vps_steps_are_unconditionally_disabled():
+    for name in ("Upload release", "Deploy and verify"):
+        step = WORKFLOW.split(f"      - name: {name}\n", 1)[1]
+        assert step.startswith("        if: ${{ false }}\n")
+
+
+def test_push_and_pull_request_keep_tests():
+    triggers = WORKFLOW.split("\njobs:\n", 1)[0]
+    for event in ("push", "pull_request"):
+        assert f"  {event}:\n    branches: [main]\n" in triggers
+    test_job = WORKFLOW.split("\n  test:\n", 1)[1].split("\n  deploy:\n", 1)[0]
+    assert "    if:" not in test_job
+    assert "python -m pytest -q" in test_job

tests/test_employee_detail_template.py

@@ -0,0 +1,35 @@
+from pathlib import Path
+
+
+def test_employee_detail_template_is_human_readable():
+    template = Path("app/templates/employee_detail.html").read_text(encoding="utf-8")
+
+    assert "Current data" not in template
+    assert "<pre class=\"code\"" not in template
+    assert ">Tabs<" not in template
+    assert "contacts.items" not in template
+    assert "contacts.contact_items" in template
+    assert "section.items" not in template
+    assert "section.list_items" in template
+    assert "Основная информация" in template
+    assert "Контакты" in template
+    assert "В новостях" in template
+    assert "employee_view.news_links" in template
+    assert "news.summary" in template
+    assert "Разделы профиля" in template
+    assert "graduation_theses" in template
+    assert "Год защиты" in template
+    assert "Parser version" not in template
+    assert "First seen" not in template
+    assert "Last seen" not in template
+    assert "Dismissed at" not in template
+    assert "Profile type" not in template
+    assert "Profile ID" not in template
+    assert "Впервые найден" in template
+    assert "Последний раз найден" in template
+    assert "Дата увольнения" in template
+    assert "Тип профиля" in template
+    assert "ID профиля" in template
+    assert "Обновить данные" in template
+    assert 'action="/admin/employees/{{ employee.id }}/refresh"' in template
+    assert "Снапшоты" in template

tests/test_parser.py

@@ -1,9 +1,124 @@
 from bs4 import BeautifulSoup
 
-from app.parser.profile import extract_person_tabs
+from app.parser.profile import enrich_sections_from_hse_widgets, extract_person_tabs, extract_sections
 from app.parser.profile_url import normalize_profile_url, parse_profile_identity
 
 
+class FakeResponse:
+    def __init__(self, payload):
+        self.payload = payload
+
+    def raise_for_status(self):
+        return None
+
+    def json(self):
+        return self.payload
+
+
+class FakeSession:
+    def __init__(self):
+        self.posts = []
+        self.gets = []
+
+    def post(self, url, **kwargs):
+        self.posts.append((url, kwargs))
+        return FakeResponse(
+            {
+                "status": "ok",
+                "result": {
+                    "more": False,
+                    "total": 1,
+                    "items": [
+                        {
+                            "id": "888959076",
+                            "type": "ARTICLE",
+                            "title": "Дублирование пакетов",
+                            "year": 2023,
+                            "language": {"name": "ru"},
+                            "status": 1,
+                            "authorsByType": {
+                                "author": [
+                                    {
+                                        "id": "568398853",
+                                        "href": "/org/persons/568398853",
+                                        "title": {"ru": "Левицкий И. А.", "en": ""},
+                                        "reverseTitle": {"ru": "И. А. Левицкий", "en": ""},
+                                    }
+                                ]
+                            },
+                            "description": {"short": {"ru": "Информационные процессы. 2023."}},
+                            "annotation": {"ru": "<p>Русская аннотация</p>"},
+                            "documents": {"DOI": {"href": "https://doi.org/10.1/test"}},
+                        }
+                    ],
+                },
+            }
+        )
+
+    def get(self, url, **kwargs):
+        self.gets.append((url, kwargs))
+        return FakeResponse(
+            {
+                "lang": "ru",
+                "success": True,
+                "data": [
+                    {
+                        "id": 1045750164,
+                        "year": 2025,
+                        "level": "Бакалавриат",
+                        "title": "Аппаратно-программный комплекс защиты сети",
+                        "rating": 8,
+                        "student": "Лесняк Владислав Евгеньевич",
+                        "learnProgram": {"title": "Информатика и вычислительная техника", "url": "https://hse.ru/ba/isct/"},
+                        "orgUnit": {"title": "МИЭМ", "url": "https://www.hse.ru/org/url/59315150"},
+                        "supervisors": [{"url": "https://www.hse.ru/org/persons/803294906", "name": "Борисов Сергей Петрович"}],
+                    }
+                ],
+            }
+        )
+
+
+class GroupedPublicationsSession(FakeSession):
+    def post(self, url, **kwargs):
+        self.posts.append((url, kwargs))
+        return FakeResponse(
+            {
+                "status": "ok",
+                "result": {
+                    "more": False,
+                    "total": 1,
+                    "groupType": 2,
+                    "items": {
+                        "year": {
+                            "header": {"ru": "по году", "en": "by year"},
+                            "criteria": {"year": []},
+                            "items": {
+                                "2011": [
+                                    {
+                                        "id": "146366790",
+                                        "type": "ARTICLE",
+                                        "title": "Развитие теории самосогласованного поля",
+                                        "year": 2011,
+                                        "description": {"short": {"ru": "Журнал физической химии 2011."}},
+                                    }
+                                ],
+                                "2012": [
+                                    {
+                                        "id": "146367323",
+                                        "type": "ARTICLE",
+                                        "title": "Self-consistent field theory investigation",
+                                        "year": 2012,
+                                        "description": {"short": {"en": "Russian Journal of Physical Chemistry A 2012."}},
+                                    }
+                                ],
+                            },
+                        }
+                    },
+                },
+            }
+        )
+
+
 def test_normalize_profile_url_supports_staff_and_org_persons():
     assert normalize_profile_url("/staff/avsergeev#sci") == "https://www.hse.ru/staff/avsergeev"
     assert normalize_profile_url("https://www.hse.ru/org/persons/123/") == "https://www.hse.ru/org/persons/123"
@@ -26,3 +141,136 @@ def test_extract_person_tabs_prefers_person_menu_addition():
 
     assert [tab["title"] for tab in tabs] == ["Домашняя страница", "Публикации"]
     assert tabs[1]["href"] == "https://www.hse.ru/staff/avsergeev#sci"
+
+
+def test_enrich_sections_from_hse_widgets_loads_publications_and_vkr():
+    soup = BeautifulSoup(
+        """
+        <script src="/n/stat/publications/dist-w/publs.js" data-author="568398853" data-widget-name="AuthorSearch"></script>
+        <script src="/n/stat/vkr/app.js" data-api-url="/n/vkr/api/" data-person-id="803294906"></script>
+        """,
+        "html.parser",
+    )
+    session = FakeSession()
+
+    sections = enrich_sections_from_hse_widgets(
+        session,
+        soup,
+        "https://www.hse.ru/org/persons/803294906",
+        {"User-Agent": "test"},
+        10,
+        [],
+    )
+
+    publications = next(section for section in sections if section["type"] == "publications")
+    theses = next(section for section in sections if section["type"] == "graduation_theses")
+
+    assert publications["publications_count"] == 1
+    assert publications["publications"][0]["url"] == "https://publications.hse.ru/view/888959076"
+    assert publications["publications"][0]["doi_url"] == "https://doi.org/10.1/test"
+    assert publications["publications"][0]["annotation"] == {"ru": "Русская аннотация"}
+    assert publications["publications"][0]["authors"][0]["is_current_employee"] is True
+    assert theses["theses_count"] == 1
+    assert theses["theses"][0]["student"] == "Лесняк Владислав Евгеньевич"
+    assert theses["theses"][0]["project_url"] == "https://www.hse.ru/edu/vkr/1045750164"
+    assert session.posts[0][0] == "https://publications.hse.ru/api/searchPubs"
+    assert session.gets[0][1]["params"] == {"supervisorId": "803294906"}
+
+
+def test_enrich_sections_from_hse_widgets_loads_grouped_publications():
+    soup = BeautifulSoup(
+        """
+        <script src="/n/stat/publications/dist-w/publs.js" data-author="133709486" data-widget-name="AuthorSearch"></script>
+        """,
+        "html.parser",
+    )
+    session = GroupedPublicationsSession()
+
+    sections = enrich_sections_from_hse_widgets(
+        session,
+        soup,
+        "https://www.hse.ru/org/persons/133709486",
+        {"User-Agent": "test"},
+        10,
+        [],
+    )
+
+    publications = next(section for section in sections if section["type"] == "publications")
+
+    assert publications["publications_count"] == 2
+    assert [item["id"] for item in publications["publications"]] == ["146366790", "146367323"]
+    assert publications["publications"][0]["url"] == "https://publications.hse.ru/view/146366790"
+    assert publications["publications"][1]["url"] == "https://publications.hse.ru/view/146367323"
+
+
+def test_news_heading_with_publications_word_does_not_absorb_widget_publications():
+    soup = BeautifulSoup(
+        """
+        <h2>Статья профессора МИЭМ вошла в число самых популярных публикаций на портале SpringerLink</h2>
+        <div class="post__text">
+          <p>Первоначально статья профессора вышла в российском журнале.</p>
+        </div>
+        <script src="/n/stat/publications/dist-w/publs.js" data-author="133709486" data-widget-name="AuthorSearch"></script>
+        """,
+        "html.parser",
+    )
+    session = FakeSession()
+
+    sections = extract_sections(soup, "https://www.hse.ru/org/persons/133709486")
+    sections = enrich_sections_from_hse_widgets(
+        session,
+        soup,
+        "https://www.hse.ru/org/persons/133709486",
+        {"User-Agent": "test"},
+        10,
+        sections,
+    )
+
+    assert sections[0]["type"] == "paragraphs"
+    assert sections[0]["title"].startswith("Статья профессора")
+    publications = [section for section in sections if section["type"] == "publications"]
+    assert len(publications) == 1
+    assert publications[0]["title"] == "Публикации и исследования"
+    assert publications[0]["publications_count"] == 1
+
+
+def test_extract_sections_parses_employee_news_links():
+    soup = BeautifulSoup(
+        """
+        <div class="b-person-data posts hidden printable" data-tab="press_links_news" tab-node="press_links_news">
+          <div class="post f8">
+            <div class="post__extra">
+              <div class="post-meta">
+                <div class="post-meta__date">
+                  <div class="post-meta__day">28</div>
+                  <div class="post-meta__month">апр.</div>
+                  <div class="post-meta__year">2026</div>
+                </div>
+              </div>
+            </div>
+            <div class="post__content">
+              <h2 class="first_child"><a class="link" href="/news/edu/1153850518.html">Как финал ВсОШ формирует кадры</a></h2>
+              <div class="post__text"><p class="with-indent">Краткое описание новости.</p></div>
+            </div>
+          </div>
+          <div class="post f8">
+            <div class="post__content">
+              <h2><a href="https://miem.hse.ru/news/1123589375.html">Партнер магистратуры</a></h2>
+            </div>
+          </div>
+        </div>
+        """,
+        "html.parser",
+    )
+
+    sections = extract_sections(soup, "https://www.hse.ru/staff/avsergeev")
+
+    assert len(sections) == 1
+    news = sections[0]
+    assert news["type"] == "news"
+    assert news["news_count"] == 2
+    assert news["news_links"][0]["title"] == "Как финал ВсОШ формирует кадры"
+    assert news["news_links"][0]["url"] == "https://www.hse.ru/news/edu/1153850518.html"
+    assert news["news_links"][0]["summary"] == "Краткое описание новости."
+    assert news["news_links"][0]["published_at"] == "2026-04-28T00:00:00+00:00"
+    assert news["news_links"][0]["published_year"] == 2026

tests/test_publication_import.py

@@ -0,0 +1,233 @@
+"""Offline checks: synthetic identities only, no settings or database access."""
+from scripts.import_publications import plan, read_rows
+from pathlib import Path
+
+
+def row(title="Test", doi="10.1234/test", year="2024", authors="Test Person"):
+    return {"Название публикации": title, "DOI": doi, "Год": year,
+            "Авторы НИУ ВШЭ": authors}
+
+
+EMPLOYEES = [{"id": 1, "full_name": "Test Person"}]
+
+
+def test_repeat_and_no_new_employees():
+    rows = [(2, row()), (3, row()), (4, row(authors="Missing Person"))]
+    actions, audit = plan(rows, EMPLOYEES, [])
+    assert len(actions) == 1 and actions[0]["result"] == "insert"
+    assert [a["reason"] for a in audit] == ["", "csv_duplicate", "employee_not_found"]
+    stored = [{"id": 11, **actions[0]["values"]}]
+    repeat, _ = plan(rows, EMPLOYEES, stored)
+    assert repeat == []
+
+
+def test_ambiguity_and_conflicts():
+    assert not plan([(2, row())], EMPLOYEES * 2, [])[0]
+    stored = [{"id": 1, "employee_id": 1, "title": "Test", "year": 2024,
+               "doi_url": "https://doi.org/10.1234/other", "raw_data": {}}]
+    assert plan([(2, row())], EMPLOYEES, stored)[1][0]["reason"] == "doi_conflict"
+    stored[0]["doi_url"] = "https://doi.org/10.1234/test"
+    assert plan([(2, row(year="2025"))], EMPLOYEES, stored)[1][0]["reason"] == "year_conflict"
+    assert plan([(2, row())], EMPLOYEES, stored * 2)[0]  # same id is one identity
+    stored.append({**stored[0], "id": 2})
+    assert plan([(2, row())], EMPLOYEES, stored)[1][0]["reason"] == "ambiguous_publication"
+
+
+def test_fill_only_preserves_metadata_and_empty_values():
+    stored = [{"id": 1, "employee_id": 1, "title": "Test", "year": 2024,
+               "doi_url": None, "raw_data": {"hse": "keep", "csv_enrichment": {"extra": "keep"}}}]
+    actions, _ = plan([(2, {**row(), "extra": "", "new": "added"})], EMPLOYEES, stored)
+    values = actions[0]["values"]
+    assert set(values) == {"doi_url", "raw_data"}
+    assert values["raw_data"]["hse"] == "keep"
+    assert values["raw_data"]["csv_enrichment"]["extra"] == "keep"
+    assert values["raw_data"]["csv_enrichment"]["new"] == "added"
+    assert "new" not in stored[0]["raw_data"]["csv_enrichment"]
+
+
+def test_missing_doi_then_doi_does_not_update_synthetic_id():
+    actions, _ = plan([(2, row(doi="")), (3, row())], EMPLOYEES, [])
+    assert len(actions) == 1 and actions[0]["result"] == "insert"
+    assert actions[0]["values"]["doi_url"] == "https://doi.org/10.1234/test"
+
+
+def test_invalid_rows_and_owner_file():
+    assert not plan([(2, row(year="oops")), (3, row(doi="bad"))], EMPLOYEES, [])[0]
+    path = Path(__file__).resolve().parents[2] / "Статьи.csv"
+    if path.exists():
+        rows = read_rows(path)
+        assert len(rows) == 1797
+        actions, audit = plan(rows, [], [])
+        assert not actions and audit and all(a["result"] == "skip" for a in audit)
+
+
+def test_cli_read_only_and_atomic_failure(tmp_path, monkeypatch):
+    import sys
+    import psycopg
+    from scripts import import_publications
+
+    source = tmp_path / "input.csv"
+    source.write_text("Название публикации;Год;DOI;Авторы НИУ ВШЭ\nFirst;2024;10.1234/first;Test Person\nSecond;2024;10.1234/second;Test Person\n", encoding="utf-8")
+
+    class Result:
+        rowcount = 1
+
+        def __init__(self, rows):
+            self.rows = rows
+
+        def fetchall(self):
+            return self.rows
+
+        def fetchone(self):
+            return self.rows[0]
+
+    class Connection:
+        def __init__(self, fail=False):
+            self.queries, self.writes = [], []
+            self.fail, self.rolled_back, self.committed = fail, False, False
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, kind, *_):
+            if kind:
+                self.rollback()
+            elif not self.rolled_back:
+                self.committed = True
+
+        def rollback(self):
+            self.rolled_back = True
+            self.writes.clear()
+
+        def execute(self, query, params=None):
+            query = query if isinstance(query, str) else query.as_string()
+            self.queries.append(query)
+            if query.startswith("SELECT id, full_name"):
+                return Result(EMPLOYEES)
+            if query.startswith("SELECT id, employee_id"):
+                return Result([])
+            if "pg_try_advisory" in query:
+                return Result([{"locked": True}])
+            if query.startswith("INSERT"):
+                self.writes.append(params)
+                if self.fail and len(self.writes) == 2:
+                    raise RuntimeError("Synthetic second-write failure")
+            return Result([])
+
+    report = tmp_path / "dry.json"
+    dry = Connection()
+    monkeypatch.setattr(psycopg, "connect", lambda *a, **k: dry)
+    monkeypatch.setattr(sys, "argv", ["import_publications", str(source), "--report", str(report)])
+    import_publications.main()
+    assert dry.rolled_back and not dry.committed and not dry.writes
+    assert not any(q.startswith(("INSERT", "UPDATE", "DELETE")) for q in dry.queries)
+    assert "READ ONLY" in dry.queries[0]
+    assert report.exists()
+
+    failed = Connection(fail=True)
+    monkeypatch.setattr(psycopg, "connect", lambda *a, **k: failed)
+    report = tmp_path / "apply.json"
+    monkeypatch.setattr(sys, "argv", ["import_publications", str(source), "--report", str(report), "--apply"])
+    import pytest
+    with pytest.raises(RuntimeError):
+        import_publications.main()
+    assert failed.rolled_back and not failed.committed and not failed.writes
+    assert not report.exists()
+
+    success = Connection()
+    monkeypatch.setattr(psycopg, "connect", lambda *a, **k: success)
+    monkeypatch.setattr(sys, "argv", ["import_publications", str(source), "--report", str(tmp_path / "success.json"), "--apply"])
+    import_publications.main()
+    assert success.committed and len(success.writes) == 2
+    assert not any(q.startswith(("DELETE", "ALTER", "CREATE")) for q in success.queries)
+
+
+def test_report_cannot_overwrite_source(tmp_path, monkeypatch):
+    import sys
+    import pytest
+    from scripts import import_publications
+    source = tmp_path / "input.csv"
+    source.write_text("unchanged", encoding="utf-8")
+    monkeypatch.setattr(sys, "argv", ["import_publications", str(source), "--report", str(source)])
+    with pytest.raises(ValueError):
+        import_publications.main()
+    assert source.read_text() == "unchanged"
+
+
+def test_duplicate_rows_fill_late_metadata_and_report_conflicts():
+    rows = [(2, {**row(), "EID (Scopus)": "", "Квартиль scopus": ""}),
+            (3, {**row(), "EID (Scopus)": "new-id", "Квартиль scopus": "Q1"}),
+            (4, {**row(), "EID (Scopus)": "different-id", "Квартиль scopus": "Q2", "UT (WoS)": "late-wos"})]
+    actions, audit = plan(rows, EMPLOYEES, [])
+    assert len(actions) == 1 and actions[0]["result"] == "insert"
+    metadata = actions[0]["values"]["raw_data"]["csv_enrichment"]
+    assert metadata["EID (Scopus)"] == "new-id" and metadata["Квартиль scopus"] == "Q1"
+    assert metadata["UT (WoS)"] == "late-wos"
+    assert audit[1]["reason"] == "merged_csv_duplicate"
+    assert audit[2]["conflict_fields"] == ["EID (Scopus)", "Квартиль scopus"]
+    assert audit[2]["result"] == "skip"
+    repeated, repeated_audit = plan(rows, EMPLOYEES, [{"id": 11, **actions[0]["values"]}])
+    assert repeated == []
+    assert repeated_audit[2]["reason"] == "metadata_conflict_first_value_kept"
+    assert "different-id" not in str(audit)
+
+
+def test_cli_apply_year_only_preserves_existing_json(tmp_path, monkeypatch):
+    import json
+    import sys
+    import psycopg
+    from scripts import import_publications
+    source = tmp_path / "input.csv"
+    fields = row()
+    source.write_text(";".join(fields) + "\n" + ";".join(fields.values()) + "\n", encoding="utf-8")
+    original_raw = {"hse": "keep", "csv_enrichment": fields.copy()}
+    stored = {"id": 11, "employee_id": 1, "title": "Test", "year": None,
+              "doi_url": "https://doi.org/10.1234/test", "raw_data": original_raw}
+    actions, _ = plan([(2, fields)], EMPLOYEES, [stored])
+    assert actions[0]["values"] == {"year": 2024}
+
+    class Result:
+        rowcount = 1
+
+        def __init__(self, rows):
+            self.rows = rows
+
+        def fetchall(self):
+            return self.rows
+
+        def fetchone(self):
+            return self.rows[0]
+
+    class Connection:
+        committed = False
+        updates = []
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, error, *_):
+            self.committed = error is None
+
+        def execute(self, query, params=None):
+            query = query if isinstance(query, str) else query.as_string()
+            if query.startswith("SELECT id, full_name"):
+                return Result(EMPLOYEES)
+            if query.startswith("SELECT id, employee_id"):
+                return Result([stored])
+            if "pg_try_advisory" in query:
+                return Result([{"locked": True}])
+            if query.startswith("UPDATE"):
+                self.updates.append((query, params))
+            return Result([])
+
+    connection = Connection()
+    report = tmp_path / "report.json"
+    monkeypatch.setattr(psycopg, "connect", lambda *a, **k: connection)
+    monkeypatch.setattr(sys, "argv", ["import_publications", str(source), "--report", str(report), "--apply"])
+    import_publications.main()
+    assert connection.committed and len(connection.updates) == 1
+    query, params = connection.updates[0]
+    assert '"year" = %s' in query and "raw_data" not in query
+    assert params == [2024, 11]
+    assert stored["raw_data"] == original_raw and stored["year"] is None
+    assert json.loads(report.read_text())["summary"]["counts"] == {"insert": 0, "update": 1, "skip": 0}

tests/test_publication_import_postgres.py

@@ -0,0 +1,280 @@
+"""Real PostgreSQL checks; opt in only inside tests/postgres/compose.yml."""
+
+import copy
+import csv
+import json
+import os
+import subprocess
+import sys
+from contextlib import contextmanager
+from pathlib import Path
+
+import pytest
+import requests
+from sqlalchemy import create_engine, select, text
+from sqlalchemy.orm import Session
+
+from app.config import Settings
+from app.db import Base
+from app.models import CrawlError, CrawlRun, Employee, EmployeePublication
+from app.services.crawler import _try_sync_employee_publications, run_crawl
+
+
+ADMIN_URL = "postgresql+psycopg://test_admin:isolated_test_admin@postgres-test:5432/miem_import_test"
+pytestmark = pytest.mark.skipif(
+    os.environ.get("MIEM_POSTGRES_INTEGRATION") != "1", reason="Requires isolated PostgreSQL Compose project"
+)
+
+
+@pytest.fixture(autouse=True)
+def isolated_environment(monkeypatch):
+    assert os.environ.get("DATABASE_URL") == ADMIN_URL, "Use the dedicated test Compose project"
+
+    def forbidden_request(*args, **kwargs):
+        raise AssertionError("External HTTP requests are forbidden in PostgreSQL integration tests")
+
+    monkeypatch.setattr(requests.sessions.Session, "request", forbidden_request)
+
+
+@contextmanager
+def database(name):
+    assert name in {"miem_import_test", "miem_crawl_test"}
+    engine = create_engine(ADMIN_URL.replace("/miem_import_test", "/" + name))
+    try:
+        with engine.connect() as connection:
+            assert connection.scalar(text("SELECT current_database()")) == name
+        Base.metadata.create_all(engine)
+        with engine.begin() as connection:
+            connection.execute(text("GRANT USAGE ON SCHEMA public TO import_test"))
+            connection.execute(text("GRANT SELECT (id, full_name) ON public.employees TO import_test"))
+            connection.execute(text("GRANT SELECT (id, employee_id, title, year, doi_url, raw_data) ON public.employee_publications TO import_test"))
+            connection.execute(text("GRANT INSERT (employee_id, title, year, doi_url, raw_data, source_hash, created_at, updated_at) ON public.employee_publications TO import_test"))
+            connection.execute(text("GRANT UPDATE (title, year, doi_url, raw_data, updated_at) ON public.employee_publications TO import_test"))
+            # Created from the actual model: not an assumed sequence name.
+            sequence = connection.scalar(text("SELECT pg_get_serial_sequence('public.employee_publications', 'id')"))
+            assert sequence == "public.employee_publications_id_seq"
+            connection.execute(text("GRANT USAGE ON SEQUENCE public.employee_publications_id_seq TO import_test"))
+            permissions = connection.execute(text("""SELECT
+                has_schema_privilege('import_test', 'public', 'CREATE'),
+                has_table_privilege('import_test', 'public.employees', 'INSERT'),
+                has_table_privilege('import_test', 'public.employee_publications', 'DELETE'),
+                has_sequence_privilege('import_test', 'public.employee_publications_id_seq', 'UPDATE')
+            """)).one()
+            assert not any(permissions)
+        with Session(engine) as session:
+            yield session, name
+    finally:
+        # Only the dedicated ephemeral test DB; never DROP DATABASE or roles.
+        Base.metadata.drop_all(engine)
+        engine.dispose()
+
+
+@pytest.fixture
+def pg_import_db():
+    with database("miem_import_test") as db:
+        yield db
+
+
+@pytest.fixture
+def pg_crawl_db():
+    with database("miem_crawl_test") as db:
+        yield db
+
+
+def employee(session, name="Synthetic One", key="one"):
+    record = Employee(full_name=name, profile_key="staff:" + key,
+                      canonical_url="https://synthetic.invalid/staff/" + key)
+    session.add(record)
+    session.commit()
+    return record
+
+
+def article(title="Synthetic article", identifier="10.1234/test", year="2024", author="Synthetic One", **metadata):
+    return {"Название публикации": title, "Год": year, "DOI": identifier,
+            "Авторы НИУ ВШЭ": author, **metadata}
+
+
+def write_csv(path, rows):
+    columns = list(dict.fromkeys(key for row in rows for key in row))
+    with path.open("w", encoding="utf-8-sig", newline="") as output:
+        writer = csv.DictWriter(output, fieldnames=columns, delimiter=";")
+        writer.writeheader()
+        writer.writerows(rows)
+
+
+def cli(tmp_path, db, rows, *, apply=False, expected_code=0):
+    _, name = db
+    source = tmp_path / "articles.csv"
+    report = tmp_path / "report.json"
+    write_csv(source, rows)
+    command = [sys.executable, str(Path(__file__).resolve().parents[1] / "scripts/import_publications.py"),
+               str(source), "--report", str(report)]
+    if apply:
+        command.append("--apply")
+    # Explicit whitelist: no production env, .env, pg_service or ~/.pgpass.
+    environment = {"PATH": os.environ["PATH"], "LANG": "C.UTF-8", "PGHOST": "postgres-test",
+                   "PGPORT": "5432", "PGDATABASE": name, "PGUSER": "import_test",
+                   "PGPASSWORD": "isolated_test_import", "PGPASSFILE": "/dev/null",
+                   "PGSERVICEFILE": "/dev/null", "PGCONNECT_TIMEOUT": "5"}
+    result = subprocess.run(command, env=environment, cwd=tmp_path, capture_output=True, text=True, timeout=60)
+    assert result.returncode == expected_code, result.stdout + result.stderr
+    assert "Synthetic One" not in result.stdout and "isolated_test_import" not in result.stdout
+    return json.loads(report.read_text()) if report.exists() else None
+
+
+def snapshot(session):
+    session.expire_all()
+    tables = {table.name: session.execute(select(table).order_by(*table.primary_key.columns)).all()
+              for table in Base.metadata.sorted_tables}
+    sequences = session.execute(text("SELECT sequencename, last_value FROM pg_sequences WHERE schemaname = 'public' ORDER BY sequencename")).all()
+    session.commit()
+    return tables, sequences
+
+
+def test_postgres_dry_run_changes_neither_rows_nor_sequences(pg_import_db, tmp_path):
+    session, _ = pg_import_db
+    employee(session)
+    before = snapshot(session)
+    report = cli(tmp_path, pg_import_db, [article()])
+    assert report["summary"]["counts"] == {"insert": 1, "update": 0, "skip": 0}
+    assert snapshot(session) == before
+
+
+def test_postgres_apply_and_reimport(pg_import_db, tmp_path):
+    session, _ = pg_import_db
+    person = employee(session)
+    existing = EmployeePublication(employee_id=person.id, title="Existing", year=2024,
+                                   source_hash="existing", raw_data={"hse": "keep"})
+    session.add(existing)
+    session.commit()
+    rows = [article(title="Existing"), article(title="New", identifier="10.1234/new"),
+            article(title="Unknown author", identifier="10.1234/unknown", author="Synthetic Missing")]
+    report = cli(tmp_path, pg_import_db, rows, apply=True)
+    assert report["summary"]["counts"] == {"insert": 1, "update": 1, "skip": 1}
+    session.expire_all()
+    assert session.query(Employee).count() == 1
+    assert session.query(EmployeePublication).count() == 2
+    assert existing.doi_url == "https://doi.org/10.1234/test" and existing.raw_data["hse"] == "keep"
+    before = snapshot(session)
+    repeated = cli(tmp_path, pg_import_db, rows, apply=True)
+    assert repeated["summary"]["counts"] == {"insert": 0, "update": 0, "skip": 3}
+    assert snapshot(session) == before
+
+
+def test_postgres_apply_failure_rolls_back_update_and_insert(pg_import_db, tmp_path):
+    session, _ = pg_import_db
+    person = employee(session)
+    existing = EmployeePublication(employee_id=person.id, title="Existing", year=2024,
+                                   source_hash="existing", raw_data={"hse": "keep"})
+    session.add(existing)
+    session.commit()
+    # Test-only constraint forces a real server error after update + successful INSERT.
+    session.execute(text("ALTER TABLE public.employee_publications ADD CONSTRAINT test_failure CHECK (title <> 'Reject')"))
+    session.commit()
+    before, _ = snapshot(session)
+    rows = [article(title="Existing"), article(title="Accepted", identifier="10.1234/accepted"),
+            article(title="Reject", identifier="10.1234/reject")]
+    assert cli(tmp_path, pg_import_db, rows, apply=True, expected_code=1) is None
+    after, _ = snapshot(session)
+    assert after == before  # nextval may advance on rollback; rows must not change.
+    assert (tmp_path / "report.json.pending").exists()
+
+
+def test_postgres_year_only_update_preserves_json(pg_import_db, tmp_path):
+    session, _ = pg_import_db
+    person = employee(session)
+    row = article()
+    raw = {"hse": {"keep": True}, "csv_enrichment": row.copy()}
+    record = EmployeePublication(employee_id=person.id, title=row["Название публикации"],
+                                 year=None, doi_url="https://doi.org/10.1234/test",
+                                 source_hash="existing", raw_data=raw)
+    session.add(record)
+    session.commit()
+    report = cli(tmp_path, pg_import_db, [row], apply=True)
+    session.expire_all()
+    assert report["summary"]["counts"]["update"] == 1
+    assert record.year == 2024 and record.raw_data == raw
+
+
+def test_postgres_late_csv_metadata_and_conflicts(pg_import_db, tmp_path):
+    session, _ = pg_import_db
+    employee(session)
+    rows = [article(), article(**{"EID (Scopus)": "late-id", "Квартиль scopus": "Q1"}),
+            article(**{"EID (Scopus)": "conflict-id", "UT (WoS)": "late-wos"})]
+    report = cli(tmp_path, pg_import_db, rows, apply=True)
+    session.expire_all()
+    assert session.query(EmployeePublication).count() == 1
+    metadata = session.query(EmployeePublication).one().raw_data["csv_enrichment"]
+    assert metadata["EID (Scopus)"] == "late-id" and metadata["Квартиль scopus"] == "Q1"
+    assert metadata["UT (WoS)"] == "late-wos"
+    assert report["summary"]["metadata_conflicts"] == 1
+    assert report["records"][2]["conflict_fields"] == ["EID (Scopus)"]
+    before = snapshot(session)
+    repeated = cli(tmp_path, pg_import_db, rows, apply=True)
+    assert repeated["summary"]["counts"]["insert"] == repeated["summary"]["counts"]["update"] == 0
+    assert snapshot(session) == before
+
+
+def test_postgres_crawler_preserves_import_and_adopts_hse_id(pg_crawl_db, tmp_path):
+    session, _ = pg_crawl_db
+    person = employee(session)
+    cli(tmp_path, pg_crawl_db, [article(), article(title="CSV only", identifier="10.1234/only")], apply=True)
+    session.expire_all()
+    ids = {p.title: p.id for p in session.query(EmployeePublication).all()}
+    run = CrawlRun(source_url="https://synthetic.invalid/persons", status="running")
+    session.add(run)
+    session.commit()
+    parsed = {"sections": [{"type": "publications", "publications": [
+        {"publication_id": "hse-one", "title": "Synthetic article", "year": 2024,
+         "doi_url": "https://doi.org/10.1234/test", "raw_data": {"id": "hse-one"}}]}]}
+    for _ in range(2):
+        _try_sync_employee_publications(session, run, person, copy.deepcopy(parsed))
+        session.commit()
+        session.expire_all()
+    assert session.query(EmployeePublication).count() == 2
+    kept = session.get(EmployeePublication, ids["Synthetic article"])
+    assert kept.publication_id == "hse-one" and "csv_enrichment" in kept.raw_data
+    assert session.get(EmployeePublication, ids["CSV only"]).raw_data["csv_enrichment"]
+    assert session.query(CrawlError).count() == 0
+
+
+@pytest.mark.parametrize("bound_id", [None, "hse-A"])
+@pytest.mark.parametrize("incoming_ids", [("hse-A", "hse-B"), ("hse-B", "hse-A")])
+def test_postgres_hse_id_conflict_does_not_block_other_employee(pg_crawl_db, tmp_path, monkeypatch, bound_id, incoming_ids):
+    session, _ = pg_crawl_db
+    first = employee(session)
+    second = employee(session, "Synthetic Two", "two")
+    cli(tmp_path, pg_crawl_db, [article(identifier="")], apply=True)
+    session.expire_all()
+    original = session.query(EmployeePublication).filter_by(employee_id=first.id).one()
+    original.publication_id = bound_id
+    original.authors = [{"id": "original-author"}]
+    session.commit()
+    fields = ("publication_id", "source_hash", "raw_data", "authors", "updated_at")
+    before = {field: copy.deepcopy(getattr(original, field)) for field in fields}
+    session.commit()
+
+    def profile(person, publications):
+        return {"source_url": person.canonical_url, "profile_type": "staff",
+                "profile_id": person.profile_key.split(":")[1], "full_name": person.full_name,
+                "parser_version": "integration-test", "tabs": [],
+                "sections": [{"type": "publications", "publications": publications}]}
+
+    profiles = {
+        first.canonical_url: profile(first, [{"publication_id": key, "title": "Synthetic article", "year": 2024,
+                                             "authors": [{"id": key}], "raw_data": {"id": key}} for key in incoming_ids]),
+        second.canonical_url: profile(second, [{"publication_id": "second-article", "title": "Other employee article",
+                                               "year": 2024, "raw_data": {"id": "second-article"}}]),
+    }
+    monkeypatch.setattr("app.services.crawler.collect_profile_links", lambda *a, **k: list(profiles))
+    monkeypatch.setattr("app.services.crawler.parse_person_profile", lambda _, url, *a, **k: copy.deepcopy(profiles[url]))
+    settings = Settings(_env_file=None, database_url=ADMIN_URL.replace("miem_import_test", "miem_crawl_test"),
+                        source_url="https://synthetic.invalid/persons", request_delay_seconds=0)
+    run = run_crawl(session, settings)
+    session.expire_all()
+    assert run.status == "completed" and run.parsed_count == 2
+    kept = session.query(EmployeePublication).filter_by(employee_id=first.id).one()
+    assert {field: getattr(kept, field) for field in fields} == before
+    assert session.query(EmployeePublication).filter_by(employee_id=second.id).one().publication_id == "second-article"
+    error = session.query(CrawlError).filter_by(crawl_run_id=run.id).one()
+    assert "Ambiguous HSE IDs" in error.message