test: prepare resource-limited isolated VPS PostgreSQL stand

Reviewed-on: #54
Reviewed-by: agent <agent@vakanaut.ru>

.env.example

@@ -10,9 +10,10 @@ CRAWL_LIMIT=
 REQUEST_TIMEOUT=30
 REQUEST_DELAY_SECONDS=1
 PARSER_USE_PLAYWRIGHT=false
+DISMISSAL_CONFIRMATION_RUNS=3
+MAX_AUTO_DISMISSALS_PER_RUN=25
 
 ADMIN_USERNAME=admin
 ADMIN_PASSWORD=change-me
 SESSION_SECRET=change-me-session-secret
 API_PORT=8000
-MCP_PORT=8001

.gitea/workflows/ci.yml

@@ -0,0 +1,104 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    env:
+      DATABASE_URL: sqlite:///./ci.db
+      PYTHONUNBUFFERED: "1"
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+      - run: python -m pip install --upgrade pip
+      - run: python -m pip install -r requirements.txt
+      - run: python -m pytest -q
+      - run: node --check app/static/admin.js
+      - run: sh -n infra/postgres-backup.sh
+      - run: sh -n infra/install-postgres-backup-cron.sh
+
+  deploy:
+    if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || github.event_name == 'workflow_dispatch'
+    needs: test
+    runs-on: ubuntu-latest
+    concurrency:
+      group: production-deploy
+      cancel-in-progress: false
+    env:
+      PROD_PATH: /srv/miem_workers
+      RELEASE_NAME: miem-workers-release-${{ github.run_id }}.tar
+    steps:
+      - uses: actions/checkout@v4
+      - name: Prepare release
+        id: prepare
+        shell: bash
+        run: |
+          set -euo pipefail
+          backend_version="$(sed -n 's/^BACKEND_VERSION = "\(.*\)"$/\1/p' app/version.py)"
+          case "$backend_version" in
+            ''|*[!0-9.]*) exit 1 ;;
+          esac
+          echo "backend_version=$backend_version" >> "$GITHUB_OUTPUT"
+          tar --exclude=.git --exclude=.env -cf "$RUNNER_TEMP/$RELEASE_NAME" .
+          printf '%s\n' "$PROD_SSH_KEY" > "$RUNNER_TEMP/deploy_key"
+          printf '%s\n' "$PROD_KNOWN_HOSTS" > "$RUNNER_TEMP/known_hosts"
+          chmod 600 "$RUNNER_TEMP/deploy_key" "$RUNNER_TEMP/known_hosts"
+        env:
+          PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }}
+          PROD_KNOWN_HOSTS: ${{ secrets.PROD_KNOWN_HOSTS }}
+      - name: Upload release
+        shell: bash
+        run: |
+          set -euo pipefail
+          scp -i "$RUNNER_TEMP/deploy_key" \
+            -o BatchMode=yes \
+            -o UserKnownHostsFile="$RUNNER_TEMP/known_hosts" \
+            "$RUNNER_TEMP/$RELEASE_NAME" \
+            "$PROD_USER@$PROD_HOST:$RELEASE_NAME"
+        env:
+          PROD_HOST: ${{ secrets.PROD_HOST }}
+          PROD_USER: ${{ secrets.PROD_USER }}
+      - name: Deploy and verify
+        shell: bash
+        run: |
+          set -euo pipefail
+          ssh -i "$RUNNER_TEMP/deploy_key" \
+            -o BatchMode=yes \
+            -o UserKnownHostsFile="$RUNNER_TEMP/known_hosts" \
+            "$PROD_USER@$PROD_HOST" \
+            "PROD_PATH='$PROD_PATH' RELEASE_NAME='$RELEASE_NAME' EXPECTED_BACKEND_VERSION='$EXPECTED_BACKEND_VERSION' bash -s" <<'REMOTE'
+          set -euo pipefail
+          cd "$PROD_PATH"
+          release="$HOME/$RELEASE_NAME"
+          test -f "$release"
+          archive_version="$(tar -xOf "$release" ./app/version.py | sed -n 's/^BACKEND_VERSION = "\(.*\)"$/\1/p')"
+          test "$archive_version" = "$EXPECTED_BACKEND_VERSION"
+          docker run --rm \
+            -v "$release:/src/release.tar:ro" \
+            -v "$PROD_PATH":/dst \
+            alpine:3.20 sh -c 'tar -xf /src/release.tar -C /dst'
+          sh /srv/miem_workers/infra/install-postgres-backup-cron.sh
+          docker compose up -d --build
+          runtime_version="$(docker compose exec -T api python -c 'from app.version import BACKEND_VERSION; print(BACKEND_VERSION)')"
+          test "$runtime_version" = "$EXPECTED_BACKEND_VERSION"
+          health="$(curl --fail --silent --show-error http://127.0.0.1:8000/api/health)"
+          case "$health" in
+            *"\"status\":\"ok\""*"\"backend_version\":\"$EXPECTED_BACKEND_VERSION\""*) ;;
+            *) echo "Unexpected health response: $health" >&2; exit 1 ;;
+          esac
+          docker compose ps
+          rm -f "$release"
+          REMOTE
+        env:
+          PROD_HOST: ${{ secrets.PROD_HOST }}
+          PROD_USER: ${{ secrets.PROD_USER }}
+          EXPECTED_BACKEND_VERSION: ${{ steps.prepare.outputs.backend_version }}

CHANGELOG.md

@@ -0,0 +1,91 @@
+# Changelog
+
+## 0.8.15
+
+- Исправлен production deploy: cron installer запускается через `sh` и не требует executable-бита.
+
+## 0.8.14
+
+- Production CD больше не создаёт backup при каждом deploy; еженедельная задача
+  сохраняет один полный PostgreSQL backup с атомарной заменой; ошибки и
+  пропущенные из-за lock запуски пишутся в system log.
+
+## 0.8.12
+
+- Передача версии в production deploy ограничена безопасным форматом.
+
+## 0.8.11
+
+- CSS-ассеты админки получают версию в URL и обновляются после релиза без ручной очистки кэша браузера.
+
+## 0.8.10
+
+- Production deploy проверяет совпадение версии release-архива и запущенного API.
+
+## 0.8.9
+
+- Контейнеры API, worker и PostgreSQL автоматически перезапускаются после остановки или перезагрузки хоста.
+
+## 0.8.8
+
+- Исправлено вертикальное выравнивание кнопки и ссылки в действиях фильтров справочника.
+
+## 0.8.7
+
+- Production CD оставляет только последний PostgreSQL backup после успешного деплоя.
+
+## 0.8.6
+
+- Исправлена загрузка release-архива production CD в домашний каталог deploy-пользователя.
+
+## 0.8.5
+
+- Исправлена адаптивная компоновка фильтров справочника и скрытие skip-link до клавиатурного фокуса.
+
+## 0.8.4
+
+- Подключены к конфигурации ограничения массового автоматического увольнения, используемые плановым обходом.
+
+## 0.8.3
+
+- Удалены устаревшие пояснение о недоступном endpoint и проверка его отсутствия.
+
+## 0.8.2
+
+- Добавлен production CD workflow с backup PostgreSQL и проверкой healthcheck после обновления.
+
+## 0.8.1
+
+- Улучшены доступность, фильтры, состояния и адаптивность административного каталога.
+
+## 0.7.8
+
+- В колонке учёной степени отображаются все найденные степени без лишнего текста.
+
+## 0.7.7
+
+- Удалена неиспользуемая интеграция обмена данными и связанная документация и тесты.
+
+## 0.7.6
+
+- Возвращён еженедельный автоматический запуск обхода сотрудников.
+
+## 0.7.5
+
+- Production Compose запускает только API и PostgreSQL.
+
+## 0.7.4
+
+- Ускорена фильтрация сотрудников по учёной степени.
+
+## 0.7.3
+
+- Восстановлена проверка профилей сотрудников и хранение истории URL профиля.
+
+## 0.7.2
+
+- В каталоге сотрудников добавлены фильтр и колонка учёной степени.
+
+## 0.7.1
+
+- Добавлена кнопка «Проверить уволенных» для принудительной сверки статуса уволенных сотрудников с текущим списком источника.

README.md

@@ -1,113 +1,152 @@
-# MIEM Employees Server
-
-Сервис собирает сотрудников МИЭМ с сайта ВШЭ, хранит карточки и историю обновлений в Postgres, показывает минимальную админку и отдает read-only MCP endpoint для ИИ-агентов.
-
-## Архитектура
-
-- `api`: FastAPI, REST API, HTML-админка, healthcheck.
+# MIEM Employees Server
+
+Сервис собирает сотрудников МИЭМ с сайта ВШЭ, хранит карточки и историю обновлений в Postgres и показывает минимальную админку.
+
+## Архитектура
+
+- `api`: FastAPI, REST API, HTML-админка и healthcheck.
 - `worker`: weekly scheduler, который запускает парсинг по `CRAWL_CRON`.
-- `mcp`: открытый HTTP MCP endpoint для ИИ-агентов.
-- `postgres`: основная БД.
-
-Парсер использует фиксированный источник сотрудников, по умолчанию `https://miem.hse.ru/persons`. Для каждой карточки сохраняются ФИО, должности, год начала работы, контакты, идентификаторы, вкладки профиля, секции, публикации, курсы, ВКР, JSON-снапшот и сжатый HTML-снапшот. Ссылки обходятся только из меню профиля самого сотрудника (`person-menu`), например `#sci`, `#teaching`, `#main`.
-
-## Переменные окружения
-
-Скопируйте `.env.example` в `.env` и поменяйте секреты:
-
-```bash
-cp .env.example .env
-```
-
-Основные настройки:
-
-- `DATABASE_URL`: строка подключения SQLAlchemy.
+- `postgres`: основная БД.
+
+Парсер использует фиксированный источник сотрудников, по умолчанию `https://miem.hse.ru/persons`. Для каждой карточки сохраняются ФИО, должности, год начала работы, контакты, идентификаторы, вкладки профиля, секции, публикации, курсы, ВКР, новости, JSON-снапшот и сжатый HTML-снапшот. Детальные публикации дополнительно нормализуются в отдельную таблицу `employee_publications`, а новости из блока «В новостях» — в `employee_news_links`. Ссылки обходятся только из меню профиля самого сотрудника (`person-menu`), например `#sci`, `#teaching`, `#main`.
+
+## Переменные окружения
+
+Скопируйте `.env.example` в `.env` и поменяйте секреты:
+
+```bash
+cp .env.example .env
+```
+
+Основные настройки:
+
+- `DATABASE_URL`: строка подключения SQLAlchemy.
 - `SOURCE_URL`: список сотрудников МИЭМ.
 - `CRAWL_CRON`: расписание в формате crontab, по умолчанию `0 3 * * 1`.
-- `CRAWL_LIMIT`: опциональный лимит профилей для тестового запуска.
-- `ADMIN_USERNAME`, `ADMIN_PASSWORD`: логин и пароль админки.
-- `SESSION_SECRET`: секрет подписи cookie.
-- `PARSER_USE_PLAYWRIGHT`: включение Playwright-рендера динамических вкладок.
+- `CRAWL_LIMIT`: опциональный лимит профилей для тестового запуска.
+- `ADMIN_USERNAME`, `ADMIN_PASSWORD`: логин и пароль админки.
+- `SESSION_SECRET`: секрет подписи cookie.
+- `PARSER_USE_PLAYWRIGHT`: включение Playwright-рендера динамических вкладок.
+- `DISMISSAL_CONFIRMATION_RUNS`: сколько последовательных проверок недоступности нужно для увольнения, по умолчанию `3`.
+- `MAX_AUTO_DISMISSALS_PER_RUN`: защитный лимит массовых автоматических увольнений за один запуск, по умолчанию `25`.
+
+## Локальный запуск
+
+```bash
+python -m venv .venv
+.venv\Scripts\activate
+pip install -r requirements.txt
+uvicorn app.main:app --reload
+```
+
+Админка: `http://localhost:8000/admin`.
+
+В админке доступны:
+
+- «Обзор»: статистика, последний добавленный сотрудник, прогресс парсинга и ручной запуск.
+- «Сотрудники»: поиск, фильтры, сортировка, пагинация и выбор колонок. По умолчанию показаны ФИО, статус, должности, дата последнего обнаружения и внешний профиль. В диалоге колонок доступны наборы «Проверка», «Контакты» и «Все поля»; ранее сохранённый выбор сохраняется.
+- «Запуски»: история обходов, ошибки и доступный индикатор прогресса.
 
-## Локальный запуск
+Все фильтры применяются кнопкой «Применить фильтры» с переходом на первую страницу. «Сбросить» очищает условия; при устаревшем номере страницы каталог возвращает первую. Подписи полей показывают выбранные условия, над таблицей указан диапазон результатов. Пояснения дат и статусов находятся под фильтрами.
 
-```bash
-python -m venv .venv
-.venv\Scripts\activate
-pip install -r requirements.txt
-uvicorn app.main:app --reload
-```
+Каталог поддерживает навигацию с клавиатуры и системный диалог колонок с Escape и возвратом фокуса. На узких экранах таблица прокручивается внутри страницы, колонка ФИО закреплена. Используется системный шрифт без внешних загрузок.
 
-Админка: `http://localhost:8000/admin`.
+Каталог формируется сервером: отдельная клиентская загрузка списка не нужна, переход показывает браузер. Пустой результат предлагает сбросить фильтры, пустая база — запустить парсинг. При ошибке базы каталог возвращает HTTP 503 и предлагает повторить загрузку с теми же фильтрами. При обновлении прогресса показывается состояние загрузки; при ошибке сохраняются последние значения с предупреждением об их актуальности и кнопкой «Повторить». Автоматические попытки продолжаются каждые 4 секунды, запрос ограничен 15 секундами; параллельные запросы не запускаются.
+
+## Docker Compose
+
+```bash
+docker compose up -d --build --remove-orphans
+```
+
+По умолчанию:
+
+- API и админка: `http://localhost:8000`
+- PostgreSQL: `postgres:5432` внутри сети Compose; порт на хост не опубликован.
 
-В админке доступны:
+Compose запускает `api`, `worker` и `postgres`. API привязан к localhost; для внешнего доступа нужен настроенный reverse proxy. REST API данных требует сессию администратора.
+
+Таблицы создаются приложением при старте. При обновлении существующей базы приложение также добавляет недостающие runtime-колонки, например `crawl_runs.skipped_count`. SQL-миграции для ручного применения лежат в `migrations/`.
+
+## Наполнение БД
+
+Основная карточка сотрудника хранится в `employees`: профиль, статус, даты обнаружения/увольнения, текущий JSON `current_data`, checksum и версия парсера. История успешных изменений сохраняется в `employee_snapshots` вместе с JSON-снимком и сжатым HTML профиля.
+
+Публикации теперь хранятся в двух видах:
+
+- краткий список остается внутри `employees.current_data.sections[].publications` для обратной совместимости;
+- детальные записи сохраняются в `employee_publications` и связываются с сотрудником через `employee_id`.
+
+`employee_publications` содержит `publication_id`, название, год, тип публикации, язык, статус, ссылку на карточку HSE Publications, DOI, внешние/document-ссылки, citation text, аннотацию, описание, авторов, raw JSON ответа `searchPubs` и `source_hash` для безопасного повторного upsert. Уникальность поддерживается по `(employee_id, publication_id)` и `(employee_id, source_hash)`, поэтому повторный crawl не должен создавать дубликаты.
 
-- `Dashboard`: общая статистика, последний добавленный сотрудник, прогресс текущего/последнего парсинга и ручной запуск.
-- `Directory`: настраиваемая таблица сотрудников с фильтрами, сортировкой, пагинацией и выбором колонок.
-- `Employees`: простая legacy-таблица сотрудников.
-- `Runs`: история запусков, ошибки и progress bar.
+Новости сотрудников также хранятся в двух видах:
+
+- краткий список остается внутри `employees.current_data.sections[].news_links`;
+- нормализованные карточки из вкладки «В новостях» сохраняются в `employee_news_links`.
+
+`employee_news_links` содержит название новости, ссылку, краткое описание, дату публикации, год публикации, raw JSON карточки и `source_hash`. Уникальность поддерживается по `(employee_id, url)` и `(employee_id, source_hash)`, поэтому повторный crawl не создает дубликаты.
 
-## Docker Compose
+## Кандидаты в диссертационный совет
 
-```bash
-docker compose up --build
-```
-
-По умолчанию:
-
-- API и админка: `http://localhost:8000`
-- MCP: `http://localhost:8001/mcp`
-- Postgres: `localhost:5432`
-
-Таблицы создаются приложением при старте. SQL-миграция для ручного применения лежит в `migrations/001_init.sql`.
-
-## Парсинг
-
-Weekly worker запускается по `CRAWL_CRON`. Ручной запуск доступен в админке на `Dashboard` и странице `Runs` или через REST:
-
-```bash
-curl -X POST http://localhost:8000/api/crawl-runs --cookie "miem_admin_session=..."
-```
-
-Алгоритм обновления:
-
-- найденные сотрудники получают статус `active` и обновленный `last_seen_at`;
-- новые сотрудники добавляются в `employees`;
-- количество новых сотрудников за запуск сохраняется в `crawl_runs.new_count`;
-- активные сотрудники, исчезнувшие из текущего списка источника, получают статус `dismissed` и `dismissed_at`;
-- каждый успешный разбор сохраняет запись в `employee_snapshots`.
-
-Во время выполнения парсинга `found_count`, `parsed_count` и `error_count` обновляются в базе. Админка опрашивает `/api/crawl-runs/latest` и показывает прогресс как `parsed_count + error_count / found_count`.
-
-## MCP
-
-Endpoint: `POST /mcp`, без авторизации на уровне приложения.
-
-Поддерживаемые tools:
-
-- `search_employees(query, status?, limit?)`
-- `get_employee(profile_id_or_url)`
-- `list_employee_publications(profile_id_or_url)`
-- `list_employee_courses(profile_id_or_url)`
-- `get_crawl_status()`
-
-Пример локального legacy-режима со статическим токеном:
-
-```bash
-curl http://localhost:8001/mcp \
-  -H "Content-Type: application/json" \
-  -d '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}'
-```
-
-Если MCP нужно ограничить, делайте это на сетевом уровне: localhost binding, VPN, firewall, reverse proxy или другой внешний контур доступа.
+Ссылка в админке открывает `/admin/council`. Страница использует действующих сотрудников и докторские степени из их профилей, а публикации — из предоставленной выгрузки `app/data/council_articles.csv`. Публикации сопоставляются только по полному ФИО при единственном совпадении; повторный DOI (или одинаковые название и год при отсутствии DOI) учитывается один раз. Порог — 10 предварительно подходящих публикаций за скользящие пять лет от текущей даты по Москве.
 
+Список **предварительный**: в CSV указаны год и квартиль Scopus/WoS, но нет точной даты, типа материала, журнала, источника квартиля, подтверждения К1 и RSCI. На границах периода год не позволяет проверить точную дату. Статьи с Q1/Q2 учитываются с пометкой источника; отсутствие кандидата не доказывает недостаточную публикационную активность. Для подтверждённого отбора нужны проверка журналов и дат и дополнительные источники публикаций. Выгрузка поставляется вместе с релизом и обновляется отдельным изменением репозитория.
+
+## Парсинг
+
+Worker запускает обход по `CRAWL_CRON`. Ручной запуск также доступен в админке на `Dashboard` и странице `Runs` или через REST:
+
+```bash
+curl -X POST http://localhost:8000/api/crawl-runs --cookie "miem_admin_session=..."
+```
+
+Алгоритм обновления:
+
+- найденные сотрудники получают статус `active` и обновленный `last_seen_at`;
+- новые сотрудники добавляются в `employees`;
+- если профиль перенесен на другой URL, он сопоставляется с прежней записью по единственному точному совпадению ФИО;
+- старые URL сохраняются в истории `employee_profile_urls`;
+- количество новых сотрудников за запуск сохраняется в `crawl_runs.new_count`;
+- публикации из HSE Publications записываются в `employee_publications`, а краткий список остается в JSON профиля;
+- новости из блока «В новостях» записываются в `employee_news_links`, а краткий список остается в JSON профиля;
+- один `404` старого профиля переводит сотрудника в статус `verification_required`, а не в `dismissed`;
+- статус `dismissed` устанавливается только после нескольких последовательных проверок `404`/`410`;
+- сетевые ошибки и ответы `5xx` не считаются подтверждением увольнения;
+- если число кандидатов на увольнение превышает защитный лимит, автоматическое увольнение приостанавливается;
+- кнопка «Проверить уволенных» сверяет только их profile_key с текущим списком источника и возвращает найденных сотрудников в `active` без обновления содержимого профиля;
+- каждый успешный новый или измененный разбор сохраняет запись в `employee_snapshots`;
+- неизмененные профили учитываются в `crawl_runs.skipped_count` и не получают новый snapshot.
+
+Во время выполнения парсинга `found_count`, `parsed_count`, `skipped_count` и `error_count` обновляются в базе. Админка опрашивает `/api/crawl-runs/latest` и показывает прогресс как `(parsed_count + skipped_count + error_count) / found_count`.
+
 ## Обслуживание
-
-```bash
+
+```bash
 docker compose logs -f api
 docker compose logs -f worker
-docker compose exec postgres pg_dump -U miem miem_workers > backup.sql
+docker compose exec postgres pg_dump -U miem miem_workers > backup.sql
 docker compose down
 ```
 
-Версия сервиса: `0.4.5`. Админка всегда показывает версии backend и frontend в footer.
+Production deploy выполняется job `deploy` workflow `CI` после успешного `test` на push в `main` или вручную. Для него нужны Actions secrets `PROD_HOST`, `PROD_USER`, `PROD_SSH_KEY` и `PROD_KNOWN_HOSTS`; `.env` и ключи в репозиторий не добавляются. PostgreSQL backup выполняется на VPS раз в неделю user cron deploy-пользователя через `infra/postgres-backup.sh`: `pg_dumpall` сохраняет весь кластер в `~/backups/postgres/postgres-latest.sql`. Новый dump сначала создаётся во временный файл и заменяет старый только после успешного завершения; предыдущие dump-файлы `miem_workers` удаляются после успешного создания нового. Одновременно выполняется только один production deploy.
+
+## Проверки
+
+После установки зависимостей из `requirements.txt` выполните:
+
+```bash
+python -m pytest -q
+node --check app/static/admin.js
+```
+
+Тесты с данными используют временную SQLite; отдельные API smoke-тесты запускают приложение с его текущей конфигурацией. Браузерная проверка: `pip install playwright`, `python -m playwright install chromium`, затем `python tests/browser_admin.py`. Она проверяет реальный рендеринг страниц, клавиатуру, диалог, фильтры и восстановление прогресса после ошибки. Скриншоты сохраняются во временную папку; путь выводится в конце.
+
+Версия сервиса: `0.10.4`. Админка всегда показывает версии backend и frontend в footer.
+
+## Разовый импорт публикаций
+
+Инструкция, минимальные права временной роли и ограниченный SSH-запуск: [docs/publication-import.md](docs/publication-import.md). По умолчанию импортер выполняет dry-run; production-применение требует отдельного решения владельца.
+
+Интеграционная проверка импорта на отдельном PostgreSQL: [docs/postgresql-import-verification.md](docs/postgresql-import-verification.md).
+
+Отдельный тестовый стенд на VPS и read-only проверка ресурсов: [docs/postgresql-import-vps.md](docs/postgresql-import-vps.md). Стенд запускает только владелец после оценки ресурсов.

app/admin.py

@@ -1,219 +1,259 @@
-from fastapi import APIRouter, BackgroundTasks, Depends, Form, Request
-from fastapi.responses import HTMLResponse, RedirectResponse
-from fastapi.templating import Jinja2Templates
-from sqlalchemy import desc, func, select
-from sqlalchemy.orm import Session
-
-from app.config import Settings, get_settings
-from app.db import SessionLocal, get_db
-from app.models import CrawlError, CrawlRun, Employee
-from app.security import SESSION_COOKIE, require_admin, sign_session, verify_admin
-from app.services.admin_data import (
-    employee_detail_payload,
-    format_admin_datetime,
-    list_employees_page,
-    run_detail_payload,
-    run_payload,
-    stats_payload,
-)
+from fastapi import APIRouter, BackgroundTasks, Depends, Form, Request
+from fastapi.responses import HTMLResponse, RedirectResponse
+from fastapi.templating import Jinja2Templates
+from sqlalchemy import desc, func, select
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.orm import Session
+
+from app.config import Settings, get_settings
+from app.db import SessionLocal, get_db
+from app.models import CrawlError, CrawlRun, Employee
+from app.security import SESSION_COOKIE, require_admin, sign_session, verify_admin
+from app.services.admin_data import (
+    employee_detail_payload,
+    format_admin_datetime,
+    list_employees_page,
+    run_detail_payload,
+    run_payload,
+    stats_payload,
+)
 from app.services.crawl_control import get_running_run, run_crawl_if_idle
-from app.services.crawler import refresh_employee
-from app.version import BACKEND_VERSION, FRONTEND_VERSION
-
-router = APIRouter(prefix="/admin")
-templates = Jinja2Templates(directory="app/templates")
-
-
-@router.get("", response_class=HTMLResponse)
-def dashboard(request: Request, db: Session = Depends(get_db), settings: Settings = Depends(get_settings)):
-    require_admin(request, settings)
-    counts = stats_payload(db)
-    counts["runs"] = db.scalar(select(func.count()).select_from(CrawlRun)) or 0
-    counts["errors"] = db.scalar(select(func.count()).select_from(CrawlError)) or 0
-    run_models = db.scalars(select(CrawlRun).order_by(desc(CrawlRun.started_at)).limit(5)).all()
-    runs = [run_payload(run) for run in run_models]
-    return _render(request, "dashboard.html", {"counts": counts, "runs": runs, "latest_run": runs[0] if runs else None})
-
-
-@router.get("/login", response_class=HTMLResponse)
-def login_form(request: Request):
-    return _render(request, "login.html", {"error": None})
-
-
-@router.post("/login")
-def login(
-    request: Request,
-    username: str = Form(...),
-    password: str = Form(...),
-    settings: Settings = Depends(get_settings),
-):
-    if not verify_admin(username, password, settings):
-        return _render(request, "login.html", {"error": "Неверный логин или пароль"}, status_code=401)
-    redirect = RedirectResponse("/admin", status_code=303)
-    redirect.set_cookie(SESSION_COOKIE, sign_session(username, settings), httponly=True, samesite="lax")
-    return redirect
-
-
-@router.post("/logout")
-def logout():
-    redirect = RedirectResponse("/admin/login", status_code=303)
-    redirect.delete_cookie(SESSION_COOKIE)
-    return redirect
-
-
-@router.get("/employees", response_class=HTMLResponse)
-def employees(
-    request: Request,
-    status: str | None = None,
-    q: str | None = None,
-    settings: Settings = Depends(get_settings),
-):
-    require_admin(request, settings)
-    return RedirectResponse("/admin/directory", status_code=303)
-
-
+from app.services.crawler import refresh_dismissed_status, refresh_employee
+from app.services.council import council_candidates
+from app.version import BACKEND_VERSION, FRONTEND_VERSION
+
+router = APIRouter(prefix="/admin")
+templates = Jinja2Templates(directory="app/templates")
+
+
+@router.get("", response_class=HTMLResponse)
+def dashboard(request: Request, db: Session = Depends(get_db), settings: Settings = Depends(get_settings)):
+    require_admin(request, settings)
+    counts = stats_payload(db)
+    counts["runs"] = db.scalar(select(func.count()).select_from(CrawlRun)) or 0
+    counts["errors"] = db.scalar(select(func.count()).select_from(CrawlError)) or 0
+    run_models = db.scalars(select(CrawlRun).order_by(desc(CrawlRun.started_at)).limit(5)).all()
+    runs = [run_payload(run) for run in run_models]
+    return _render(request, "dashboard.html", {"counts": counts, "runs": runs, "latest_run": runs[0] if runs else None})
+
+
+@router.get("/login", response_class=HTMLResponse)
+def login_form(request: Request):
+    return _render(request, "login.html", {"error": None})
+
+
+@router.post("/login")
+def login(
+    request: Request,
+    username: str = Form(...),
+    password: str = Form(...),
+    settings: Settings = Depends(get_settings),
+):
+    if not verify_admin(username, password, settings):
+        return _render(request, "login.html", {"error": "Неверный логин или пароль"}, status_code=401)
+    redirect = RedirectResponse("/admin", status_code=303)
+    redirect.set_cookie(SESSION_COOKIE, sign_session(username, settings), httponly=True, samesite="lax")
+    return redirect
+
+
+@router.post("/logout")
+def logout():
+    redirect = RedirectResponse("/admin/login", status_code=303)
+    redirect.delete_cookie(SESSION_COOKIE)
+    return redirect
+
+
+@router.get("/employees", response_class=HTMLResponse)
+def employees(
+    request: Request,
+    status: str | None = None,
+    q: str | None = None,
+    settings: Settings = Depends(get_settings),
+):
+    require_admin(request, settings)
+    return RedirectResponse("/admin/directory", status_code=303)
+
+
 @router.get("/directory", response_class=HTMLResponse)
 def directory(
-    request: Request,
-    status: str | None = None,
-    q: str | None = None,
-    started_from: str | None = None,
+    request: Request,
+    status: str | None = None,
+    q: str | None = None,
+    started_from: str | None = None,
     started_to: str | None = None,
     has_email: str | None = None,
-    sort: str = "full_name",
-    direction: str = "asc",
-    limit: int = 50,
-    offset: int = 0,
-    db: Session = Depends(get_db),
-    settings: Settings = Depends(get_settings),
-):
-    require_admin(request, settings)
-    parsed_started_from = _parse_date(started_from)
-    parsed_started_to = _parse_date(started_to)
+    has_academic_degree: str | None = None,
+    sort: str = "full_name",
+    direction: str = "asc",
+    limit: int = 50,
+    offset: int = 0,
+    db: Session = Depends(get_db),
+    settings: Settings = Depends(get_settings),
+):
+    require_admin(request, settings)
+    parsed_started_from = _parse_date(started_from)
+    parsed_started_to = _parse_date(started_to)
     parsed_has_email = None if has_email in (None, "") else has_email == "true"
-    page = list_employees_page(
-        db,
-        status=status,
-        q=q,
-        started_from=parsed_started_from,
-        started_to=parsed_started_to,
-        has_email=parsed_has_email,
-        sort=sort,
-        direction=direction,
-        limit=limit,
-        offset=offset,
-    )
-    return _render(
-        request,
-        "directory.html",
-        {
-            "page": page,
-            "filters": {
-                "status": status or "",
-                "q": q or "",
-                "started_from": started_from or "",
-                "started_to": started_to or "",
+    parsed_has_academic_degree = None if has_academic_degree in (None, "") else has_academic_degree == "true"
+    try:
+        page = list_employees_page(
+            db,
+            status=status,
+            q=q,
+            started_from=parsed_started_from,
+            started_to=parsed_started_to,
+            has_email=parsed_has_email,
+            has_academic_degree=parsed_has_academic_degree,
+            sort=sort,
+            direction=direction,
+            limit=limit,
+            offset=offset,
+        )
+    except SQLAlchemyError:
+        db.rollback()
+        return _render(request, "directory_error.html", {}, status_code=503)
+    return _render(
+        request,
+        "directory.html",
+        {
+            "page": page,
+            "filters": {
+                "status": status or "",
+                "q": q or "",
+                "started_from": started_from or "",
+                "started_to": started_to or "",
                 "has_email": has_email or "",
-                "sort": sort,
-                "direction": direction,
-                "limit": page["limit"],
-                "offset": offset,
-            },
-        },
+                "has_academic_degree": has_academic_degree or "",
+                "sort": sort,
+                "direction": direction,
+                "limit": page["limit"],
+                "offset": page["offset"],
+            },
+        },
     )
 
 
+@router.get("/council", response_class=HTMLResponse)
+def council(request: Request, db: Session = Depends(get_db), settings: Settings = Depends(get_settings)):
+    require_admin(request, settings)
+    try:
+        result = council_candidates(db)
+    except (SQLAlchemyError, OSError, ValueError, KeyError):
+        db.rollback()
+        return _render(request, "council.html", {"error": True}, status_code=503)
+    return _render(request, "council.html", {"error": False, **result})
+
+
 @router.get("/employees/{employee_id}", response_class=HTMLResponse)
-def employee_detail(
-    employee_id: int,
-    request: Request,
-    db: Session = Depends(get_db),
-    settings: Settings = Depends(get_settings),
-):
-    require_admin(request, settings)
-    employee = db.get(Employee, employee_id)
-    if not employee:
-        return RedirectResponse("/admin/employees", status_code=303)
-    snapshots = [
-        {
-            "captured_display": format_admin_datetime(snapshot.captured_at),
-            "checksum": snapshot.checksum,
-            "parser_version": snapshot.parser_version,
-        }
-        for snapshot in sorted(employee.snapshots, key=lambda item: item.captured_at, reverse=True)[:20]
-    ]
-    return _render(
-        request,
-        "employee_detail.html",
-        {
-            "employee": employee,
-            "employee_view": employee_detail_payload(employee),
-            "snapshots": snapshots,
-            "refresh_status": request.query_params.get("refresh_status"),
-        },
-    )
-
-
-@router.post("/employees/{employee_id}/refresh")
-def refresh_employee_detail(
-    employee_id: int,
-    request: Request,
-    db: Session = Depends(get_db),
-    settings: Settings = Depends(get_settings),
-):
-    require_admin(request, settings)
-    employee = db.get(Employee, employee_id)
-    if not employee:
-        return RedirectResponse("/admin/directory", status_code=303)
-    run = refresh_employee(db, employee, settings)
-    status = "success" if run.status == "completed" else "error"
-    return RedirectResponse(f"/admin/employees/{employee_id}?refresh_status={status}", status_code=303)
-
-
-@router.get("/runs", response_class=HTMLResponse)
-def runs(request: Request, db: Session = Depends(get_db), settings: Settings = Depends(get_settings)):
-    require_admin(request, settings)
-    run_models = db.scalars(select(CrawlRun).order_by(desc(CrawlRun.started_at)).limit(50)).all()
-    items = [run_payload(run) for run in run_models]
-    errors = db.scalars(select(CrawlError).order_by(desc(CrawlError.created_at)).limit(50)).all()
-    return _render(request, "runs.html", {"runs": items, "errors": errors})
-
-
-@router.get("/runs/{run_id}", response_class=HTMLResponse)
-def run_detail(
-    run_id: int,
-    request: Request,
-    db: Session = Depends(get_db),
-    settings: Settings = Depends(get_settings),
-):
-    require_admin(request, settings)
-    run = db.get(CrawlRun, run_id)
-    if not run:
-        return RedirectResponse("/admin/runs", status_code=303)
-    return _render(request, "run_detail.html", {"run": run_detail_payload(db, run)})
-
-
-@router.post("/runs")
-def trigger_run(
-    request: Request,
-    background_tasks: BackgroundTasks,
-    db: Session = Depends(get_db),
-    settings: Settings = Depends(get_settings),
-):
-    require_admin(request, settings)
-    if get_running_run(db):
-        return RedirectResponse("/admin/runs", status_code=303)
-
-    def _crawl() -> None:
-        with SessionLocal() as db:
-            run_crawl_if_idle(db, settings)
-
-    background_tasks.add_task(_crawl)
-    return RedirectResponse("/admin/runs", status_code=303)
-
-
+def employee_detail(
+    employee_id: int,
+    request: Request,
+    db: Session = Depends(get_db),
+    settings: Settings = Depends(get_settings),
+):
+    require_admin(request, settings)
+    employee = db.get(Employee, employee_id)
+    if not employee:
+        return RedirectResponse("/admin/employees", status_code=303)
+    snapshots = [
+        {
+            "captured_display": format_admin_datetime(snapshot.captured_at),
+            "checksum": snapshot.checksum,
+            "parser_version": snapshot.parser_version,
+        }
+        for snapshot in sorted(employee.snapshots, key=lambda item: item.captured_at, reverse=True)[:20]
+    ]
+    return _render(
+        request,
+        "employee_detail.html",
+        {
+            "employee": employee,
+            "employee_view": employee_detail_payload(employee),
+            "snapshots": snapshots,
+            "refresh_status": request.query_params.get("refresh_status"),
+        },
+    )
+
+
+@router.post("/employees/{employee_id}/refresh")
+def refresh_employee_detail(
+    employee_id: int,
+    request: Request,
+    db: Session = Depends(get_db),
+    settings: Settings = Depends(get_settings),
+):
+    require_admin(request, settings)
+    employee = db.get(Employee, employee_id)
+    if not employee:
+        return RedirectResponse("/admin/directory", status_code=303)
+    run = refresh_employee(db, employee, settings)
+    status = "success" if run.status == "completed" else "error"
+    return RedirectResponse(f"/admin/employees/{employee_id}?refresh_status={status}", status_code=303)
+
+
+@router.get("/runs", response_class=HTMLResponse)
+def runs(request: Request, db: Session = Depends(get_db), settings: Settings = Depends(get_settings)):
+    require_admin(request, settings)
+    run_models = db.scalars(select(CrawlRun).order_by(desc(CrawlRun.started_at)).limit(50)).all()
+    items = [run_payload(run) for run in run_models]
+    errors = db.scalars(select(CrawlError).order_by(desc(CrawlError.created_at)).limit(50)).all()
+    return _render(request, "runs.html", {"runs": items, "errors": errors})
+
+
+@router.get("/runs/{run_id}", response_class=HTMLResponse)
+def run_detail(
+    run_id: int,
+    request: Request,
+    db: Session = Depends(get_db),
+    settings: Settings = Depends(get_settings),
+):
+    require_admin(request, settings)
+    run = db.get(CrawlRun, run_id)
+    if not run:
+        return RedirectResponse("/admin/runs", status_code=303)
+    return _render(request, "run_detail.html", {"run": run_detail_payload(db, run)})
+
+
+@router.post("/runs")
+def trigger_run(
+    request: Request,
+    background_tasks: BackgroundTasks,
+    db: Session = Depends(get_db),
+    settings: Settings = Depends(get_settings),
+):
+    require_admin(request, settings)
+    if get_running_run(db):
+        return RedirectResponse("/admin/runs", status_code=303)
+
+    def _crawl() -> None:
+        with SessionLocal() as db:
+            run_crawl_if_idle(db, settings)
+
+    background_tasks.add_task(_crawl)
+    return RedirectResponse("/admin/runs", status_code=303)
+
+
 @router.post("/crawl-now")
 def crawl_now(
+    request: Request,
+    background_tasks: BackgroundTasks,
+    db: Session = Depends(get_db),
+    settings: Settings = Depends(get_settings),
+):
+    require_admin(request, settings)
+    if get_running_run(db):
+        return RedirectResponse("/admin", status_code=303)
+
+    def _crawl() -> None:
+        with SessionLocal() as db:
+            run_crawl_if_idle(db, settings)
+
+    background_tasks.add_task(_crawl)
+    return RedirectResponse("/admin", status_code=303)
+
+
+@router.post("/dismissed/refresh")
+def refresh_dismissed(
     request: Request,
     background_tasks: BackgroundTasks,
     db: Session = Depends(get_db),
@@ -223,30 +263,30 @@ def crawl_now(
     if get_running_run(db):
         return RedirectResponse("/admin", status_code=303)
 
-    def _crawl() -> None:
+    def _refresh() -> None:
         with SessionLocal() as db:
-            run_crawl_if_idle(db, settings)
+            refresh_dismissed_status(db, settings)
 
-    background_tasks.add_task(_crawl)
+    background_tasks.add_task(_refresh)
     return RedirectResponse("/admin", status_code=303)
 
 
 def _render(request: Request, template: str, context: dict, status_code: int = 200) -> HTMLResponse:
-    payload = {
-        "request": request,
-        "backend_version": BACKEND_VERSION,
-        "frontend_version": FRONTEND_VERSION,
-        **context,
-    }
-    return templates.TemplateResponse(request, template, payload, status_code=status_code)
-
-
-def _parse_date(value: str | None):
-    if not value:
-        return None
-    try:
-        from datetime import date
-
-        return date.fromisoformat(value)
-    except ValueError:
-        return None
+    payload = {
+        "request": request,
+        "backend_version": BACKEND_VERSION,
+        "frontend_version": FRONTEND_VERSION,
+        **context,
+    }
+    return templates.TemplateResponse(request, template, payload, status_code=status_code)
+
+
+def _parse_date(value: str | None):
+    if not value:
+        return None
+    try:
+        from datetime import date
+
+        return date.fromisoformat(value)
+    except ValueError:
+        return None

app/api.py

@@ -28,6 +28,7 @@ def list_employees(
     started_from: date | None = None,
     started_to: date | None = None,
     has_email: bool | None = None,
+    has_academic_degree: bool | None = None,
     sort: str = "full_name",
     direction: str = "asc",
     limit: int = 50,
@@ -43,6 +44,7 @@ def list_employees(
         started_from=started_from,
         started_to=started_to,
         has_email=has_email,
+        has_academic_degree=has_academic_degree,
         sort=sort,
         direction=direction,
         limit=limit,

app/config.py

@@ -13,6 +13,8 @@ class Settings(BaseSettings):
     request_timeout: int = 30
     request_delay_seconds: float = 1.0
     parser_use_playwright: bool = False
+    dismissal_confirmation_runs: int = Field(default=3, ge=1)
+    max_auto_dismissals_per_run: int = Field(default=25, ge=1)
 
     admin_username: str = "admin"
     admin_password: str = "admin"

app/db.py

@@ -1,6 +1,6 @@
 from collections.abc import Generator
 
-from sqlalchemy import create_engine
+from sqlalchemy import create_engine, inspect, text
 from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
 
 from app.config import get_settings
@@ -25,6 +25,52 @@ def init_db() -> None:
     import app.models  # noqa: F401
 
     Base.metadata.create_all(bind=engine)
+    _ensure_runtime_schema()
+
+
+def _ensure_runtime_schema() -> None:
+    import app.models as models
+
+    inspector = inspect(engine)
+    table_names = set(inspector.get_table_names())
+    if "employees" in table_names and "employee_publications" not in table_names:
+        models.EmployeePublication.__table__.create(bind=engine, checkfirst=True)
+        inspector = inspect(engine)
+        table_names = set(inspector.get_table_names())
+    if "employees" in table_names and "employee_news_links" not in table_names:
+        models.EmployeeNewsLink.__table__.create(bind=engine, checkfirst=True)
+        inspector = inspect(engine)
+        table_names = set(inspector.get_table_names())
+    if "employees" in table_names and "employee_profile_urls" not in table_names:
+        models.EmployeeProfileUrl.__table__.create(bind=engine, checkfirst=True)
+        inspector = inspect(engine)
+    if "employees" in table_names:
+        employee_columns = {column["name"] for column in inspector.get_columns("employees")}
+        missing_columns = []
+        if "profile_unavailable_streak" not in employee_columns:
+            missing_columns.append("profile_unavailable_streak INTEGER NOT NULL DEFAULT 0")
+        if "last_profile_check_at" not in employee_columns:
+            missing_columns.append("last_profile_check_at TIMESTAMPTZ")
+        if "has_academic_degree" not in employee_columns:
+            missing_columns.append("has_academic_degree BOOLEAN NOT NULL DEFAULT FALSE")
+        if missing_columns:
+            with engine.begin() as connection:
+                for column in missing_columns:
+                    connection.execute(text(f"ALTER TABLE employees ADD COLUMN {column}"))
+                if engine.dialect.name == "postgresql":
+                    connection.execute(
+                        text(
+                            "UPDATE employees SET has_academic_degree = "
+                            "COALESCE(current_data::text ~* '(кандидат|доктор).{0,80}наук|ph\\.?[[:space:]]*d\\.?', FALSE)"
+                        )
+                    )
+                connection.execute(text("CREATE INDEX IF NOT EXISTS ix_employees_has_academic_degree ON employees (has_academic_degree)"))
+    if "crawl_runs" not in table_names:
+        return
+    crawl_run_columns = {column["name"] for column in inspector.get_columns("crawl_runs")}
+    if "skipped_count" not in crawl_run_columns:
+        with engine.begin() as connection:
+            connection.execute(text("ALTER TABLE crawl_runs ADD COLUMN skipped_count INTEGER NOT NULL DEFAULT 0"))
 
 
 def get_db() -> Generator[Session, None, None]:

app/main.py

@@ -4,14 +4,12 @@ from fastapi.staticfiles import StaticFiles
 from app.admin import router as admin_router
 from app.api import router as api_router
 from app.db import init_db
-from app.mcp import router as mcp_router
 from app.version import BACKEND_VERSION
 
 app = FastAPI(title="MIEM Employees", version=BACKEND_VERSION)
 app.mount("/static", StaticFiles(directory="app/static"), name="static")
 app.include_router(api_router)
 app.include_router(admin_router)
-app.include_router(mcp_router)
 
 
 @app.on_event("startup")

app/mcp.py

@@ -1,217 +0,0 @@
-import json
-
-from fastapi import APIRouter, Depends, Request
-from sqlalchemy import desc, or_, select
-from sqlalchemy.orm import Session
-
-from app.db import get_db
-from app.models import CrawlRun, Employee
-from app.services.admin_data import run_detail_payload
-from app.services.dataset_versions import service_info_payload, sync_employees_payload
-from app.version import BACKEND_VERSION
-
-router = APIRouter(prefix="/mcp")
-PROTOCOL_VERSION = "2024-11-05"
-SERVICE_NAME = "miem-employees"
-
-
-TOOLS = [
-    {
-        "name": "get_service_info",
-        "description": "Return service metadata, supported tools, and current dataset version.",
-        "inputSchema": {"type": "object", "properties": {}},
-    },
-    {
-        "name": "sync_employees",
-        "description": "Synchronize employees by dataset hash. Returns a full snapshot or a delta from client_hash.",
-        "inputSchema": {
-            "type": "object",
-            "properties": {
-                "client_hash": {"type": "string"},
-                "include_data": {"type": "boolean", "default": True},
-            },
-        },
-    },
-    {
-        "name": "search_employees",
-        "description": "Search MIEM employees by name or profile URL.",
-        "inputSchema": {
-            "type": "object",
-            "properties": {
-                "query": {"type": "string"},
-                "status": {"type": "string", "enum": ["active", "dismissed"]},
-                "limit": {"type": "integer", "default": 20},
-            },
-            "required": ["query"],
-        },
-    },
-    {
-        "name": "get_employee",
-        "description": "Get one employee by profile id, profile key, or canonical URL.",
-        "inputSchema": {"type": "object", "properties": {"profile_id_or_url": {"type": "string"}}, "required": ["profile_id_or_url"]},
-    },
-    {
-        "name": "list_employee_publications",
-        "description": "List publications parsed from an employee profile.",
-        "inputSchema": {"type": "object", "properties": {"profile_id_or_url": {"type": "string"}}, "required": ["profile_id_or_url"]},
-    },
-    {
-        "name": "list_employee_courses",
-        "description": "List teaching courses parsed from an employee profile.",
-        "inputSchema": {"type": "object", "properties": {"profile_id_or_url": {"type": "string"}}, "required": ["profile_id_or_url"]},
-    },
-    {
-        "name": "get_crawl_status",
-        "description": "Return the latest crawl run status.",
-        "inputSchema": {"type": "object", "properties": {}},
-    },
-    {
-        "name": "get_crawl_run_details",
-        "description": "Return detailed employee changes and errors for one crawl run.",
-        "inputSchema": {
-            "type": "object",
-            "properties": {"run_id": {"type": "integer"}},
-            "required": ["run_id"],
-        },
-    },
-]
-
-
-@router.post("")
-async def mcp_http(
-    request: Request,
-    db: Session = Depends(get_db),
-) -> dict:
-    payload = await request.json()
-    method = payload.get("method")
-    request_id = payload.get("id")
-    params = payload.get("params") or {}
-
-    try:
-        if method == "initialize":
-            result = {
-                "protocolVersion": PROTOCOL_VERSION,
-                "serverInfo": {"name": SERVICE_NAME, "version": BACKEND_VERSION},
-                "capabilities": {"tools": {}},
-            }
-        elif method == "tools/list":
-            result = {"tools": TOOLS}
-        elif method == "tools/call":
-            result = _call_tool(db, params.get("name"), params.get("arguments") or {})
-        else:
-            return {"jsonrpc": "2.0", "id": request_id, "error": {"code": -32601, "message": "Method not found"}}
-        return {"jsonrpc": "2.0", "id": request_id, "result": result}
-    except Exception as exc:
-        return {"jsonrpc": "2.0", "id": request_id, "error": {"code": -32000, "message": str(exc)}}
-
-
-def _call_tool(db: Session, name: str, arguments: dict) -> dict:
-    if name == "get_service_info":
-        return _tool_response(
-            service_info_payload(
-                db,
-                tools=TOOLS,
-                service_name=SERVICE_NAME,
-                backend_version=BACKEND_VERSION,
-                protocol_version=PROTOCOL_VERSION,
-            )
-        )
-    if name == "sync_employees":
-        return _tool_response(
-            sync_employees_payload(
-                db,
-                client_hash=arguments.get("client_hash"),
-                include_data=bool(arguments.get("include_data", True)),
-            )
-        )
-    if name == "search_employees":
-        return _tool_response(_search_employees(db, arguments))
-    if name == "get_employee":
-        employee = _find_employee(db, arguments["profile_id_or_url"])
-        return _tool_response(_employee_payload(employee) if employee else {"error": "not_found"})
-    if name == "list_employee_publications":
-        employee = _find_employee(db, arguments["profile_id_or_url"])
-        return _tool_response(_collect_section_items(employee, "publications"))
-    if name == "list_employee_courses":
-        employee = _find_employee(db, arguments["profile_id_or_url"])
-        return _tool_response(_collect_section_items(employee, "courses_by_year"))
-    if name == "get_crawl_status":
-        run = db.scalar(select(CrawlRun).order_by(desc(CrawlRun.started_at)).limit(1))
-        return _tool_response(_run_payload(run) if run else {"status": "never_run"})
-    if name == "get_crawl_run_details":
-        run = db.get(CrawlRun, int(arguments["run_id"]))
-        return _tool_response(run_detail_payload(db, run) if run else {"error": "not_found"})
-    raise ValueError(f"Unknown tool: {name}")
-
-
-def _search_employees(db: Session, arguments: dict) -> list[dict]:
-    query = arguments.get("query", "")
-    limit = min(int(arguments.get("limit") or 20), 100)
-    stmt = select(Employee)
-    if arguments.get("status"):
-        stmt = stmt.where(Employee.status == arguments["status"])
-    if query:
-        pattern = f"%{query}%"
-        stmt = stmt.where(or_(Employee.full_name.ilike(pattern), Employee.canonical_url.ilike(pattern)))
-    employees = db.scalars(stmt.order_by(Employee.full_name).limit(limit)).all()
-    return [_employee_payload(employee, include_data=False) for employee in employees]
-
-
-def _find_employee(db: Session, value: str) -> Employee | None:
-    pattern = value.strip()
-    stmt = select(Employee).where(
-        or_(
-            Employee.profile_key == pattern,
-            Employee.profile_id == pattern,
-            Employee.canonical_url == pattern,
-            Employee.canonical_url.ilike(f"%{pattern}%"),
-        )
-    )
-    return db.scalar(stmt.limit(1))
-
-
-def _collect_section_items(employee: Employee | None, section_type: str) -> dict:
-    if not employee or not employee.current_data:
-        return {"items": []}
-    items = []
-    for section in employee.current_data.get("sections") or []:
-        if section.get("type") != section_type:
-            continue
-        if section_type == "publications":
-            items.extend(section.get("publications") or [])
-        elif section_type == "courses_by_year":
-            items.extend(section.get("courses") or [])
-    return {"employee": _employee_payload(employee, include_data=False), "items": items}
-
-
-def _employee_payload(employee: Employee, include_data: bool = True) -> dict:
-    payload = {
-        "profile_key": employee.profile_key,
-        "profile_id": employee.profile_id,
-        "full_name": employee.full_name,
-        "status": employee.status,
-        "canonical_url": employee.canonical_url,
-        "last_seen_at": employee.last_seen_at.isoformat() if employee.last_seen_at else None,
-        "dismissed_at": employee.dismissed_at.isoformat() if employee.dismissed_at else None,
-    }
-    if include_data:
-        payload["data"] = employee.current_data
-    return payload
-
-
-def _run_payload(run: CrawlRun) -> dict:
-    return {
-        "id": run.id,
-        "status": run.status,
-        "source_url": run.source_url,
-        "started_at": run.started_at.isoformat() if run.started_at else None,
-        "finished_at": run.finished_at.isoformat() if run.finished_at else None,
-        "found_count": run.found_count,
-        "parsed_count": run.parsed_count,
-        "error_count": run.error_count,
-        "dismissed_count": run.dismissed_count,
-    }
-
-
-def _tool_response(data: object) -> dict:
-    return {"content": [{"type": "text", "text": json.dumps(data, ensure_ascii=False, default=str)}]}

app/models.py

@@ -1,6 +1,6 @@
 from datetime import datetime, timezone
 
-from sqlalchemy import DateTime, ForeignKey, Index, Integer, LargeBinary, String, Text, UniqueConstraint
+from sqlalchemy import Boolean, DateTime, ForeignKey, Index, Integer, LargeBinary, String, Text, UniqueConstraint
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 from sqlalchemy.types import JSON
@@ -21,6 +21,7 @@ class Employee(Base):
         UniqueConstraint("profile_key", name="uq_employees_profile_key"),
         Index("ix_employees_full_name", "full_name"),
         Index("ix_employees_status", "status"),
+        Index("ix_employees_has_academic_degree", "has_academic_degree"),
     )
 
     id: Mapped[int] = mapped_column(Integer, primary_key=True)
@@ -33,17 +34,39 @@ class Employee(Base):
     first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)
     last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)
     dismissed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    profile_unavailable_streak: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
+    last_profile_check_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     parser_version: Mapped[str | None] = mapped_column(String(32))
     current_data: Mapped[dict | None] = mapped_column(json_type)
+    has_academic_degree: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
     current_checksum: Mapped[str | None] = mapped_column(String(64))
     created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)
     updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow, nullable=False)
 
     snapshots: Mapped[list["EmployeeSnapshot"]] = relationship(back_populates="employee")
+    profile_urls: Mapped[list["EmployeeProfileUrl"]] = relationship(back_populates="employee", cascade="all, delete-orphan")
     tabs: Mapped[list["ProfileTab"]] = relationship(back_populates="employee", cascade="all, delete-orphan")
+    publications: Mapped[list["EmployeePublication"]] = relationship(back_populates="employee", cascade="all, delete-orphan")
+    news_links: Mapped[list["EmployeeNewsLink"]] = relationship(back_populates="employee", cascade="all, delete-orphan")
     crawl_run_changes: Mapped[list["CrawlRunEmployeeChange"]] = relationship(back_populates="employee")
 
 
+class EmployeeProfileUrl(Base):
+    __tablename__ = "employee_profile_urls"
+    __table_args__ = (
+        UniqueConstraint("employee_id", "url", name="uq_employee_profile_urls_employee_url"),
+        Index("ix_employee_profile_urls_employee_id", "employee_id"),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    employee_id: Mapped[int] = mapped_column(ForeignKey("employees.id", ondelete="CASCADE"), nullable=False)
+    url: Mapped[str] = mapped_column(Text, nullable=False)
+    first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)
+    last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)
+
+    employee: Mapped[Employee] = relationship(back_populates="profile_urls")
+
+
 class EmployeeSnapshot(Base):
     __tablename__ = "employee_snapshots"
     __table_args__ = (Index("ix_employee_snapshots_employee_id", "employee_id"),)
@@ -60,6 +83,68 @@ class EmployeeSnapshot(Base):
     employee: Mapped[Employee] = relationship(back_populates="snapshots")
 
 
+class EmployeePublication(Base):
+    __tablename__ = "employee_publications"
+    __table_args__ = (
+        UniqueConstraint("employee_id", "publication_id", name="uq_employee_publications_employee_publication"),
+        UniqueConstraint("employee_id", "source_hash", name="uq_employee_publications_employee_source_hash"),
+        Index("ix_employee_publications_employee_id", "employee_id"),
+        Index("ix_employee_publications_publication_id", "publication_id"),
+        Index("ix_employee_publications_doi_url", "doi_url"),
+        Index("ix_employee_publications_year", "year"),
+        Index("ix_employee_publications_publication_type", "publication_type"),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    employee_id: Mapped[int] = mapped_column(ForeignKey("employees.id", ondelete="CASCADE"), nullable=False)
+    publication_id: Mapped[str | None] = mapped_column(String(64))
+    title: Mapped[str] = mapped_column(Text, nullable=False)
+    year: Mapped[int | None] = mapped_column(Integer)
+    publication_type: Mapped[str | None] = mapped_column(String(64))
+    language: Mapped[str | None] = mapped_column(String(16))
+    status: Mapped[int | None] = mapped_column(Integer)
+    url: Mapped[str | None] = mapped_column(Text)
+    doi_url: Mapped[str | None] = mapped_column(Text)
+    other_url: Mapped[str | None] = mapped_column(Text)
+    document_url: Mapped[str | None] = mapped_column(Text)
+    citation_text: Mapped[str | None] = mapped_column(Text)
+    annotation: Mapped[dict | None] = mapped_column(json_type)
+    description: Mapped[dict | None] = mapped_column(json_type)
+    authors: Mapped[list | None] = mapped_column(json_type)
+    raw_data: Mapped[dict | None] = mapped_column(json_type)
+    source_hash: Mapped[str] = mapped_column(String(64), nullable=False)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow, nullable=False)
+
+    employee: Mapped[Employee] = relationship(back_populates="publications")
+
+
+class EmployeeNewsLink(Base):
+    __tablename__ = "employee_news_links"
+    __table_args__ = (
+        UniqueConstraint("employee_id", "url", name="uq_employee_news_links_employee_url"),
+        UniqueConstraint("employee_id", "source_hash", name="uq_employee_news_links_employee_source_hash"),
+        Index("ix_employee_news_links_employee_id", "employee_id"),
+        Index("ix_employee_news_links_url", "url"),
+        Index("ix_employee_news_links_published_at", "published_at"),
+        Index("ix_employee_news_links_published_year", "published_year"),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    employee_id: Mapped[int] = mapped_column(ForeignKey("employees.id", ondelete="CASCADE"), nullable=False)
+    title: Mapped[str] = mapped_column(Text, nullable=False)
+    url: Mapped[str | None] = mapped_column(Text)
+    summary: Mapped[str | None] = mapped_column(Text)
+    published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    published_year: Mapped[int | None] = mapped_column(Integer)
+    source_hash: Mapped[str] = mapped_column(String(64), nullable=False)
+    raw_data: Mapped[dict | None] = mapped_column(json_type)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow, nullable=False)
+
+    employee: Mapped[Employee] = relationship(back_populates="news_links")
+
+
 class CrawlRun(Base):
     __tablename__ = "crawl_runs"
 
@@ -70,13 +155,13 @@ class CrawlRun(Base):
     finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     found_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
     parsed_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
+    skipped_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
     new_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
     error_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
     dismissed_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
     message: Mapped[str | None] = mapped_column(Text)
 
     employee_changes: Mapped[list["CrawlRunEmployeeChange"]] = relationship(back_populates="crawl_run")
-    dataset_versions: Mapped[list["DatasetVersion"]] = relationship(back_populates="crawl_run")
 
 
 class CrawlRunEmployeeChange(Base):
@@ -137,40 +222,22 @@ class ParserSource(Base):
     created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)
 
 
-class DatasetVersion(Base):
-    __tablename__ = "dataset_versions"
+class ParseResourceCache(Base):
+    __tablename__ = "parse_resource_cache"
     __table_args__ = (
-        UniqueConstraint("hash", name="uq_dataset_versions_hash"),
-        Index("ix_dataset_versions_created_at", "created_at"),
+        UniqueConstraint("profile_key", "resource_key", "request_fingerprint", name="uq_parse_resource_cache_resource"),
+        Index("ix_parse_resource_cache_profile_key", "profile_key"),
     )
 
     id: Mapped[int] = mapped_column(Integer, primary_key=True)
-    hash: Mapped[str] = mapped_column(String(64), nullable=False)
-    previous_hash: Mapped[str | None] = mapped_column(String(64))
-    crawl_run_id: Mapped[int | None] = mapped_column(ForeignKey("crawl_runs.id"))
-    employee_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
-    active_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
-    dismissed_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
-    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)
-
-    crawl_run: Mapped[CrawlRun | None] = relationship(back_populates="dataset_versions")
-    items: Mapped[list["DatasetVersionItem"]] = relationship(back_populates="dataset_version", cascade="all, delete-orphan")
-
-
-class DatasetVersionItem(Base):
-    __tablename__ = "dataset_version_items"
-    __table_args__ = (
-        UniqueConstraint("dataset_version_id", "profile_key", name="uq_dataset_version_items_version_profile"),
-        Index("ix_dataset_version_items_hash", "dataset_version_id"),
-        Index("ix_dataset_version_items_profile_key", "profile_key"),
-    )
-
-    id: Mapped[int] = mapped_column(Integer, primary_key=True)
-    dataset_version_id: Mapped[int] = mapped_column(ForeignKey("dataset_versions.id"), nullable=False)
     profile_key: Mapped[str] = mapped_column(String(255), nullable=False)
-    employee_id: Mapped[int | None] = mapped_column(ForeignKey("employees.id"))
-    status: Mapped[str] = mapped_column(String(32), nullable=False)
-    checksum: Mapped[str] = mapped_column(String(64), nullable=False)
-
-    dataset_version: Mapped[DatasetVersion] = relationship(back_populates="items")
-    employee: Mapped[Employee | None] = relationship()
+    resource_key: Mapped[str] = mapped_column(String(255), nullable=False)
+    method: Mapped[str] = mapped_column(String(16), nullable=False)
+    url: Mapped[str] = mapped_column(Text, nullable=False)
+    request_fingerprint: Mapped[str] = mapped_column(String(64), nullable=False)
+    etag: Mapped[str | None] = mapped_column(Text)
+    last_modified: Mapped[str | None] = mapped_column(Text)
+    body_hash: Mapped[str] = mapped_column(String(64), nullable=False)
+    body_snapshot: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
+    parser_version: Mapped[str | None] = mapped_column(String(32))
+    fetched_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)

app/parser/profile.py

@@ -1,4 +1,7 @@
+import hashlib
+import json
 import re
+from datetime import datetime, timezone
 from urllib.parse import urljoin
 
 from bs4 import BeautifulSoup, NavigableString, Tag
@@ -99,6 +102,8 @@ def extract_person_header(soup: BeautifulSoup, source_url: str) -> dict:
 def extract_sections(soup: BeautifulSoup, source_url: str) -> list[dict]:
     sections = []
     for h2 in soup.select("h2"):
+        if h2.find_parent(class_="post") or h2.find_parent(attrs={"data-tab": "press_links_news"}):
+            continue
         title = normalize_ws(h2.get_text(" ", strip=True))
         if not title or "расписание занятий" in title.lower():
             continue
@@ -140,6 +145,21 @@ def extract_sections(soup: BeautifulSoup, source_url: str) -> list[dict]:
             if section_type in {"generic", "paragraphs"}:
                 section["type"] = "year_blocks"
         sections.append(section)
+    news_links = _parse_news_links(soup, source_url)
+    if news_links:
+        sections.append(
+            {
+                "title": "В новостях",
+                "slug": "v_novostyah",
+                "type": "news",
+                "raw_text": "",
+                "paragraphs": [],
+                "items": [item["title"] for item in news_links if item.get("title")],
+                "links": [{"text": item["title"], "url": item["url"]} for item in news_links if item.get("title") and item.get("url")],
+                "news_count": len(news_links),
+                "news_links": news_links,
+            }
+        )
     return sections
 
 
@@ -149,22 +169,42 @@ def parse_person_profile(
     headers: dict[str, str],
     timeout: int,
     use_playwright: bool = False,
+    resource_cache=None,
 ) -> dict | None:
     normalized_url = normalize_profile_url(source_url)
     if not normalized_url:
         return None
-    response = session.get(normalized_url, headers=headers, timeout=timeout)
-    response.raise_for_status()
-    html = response.text
+    profile_type, profile_id = parse_profile_identity(normalized_url)
+    cache_profile_key = f"{profile_type}:{profile_id}"
+    resource_manifest = []
+    html = _fetch_text(
+        session,
+        normalized_url,
+        headers,
+        timeout,
+        resource_cache=resource_cache,
+        profile_key=cache_profile_key,
+        resource_key="main-html",
+        resource_manifest=resource_manifest,
+    )
     if use_playwright:
         html = _render_with_playwright(normalized_url, html)
 
     soup = BeautifulSoup(html, "html.parser")
-    profile_type, profile_id = parse_profile_identity(normalized_url)
     header = extract_person_header(soup, normalized_url)
     tabs = extract_person_tabs(soup, normalized_url)
     sections = extract_sections(soup, normalized_url)
-    sections = enrich_sections_from_hse_widgets(session, soup, normalized_url, headers, timeout, sections)
+    sections = enrich_sections_from_hse_widgets(
+        session,
+        soup,
+        normalized_url,
+        headers,
+        timeout,
+        sections,
+        resource_cache=resource_cache,
+        profile_key=cache_profile_key,
+        resource_manifest=resource_manifest,
+    )
     internal_links = [tab["href"] for tab in tabs if tab.get("href")]
 
     return {
@@ -181,6 +221,7 @@ def parse_person_profile(
         "employee_internal_links": internal_links,
         "parser_version": BACKEND_VERSION,
         "_html": html,
+        "_resource_manifest": resource_manifest,
     }
 
 
@@ -191,13 +232,33 @@ def enrich_sections_from_hse_widgets(
     headers: dict[str, str],
     timeout: int,
     sections: list[dict],
+    resource_cache=None,
+    profile_key: str | None = None,
+    resource_manifest: list[dict] | None = None,
 ) -> list[dict]:
     enriched = list(sections)
-    publications = _load_widget_publications(session, soup, headers, timeout)
+    publications = _load_widget_publications(
+        session,
+        soup,
+        headers,
+        timeout,
+        resource_cache=resource_cache,
+        profile_key=profile_key,
+        resource_manifest=resource_manifest,
+    )
     if publications:
         enriched = _upsert_publications_section(enriched, publications)
 
-    theses = _load_widget_graduation_theses(session, soup, source_url, headers, timeout)
+    theses = _load_widget_graduation_theses(
+        session,
+        soup,
+        source_url,
+        headers,
+        timeout,
+        resource_cache=resource_cache,
+        profile_key=profile_key,
+        resource_manifest=resource_manifest,
+    )
     if theses:
         enriched = _upsert_graduation_theses_section(enriched, theses)
     return enriched
@@ -226,7 +287,16 @@ def _render_with_playwright(source_url: str, fallback_html: str) -> str:
         return fallback_html
 
 
-def _load_widget_publications(session: Session, soup: BeautifulSoup, headers: dict[str, str], timeout: int) -> list[dict]:
+def _load_widget_publications(
+    session: Session,
+    soup: BeautifulSoup,
+    headers: dict[str, str],
+    timeout: int,
+    *,
+    resource_cache=None,
+    profile_key: str | None = None,
+    resource_manifest: list[dict] | None = None,
+) -> list[dict]:
     script = soup.select_one('script[data-widget-name="AuthorSearch"][data-author]')
     if not script:
         return []
@@ -251,14 +321,29 @@ def _load_widget_publications(session: Session, soup: BeautifulSoup, headers: di
             },
         }
         try:
-            response = session.post(
-                "https://publications.hse.ru/api/searchPubs",
-                json=payload,
-                headers=headers,
-                timeout=timeout,
-            )
-            response.raise_for_status()
-            data = response.json()
+            if resource_cache and profile_key:
+                text = _fetch_text(
+                    session,
+                    "https://publications.hse.ru/api/searchPubs",
+                    headers,
+                    timeout,
+                    resource_cache=resource_cache,
+                    profile_key=profile_key,
+                    resource_key=f"publications-page-{page_id}",
+                    resource_manifest=resource_manifest,
+                    method="POST",
+                    json_payload=payload,
+                )
+                data = json.loads(text)
+            else:
+                response = session.post(
+                    "https://publications.hse.ru/api/searchPubs",
+                    json=payload,
+                    headers=headers,
+                    timeout=timeout,
+                )
+                response.raise_for_status()
+                data = response.json()
         except Exception:
             return publications
 
@@ -266,7 +351,7 @@ def _load_widget_publications(session: Session, soup: BeautifulSoup, headers: di
         items = _extract_publication_items(result)
         if not items:
             break
-        publications.extend(_normalize_publication_item(item) for item in items)
+        publications.extend(_normalize_publication_item(item, author_id) for item in items)
 
         total = int(result.get("total") or 0)
         if not result.get("more") and len(publications) >= total:
@@ -309,6 +394,10 @@ def _load_widget_graduation_theses(
     source_url: str,
     headers: dict[str, str],
     timeout: int,
+    *,
+    resource_cache=None,
+    profile_key: str | None = None,
+    resource_manifest: list[dict] | None = None,
 ) -> list[dict]:
     script = soup.select_one('script[src*="/n/stat/vkr/app.js"][data-person-id]')
     if not script:
@@ -320,14 +409,30 @@ def _load_widget_graduation_theses(
 
     request_headers = {**headers, "x-portal-language": "ru"}
     try:
-        response = session.get(
-            urljoin(source_url, api_url),
-            params={"supervisorId": person_id},
-            headers=request_headers,
-            timeout=timeout,
-        )
-        response.raise_for_status()
-        data = response.json()
+        url = urljoin(source_url, api_url)
+        params = {"supervisorId": person_id}
+        if resource_cache and profile_key:
+            text = _fetch_text(
+                session,
+                url,
+                request_headers,
+                timeout,
+                resource_cache=resource_cache,
+                profile_key=profile_key,
+                resource_key="graduation-theses",
+                resource_manifest=resource_manifest,
+                params=params,
+            )
+            data = json.loads(text)
+        else:
+            response = session.get(
+                url,
+                params=params,
+                headers=request_headers,
+                timeout=timeout,
+            )
+            response.raise_for_status()
+            data = response.json()
     except Exception:
         return []
 
@@ -488,20 +593,126 @@ def _parse_vkr_items(nodes: list) -> list[str]:
     return [item for item in dict.fromkeys(items) if item]
 
 
-def _normalize_publication_item(item: dict) -> dict:
+def _parse_news_links(soup: BeautifulSoup, source_url: str) -> list[dict]:
+    news = []
+    for post in soup.select('[data-tab="press_links_news"] .post'):
+        if not isinstance(post, Tag):
+            continue
+        anchor = post.select_one(".post__content h2 a[href], h2 a[href], a[href]")
+        title = normalize_ws(anchor.get_text(" ", strip=True)) if anchor else ""
+        href = normalize_ws(anchor.get("href")) if anchor else ""
+        summary_node = post.select_one(".post__text")
+        summary = normalize_ws(summary_node.get_text(" ", strip=True)) if summary_node else ""
+        published_at = _parse_post_date(post)
+        if not title and not href:
+            continue
+        item = {
+            "title": title or href,
+            "url": urljoin(source_url, href) if href else None,
+            "summary": summary or None,
+            "published_at": published_at.isoformat() if published_at else None,
+            "published_year": published_at.year if published_at else _int_or_none(normalize_ws(_select_text(post, ".post-meta__year"))),
+            "raw_data": {
+                "title": title or href,
+                "url": href or None,
+                "summary": summary or None,
+                "date_text": normalize_ws(_select_text(post, ".post-meta__date")),
+            },
+        }
+        news.append(item)
+    return _dedupe_news_links(news)
+
+
+def _select_text(node: Tag, selector: str) -> str:
+    selected = node.select_one(selector)
+    return selected.get_text(" ", strip=True) if selected else ""
+
+
+def _parse_post_date(post: Tag) -> datetime | None:
+    day = _int_or_none(normalize_ws(_select_text(post, ".post-meta__day")))
+    month = _month_number(normalize_ws(_select_text(post, ".post-meta__month")))
+    year = _int_or_none(normalize_ws(_select_text(post, ".post-meta__year")))
+    if not day or not month or not year:
+        return None
+    try:
+        return datetime(year, month, day, tzinfo=timezone.utc)
+    except ValueError:
+        return None
+
+
+def _month_number(value: str) -> int | None:
+    lowered = value.lower().strip(".")
+    months = {
+        "янв": 1,
+        "январь": 1,
+        "января": 1,
+        "фев": 2,
+        "февр": 2,
+        "февраль": 2,
+        "февраля": 2,
+        "март": 3,
+        "мар": 3,
+        "марта": 3,
+        "апр": 4,
+        "апрель": 4,
+        "апреля": 4,
+        "май": 5,
+        "мая": 5,
+        "июнь": 6,
+        "июня": 6,
+        "июль": 7,
+        "июля": 7,
+        "авг": 8,
+        "август": 8,
+        "августа": 8,
+        "сент": 9,
+        "сен": 9,
+        "сентябрь": 9,
+        "сентября": 9,
+        "окт": 10,
+        "октябрь": 10,
+        "октября": 10,
+        "нояб": 11,
+        "ноябрь": 11,
+        "ноября": 11,
+        "дек": 12,
+        "декабрь": 12,
+        "декабря": 12,
+    }
+    return months.get(lowered)
+
+
+def _normalize_publication_item(item: dict, current_author_id: str | None = None) -> dict:
     publication_id = str(item.get("id") or "").strip()
     title = _html_to_text(item.get("title"))
-    year = item.get("year")
+    year = _int_or_none(item.get("year"))
     publication_type = str(item.get("type") or "").strip() or None
     description = item.get("description") if isinstance(item.get("description"), dict) else {}
     short_description = _localized_value(description.get("short")) or _localized_value(description.get("shortLeft"))
+    documents = item.get("documents") if isinstance(item.get("documents"), dict) else {}
+    language = item.get("language") if isinstance(item.get("language"), dict) else {}
+    annotation = _localized_text_map(item.get("annotation"))
+    authors = _normalize_publication_authors(item.get("authorsByType"), current_author_id)
+    citation_text = normalize_ws(str(description.get("main") or "")) or _build_publication_citation(title, authors, year)
     text = normalize_ws(" ".join(part for part in [title, str(year or ""), short_description] if part))
     return {
         "id": publication_id or None,
+        "publication_id": publication_id or None,
         "title": title or publication_id,
         "year": year,
         "type": publication_type,
+        "publication_type": publication_type,
+        "language": normalize_ws(language.get("name")) or None,
+        "status": _int_or_none(item.get("status")),
         "url": f"https://publications.hse.ru/view/{publication_id}" if publication_id else None,
+        "doi_url": _document_href(documents, "DOI"),
+        "other_url": _document_href(documents, "OTHER_URL"),
+        "document_url": _document_href(documents, "DOCUMENT"),
+        "citation_text": citation_text or None,
+        "annotation": annotation,
+        "description": description or None,
+        "authors": authors,
+        "raw_data": item,
         "text": text or title or publication_id,
     }
 
@@ -594,16 +805,84 @@ def _dedupe_publications(items: list[dict]) -> list[dict]:
     return unique
 
 
+def _dedupe_news_links(items: list[dict]) -> list[dict]:
+    seen = set()
+    unique = []
+    for item in items:
+        key = item.get("url") or item.get("title")
+        if key and key not in seen:
+            seen.add(key)
+            unique.append(item)
+    return unique
+
+
 def _html_to_text(value: object) -> str:
     return normalize_ws(BeautifulSoup(str(value or ""), "html.parser").get_text(" ", strip=True))
 
 
+def _localized_text_map(value: object) -> dict[str, str]:
+    if not isinstance(value, dict):
+        return {}
+    localized = {}
+    for key in ("ru", "en", "publ"):
+        text = _html_to_text(value.get(key))
+        if text:
+            localized[key] = text
+    return localized
+
+
 def _localized_value(value: object) -> str:
     if isinstance(value, dict):
         return normalize_ws(value.get("ru") or value.get("publ") or value.get("en"))
     return normalize_ws(str(value or ""))
 
 
+def _normalize_publication_authors(value: object, current_author_id: str | None) -> list[dict]:
+    if not isinstance(value, dict):
+        return []
+    authors = []
+    for author in value.get("author") or []:
+        if not isinstance(author, dict):
+            continue
+        title = author.get("title") if isinstance(author.get("title"), dict) else {}
+        reverse_title = author.get("reverseTitle") if isinstance(author.get("reverseTitle"), dict) else {}
+        author_id = normalize_ws(author.get("id"))
+        href = normalize_ws(author.get("href"))
+        authors.append(
+            {
+                "id": author_id or None,
+                "href": urljoin("https://www.hse.ru", href) if href else None,
+                "title_ru": _html_to_text(title.get("ru")),
+                "title_en": _html_to_text(title.get("en")),
+                "reverse_title_ru": _html_to_text(reverse_title.get("ru")),
+                "reverse_title_en": _html_to_text(reverse_title.get("en")),
+                "alt_name": normalize_ws(author.get("altName")) or None,
+                "other_name": normalize_ws(author.get("otherName")) or None,
+                "is_current_employee": bool(current_author_id and author_id == current_author_id),
+            }
+        )
+    return authors
+
+
+def _document_href(documents: dict, key: str) -> str | None:
+    document = documents.get(key)
+    if not isinstance(document, dict):
+        return None
+    return normalize_ws(document.get("href")) or None
+
+
+def _build_publication_citation(title: str, authors: list[dict], year: int | None) -> str:
+    author_names = [author.get("title_ru") or author.get("title_en") or author.get("alt_name") for author in authors]
+    return normalize_ws(". ".join(part for part in [", ".join(filter(None, author_names)), title, str(year or "")] if part))
+
+
+def _int_or_none(value: object) -> int | None:
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
 def _slugify(value: str) -> str:
     cleaned = re.sub(r"[^\w\s-]", "", value.lower(), flags=re.UNICODE)
     return re.sub(r"[-\s]+", "_", cleaned).strip("_") or "section"
@@ -629,3 +908,62 @@ def _dedupe_dicts(items: list[dict]) -> list[dict]:
             seen.add(key)
             unique.append(item)
     return unique
+
+
+def _fetch_text(
+    session: Session,
+    url: str,
+    headers: dict[str, str],
+    timeout: int,
+    *,
+    resource_cache=None,
+    profile_key: str | None = None,
+    resource_key: str,
+    resource_manifest: list[dict] | None,
+    method: str = "GET",
+    json_payload: object | None = None,
+    params: dict | None = None,
+) -> str:
+    if resource_cache and profile_key:
+        cached = resource_cache.fetch_text(
+            session,
+            profile_key=profile_key,
+            resource_key=resource_key,
+            method=method,
+            url=url,
+            headers=headers,
+            timeout=timeout,
+            json_payload=json_payload,
+            params=params,
+        )
+        if resource_manifest is not None:
+            resource_manifest.append(
+                {
+                    "resource_key": resource_key,
+                    "method": method,
+                    "url": url,
+                    "body_hash": cached.body_hash,
+                    "from_cache": cached.from_cache,
+                    "status_code": cached.status_code,
+                }
+            )
+        return cached.text
+
+    if method.upper() == "POST":
+        response = session.post(url, json=json_payload, headers=headers, timeout=timeout, params=params)
+    else:
+        response = session.get(url, headers=headers, timeout=timeout, params=params)
+    response.raise_for_status()
+    text = response.text
+    if resource_manifest is not None:
+        resource_manifest.append(
+            {
+                "resource_key": resource_key,
+                "method": method,
+                "url": url,
+                "body_hash": hashlib.sha256(text.encode("utf-8")).hexdigest(),
+                "from_cache": False,
+                "status_code": response.status_code,
+            }
+        )
+    return text

app/services/academic_degrees.py

@@ -0,0 +1,33 @@
+import re
+from typing import Any
+
+
+_PATTERN = re.compile(r"\b(?:кандидат|доктор)(?:\s+[\w-]+){0,12}\s+наук\b|\bph\.?\s*d\.?(?!\w)", re.IGNORECASE)
+
+
+def academic_degrees(data: dict[str, Any] | None) -> list[str]:
+    degrees = []
+    seen = set()
+    for section in (data or {}).get("sections") or []:
+        if not isinstance(section, dict) or not re.search(
+            r"образован|степен|academic degree|education", str(section.get("title") or ""), re.IGNORECASE
+        ):
+            continue
+        values = [
+            *(entry.get("text") for entry in section.get("year_entries") or [] if isinstance(entry, dict)),
+            *(section.get("paragraphs") or []),
+            *(section.get("items") or []),
+            section.get("raw_text"),
+        ]
+        table = section.get("table") or {}
+        for row in table.get("rows") or []:
+            if isinstance(row, dict):
+                values.extend(row.get("cells") or [])
+        for value in values:
+            text = str(value or "").strip()
+            for match in _PATTERN.finditer(text):
+                degree = match.group(0).strip()
+                if degree.casefold() not in seen:
+                    seen.add(degree.casefold())
+                    degrees.append(degree)
+    return degrees

app/services/admin_data.py

@@ -8,7 +8,8 @@ from zoneinfo import ZoneInfo
 from sqlalchemy import Select, Text, and_, desc, func, or_, select
 from sqlalchemy.orm import Session
 
-from app.models import CrawlError, CrawlRun, CrawlRunEmployeeChange, Employee
+from app.models import CrawlError, CrawlRun, CrawlRunEmployeeChange, Employee, EmployeeNewsLink
+from app.services.academic_degrees import academic_degrees
 
 EMPLOYEE_SORTS = {
     "full_name": Employee.full_name,
@@ -19,14 +20,15 @@ EMPLOYEE_SORTS = {
     "hse_start_year": Employee.current_data["hse_start_year"].as_integer(),
 }
 
-
 def employee_display_payload(employee: Employee) -> dict[str, Any]:
     data = _as_dict(employee.current_data)
     contacts = _as_dict(data.get("contacts"))
     sections = _as_list(data.get("sections"))
+    stored_news_links = _stored_news_links(employee)
     positions = _clean_list(data.get("positions"))
     emails = _clean_list(contacts.get("emails"))
     phones = _clean_list(contacts.get("phones"))
+    degree_values = academic_degrees(data)
     return {
         "id": employee.id,
         "full_name": employee.full_name,
@@ -41,8 +43,10 @@ def employee_display_payload(employee: Employee) -> dict[str, Any]:
         "phones": phones,
         "phone_text": ", ".join(phones),
         "address": contacts.get("address"),
+        "academic_degree_text": "; ".join(degree_values),
         "publications_count": _count_section_items(sections, "publications"),
         "courses_count": _count_section_items(sections, "courses_by_year"),
+        "news_count": len(stored_news_links) or _count_section_items(sections, "news"),
         "first_seen_at": employee.first_seen_at.isoformat() if employee.first_seen_at else None,
         "last_seen_at": employee.last_seen_at.isoformat() if employee.last_seen_at else None,
         "dismissed_at": employee.dismissed_at.isoformat() if employee.dismissed_at else None,
@@ -67,6 +71,7 @@ def employee_detail_payload(employee: Employee) -> dict[str, Any]:
             "contact_items": _normalize_contact_items(contacts.get("items")),
         },
         "external_ids": _normalize_external_ids(data.get("external_ids")),
+        "news_links": _detail_news_links(employee, data),
         "sections": [_normalize_section(section) for section in _as_list(data.get("sections"))],
     }
 
@@ -78,6 +83,7 @@ def build_employee_query(
     started_from: date | None = None,
     started_to: date | None = None,
     has_email: bool | None = None,
+    has_academic_degree: bool | None = None,
 ) -> Select[tuple[Employee]]:
     stmt = select(Employee)
     filters = []
@@ -94,6 +100,8 @@ def build_employee_query(
         filters.append(Employee.current_data.cast(Text).ilike("%@%"))
     elif has_email is False:
         filters.append(or_(Employee.current_data.is_(None), ~Employee.current_data.cast(Text).ilike("%@%")))
+    if has_academic_degree is not None:
+        filters.append(Employee.has_academic_degree.is_(has_academic_degree))
     if filters:
         stmt = stmt.where(and_(*filters))
     return stmt
@@ -107,6 +115,7 @@ def list_employees_page(
     started_from: date | None = None,
     started_to: date | None = None,
     has_email: bool | None = None,
+    has_academic_degree: bool | None = None,
     sort: str = "full_name",
     direction: str = "asc",
     limit: int = 50,
@@ -120,8 +129,11 @@ def list_employees_page(
         started_from=started_from,
         started_to=started_to,
         has_email=has_email,
+        has_academic_degree=has_academic_degree,
     )
     total = db.scalar(select(func.count()).select_from(base_stmt.subquery())) or 0
+    if offset >= total:
+        offset = 0
     sort_column = EMPLOYEE_SORTS.get(sort, Employee.full_name)
     order = desc(sort_column) if direction == "desc" else sort_column
     employees = db.scalars(base_stmt.order_by(order).limit(limit).offset(offset)).all()
@@ -142,6 +154,10 @@ def stats_payload(db: Session) -> dict[str, Any]:
     return {
         "total": db.scalar(select(func.count()).select_from(Employee)) or 0,
         "active": db.scalar(select(func.count()).select_from(Employee).where(Employee.status == "active")) or 0,
+        "verification_required": db.scalar(
+            select(func.count()).select_from(Employee).where(Employee.status == "verification_required")
+        )
+        or 0,
         "dismissed": db.scalar(select(func.count()).select_from(Employee).where(Employee.status == "dismissed")) or 0,
         "new_in_last_run": latest_run.new_count if latest_run else 0,
         "latest_added": employee_display_payload(latest_added) if latest_added else None,
@@ -153,7 +169,7 @@ def stats_payload(db: Session) -> dict[str, Any]:
 def run_payload(run: CrawlRun | None) -> dict[str, Any] | None:
     if not run:
         return None
-    processed = run.parsed_count + run.error_count
+    processed = run.parsed_count + run.skipped_count + run.error_count
     percent = round((processed / run.found_count) * 100, 1) if run.found_count else 0
     return {
         "id": run.id,
@@ -166,6 +182,7 @@ def run_payload(run: CrawlRun | None) -> dict[str, Any] | None:
         "finished_display": format_admin_datetime(run.finished_at),
         "found_count": run.found_count,
         "parsed_count": run.parsed_count,
+        "skipped_count": run.skipped_count,
         "new_count": run.new_count,
         "error_count": run.error_count,
         "dismissed_count": run.dismissed_count,
@@ -184,7 +201,7 @@ def run_detail_payload(db: Session, run: CrawlRun | None) -> dict[str, Any] | No
         .order_by(CrawlRunEmployeeChange.created_at, CrawlRunEmployeeChange.id)
     ).all()
     errors = db.scalars(select(CrawlError).where(CrawlError.crawl_run_id == run.id).order_by(CrawlError.created_at)).all()
-    grouped_changes = {"new": [], "missing_from_source": [], "dismissed": []}
+    grouped_changes = {"new": [], "missing_from_source": [], "verification_required": [], "dismissed": []}
     for change in changes:
         grouped_changes.setdefault(change.change_type, []).append(_change_payload(change))
     return {
@@ -211,7 +228,7 @@ def format_admin_datetime(value: Any) -> str:
 
 
 def _employee_status_display(status: str | None) -> str:
-    labels = {"active": "Работает", "dismissed": "Уволен"}
+    labels = {"active": "Работает", "verification_required": "Требует проверки", "dismissed": "Уволен"}
     return labels.get(status or "", status or "Не указано")
 
 
@@ -275,6 +292,8 @@ def _count_section_items(sections: list[dict[str, Any]], section_type: str) -> i
             total += len(section.get("publications") or section.get("items") or [])
         elif section_type == "courses_by_year":
             total += len(section.get("courses") or [])
+        elif section_type == "news":
+            total += len(section.get("news_links") or section.get("items") or [])
     return total
 
 
@@ -347,6 +366,8 @@ def _normalize_section(section: Any) -> dict[str, Any]:
         "year_entries": _normalize_year_entries(section.get("year_entries")),
         "publications": _normalize_publications(section.get("publications")),
         "publications_count": section.get("publications_count"),
+        "news_links": _normalize_news_links(section.get("news_links")),
+        "news_count": section.get("news_count"),
         "theses": _normalize_theses(section.get("theses")),
         "theses_count": section.get("theses_count"),
         "academic_year": section.get("academic_year"),
@@ -369,6 +390,77 @@ def _normalize_links(items: Any) -> list[dict[str, str | None]]:
     return normalized
 
 
+def _stored_news_links(employee: Employee) -> list[dict[str, Any]]:
+    return [_stored_news_link_payload(item) for item in sorted(employee.news_links, key=_news_link_sort_key)]
+
+
+def _news_link_sort_key(item: EmployeeNewsLink) -> tuple:
+    timestamp = item.published_at.timestamp() if item.published_at else 0
+    return (-timestamp, item.title or "", item.id)
+
+
+def _stored_news_link_payload(item: EmployeeNewsLink) -> dict[str, Any]:
+    return {
+        "title": item.title,
+        "url": item.url,
+        "summary": item.summary,
+        "published_at": item.published_at.isoformat() if item.published_at else None,
+        "published_year": item.published_year,
+        "published_display": format_admin_date(item.published_at) if item.published_at else str(item.published_year or ""),
+    }
+
+
+def _detail_news_links(employee: Employee, data: dict[str, Any]) -> list[dict[str, Any]]:
+    stored = _stored_news_links(employee)
+    if stored:
+        return stored
+    for section in _as_list(data.get("sections")):
+        if isinstance(section, dict) and section.get("type") == "news":
+            return _normalize_news_links(section.get("news_links"))
+    return []
+
+
+def format_admin_date(value: Any) -> str:
+    if not value:
+        return ""
+    if isinstance(value, str):
+        try:
+            value = datetime.fromisoformat(value.replace("Z", "+00:00"))
+        except ValueError:
+            return value
+    if not isinstance(value, datetime):
+        return str(value)
+    if value.tzinfo:
+        value = value.astimezone(ZoneInfo("Europe/Moscow"))
+    return value.strftime("%d.%m.%Y")
+
+
+def _normalize_news_links(items: Any) -> list[dict[str, Any]]:
+    normalized = []
+    if not isinstance(items, list):
+        return normalized
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        title = str(item.get("title") or item.get("url") or "").strip()
+        url = str(item.get("url") or "").strip()
+        summary = str(item.get("summary") or "").strip()
+        published_at = str(item.get("published_at") or "").strip()
+        published_year = item.get("published_year")
+        if title or url:
+            normalized.append(
+                {
+                    "title": title or url,
+                    "url": url or None,
+                    "summary": summary or None,
+                    "published_at": published_at or None,
+                    "published_year": published_year,
+                    "published_display": format_admin_date(published_at) if published_at else str(published_year or ""),
+                }
+            )
+    return normalized
+
+
 def _normalize_year_entries(items: Any) -> list[dict[str, Any]]:
     normalized = []
     if not isinstance(items, list):

app/services/council.py

@@ -0,0 +1,117 @@
+"""Preliminary council candidates from the supplied publication export."""
+
+import calendar
+import csv
+import re
+from collections import Counter, defaultdict
+from datetime import date, datetime
+from functools import lru_cache
+from pathlib import Path
+from urllib.parse import quote
+from zoneinfo import ZoneInfo
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from app.models import Employee
+from app.services.academic_degrees import academic_degrees
+
+
+EXPORT = Path(__file__).resolve().parents[1] / "data" / "council_articles.csv"
+DOCTOR = re.compile(r"^доктор\b.+\bнаук$", re.IGNORECASE)
+DOI = re.compile(r"10\.\d{4,9}/\S+", re.IGNORECASE)
+
+
+def _name(value: str) -> str:
+    return " ".join(value.casefold().replace("ё", "е").split())
+
+
+def _doi(value: str) -> str:
+    value = value.strip().lower()
+    for prefix in ("https://doi.org/", "http://doi.org/", "doi:"):
+        if value.startswith(prefix):
+            value = value[len(prefix):]
+    return value
+
+
+def period(today: date) -> tuple[date, date]:
+    year = today.year - 5
+    return date(year, today.month, min(today.day, calendar.monthrange(year, today.month)[1])), today
+
+
+@lru_cache(maxsize=1)
+def _articles_by_author() -> dict[str, list[dict]]:
+    articles = defaultdict(list)
+    with EXPORT.open(encoding="utf-8-sig", newline="") as source:
+        for row_number, row in enumerate(csv.DictReader(source, delimiter=";"), 2):
+            sources = []
+            for label, quartile, identifier in (
+                ("Scopus", "Квартиль scopus", "EID (Scopus)"),
+                ("Web of Science", "Квартиль wos", "UT (WoS)"),
+            ):
+                value = (row[quartile] or "").strip().upper()
+                if value in {"Q1", "Q2"}:
+                    sources.append({"name": label, "quartile": value, "identifier": (row[identifier] or "").strip()})
+            if not sources:
+                continue
+            try:
+                year = int(row["Год"])
+            except (TypeError, ValueError):
+                continue
+            title = (row["Название публикации"] or "").strip()
+            if not title:
+                continue
+            doi = _doi(row["DOI"] or "")
+            article = {
+                "title": title,
+                "year": year,
+                "doi": doi,
+                "doi_url": f"https://doi.org/{quote(doi, safe='/')}" if DOI.fullmatch(doi) else None,
+                "sources": sources,
+                "row_number": row_number,
+            }
+            for author in set(_name(value) for value in (row["Авторы НИУ ВШЭ"] or "").split(";")) - {""}:
+                articles[author].append(article)
+    return articles
+
+
+def _unique_articles(articles: list[dict], start: date, end: date) -> list[dict]:
+    unique = {}
+    for article in articles:
+        if not start.year <= article["year"] <= end.year:
+            continue
+        key = article["doi"] or (_name(article["title"]), article["year"])
+        if key not in unique:
+            unique[key] = {**article, "sources": list(article["sources"])}
+        else:
+            known = {item["name"] for item in unique[key]["sources"]}
+            unique[key]["sources"].extend(
+                item for item in article["sources"] if item["name"] not in known
+            )
+    return sorted(unique.values(), key=lambda item: (-len(item["sources"]), -item["year"], item["title"].casefold()))
+
+
+def council_candidates(db: Session, today: date | None = None) -> dict:
+    today = today or datetime.now(ZoneInfo("Europe/Moscow")).date()
+    start, end = period(today)
+    employees = db.scalars(select(Employee).where(Employee.status == "active")).all()
+    name_counts = Counter(_name(employee.full_name or "") for employee in employees)
+    by_author = _articles_by_author()
+    candidates = []
+    for employee in employees:
+        author = _name(employee.full_name or "")
+        if not author or name_counts[author] != 1:
+            continue
+        try:
+            degrees = academic_degrees(employee.current_data if isinstance(employee.current_data, dict) else {})
+        except (AttributeError, TypeError):
+            continue
+        degree = next((value for value in degrees if DOCTOR.fullmatch(value)), None)
+        if not degree:
+            continue
+        articles = _unique_articles(by_author.get(author, []), start, end)
+        if len(articles) >= 10:
+            candidates.append({"id": employee.id, "name": employee.full_name, "degree": degree,
+                               "count": len(articles), "articles": articles})
+    candidates.sort(key=lambda item: (-item["count"], item["name"].casefold()))
+    return {"candidates": candidates, "start": start, "end": end, "total": len(candidates)}

app/services/dataset_versions.py

@@ -1,227 +0,0 @@
-import hashlib
-import json
-from dataclasses import dataclass
-
-from sqlalchemy import desc, select
-from sqlalchemy.orm import Session
-
-from app.models import DatasetVersion, DatasetVersionItem, Employee
-
-
-@dataclass(frozen=True)
-class EmployeeMarker:
-    profile_key: str
-    employee_id: int | None
-    status: str
-    checksum: str
-
-
-def get_or_create_current_version(db: Session, *, crawl_run_id: int | None = None) -> DatasetVersion:
-    employees = db.scalars(select(Employee).order_by(Employee.profile_key)).all()
-    markers = [_employee_marker(employee) for employee in employees]
-    dataset_hash = _dataset_hash(markers)
-    latest = get_latest_version(db)
-    if latest and latest.hash == dataset_hash:
-        return latest
-
-    active_count = sum(1 for marker in markers if marker.status == "active")
-    dismissed_count = sum(1 for marker in markers if marker.status == "dismissed")
-    version = DatasetVersion(
-        hash=dataset_hash,
-        previous_hash=latest.hash if latest else None,
-        crawl_run_id=crawl_run_id,
-        employee_count=len(markers),
-        active_count=active_count,
-        dismissed_count=dismissed_count,
-    )
-    db.add(version)
-    db.flush()
-    for marker in markers:
-        db.add(
-            DatasetVersionItem(
-                dataset_version_id=version.id,
-                profile_key=marker.profile_key,
-                employee_id=marker.employee_id,
-                status=marker.status,
-                checksum=marker.checksum,
-            )
-        )
-    db.flush()
-    return version
-
-
-def get_latest_version(db: Session) -> DatasetVersion | None:
-    return db.scalar(select(DatasetVersion).order_by(desc(DatasetVersion.created_at), desc(DatasetVersion.id)).limit(1))
-
-
-def get_version_by_hash(db: Session, dataset_hash: str | None) -> DatasetVersion | None:
-    if not dataset_hash:
-        return None
-    return db.scalar(select(DatasetVersion).where(DatasetVersion.hash == dataset_hash).limit(1))
-
-
-def service_info_payload(db: Session, *, tools: list[dict], service_name: str, backend_version: str, protocol_version: str) -> dict:
-    version = get_or_create_current_version(db)
-    db.commit()
-    return {
-        "service_name": service_name,
-        "backend_version": backend_version,
-        "protocolVersion": protocol_version,
-        "tools": tools,
-        "dataset": _version_payload(version),
-    }
-
-
-def sync_employees_payload(db: Session, *, client_hash: str | None = None, include_data: bool = True) -> dict:
-    current = get_or_create_current_version(db)
-    db.commit()
-    if not client_hash:
-        return _full_sync_payload(db, current, include_data=include_data, reason=None)
-    if client_hash == current.hash:
-        return {
-            "mode": "delta",
-            "from_hash": client_hash,
-            "to_hash": current.hash,
-            "dataset": _version_payload(current),
-            "changes": {"added": [], "updated": [], "dismissed": [], "removed": []},
-        }
-
-    previous = get_version_by_hash(db, client_hash)
-    if not previous:
-        return _full_sync_payload(db, current, include_data=include_data, reason="unknown_client_hash", from_hash=client_hash)
-
-    return _delta_sync_payload(db, previous, current, include_data=include_data)
-
-
-def _full_sync_payload(
-    db: Session,
-    current: DatasetVersion,
-    *,
-    include_data: bool,
-    reason: str | None,
-    from_hash: str | None = None,
-) -> dict:
-    employees = db.scalars(select(Employee).order_by(Employee.profile_key)).all()
-    payload = {
-        "mode": "full",
-        "from_hash": from_hash,
-        "to_hash": current.hash,
-        "dataset": _version_payload(current),
-        "items": [_employee_payload(employee, include_data=include_data) for employee in employees],
-    }
-    if reason:
-        payload["reason"] = reason
-    return payload
-
-
-def _delta_sync_payload(db: Session, previous: DatasetVersion, current: DatasetVersion, *, include_data: bool) -> dict:
-    previous_items = _items_by_profile_key(previous)
-    current_items = _items_by_profile_key(current)
-    employees = {employee.profile_key: employee for employee in db.scalars(select(Employee)).all()}
-    added = []
-    updated = []
-    dismissed = []
-    removed = []
-
-    for profile_key, current_item in sorted(current_items.items()):
-        previous_item = previous_items.get(profile_key)
-        employee = employees.get(profile_key)
-        if not previous_item:
-            if employee:
-                added.append(_employee_payload(employee, include_data=include_data))
-            continue
-        if previous_item.checksum == current_item.checksum and previous_item.status == current_item.status:
-            continue
-        if current_item.status == "dismissed":
-            dismissed.append(_tombstone(profile_key, current_item.status, employee))
-        elif employee:
-            updated.append(_employee_payload(employee, include_data=include_data))
-
-    for profile_key, previous_item in sorted(previous_items.items()):
-        if profile_key not in current_items:
-            removed.append(_tombstone(profile_key, "removed", employees.get(profile_key), checksum=previous_item.checksum))
-
-    return {
-        "mode": "delta",
-        "from_hash": previous.hash,
-        "to_hash": current.hash,
-        "dataset": _version_payload(current),
-        "changes": {
-            "added": added,
-            "updated": updated,
-            "dismissed": dismissed,
-            "removed": removed,
-        },
-    }
-
-
-def _items_by_profile_key(version: DatasetVersion) -> dict[str, DatasetVersionItem]:
-    return {item.profile_key: item for item in version.items}
-
-
-def _version_payload(version: DatasetVersion) -> dict:
-    return {
-        "hash": version.hash,
-        "previous_hash": version.previous_hash,
-        "created_at": version.created_at.isoformat() if version.created_at else None,
-        "crawl_run_id": version.crawl_run_id,
-        "employee_count": version.employee_count,
-        "active_count": version.active_count,
-        "dismissed_count": version.dismissed_count,
-    }
-
-
-def _employee_marker(employee: Employee) -> EmployeeMarker:
-    return EmployeeMarker(
-        profile_key=employee.profile_key,
-        employee_id=employee.id,
-        status=employee.status,
-        checksum=employee.current_checksum or _payload_hash(employee.current_data or {}),
-    )
-
-
-def _dataset_hash(markers: list[EmployeeMarker]) -> str:
-    payload = [
-        {"profile_key": marker.profile_key, "status": marker.status, "checksum": marker.checksum}
-        for marker in sorted(markers, key=lambda item: item.profile_key)
-    ]
-    return _payload_hash(payload)
-
-
-def _payload_hash(value: object) -> str:
-    payload = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str)
-    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
-
-
-def _employee_payload(employee: Employee, *, include_data: bool) -> dict:
-    payload = {
-        "profile_key": employee.profile_key,
-        "profile_id": employee.profile_id,
-        "full_name": employee.full_name,
-        "status": employee.status,
-        "canonical_url": employee.canonical_url,
-        "last_seen_at": employee.last_seen_at.isoformat() if employee.last_seen_at else None,
-        "dismissed_at": employee.dismissed_at.isoformat() if employee.dismissed_at else None,
-        "checksum": employee.current_checksum or _payload_hash(employee.current_data or {}),
-    }
-    if include_data:
-        payload["data"] = employee.current_data
-    return payload
-
-
-def _tombstone(profile_key: str, status: str, employee: Employee | None, *, checksum: str | None = None) -> dict:
-    payload = {
-        "profile_key": profile_key,
-        "status": status,
-        "checksum": checksum or (employee.current_checksum if employee else None),
-    }
-    if employee:
-        payload.update(
-            {
-                "profile_id": employee.profile_id,
-                "full_name": employee.full_name,
-                "canonical_url": employee.canonical_url,
-                "dismissed_at": employee.dismissed_at.isoformat() if employee.dismissed_at else None,
-            }
-        )
-    return payload

app/services/resource_cache.py

@@ -0,0 +1,147 @@
+from __future__ import annotations
+
+import gzip
+import hashlib
+import json
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Any
+
+import requests
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from app.models import ParseResourceCache
+from app.version import BACKEND_VERSION
+
+
+@dataclass(frozen=True)
+class CachedResource:
+    text: str
+    body_hash: str
+    from_cache: bool
+    status_code: int
+
+
+class ResourceCache:
+    def __init__(self, db: Session):
+        self.db = db
+
+    def fetch_text(
+        self,
+        session: requests.Session,
+        *,
+        profile_key: str,
+        resource_key: str,
+        method: str,
+        url: str,
+        headers: dict[str, str],
+        timeout: int,
+        json_payload: Any | None = None,
+        params: dict[str, Any] | None = None,
+    ) -> CachedResource:
+        method = method.upper()
+        fingerprint = _request_fingerprint(method=method, url=url, json_payload=json_payload, params=params)
+        cached = self.db.scalar(
+            select(ParseResourceCache).where(
+                ParseResourceCache.profile_key == profile_key,
+                ParseResourceCache.resource_key == resource_key,
+                ParseResourceCache.request_fingerprint == fingerprint,
+            )
+        )
+
+        request_headers = dict(headers)
+        if cached:
+            if cached.etag:
+                request_headers["If-None-Match"] = cached.etag
+            if cached.last_modified:
+                request_headers["If-Modified-Since"] = cached.last_modified
+
+        response = _send(
+            session,
+            method=method,
+            url=url,
+            headers=request_headers,
+            timeout=timeout,
+            json_payload=json_payload,
+            params=params,
+        )
+        if response.status_code == 304 and cached:
+            cached.fetched_at = datetime.now(timezone.utc)
+            self.db.flush()
+            return CachedResource(
+                text=gzip.decompress(cached.body_snapshot).decode("utf-8"),
+                body_hash=cached.body_hash,
+                from_cache=True,
+                status_code=response.status_code,
+            )
+
+        response.raise_for_status()
+        text = response.text
+        body_hash = _body_hash(text)
+        etag = response.headers.get("ETag") if hasattr(response, "headers") else None
+        last_modified = response.headers.get("Last-Modified") if hasattr(response, "headers") else None
+
+        if cached:
+            cached.method = method
+            cached.url = url
+            cached.etag = etag
+            cached.last_modified = last_modified
+            cached.body_hash = body_hash
+            cached.body_snapshot = gzip.compress(text.encode("utf-8"))
+            cached.parser_version = BACKEND_VERSION
+            cached.fetched_at = datetime.now(timezone.utc)
+        else:
+            self.db.add(
+                ParseResourceCache(
+                    profile_key=profile_key,
+                    resource_key=resource_key,
+                    method=method,
+                    url=url,
+                    request_fingerprint=fingerprint,
+                    etag=etag,
+                    last_modified=last_modified,
+                    body_hash=body_hash,
+                    body_snapshot=gzip.compress(text.encode("utf-8")),
+                    parser_version=BACKEND_VERSION,
+                    fetched_at=datetime.now(timezone.utc),
+                )
+            )
+        self.db.flush()
+        return CachedResource(text=text, body_hash=body_hash, from_cache=False, status_code=response.status_code)
+
+
+def _send(
+    session: requests.Session,
+    *,
+    method: str,
+    url: str,
+    headers: dict[str, str],
+    timeout: int,
+    json_payload: Any | None,
+    params: dict[str, Any] | None,
+) -> requests.Response:
+    if method == "POST":
+        return session.post(url, json=json_payload, headers=headers, timeout=timeout, params=params)
+    return session.get(url, headers=headers, timeout=timeout, params=params)
+
+
+def _request_fingerprint(
+    *,
+    method: str,
+    url: str,
+    json_payload: Any | None,
+    params: dict[str, Any] | None,
+) -> str:
+    payload = {
+        "method": method,
+        "url": url,
+        "json": json_payload,
+        "params": params,
+    }
+    encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+    return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
+
+
+def _body_hash(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()

app/static/admin.js

@@ -3,13 +3,15 @@
     "full_name",
     "status",
     "positions",
-    "hse_start_year",
-    "email",
     "last_seen_at",
-    "dismissed_at",
     "profile",
   ];
   const storageKey = "miem.directory.columns";
+  const columnPresets = {
+    review: ["full_name", "status", "academic_degree", "last_seen_at", "profile"],
+    contacts: ["full_name", "status", "email", "phone", "address", "profile"],
+    full: ["full_name", "status", "positions", "hse_start_year", "email", "phone", "address", "academic_degree", "publications_count", "courses_count", "news_count", "first_seen_at", "last_seen_at", "dismissed_at", "profile"],
+  };
 
   function readColumns() {
     try {
@@ -39,18 +41,33 @@
     if (!document.querySelector("[data-directory-table]")) return;
     let columns = readColumns();
     const modal = document.querySelector("[data-columns-modal]");
+    const trigger = document.querySelector("[data-columns-open]");
     applyColumns(columns);
 
-    document.querySelectorAll("[data-columns-open]").forEach((button) => {
-      button.addEventListener("click", () => {
-        modal.hidden = false;
-      });
+    trigger.addEventListener("click", () => {
+      modal.showModal();
+      trigger.setAttribute("aria-expanded", "true");
+      modal.querySelector("input")?.focus();
     });
     document.querySelectorAll("[data-columns-close]").forEach((button) => {
       button.addEventListener("click", () => {
-        modal.hidden = true;
+        modal.close();
       });
     });
+    document.querySelectorAll("[data-columns-preset]").forEach((button) => {
+      button.addEventListener("click", () => {
+        columns = columnPresets[button.dataset.columnsPreset];
+        writeColumns(columns);
+        applyColumns(columns);
+      });
+    });
+    modal.addEventListener("click", (event) => {
+      if (event.target === modal) modal.close();
+    });
+    modal.addEventListener("close", () => {
+      trigger.setAttribute("aria-expanded", "false");
+      trigger.focus();
+    });
     document.querySelectorAll("[data-column-toggle]").forEach((checkbox) => {
       checkbox.addEventListener("change", () => {
         columns = Array.from(document.querySelectorAll("[data-column-toggle]:checked")).map((item) => item.value);
@@ -61,58 +78,70 @@
     });
   }
 
-  function setupClickableRows() {
-    const openRow = (row) => {
-      window.location.href = row.dataset.rowHref;
-    };
-
-    document.querySelectorAll("[data-row-href]").forEach((row) => {
-      row.addEventListener("click", (event) => {
-        if (event.target.closest("a, button, input, select, label")) return;
-        openRow(row);
-      });
-      row.addEventListener("keydown", (event) => {
-        if (!["Enter", " "].includes(event.key)) return;
-        if (event.target.closest("a, button, input, select, label")) return;
-        event.preventDefault();
-        openRow(row);
-      });
-    });
-  }
-
   function setupProgress() {
     const panel = document.querySelector("[data-progress-panel]");
     if (!panel) return;
+    const state = document.createElement("p");
+    state.className = "progress-panel__empty";
+    state.setAttribute("role", "status");
+    panel.append(state);
+    let pending = false;
 
     const update = (run) => {
       if (!run) return;
       const status = document.querySelector("[data-progress-status]");
       const processed = document.querySelector("[data-progress-processed]");
       const found = document.querySelector("[data-progress-found]");
+      const skipped = document.querySelector("[data-progress-skipped]");
       const errors = document.querySelector("[data-progress-errors]");
       const fill = document.querySelector("[data-progress-fill]");
       const percent = document.querySelector("[data-progress-percent]");
+      const error = document.querySelector("[data-progress-error]");
       if (status) status.textContent = run.status_display || run.status;
       if (processed) processed.textContent = run.processed_count;
       if (found) found.textContent = run.found_count;
+      if (skipped) skipped.textContent = run.skipped_count;
       if (errors) errors.textContent = run.error_count;
       if (fill) fill.style.width = `${run.progress_percent}%`;
       if (percent) percent.textContent = run.progress_percent;
+      if (fill) {
+        fill.parentElement.setAttribute("aria-valuenow", run.progress_percent);
+        fill.parentElement.setAttribute("aria-valuetext", `${run.progress_percent}%`);
+      }
+      if (error) error.hidden = true;
     };
 
     const poll = async () => {
+      if (pending) return true;
+      pending = true;
+      state.textContent = "Обновляем прогресс…";
+      panel.setAttribute("aria-busy", "true");
       try {
-        const response = await fetch("/api/crawl-runs/latest", { credentials: "same-origin" });
-        if (!response.ok) return false;
+        const response = await fetch("/api/crawl-runs/latest", {
+          credentials: "same-origin",
+          signal: AbortSignal.timeout(15000),
+        });
+        if (!response.ok) throw new Error("progress request failed");
         const data = await response.json();
         const run = data.running || data.latest;
         update(run);
+        state.textContent = run ? "Прогресс обновлён" : "Запусков пока нет. Запустите парсинг.";
+        const error = document.querySelector("[data-progress-error]");
+        if (error) error.hidden = true;
         return Boolean(data.running);
       } catch (_error) {
-        return false;
+        const error = document.querySelector("[data-progress-error]");
+        if (error) error.hidden = false;
+        state.textContent = "Показаны последние полученные данные. Прогресс может быть устаревшим.";
+        return true;
+      } finally {
+        pending = false;
+        panel.setAttribute("aria-busy", "false");
       }
     };
 
+    document.querySelectorAll("[data-progress-retry]").forEach((button) => button.addEventListener("click", poll));
+
     const interval = window.setInterval(async () => {
       const keepGoing = await poll();
       if (!keepGoing) window.clearInterval(interval);
@@ -120,6 +149,5 @@
   }
 
   setupColumns();
-  setupClickableRows();
   setupProgress();
 })();

app/templates/base.html

@@ -4,21 +4,23 @@
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <title>{% block title %}MIEM Employees{% endblock %}</title>
-    <link rel="stylesheet" href="/static/admin.css">
+    <link rel="stylesheet" href="/static/admin.css?v={{ frontend_version }}">
   </head>
   <body class="admin">
+    <a class="admin__skip-link" href="#main-content">Перейти к содержимому</a>
     <header class="admin__header">
       <h1 class="admin__brand"><a class="admin__brand-link" href="/admin">MIEM Employees</a></h1>
       <nav class="admin__nav">
-        <a class="admin__link" href="/admin">Обзор</a>
-        <a class="admin__link" href="/admin/directory">Сотрудники</a>
-        <a class="admin__link" href="/admin/runs">Запуски</a>
+        <a class="admin__link" href="/admin"{% if request.url.path == "/admin" %} aria-current="page"{% endif %}>Обзор</a>
+        <a class="admin__link" href="/admin/directory"{% if request.url.path == "/admin/directory" %} aria-current="page"{% endif %}>Сотрудники</a>
+        <a class="admin__link" href="/admin/council"{% if request.url.path == "/admin/council" %} aria-current="page"{% endif %}>Диссертационный совет</a>
+        <a class="admin__link" href="/admin/runs"{% if request.url.path.startswith("/admin/runs") %} aria-current="page"{% endif %}>Запуски</a>
         <form method="post" action="/admin/logout">
           <button class="button button--ghost" type="submit">Выйти</button>
         </form>
       </nav>
     </header>
-    <main class="admin__main">
+    <main class="admin__main" id="main-content">
       {% block content %}{% endblock %}
     </main>
     <footer class="admin__footer">

app/templates/council.html

@@ -0,0 +1,72 @@
+{% extends "base.html" %}
+{% block title %}Кандидаты в диссертационный совет · MIEM Employees{% endblock %}
+{% block content %}
+<section class="council">
+  <header class="council__hero">
+    <div class="council__hero-copy">
+      <h2 class="council__title">Кандидаты в диссертационный совет</h2>
+      <p class="council__intro">Сотрудники с докторской степенью в профиле и публикациями с отметками Q1 или Q2 в имеющейся выгрузке. Список помогает начать проверку состава комиссии.</p>
+    </div>
+    {% if not error %}<p class="council__total"><strong>{{ total }}</strong> предварительно подходят</p>{% endif %}
+  </header>
+
+  {% if error %}
+  <div class="council__message" role="alert">
+    <h3>Не удалось загрузить список</h3>
+    <p>Проверьте доступность базы сотрудников и файла публикаций, затем обновите страницу.</p>
+  </div>
+  {% else %}
+  <div class="council__criteria" aria-label="Правила предварительного отбора">
+    <p><strong>10 и более</strong><span>различных публикаций</span></p>
+    <p><strong>{{ start.strftime('%d.%m.%Y') }} — {{ end.strftime('%d.%m.%Y') }}</strong><span>скользящие пять лет</span></p>
+    <p><strong>К1 · RSCI · Q1/Q2</strong><span>итоговые основания отбора</span></p>
+  </div>
+  <div class="council__notice" role="note">
+    <strong>Предварительный список.</strong> Сейчас учитываются только отметки Q1/Q2 в CSV для Scopus и Web of Science. В нём нет названия журнала, точной даты статьи, типа материала и подтверждения К1 или RSCI. Год на границе периода и принадлежность к журналу нужно проверить. Сотрудники с публикациями только в К1 или RSCI могут отсутствовать.
+  </div>
+
+  <div class="council__list-heading">
+    <h3>Сотрудники</h3>
+    <p>Откройте строку, чтобы увидеть публикации и основания.</p>
+  </div>
+  {% if candidates %}
+  <div class="council__list">
+    {% for candidate in candidates %}
+    <details class="council__candidate">
+      <summary class="council__candidate-summary">
+        <span class="council__candidate-identity"><strong>{{ candidate.name }}</strong><small>{{ candidate.degree }}</small></span>
+        <span class="council__candidate-count"><strong>{{ candidate.count }}</strong><small>публикаций по выгрузке</small></span>
+        <span class="council__candidate-cue" aria-hidden="true"></span>
+      </summary>
+      <div class="council__candidate-body">
+        <a class="admin__link" href="/admin/employees/{{ candidate.id }}">Открыть профиль сотрудника</a>
+        <ol class="council__articles">
+          {% for article in candidate.articles %}
+          <li class="council__article">
+            <div class="council__article-heading">
+              <strong>{{ article.title }}</strong>
+              <span>{{ article.year }}</span>
+            </div>
+            <div class="council__article-meta">
+              {% for source in article.sources %}
+              <span class="council__source">{{ source.name }} {{ source.quartile }}{% if source.identifier %} · {{ source.identifier }}{% else %} · ID не указан{% endif %}</span>
+              {% endfor %}
+              {% if article.doi_url %}<a class="admin__link" href="{{ article.doi_url }}" target="_blank" rel="noopener noreferrer">DOI</a>{% endif %}
+              <span>Строка {{ article.row_number }} в CSV</span>
+            </div>
+          </li>
+          {% endfor %}
+        </ol>
+      </div>
+    </details>
+    {% endfor %}
+  </div>
+  {% else %}
+  <div class="council__message">
+    <h4>Кандидаты пока не найдены</h4>
+    <p>По доступной выгрузке нет действующих сотрудников с докторской степенью, указанной в профиле, и десятью предварительно подходящими публикациями. Это не означает, что таких сотрудников нет.</p>
+  </div>
+  {% endif %}
+  {% endif %}
+</section>
+{% endblock %}

app/templates/dashboard.html

@@ -1,62 +1,70 @@
-{% extends "base.html" %}
-{% block title %}Обзор · MIEM Employees{% endblock %}
-{% block content %}
-<section class="admin__grid">
-  <a class="metric metric--link" href="/admin/directory"><span class="metric__label">Всего в базе</span><span class="metric__value">{{ counts.total }}</span></a>
-  <a class="metric metric--link" href="/admin/directory?status=active"><span class="metric__label">Работают</span><span class="metric__value">{{ counts.active }}</span></a>
-  <a class="metric metric--link" href="{% if latest_run %}/admin/runs/{{ latest_run.id }}#new-employees{% else %}/admin/runs{% endif %}"><span class="metric__label">Новые за запуск</span><span class="metric__value">{{ counts.new_in_last_run }}</span></a>
-  <a class="metric metric--link" href="/admin/directory?status=dismissed"><span class="metric__label">Уволены</span><span class="metric__value">{{ counts.dismissed }}</span></a>
-</section>
-<section class="stats-strip">
-  <div class="stats-strip__item">
-    <span class="stats-strip__label">Последний добавленный</span>
-    {% if counts.latest_added %}
-    <a class="stats-strip__value" href="/admin/employees/{{ counts.latest_added.id }}">{{ counts.latest_added.full_name or counts.latest_added.canonical_url }}</a>
-    {% else %}
-    <span class="stats-strip__value">Сотрудников пока нет</span>
-    {% endif %}
-  </div>
-  <a class="stats-strip__item stats-strip__item--link" href="/admin/runs">
-    <span class="stats-strip__label">Запуски</span>
-    <span class="stats-strip__value">{{ counts.runs }}</span>
-  </a>
-  <div class="stats-strip__item">
-    <span class="stats-strip__label">Ошибки</span>
-    <span class="stats-strip__value">{{ counts.errors }}</span>
-  </div>
-</section>
-<section class="panel progress-panel" data-progress-panel>
-  <div class="progress-panel__header">
-    <h2 class="panel__title">Прогресс парсинга</h2>
-    <form method="post" action="/admin/crawl-now">
-      <button class="button" type="submit">Запустить парсинг</button>
-    </form>
-  </div>
-  {% set run = counts.current_running_run or latest_run %}
-  <div class="progress-panel__body" data-progress-body>
-    <div class="progress-panel__meta">
-      <span data-progress-status>{{ run.status_display if run else "Ожидание" }}</span>
-      <span>обработано: <span data-progress-processed>{{ run.processed_count if run else 0 }}</span> / <span data-progress-found>{{ run.found_count if run else 0 }}</span></span>
-      <span>ошибок: <span data-progress-errors>{{ run.error_count if run else 0 }}</span></span>
-    </div>
-    <div class="progress-bar" aria-label="Parsing progress">
-      <div class="progress-bar__fill" data-progress-fill style="width: {{ run.progress_percent if run else 0 }}%"></div>
-    </div>
-    <div class="progress-panel__percent"><span data-progress-percent>{{ run.progress_percent if run else 0 }}</span>%</div>
-  </div>
-</section>
-<section class="panel">
-  <h2 class="panel__title">Последние запуски</h2>
-  <table class="table">
-    <thead><tr><th class="table__head">ID</th><th class="table__head">Статус</th><th class="table__head">Обработано</th><th class="table__head">Ошибки</th><th class="table__head">Старт</th></tr></thead>
-    <tbody>
-      {% for run in runs %}
-      <tr class="table__row" onclick="window.location.href='/admin/runs/{{ run.id }}'" onkeydown="if (event.key === 'Enter' || event.key === ' ') { event.preventDefault(); window.location.href='/admin/runs/{{ run.id }}'; }" role="link" tabindex="0"><td class="table__cell">{{ run.id }}</td><td class="table__cell">{{ run.status_display }}</td><td class="table__cell">{{ run.parsed_count }}</td><td class="table__cell">{{ run.error_count }}</td><td class="table__cell">{{ run.started_display }}</td></tr>
-      {% endfor %}
-    </tbody>
-  </table>
-</section>
-{% endblock %}
-{% block scripts %}
-<script src="/static/admin.js"></script>
-{% endblock %}
+{% extends "base.html" %}
+{% block title %}Обзор · MIEM Employees{% endblock %}
+{% block content %}
+<section class="admin__grid">
+  <a class="metric metric--link" href="/admin/directory"><span class="metric__label">Всего в базе</span><span class="metric__value">{{ counts.total }}</span></a>
+  <a class="metric metric--link" href="/admin/directory?status=active"><span class="metric__label">Работают</span><span class="metric__value">{{ counts.active }}</span></a>
+  <a class="metric metric--link" href="/admin/directory?status=verification_required"><span class="metric__label">Требуют проверки</span><span class="metric__value">{{ counts.verification_required }}</span></a>
+  <a class="metric metric--link" href="{% if latest_run %}/admin/runs/{{ latest_run.id }}#new-employees{% else %}/admin/runs{% endif %}"><span class="metric__label">Новые за запуск</span><span class="metric__value">{{ counts.new_in_last_run }}</span></a>
+  <a class="metric metric--link" href="/admin/directory?status=dismissed"><span class="metric__label">Уволены</span><span class="metric__value">{{ counts.dismissed }}</span></a>
+</section>
+<section class="stats-strip">
+  <div class="stats-strip__item">
+    <span class="stats-strip__label">Последний добавленный</span>
+    {% if counts.latest_added %}
+    <a class="stats-strip__value" href="/admin/employees/{{ counts.latest_added.id }}">{{ counts.latest_added.full_name or counts.latest_added.canonical_url }}</a>
+    {% else %}
+    <span class="stats-strip__value">Сотрудников пока нет</span>
+    {% endif %}
+  </div>
+  <a class="stats-strip__item stats-strip__item--link" href="/admin/runs">
+    <span class="stats-strip__label">Запуски</span>
+    <span class="stats-strip__value">{{ counts.runs }}</span>
+  </a>
+  <div class="stats-strip__item">
+    <span class="stats-strip__label">Ошибки</span>
+    <span class="stats-strip__value">{{ counts.errors }}</span>
+  </div>
+</section>
+<section class="panel progress-panel" data-progress-panel>
+  <div class="progress-panel__header">
+    <h2 class="panel__title">Прогресс парсинга</h2>
+    <div class="progress-panel__actions">
+      <form method="post" action="/admin/crawl-now">
+        <button class="button" type="submit">Запустить парсинг</button>
+      </form>
+      <form method="post" action="/admin/dismissed/refresh">
+        <button class="button button--secondary" type="submit">Проверить уволенных</button>
+      </form>
+    </div>
+  </div>
+  {% set run = counts.current_running_run or latest_run %}
+  <div class="progress-panel__body" data-progress-body>
+    <div class="progress-panel__meta">
+      <span data-progress-status>{{ run.status_display if run else "Ожидание" }}</span>
+      <span>обработано: <span data-progress-processed>{{ run.processed_count if run else 0 }}</span> / <span data-progress-found>{{ run.found_count if run else 0 }}</span></span>
+      <span>без изменений: <span data-progress-skipped>{{ run.skipped_count if run else 0 }}</span></span>
+      <span>ошибок: <span data-progress-errors>{{ run.error_count if run else 0 }}</span></span>
+    </div>
+    <div class="progress-bar" role="progressbar" aria-label="Прогресс парсинга" aria-valuemin="0" aria-valuemax="100" aria-valuenow="{{ run.progress_percent if run else 0 }}" aria-valuetext="{{ run.progress_percent if run else 0 }}%">
+      <div class="progress-bar__fill" data-progress-fill style="width: {{ run.progress_percent if run else 0 }}%"></div>
+    </div>
+    <div class="progress-panel__percent"><span data-progress-percent>{{ run.progress_percent if run else 0 }}</span>%</div>
+    <p class="progress-panel__error" data-progress-error role="status" hidden>Не удалось обновить прогресс. <button class="button button--ghost button--compact" type="button" data-progress-retry>Повторить</button></p>
+  </div>
+</section>
+<section class="panel">
+  <h2 class="panel__title">Последние запуски</h2>
+  <table class="table">
+    <thead><tr><th class="table__head">ID</th><th class="table__head">Статус</th><th class="table__head">Обработано</th><th class="table__head">Без изменений</th><th class="table__head">Ошибки</th><th class="table__head">Старт</th></tr></thead>
+    <tbody>
+      {% for run in runs %}
+      <tr class="table__row"><td class="table__cell"><a class="admin__link" href="/admin/runs/{{ run.id }}">{{ run.id }}</a></td><td class="table__cell">{{ run.status_display }}</td><td class="table__cell">{{ run.parsed_count }}</td><td class="table__cell">{{ run.skipped_count }}</td><td class="table__cell">{{ run.error_count }}</td><td class="table__cell">{{ run.started_display }}</td></tr>
+      {% endfor %}
+    </tbody>
+  </table>
+</section>
+{% endblock %}
+{% block scripts %}
+<script src="/static/admin.js"></script>
+{% endblock %}

app/templates/directory.html

@@ -5,43 +5,70 @@
   <div class="directory__header">
     <div>
       <h2 class="directory__title">Сотрудники</h2>
-      <p class="directory__summary">Найдено: {{ page.total }}</p>
+      {% set range_start = page.offset + 1 if page.total else 0 %}
+      {% set range_end = [page.offset + page.employees|length, page.total]|min %}
+      <p class="directory__summary" role="status">Показаны {{ range_start }}–{{ range_end }} из {{ page.total }}</p>
     </div>
-    <button class="button" type="button" data-columns-open>Колонки</button>
+    <button class="button" type="button" data-columns-open aria-controls="columns-dialog" aria-expanded="false">Настроить колонки</button>
   </div>
 
   <form class="directory__filters" method="get" action="/admin/directory">
-    <input class="directory__input" name="q" value="{{ filters.q }}" placeholder="ФИО или ссылка">
-    <select class="directory__input" name="status">
+    <fieldset class="directory__filter-group directory__filter-group--search">
+      <legend class="directory__filter-legend">Поиск</legend>
+      <label class="directory__field">ФИО или ссылка<input class="directory__input" name="q" value="{{ filters.q }}" placeholder="Например, Иванов или hse.ru"></label>
+    </fieldset>
+    <fieldset class="directory__filter-group">
+      <legend class="directory__filter-legend">Статус и данные</legend>
+      <label class="directory__field">Статус<select class="directory__input" name="status">
       <option value="" {% if not filters.status %}selected{% endif %}>Все статусы</option>
       <option value="active" {% if filters.status == "active" %}selected{% endif %}>Работает</option>
+      <option value="verification_required" {% if filters.status == "verification_required" %}selected{% endif %}>Требует проверки</option>
       <option value="dismissed" {% if filters.status == "dismissed" %}selected{% endif %}>Уволен</option>
-    </select>
-    <select class="directory__input" name="has_email">
+      </select></label>
+      <label class="directory__field">Email<select class="directory__input" name="has_email">
       <option value="" {% if not filters.has_email %}selected{% endif %}>Любой email</option>
       <option value="true" {% if filters.has_email == "true" %}selected{% endif %}>Есть email</option>
       <option value="false" {% if filters.has_email == "false" %}selected{% endif %}>Нет email</option>
-    </select>
-    <input class="directory__input" type="date" name="started_from" value="{{ filters.started_from }}" aria-label="Впервые найден с">
-    <input class="directory__input" type="date" name="started_to" value="{{ filters.started_to }}" aria-label="Впервые найден по">
-    <select class="directory__input" name="sort">
+      </select></label>
+      <label class="directory__field">Учёная степень<select class="directory__input" name="has_academic_degree">
+      <option value="" {% if not filters.has_academic_degree %}selected{% endif %}>Любая учёная степень</option>
+      <option value="true" {% if filters.has_academic_degree == "true" %}selected{% endif %}>Есть учёная степень</option>
+      <option value="false" {% if filters.has_academic_degree == "false" %}selected{% endif %}>Нет учёной степени</option>
+      </select></label>
+    </fieldset>
+    <fieldset class="directory__filter-group">
+      <legend class="directory__filter-legend">Период обнаружения</legend>
+      <label class="directory__field">От<input class="directory__input" type="date" name="started_from" value="{{ filters.started_from }}"></label>
+      <label class="directory__field">До<input class="directory__input" type="date" name="started_to" value="{{ filters.started_to }}"></label>
+    </fieldset>
+    <fieldset class="directory__filter-group">
+      <legend class="directory__filter-legend">Сортировка и страница</legend>
+      <label class="directory__field">Сортировать по<select class="directory__input" name="sort">
       {% for value, label in [("full_name", "ФИО"), ("status", "Статус"), ("hse_start_year", "Год начала"), ("first_seen_at", "Впервые найден"), ("last_seen_at", "Последний раз найден"), ("dismissed_at", "Дата увольнения")] %}
-      <option value="{{ value }}" {% if filters.sort == value %}selected{% endif %}>Сортировка: {{ label }}</option>
+      <option value="{{ value }}" {% if filters.sort == value %}selected{% endif %}>{{ label }}</option>
       {% endfor %}
-    </select>
-    <select class="directory__input" name="direction">
+      </select></label>
+      <label class="directory__field">Направление<select class="directory__input" name="direction">
       <option value="asc" {% if filters.direction == "asc" %}selected{% endif %}>По возрастанию</option>
       <option value="desc" {% if filters.direction == "desc" %}selected{% endif %}>По убыванию</option>
-    </select>
-    <select class="directory__input" name="limit" onchange="this.form.offset.value = 0; this.form.submit()">
+      </select></label>
+      <label class="directory__field">Сотрудников на странице<select class="directory__input" name="limit">
       {% for value in [25, 50, 100] %}
-      <option value="{{ value }}" {% if filters.limit == value %}selected{% endif %}>На странице: {{ value }}</option>
+      <option value="{{ value }}" {% if filters.limit == value %}selected{% endif %}>{{ value }}</option>
       {% endfor %}
-    </select>
-    <input type="hidden" name="offset" value="{{ filters.offset }}">
-    <button class="button" type="submit">Применить</button>
+      </select></label>
+    </fieldset>
+    <div class="directory__filter-actions">
+      <button class="button" type="submit">Применить фильтры</button>
+      <a class="button button--ghost" href="/admin/directory">Сбросить</a>
+    </div>
   </form>
 
+  <details class="directory__help">
+    <summary>Что означают даты и статусы</summary>
+    <p>«Год начала» — начало работы в ВШЭ. «Впервые найден» и «Последний раз найден» — даты обнаружения сотрудника в списке источника, а не даты приёма и увольнения. «Требует проверки» означает, что профиль недоступен и увольнение ещё не подтверждено.</p>
+  </details>
+
   <div class="directory__table-wrap">
     <table class="directory-table" data-directory-table>
       <thead>
@@ -53,8 +80,10 @@
           <th class="directory-table__head" data-column="email">Email</th>
           <th class="directory-table__head" data-column="phone">Телефон</th>
           <th class="directory-table__head" data-column="address">Адрес</th>
+          <th class="directory-table__head" data-column="academic_degree">Учёная степень</th>
           <th class="directory-table__head" data-column="publications_count">Публикации</th>
           <th class="directory-table__head" data-column="courses_count">Курсы</th>
+          <th class="directory-table__head" data-column="news_count">Новости</th>
           <th class="directory-table__head" data-column="first_seen_at">Впервые найден</th>
           <th class="directory-table__head" data-column="last_seen_at">Последний раз найден</th>
           <th class="directory-table__head" data-column="dismissed_at">Дата увольнения</th>
@@ -63,23 +92,33 @@
       </thead>
       <tbody>
         {% for employee in page.employees %}
-        <tr class="directory-table__row" data-row-href="/admin/employees/{{ employee.id }}">
-          <td class="directory-table__cell" data-column="full_name">{{ employee.full_name or "Без имени" }}</td>
-          <td class="directory-table__cell" data-column="status"><span class="badge {% if employee.status == "dismissed" %}badge--dismissed{% endif %}">{{ employee.status_display }}</span></td>
+        <tr class="directory-table__row">
+          <td class="directory-table__cell" data-column="full_name"><a class="admin__link" href="/admin/employees/{{ employee.id }}">{{ employee.full_name or "Без имени" }}</a></td>
+          <td class="directory-table__cell" data-column="status"><span class="badge {% if employee.status == "dismissed" %}badge--dismissed{% elif employee.status == "verification_required" %}badge--verification{% endif %}">{{ employee.status_display }}</span></td>
           <td class="directory-table__cell" data-column="positions">{{ employee.positions_text }}</td>
           <td class="directory-table__cell" data-column="hse_start_year">{{ employee.hse_start_year or "" }}</td>
           <td class="directory-table__cell" data-column="email">{{ employee.email_text }}</td>
           <td class="directory-table__cell" data-column="phone">{{ employee.phone_text }}</td>
           <td class="directory-table__cell" data-column="address">{{ employee.address or "" }}</td>
+          <td class="directory-table__cell" data-column="academic_degree">{{ employee.academic_degree_text }}</td>
           <td class="directory-table__cell" data-column="publications_count">{{ employee.publications_count }}</td>
           <td class="directory-table__cell" data-column="courses_count">{{ employee.courses_count }}</td>
+          <td class="directory-table__cell" data-column="news_count">{{ employee.news_count }}</td>
           <td class="directory-table__cell" data-column="first_seen_at">{{ employee.first_seen_display }}</td>
           <td class="directory-table__cell" data-column="last_seen_at">{{ employee.last_seen_display }}</td>
           <td class="directory-table__cell" data-column="dismissed_at">{{ employee.dismissed_display }}</td>
-          <td class="directory-table__cell" data-column="profile"><a class="admin__link" href="{{ employee.canonical_url }}">Открыть</a></td>
+          <td class="directory-table__cell" data-column="profile"><a class="admin__link" href="{{ employee.canonical_url }}" target="_blank" rel="noopener">Внешний профиль</a></td>
         </tr>
         {% else %}
-        <tr><td class="directory-table__empty" colspan="13">По этим фильтрам сотрудники не найдены.</td></tr>
+        <tr><td class="directory-table__empty" colspan="15">
+          <strong>Сотрудники не найдены</strong>
+          {% if filters.q or filters.status or filters.has_email or filters.has_academic_degree or filters.started_from or filters.started_to %}
+          <span>Попробуйте изменить запрос или сбросить фильтры.</span>
+          <a class="admin__link" href="/admin/directory">Сбросить фильтры</a>
+          {% else %}
+          <span>Запустите парсинг, чтобы загрузить сотрудников.</span>
+          {% endif %}
+        </td></tr>
         {% endfor %}
       </tbody>
     </table>
@@ -98,20 +137,25 @@
   </div>
 </section>
 
-<div class="columns-modal" data-columns-modal hidden>
-  <div class="columns-modal__backdrop" data-columns-close></div>
-  <section class="columns-modal__panel" aria-label="Настройка колонок">
+<dialog class="columns-modal" id="columns-dialog" data-columns-modal aria-labelledby="columns-dialog-title">
+  <section class="columns-modal__panel">
     <div class="columns-modal__header">
-      <h3 class="columns-modal__title">Отображаемые колонки</h3>
+      <h3 class="columns-modal__title" id="columns-dialog-title">Отображаемые колонки</h3>
       <button class="button button--ghost" type="button" data-columns-close>Закрыть</button>
     </div>
     <div class="columns-modal__grid">
-      {% for key, label in [("full_name", "ФИО"), ("status", "Статус"), ("positions", "Должности"), ("hse_start_year", "Год начала"), ("email", "Email"), ("phone", "Телефон"), ("address", "Адрес"), ("publications_count", "Публикации"), ("courses_count", "Курсы"), ("first_seen_at", "Впервые найден"), ("last_seen_at", "Последний раз найден"), ("dismissed_at", "Дата увольнения"), ("profile", "Профиль")] %}
+      {% for key, label in [("full_name", "ФИО"), ("status", "Статус"), ("positions", "Должности"), ("hse_start_year", "Год начала"), ("email", "Email"), ("phone", "Телефон"), ("address", "Адрес"), ("academic_degree", "Учёная степень"), ("publications_count", "Публикации"), ("courses_count", "Курсы"), ("news_count", "Новости"), ("first_seen_at", "Впервые найден"), ("last_seen_at", "Последний раз найден"), ("dismissed_at", "Дата увольнения"), ("profile", "Профиль")] %}
       <label class="columns-modal__option"><input class="columns-modal__checkbox" type="checkbox" value="{{ key }}" data-column-toggle> {{ label }}</label>
       {% endfor %}
     </div>
+    <fieldset class="columns-modal__presets">
+      <legend class="columns-modal__preset-label">Быстрый набор</legend>
+      <button class="button button--ghost button--compact" type="button" data-columns-preset="review">Проверка</button>
+      <button class="button button--ghost button--compact" type="button" data-columns-preset="contacts">Контакты</button>
+      <button class="button button--ghost button--compact" type="button" data-columns-preset="full">Все поля</button>
+    </fieldset>
   </section>
-</div>
+</dialog>
 {% endblock %}
 {% block scripts %}
 <script src="/static/admin.js"></script>

app/templates/directory_error.html

@@ -0,0 +1,9 @@
+{% extends "base.html" %}
+{% block title %}Каталог недоступен · MIEM Employees{% endblock %}
+{% block content %}
+<section class="panel">
+  <h2 class="panel__title">Не удалось загрузить сотрудников</h2>
+  <p>База данных временно недоступна. Повторите запрос; выбранные фильтры сохранятся.</p>
+  <a class="button" href="{{ request.url.path }}{% if request.url.query %}?{{ request.url.query }}{% endif %}">Повторить загрузку</a>
+</section>
+{% endblock %}

app/templates/employee_detail.html

@@ -104,6 +104,25 @@
   </section>
   {% endif %}
 
+  {% if employee_view.news_links %}
+  <section class="employee-card__section">
+    <h3 class="employee-section__title">В новостях</h3>
+    <ul class="employee-card__list">
+      {% for news in employee_view.news_links %}
+      <li class="employee-card__list-item">
+        {% if news.published_display %}<div class="employee-section__meta"><span class="employee-section__meta-item">{{ news.published_display }}</span></div>{% endif %}
+        {% if news.url %}
+        <a class="admin__link" href="{{ news.url }}">{{ news.title }}</a>
+        {% else %}
+        {{ news.title }}
+        {% endif %}
+        {% if news.summary %}<div class="employee-section__text">{{ news.summary }}</div>{% endif %}
+      </li>
+      {% endfor %}
+    </ul>
+  </section>
+  {% endif %}
+
   <section class="employee-card__section">
     <h3 class="employee-section__title">Разделы профиля</h3>
     {% if employee_view.sections %}

app/templates/login.html

@@ -4,7 +4,7 @@
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <title>Вход · MIEM Employees</title>
-    <link rel="stylesheet" href="/static/admin.css">
+    <link rel="stylesheet" href="/static/admin.css?v={{ frontend_version }}">
   </head>
   <body class="admin">
     <main class="admin__main">

app/templates/run_detail.html

@@ -12,6 +12,7 @@
   <div class="stats-strip">
     <div class="stats-strip__item"><span class="stats-strip__label">Найдено</span><span class="stats-strip__value">{{ run.found_count }}</span></div>
     <div class="stats-strip__item"><span class="stats-strip__label">Обработано</span><span class="stats-strip__value">{{ run.parsed_count }}</span></div>
+    <div class="stats-strip__item"><span class="stats-strip__label">Без изменений</span><span class="stats-strip__value">{{ run.skipped_count }}</span></div>
     <div class="stats-strip__item"><span class="stats-strip__label">Новые</span><span class="stats-strip__value">{{ run.new_count }}</span></div>
     <div class="stats-strip__item"><span class="stats-strip__label">Потеряшки</span><span class="stats-strip__value">{{ run.changes.missing_from_source | length }}</span></div>
     <div class="stats-strip__item"><span class="stats-strip__label">Уволены</span><span class="stats-strip__value">{{ run.dismissed_count }}</span></div>
@@ -22,7 +23,7 @@
   {% endif %}
 </section>
 
-{% for group, title in [("new", "Новые сотрудники"), ("missing_from_source", "Потеряшки"), ("dismissed", "Уволенные")] %}
+{% for group, title in [("new", "Новые сотрудники"), ("missing_from_source", "Потеряшки"), ("verification_required", "Требуют проверки"), ("dismissed", "Уволенные")] %}
 <section class="panel"{% if group == "new" %} id="new-employees"{% endif %}>
   <h2 class="panel__title">{{ title }}</h2>
   {% set items = run.changes[group] %}

app/templates/runs.html

@@ -8,37 +8,41 @@
   </div>
   {% set run = runs[0] if runs else none %}
   {% if run %}
-  {% set processed = run.parsed_count + run.error_count %}
+  {% set processed = run.parsed_count + run.skipped_count + run.error_count %}
   {% set percent = ((processed / run.found_count) * 100) | round(1) if run.found_count else 0 %}
   <div class="progress-panel" data-progress-panel>
     <div class="progress-panel__meta">
       <span data-progress-status>{{ run.status_display }}</span>
       <span>обработано: <span data-progress-processed>{{ processed }}</span> / <span data-progress-found>{{ run.found_count }}</span></span>
+      <span>без изменений: <span data-progress-skipped>{{ run.skipped_count }}</span></span>
       <span>ошибок: <span data-progress-errors>{{ run.error_count }}</span></span>
     </div>
-    <div class="progress-bar" aria-label="Parsing progress">
+    <div class="progress-bar" role="progressbar" aria-label="Прогресс парсинга" aria-valuemin="0" aria-valuemax="100" aria-valuenow="{{ percent }}" aria-valuetext="{{ percent }}%">
       <div class="progress-bar__fill" data-progress-fill style="width: {{ percent }}%"></div>
     </div>
   <div class="progress-panel__percent"><span data-progress-percent>{{ percent }}</span>%</div>
+  <p class="progress-panel__error" data-progress-error role="status" hidden>Не удалось обновить прогресс. <button class="button button--ghost button--compact" type="button" data-progress-retry>Повторить</button></p>
   </div>
   {% else %}
   <div class="progress-panel" data-progress-panel>
     <div class="progress-panel__meta">
       <span data-progress-status>Ожидание</span>
       <span>обработано: <span data-progress-processed>0</span> / <span data-progress-found>0</span></span>
+      <span>без изменений: <span data-progress-skipped>0</span></span>
       <span>ошибок: <span data-progress-errors>0</span></span>
     </div>
-    <div class="progress-bar" aria-label="Parsing progress">
+    <div class="progress-bar" role="progressbar" aria-label="Прогресс парсинга" aria-valuemin="0" aria-valuemax="100" aria-valuenow="0" aria-valuetext="0%">
       <div class="progress-bar__fill" data-progress-fill style="width: 0%"></div>
     </div>
     <div class="progress-panel__percent"><span data-progress-percent>0</span>%</div>
+    <p class="progress-panel__error" data-progress-error role="status" hidden>Не удалось обновить прогресс. <button class="button button--ghost button--compact" type="button" data-progress-retry>Повторить</button></p>
   </div>
   {% endif %}
   <table class="table">
-    <thead><tr><th class="table__head">ID</th><th class="table__head">Статус</th><th class="table__head">Найдено</th><th class="table__head">Обработано</th><th class="table__head">Новые</th><th class="table__head">Ошибки</th><th class="table__head">Уволены</th><th class="table__head">Старт</th></tr></thead>
+    <thead><tr><th class="table__head">ID</th><th class="table__head">Статус</th><th class="table__head">Найдено</th><th class="table__head">Обработано</th><th class="table__head">Без изменений</th><th class="table__head">Новые</th><th class="table__head">Ошибки</th><th class="table__head">Уволены</th><th class="table__head">Старт</th></tr></thead>
     <tbody>
       {% for run in runs %}
-      <tr class="table__row" onclick="window.location.href='/admin/runs/{{ run.id }}'" onkeydown="if (event.key === 'Enter' || event.key === ' ') { event.preventDefault(); window.location.href='/admin/runs/{{ run.id }}'; }" role="link" tabindex="0"><td class="table__cell">{{ run.id }}</td><td class="table__cell">{{ run.status_display }}</td><td class="table__cell">{{ run.found_count }}</td><td class="table__cell">{{ run.parsed_count }}</td><td class="table__cell">{{ run.new_count }}</td><td class="table__cell">{{ run.error_count }}</td><td class="table__cell">{{ run.dismissed_count }}</td><td class="table__cell">{{ run.started_display }}</td></tr>
+      <tr class="table__row"><td class="table__cell"><a class="admin__link" href="/admin/runs/{{ run.id }}">{{ run.id }}</a></td><td class="table__cell">{{ run.status_display }}</td><td class="table__cell">{{ run.found_count }}</td><td class="table__cell">{{ run.parsed_count }}</td><td class="table__cell">{{ run.skipped_count }}</td><td class="table__cell">{{ run.new_count }}</td><td class="table__cell">{{ run.error_count }}</td><td class="table__cell">{{ run.dismissed_count }}</td><td class="table__cell">{{ run.started_display }}</td></tr>
       {% endfor %}
     </tbody>
   </table>

app/version.py

@@ -1,3 +1,3 @@
-APP_VERSION = "0.5.0"
-FRONTEND_VERSION = "0.5.0"
-BACKEND_VERSION = "0.5.0"
+APP_VERSION = "0.10.4"
+FRONTEND_VERSION = "0.10.4"
+BACKEND_VERSION = "0.10.4"

app/worker.py

@@ -17,7 +17,14 @@ def crawl_once() -> None:
     settings = get_settings()
     with SessionLocal() as db:
         run = run_crawl(db, settings)
-        logger.info("crawl finished: id=%s status=%s parsed=%s errors=%s", run.id, run.status, run.parsed_count, run.error_count)
+        logger.info(
+            "crawl finished: id=%s status=%s parsed=%s skipped=%s errors=%s",
+            run.id,
+            run.status,
+            run.parsed_count,
+            run.skipped_count,
+            run.error_count,
+        )
 
 
 def main() -> None:

docker-compose.yml

@@ -1,6 +1,7 @@
 services:
   postgres:
     image: postgres:16-alpine
+    restart: always
     environment:
       POSTGRES_DB: ${POSTGRES_DB:-miem_workers}
       POSTGRES_USER: ${POSTGRES_USER:-miem}
@@ -15,6 +16,7 @@ services:
 
   api:
     build: .
+    restart: always
     command: uvicorn app.main:app --host 0.0.0.0 --port 8000
     env_file: .env
     environment:
@@ -27,6 +29,7 @@ services:
 
   worker:
     build: .
+    restart: always
     command: python -m app.worker
     env_file: .env
     environment:
@@ -35,17 +38,5 @@ services:
       postgres:
         condition: service_healthy
 
-  mcp:
-    build: .
-    command: uvicorn app.main:app --host 0.0.0.0 --port 8000
-    env_file: .env
-    environment:
-      DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-miem}:${POSTGRES_PASSWORD:-miem_password}@postgres:5432/${POSTGRES_DB:-miem_workers}
-    ports:
-      - "127.0.0.1:${MCP_PORT:-8001}:8000"
-    depends_on:
-      postgres:
-        condition: service_healthy
-
 volumes:
   postgres_data:

docs/postgresql-import-verification.md

@@ -0,0 +1,99 @@
+# Проверка импорта на изолированном PostgreSQL
+
+Запускается средствами проекта: pytest, SQLAlchemy, psycopg и отдельный Docker
+Compose. Не использует production Compose, `.env`, SSH или данные сотрудников.
+PostgreSQL 16 и runner работают в отдельной `internal`-сети без опубликованных
+портов. Данные PostgreSQL находятся в tmpfs, без постоянных volumes. Базы:
+`miem_import_test` и `miem_crawl_test`. Все сотрудники, статьи и credentials
+синтетические; тестовые пароли непригодны для реальных систем.
+
+Dockerfile копирует только нужный код и requirements. Его отдельный dockerignore
+исключает `.git`, `.env`, файлы ключей/credentials, CSV владельца, данные
+`app/data` и остальные файлы workspace. Runtime не получает переменные хоста.
+CLI subprocess дополнительно получает только явно заданные тестовые libpq
+параметры; `.pgpass`/service-файлы отключены. Не изменяйте адрес `postgres-test`
+или имена тестовых БД для подключения к существующему серверу.
+
+Синтетическая роль `import_test` создаётся только init-скриптом одноразового
+тестового кластера. Fixture выдаёт ей документированные column grants и USAGE
+sequence. Проверяется отсутствие CREATE схемы, INSERT сотрудников, DELETE
+публикаций и sequence UPDATE. Это не создание временного доступа к рабочей БД.
+Таблицы создаются моделью приложения и очищаются только в тестовых БД.
+
+HTTP заблокирован test fixture; входные профили полного обхода подставляются
+локально. Это проверяет настоящий SQL, транзакции, savepoint и сериализацию JSON,
+но не парсинг сайта ВШЭ. Fixtures SQLite и подмена psycopg для интеграционных
+тестов не используются.
+
+## Что проверяется
+
+1. Dry-run: все строки всех таблиц и состояния sequences совпадают до/после.
+2. Apply: одна вставка, одно обновление, пропуск отсутствующего сотрудника.
+3. Повторный импорт: ноль INSERT/UPDATE, неизменные таблицы/sequences.
+4. CHECK constraint тестовой БД вызывает ошибку третьего действия после UPDATE
+   и успешного INSERT; все строки возвращаются в исходное состояние. Sequence
+   может продвинуться при rollback — это ожидаемое поведение PostgreSQL.
+5. Обновление только года без raw_data сохраняет существующий JSONB.
+6. Поздние строки CSV дополняют метаданные; первое непустое значение сохраняется
+   при конфликте, отчёт содержит названия конфликтующих полей. Повтор идемпотентен.
+7. Два последовательных crawl сохраняют CSV-статью, отсутствующую у ВШЭ;
+   появившаяся статья получает HSE ID без второй записи и потери метаданных.
+8. Полный `run_crawl` обрабатывает двух сотрудников. Два HSE ID на одну CSV-запись
+   откатывают savepoint первого, сохраняя JSONB, авторов, ID, хеш и updated_at;
+   публикация второго сохраняется. Проверяются обе очередности ID и уже
+   установленная привязка; неоднозначность отражается в CrawlError.
+
+Всего 10 PostgreSQL-тестов (восьмой сценарий параметризован). В обычном
+`python -m pytest -q` они пропускаются. Opt-in `MIEM_POSTGRES_INTEGRATION=1`
+разрешён только с точным тестовым DATABASE_URL Compose-проекта.
+
+## Команды владельцу
+
+Выполняйте на локальной машине с Docker Engine/Compose v2 из корня репозитория
+после независимого ревью. Команды явно выбирают локальный Unix socket и очищают
+наследуемое окружение. Для Docker Desktop/Windows используйте Linux/WSL с
+доступным локальным `/var/run/docker.sock`. Для отдельного стенда на VPS используйте
+[отдельную инструкцию](postgresql-import-vps.md) и `compose.vps.yml`: этот локальный
+Compose не предназначен для запуска на сервере с работающими сервисами.
+Первый шаг:
+
+```sh
+env -i PATH="$PATH" docker --host unix:///var/run/docker.sock version
+```
+
+Ожидается exit code 0 и секции `Client` и `Server`. `command not found`,
+`permission denied` или только Client с ошибкой подключения означает, что
+до запуска этого варианта нужен локальный Docker. При его отсутствии сначала
+выполните только read-only диагностику из инструкции VPS.
+
+Дальнейшие точные команды сохранены здесь для воспроизводимого запуска:
+
+```sh
+env -i PATH="$PATH" docker --host unix:///var/run/docker.sock compose --env-file /dev/null --project-name miem-publication-integration -f tests/postgres/compose.yml config --quiet
+env -i PATH="$PATH" docker --host unix:///var/run/docker.sock compose --env-file /dev/null --project-name miem-publication-integration -f tests/postgres/compose.yml build tests
+env -i PATH="$PATH" docker --host unix:///var/run/docker.sock compose --env-file /dev/null --project-name miem-publication-integration -f tests/postgres/compose.yml up --abort-on-container-exit --exit-code-from tests
+```
+
+Первые две команды проверяют Compose и собирают runner с уже используемыми
+зависимостями. Сборка/скачивание образов требуют registry/PyPI, но тестовая
+runtime-сеть не имеет внешнего доступа и запросов к ВШЭ нет. При успехе третьей
+команды ожидаются `10 passed` и exit code 0. Фактическую совместимость образа,
+column grants и PostgreSQL можно считать подтверждённой **только после такого
+выполнения**, не по unit-тестам или статическому чтению файлов.
+
+После сохранения вывода тестов удалите именно этот тестовый проект:
+
+```sh
+env -i PATH="$PATH" docker --host unix:///var/run/docker.sock compose --env-file /dev/null --project-name miem-publication-integration -f tests/postgres/compose.yml down
+```
+
+Это уничтожит только синтетические данные в tmpfs тестового PostgreSQL. Не
+используйте `docker system prune`, production Compose или удаление рабочих
+volumes. Перед повторным прогоном выполните этот же `down`, чтобы следующий
+кластер начинался с чистого init.sql; не запускайте два прогона одновременно.
+
+## Статус подготовки
+
+В среде подготовки Docker отсутствует; настоящие PostgreSQL-тесты не запускались.
+Локально доступны только unit-тесты, проверка Python и статические проверки.
+Production/VPS и доступ к рабочей БД не изменялись.

docs/postgresql-import-vps.md

@@ -0,0 +1,153 @@
+# PostgreSQL-проверка на отдельном стенде VPS
+
+Команды выполняет владелец под root. Подготовка не подключается к VPS и не
+запускает стенд. Не используйте production Compose/`.env`, Docker context,
+volumes, сети, mounts или production credentials. Никаких production-ролей БД,
+SSH-ключей и правил firewall эти команды не создают и не изменяют.
+
+## Сначала только read-only проверка
+
+Следующий блок не создаёт контейнеров, не скачивает образов и не меняет данные.
+Docker CLI использует локальный Unix socket и очищенное окружение; конфигурация
+CLI указывает на отдельный каталог, который здесь не создаётся.
+
+```sh
+free -h
+nproc
+uptime
+df -h / /var/lib/docker
+env -i PATH="$PATH" docker --config /root/miem-publication-integration/docker-config --host unix:///var/run/docker.sock info --format 'CPUs={{.NCPU}} RAM_bytes={{.MemTotal}} cgroup={{.CgroupVersion}} DockerRootDir={{.DockerRootDir}} warnings={{json .Warnings}}'
+env -i PATH="$PATH" docker --config /root/miem-publication-integration/docker-config --host unix:///var/run/docker.sock compose version
+env -i PATH="$PATH" docker --config /root/miem-publication-integration/docker-config --host unix:///var/run/docker.sock stats --no-stream --format 'table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.PIDs}}'
+env -i PATH="$PATH" docker --config /root/miem-publication-integration/docker-config --host unix:///var/run/docker.sock image inspect postgres:16-alpine python:3.12-slim miem-publication-integration-postgres:pr55 miem-publication-integration-runner:pr55 --format '{{.RepoTags}} id={{.Id}} revision={{index .Config.Labels "org.opencontainers.image.revision"}}'
+```
+
+Как читать результат:
+
+- `free`: нужна колонка **available**, а не только free. Предварительный запас —
+  не менее 2 GiB: стенд ограничен суммарно 1 GiB, ещё минимум 1 GiB остаётся
+  системе/сервисам. Это ориентир, решение зависит от их нагрузки и пиков.
+- `nproc` и `uptime`: сравните число CPU и load average. Постоянный load на уровне
+  числа CPU или выше — основание отложить тест, а не останавливать сервисы.
+- `df`: желательно не менее 4 GiB свободно для загрузки образов и запаса работы
+  Docker. Если DockerRootDir отличается от `/var/lib/docker`, отдельно выполните
+  `df -h` для показанного каталога. Не используйте prune для освобождения места.
+- `stats`: показывает потребление уже работающих контейнеров, но не их пиковые
+  требования. Docker warnings о недоступных memory/swap/cgroup limits нужно
+  устранить до запуска; без действующих лимитов этот вариант не запускать.
+- `image inspect`: существующие образы показывают tags, ID и revision.
+  `No such image` означает отсутствие образа, **не** разрешение на pull/build.
+  Для запуска нужны два образа `miem-publication-integration-*:pr55` с revision
+  проверенного commit PR #55. Base-образы нужны только для сборки, не для самого
+  запуска уже подготовленного стенда.
+
+Пришлите только этот диагностический вывод. Не присылайте `docker inspect`
+production-контейнеров целиком, env, конфигурации подключения или секреты.
+До оценки ресурсов и завершения независимого review остальные действия ниже
+не выполнять.
+
+## Изоляция и лимиты подготовленного стенда
+
+Используется только `tests/postgres/compose.vps.yml`, не override production или
+локального Compose. Project name фиксирован:
+`miem-publication-integration-vps`.
+
+| Ресурс | Лимит / область |
+|---|---|
+| PostgreSQL | Отдельный контейнер `postgres-test`, 0.5 CPU, 512 MiB RAM, без дополнительного swap, 128 процессов |
+| PostgreSQL настройки | shared_buffers 64 MiB, max_connections 12 |
+| Test runner | 0.5 CPU, 512 MiB RAM, без дополнительного swap, 128 процессов; UID 65534, read-only rootfs, no-new-privileges, без capabilities |
+| Данные PostgreSQL | Только named volume `miem-publication-integration-vps_pgdata`; local driver tmpfs до 256 MiB |
+| Временные файлы runner | tmpfs `/tmp` до 128 MiB; входят в memory limit контейнера |
+| Сеть | Только `miem-publication-integration-vps_isolated`, internal; опубликованных портов нет |
+| Логи | json-file, до 5 MiB на контейнер, один файл |
+
+Volume с tmpfs выбран намеренно: отдельное имя для адресной очистки и предел
+размера синтетических данных. Данные не сохраняются после остановки/размонтирования.
+Образы содержат только код и синтетический init.sql. Host bind mounts,
+Docker socket, production networks и env_file отсутствуют. Docker socket
+доступен только CLI владельца на хосте, ни одному контейнеру он не передаётся.
+Нет restart policy. Pull запрещён; ресурсы не создаются автоматически при
+отсутствии заранее подготовленного образа.
+
+Лимиты процесса сборки/загрузки образов отличаются от runtime: Compose CPU/RAM
+limits **не ограничивают Docker build**. Поэтому на VPS с рабочими сервисами
+не выполняйте незапланированную сборку. Если готовых образов нет, отдельно
+согласуйте место сборки и окно загрузки после read-only диагностики.
+
+## Подготовка образов после review (не запуск стенда)
+
+На отдельной согласованной машине сборки с Docker, из проверенной копии PR #55,
+укажите полный SHA проверенного commit в переменной `TEST_COMMIT` (не секрет).
+Сначала `git status --short` должен быть пустым, `git rev-parse HEAD` — совпадать
+с TEST_COMMIT. Это не команды для немедленного выполнения на VPS.
+
+```sh
+git status --short
+git rev-parse HEAD
+env -i PATH="$PATH" docker --host unix:///var/run/docker.sock build --label org.opencontainers.image.revision="$TEST_COMMIT" -t miem-publication-integration-runner:pr55 -f tests/postgres/Dockerfile .
+env -i PATH="$PATH" docker --host unix:///var/run/docker.sock build --label org.opencontainers.image.revision="$TEST_COMMIT" -t miem-publication-integration-postgres:pr55 -f tests/postgres/Dockerfile.postgres tests/postgres
+env -i PATH="$PATH" docker --host unix:///var/run/docker.sock image save --output miem-publication-integration-images.tar miem-publication-integration-postgres:pr55 miem-publication-integration-runner:pr55
+sha256sum miem-publication-integration-images.tar tests/postgres/compose.vps.yml
+```
+
+Dockerfile-specific ignore-файлы исключают секреты, реальные CSV и остальные
+файлы workspace из обоих контекстов сборки. Передайте владельцу только архив
+этих двух образов и `compose.vps.yml` с проверенными SHA256, без полного workspace
+и `.env`. Транспорт выбирает владелец; инструкция не получает SSH/production
+credentials. Base-образы могут скачиваться только на согласованной машине сборки.
+
+На VPS после оценки ресурсов/отдельного разрешения владельца: файлы размещаются
+в `/root/miem-publication-integration`, Compose переименован в `compose.vps.yml`,
+архив — в `images.tar`. Проверка checksum должна совпасть с источником.
+
+```sh
+install -d -m 700 /root/miem-publication-integration /root/miem-publication-integration/docker-config
+sha256sum /root/miem-publication-integration/images.tar /root/miem-publication-integration/compose.vps.yml
+env -i PATH="$PATH" docker --config /root/miem-publication-integration/docker-config --host unix:///var/run/docker.sock image load --input /root/miem-publication-integration/images.tar
+env -i PATH="$PATH" docker --config /root/miem-publication-integration/docker-config --host unix:///var/run/docker.sock image inspect miem-publication-integration-postgres:pr55 miem-publication-integration-runner:pr55 --format '{{.RepoTags}} id={{.Id}} revision={{index .Config.Labels "org.opencontainers.image.revision"}}'
+env -i PATH="$PATH" docker --config /root/miem-publication-integration/docker-config --host unix:///var/run/docker.sock compose --env-file /dev/null --project-name miem-publication-integration-vps -f /root/miem-publication-integration/compose.vps.yml config --quiet
+```
+
+Проверьте отсутствие уже существующих ресурсов с этим project name. Если они
+есть, выясните их назначение; не запускайте второй стенд и не удаляйте чужие
+ресурсы. Следующие команды только читают labels Docker Compose:
+
+```sh
+env -i PATH="$PATH" docker --config /root/miem-publication-integration/docker-config --host unix:///var/run/docker.sock ps -a --filter label=com.docker.compose.project=miem-publication-integration-vps
+env -i PATH="$PATH" docker --config /root/miem-publication-integration/docker-config --host unix:///var/run/docker.sock network ls --filter label=com.docker.compose.project=miem-publication-integration-vps
+env -i PATH="$PATH" docker --config /root/miem-publication-integration/docker-config --host unix:///var/run/docker.sock volume ls --filter label=com.docker.compose.project=miem-publication-integration-vps
+```
+
+## Запуск и адресная очистка (пока не выполнять)
+
+Только после согласования владельцем ресурсов и проверенного состава стенда:
+
+```sh
+env -i PATH="$PATH" docker --config /root/miem-publication-integration/docker-config --host unix:///var/run/docker.sock compose --env-file /dev/null --project-name miem-publication-integration-vps -f /root/miem-publication-integration/compose.vps.yml up --no-build --pull never --abort-on-container-exit --exit-code-from tests
+```
+
+Ожидаемый успех — `10 passed`, exit code 0. Сохраните вывод/код выхода и логи:
+
+```sh
+env -i PATH="$PATH" docker --config /root/miem-publication-integration/docker-config --host unix:///var/run/docker.sock compose --env-file /dev/null --project-name miem-publication-integration-vps -f /root/miem-publication-integration/compose.vps.yml logs --no-color > /root/miem-publication-integration/test.log
+```
+
+**Следующая команда удаляет контейнеры, внутреннюю сеть и named volume именно
+этого теста; все его синтетические данные будут потеряны.** Перед ней сохраните
+отчёт. Она не затрагивает ресурсы других Compose-проектов:
+
+```sh
+env -i PATH="$PATH" docker --config /root/miem-publication-integration/docker-config --host unix:///var/run/docker.sock compose --env-file /dev/null --project-name miem-publication-integration-vps -f /root/miem-publication-integration/compose.vps.yml down --volumes
+```
+
+Не используйте `--remove-orphans`, `docker system prune`, общие rm/rmi/volume prune
+или down рабочего проекта. Base-образы и образы теста оставьте: они могут быть
+общими для других проверок. Архив и журнал также не удаляются этой командой.
+Для подтверждения очистки повторите три read-only команды с project label:
+в выводе должны остаться только заголовки.
+
+## Статус
+
+VPS-стенд не запускался, Docker/Compose-проверка не выполнялась в среде
+подготовки из-за отсутствия Docker. PostgreSQL-интеграция ещё не подтверждена.

docs/publication-import.md

@@ -0,0 +1,312 @@
+# Разовый импорт публикаций
+
+Инструмент: `scripts/import_publications.py`. Он не импортирует приложение, не читает
+`.env`, не вызывает `init_db`, не создаёт таблиц и не удаляет записи. Подключение
+берётся только из libpq environment/service, настроенных владельцем. Production в
+рамках подготовки не проверялся и не изменялся.
+
+Локальные проверки: синтетические сотрудники, повторный запуск, дубликаты,
+неоднозначности, конфликты DOI/года, сохранение заполненных значений и
+транзакционный путь на тестовом соединении (read-only / rollback при второй
+ошибочной вставке / успешный commit). Прогон на настоящем PostgreSQL и проверка
+grants не выполнялись; первый owner-run dry-run с read-only ролью обязателен.
+
+## Вход и сопоставление
+
+Файл владельца `Статьи.csv`: UTF-8 с возможным BOM, разделитель `;`, 1797 записей,
+SHA256 `1f236be56d99677b1403df29a4c1d3a7e6d10cb90bd355d678201a027921b0ce`.
+Обязательные колонки: `Название публикации`, `Год`, `DOI`, `Авторы НИУ ВШЭ`.
+Номер строки в отчёте — номер CSV-записи с заголовком, а не физической строки.
+
+Сотрудник сопоставляется по полному ФИО: Unicode NFKC, регистр, пробелы и ё/е.
+Учитываются все существующие статусы; новые сотрудники не создаются. Ноль
+совпадений → `employee_not_found`, несколько → `ambiguous_employee`. Инициалы,
+переводы, перестановки слов и приблизительные совпадения не используются.
+
+Публикация сопоставляется **внутри конкретного сотрудника** по нормализованному DOI
+или полному названию и году. Отсутствующий год БД допускает заполнение при
+совпадении названия. Если эти способы указывают на разные записи, импорт
+пропускается (`ambiguous_publication`). Разные заполненные DOI или годы —
+`doi_conflict` / `year_conflict`. Существующие дубликаты не объединяются и не
+удаляются. Одна статья у разных сотрудников — предусмотренные моделью авторские
+связи, а не дубли внутри карточки.
+
+Повтор CSV внутри сотрудника не создаёт записи. Каждая поздняя строка дополняет
+ещё пустые метаданные; первый непустой источник имеет приоритет, в том числе
+ранее сохранённое значение БД. Повтор без изменений — `csv_duplicate`, дополнение
+запланированной вставки — `merged_csv_duplicate`. При конфликте непустых
+метаданных первое значение сохраняется; остальные пустые поля всё равно
+дополняются. Отчёт содержит `metadata_conflict_first_value_kept`, список
+`conflict_fields` (только названия полей, без значений) и сводный счётчик
+`metadata_conflicts`. Конфликт DOI/года идентификации по-прежнему пропускает
+всю связь. Если строка без DOI была запланирована первой, следующая строка с
+DOI может дополнить ту же вставку. При повторном запуске уже заполненные записи дают
+`already_present`. Хеш новой записи детерминирован по сотруднику и DOI либо
+названию/году. `source_hash` существующих записей не меняется.
+
+Пустые ячейки ничего не стирают. Пустые название/авторы, некорректный год
+(допустимо 1000–9999) или DOI пропускаются как `invalid_fields`. Повреждённая
+структура CSV прерывает весь запуск до подключения. Несовместимый JSON БД
+пропускается (`unsupported_raw_data`, `unsupported_csv_metadata`).
+
+## Таблицы, столбцы и права
+
+| Объект | Операция / столбцы |
+|---|---|
+| `public.employees` | Только SELECT: `id`, `full_name` |
+| `public.employee_publications` | SELECT: `id`, `employee_id`, `title`, `year`, `doi_url`, `raw_data` |
+| Новая публикация | INSERT: `employee_id`, `title`, `year`, `doi_url` при наличии, `raw_data`, `source_hash`, `created_at`, `updated_at` |
+| Существующая публикация | UPDATE: только пустые `title`, `year`, `doi_url`; `raw_data`; служебный `updated_at` |
+| Sequence идентификатора публикации | USAGE для `nextval`; имя определяется `pg_get_serial_sequence`, обычно `public.employee_publications_id_seq` |
+
+Остальные таблицы и sequences не затрагиваются. В `raw_data` сохраняются прежние
+ключи; добавляется/дополняется `csv_enrichment` только отсутствующими или пустыми
+значениями. Уже заполненные CSV-метаданные не заменяются. Поля `authors`,
+`publication_id`, тип материала, язык, аннотация, ссылки HSE и статус не меняются.
+CSV содержит **только авторов НИУ ВШЭ**, поэтому он не считается полным составом
+авторов и не заменяет `authors`. Идентификаторы Scopus/WoS, квартиль и остальные
+непустые исходные ячейки сохраняются в `raw_data.csv_enrichment` без интерпретации.
+Карточки `employees.current_data` и snapshots не изменяются; отчёт и данные
+таблицы не означают обновления всех представлений интерфейса.
+
+Минимальные собственные grants роли: CONNECT к БД, USAGE схемы public,
+перечисленные SELECT/INSERT/UPDATE по столбцам, USAGE найденной sequence.
+Не нужны DELETE, TRUNCATE, CREATE, ALTER, ownership, superuser, доступ к прочим
+таблицам и sequence UPDATE/setval. Роль новая, без членства в других ролях.
+Права PUBLIC всё равно наследуются: владелец должен отдельно проверить их;
+этот набор команд не меняет общие права PUBLIC и не гарантирует изоляцию при
+уже существующих широких общих grants. В частности TEMP может быть выдан PUBLIC.
+
+## Применение и ограничения
+
+Без `--apply` используется PostgreSQL `SERIALIZABLE READ ONLY`, без INSERT,
+UPDATE и nextval, с rollback. С `--apply` весь план применяется одной транзакцией
+SERIALIZABLE. Advisory lock предотвращает одновременное применение этим
+импортером. Таймаут запроса — 60 секунд, ожидания блокировок — 5 секунд.
+Ошибка записи/конфликт сериализации откатывает все изменения таблиц; автоматических
+повторов нет. Sequence может получить пропуски при откате — это нормальное
+поведение PostgreSQL. Изменение входного файла во время запуска прерывает импорт.
+
+JSON-отчёт содержит количества insert/update/skip, причины и пары
+`номер CSV-записи — employee_id`, без ФИО, названий статей и секретов. Счётчики
+относятся к авторским связям, а не уникальным статьям. Консоль выводит только
+сводку. Отчёт сначала пишется в `.pending` до commit, после успеха переименовывается.
+Если остался `.pending`, следующий запуск блокируется: сначала проверьте исход
+транзакции. Отчёт `.pending` не доказывает commit; ошибка переименования после
+commit также возможна. Безопасный повторный dry-run позволяет проверить состояние.
+При необходимости владелец перемещает `.pending` в архив перед повтором.
+
+**До применения нужен свежий backup БД.** Во время импорта владелец должен
+исключить конкурентные записи публикаций на время транзакции, согласовав окно
+работы; эта подготовка не отключает crawler. Не выполняйте production apply до
+независимого ревью и развёртывания исправления crawler из этого PR.
+
+До исправления `_upsert_employee` запускал синхронизацию публикаций даже для
+неизменённой карточки. При пустом списке публикаций или отсутствии таблицы
+синхронизация пропускалась. При непустом списке `_sync_employee_publications`
+удалял все записи сотрудника, чей `source_hash` отсутствовал среди хешей ответа
+ВШЭ, включая CSV-вставки. Найденные по `publication_id`/хешу записи полностью
+получали новый `raw_data`, поэтому теряли CSV-метаданные даже без удаления.
+
+Исправление сохраняет записи с dict `raw_data.csv_enrichment` при очистке
+устаревших публикаций. При обновлении сохраняется этот ключ и заполненные
+title/year/doi_url, если ВШЭ вернул пустые значения. Непустые значения ВШЭ
+продолжают обновлять канонические поля, а исходные CSV-значения остаются в
+метаданных. Если статья впервые появилась у ВШЭ, crawler сопоставляет её с
+импортированной записью по DOI или названию/году до новой вставки, сохраняет id
+и добавляет HSE publication_id. Конфликт/неоднозначность откатывает весь savepoint
+синхронизации публикаций и регистрирует CrawlError; карточка сотрудника продолжает
+обрабатываться. Удаление обычных устаревших HSE-записей остаётся прежним. Здесь
+не приведены команды рестарта или отключения сервисов.
+
+Уже установленный HSE publication_id CSV-записи нельзя заменить другим HSE ID,
+даже когда DOI/название/год совпадают. Это проверяется перед изменением полей
+при любом пути сопоставления (ID, source_hash, CSV-поля). Поэтому две HSE-статьи
+не могут по очереди перезаписать одну CSV-запись: весь savepoint откатывается
+независимо от порядка статей. Отсутствующий ID во входе не очищает существующую
+CSV-привязку. Отчёт обхода (`errors` в деталях запуска) содержит CrawlError с
+идентификатором записи БД и конфликтующими HSE ID, без ФИО и названий статей.
+
+## Команды владельцу (не выполнялись)
+
+Ниже — подготовка разового доступа после ревью. Выполняет владелец на сервере.
+Рабочая директория — проверенная копия этого PR; файлы программы и CSV должны
+быть защищены от записи временным SSH-пользователем. Пароли/ключи не включать
+в репозиторий или чат. Имена контейнеров соответствуют текущему Compose-проекту;
+владелец проверяет их локально перед выполнением.
+
+Сначала резервная копия в приватный каталог владельца:
+
+```sh
+install -d -m 700 /var/backups/miem-publication-import
+docker exec miem_workers-postgres-1 pg_dump -U miem -d miem_workers -Fc -f /tmp/publications-before.dump
+docker cp miem_workers-postgres-1:/tmp/publications-before.dump /var/backups/miem-publication-import/before.dump
+chmod 600 /var/backups/miem-publication-import/before.dump
+```
+
+При повторной операции используйте новый путь backup; не перезаписывайте нужную
+предыдущую копию. Проверьте exit codes и архив `pg_restore --list` до выдачи write.
+
+Откройте psql владельцем БД:
+
+```sh
+docker exec -it miem_workers-postgres-1 psql -U miem -d miem_workers
+```
+
+Выполните SQL (без пароля в SQL/истории):
+
+```sql
+\set ON_ERROR_STOP on
+SELECT format('CREATE ROLE miem_publication_import LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS CONNECTION LIMIT 1 VALID UNTIL %L', CURRENT_TIMESTAMP + interval '24 hours') \gexec
+\password miem_publication_import
+GRANT CONNECT ON DATABASE miem_workers TO miem_publication_import;
+GRANT USAGE ON SCHEMA public TO miem_publication_import;
+GRANT SELECT (id, full_name) ON public.employees TO miem_publication_import;
+GRANT SELECT (id, employee_id, title, year, doi_url, raw_data) ON public.employee_publications TO miem_publication_import;
+-- Сначала выдаются только read-права для dry-run.
+SELECT pg_get_serial_sequence('public.employee_publications', 'id');
+SELECT has_schema_privilege('miem_publication_import', 'public', 'CREATE') AS unexpected_create,
+       has_table_privilege('miem_publication_import', 'public.employees', 'INSERT') AS unexpected_employee_insert,
+       has_table_privilege('miem_publication_import', 'public.employee_publications', 'DELETE') AS unexpected_delete;
+```
+
+Три проверки должны вернуть false. Не выдавайте write-права при неожиданном
+доступе. Если sequence не найдена, выясните фактический механизм id перед импортом.
+Срок роли ограничивает парольную авторизацию, но не заменяет отзыв доступа и
+завершение открытых сессий.
+
+После принятого dry-run, непосредственно перед разрешённым apply:
+
+```sql
+GRANT INSERT (employee_id, title, year, doi_url, raw_data, source_hash, created_at, updated_at) ON public.employee_publications TO miem_publication_import;
+GRANT UPDATE (title, year, doi_url, raw_data, updated_at) ON public.employee_publications TO miem_publication_import;
+SELECT format('GRANT USAGE ON SEQUENCE %s TO miem_publication_import', pg_get_serial_sequence('public.employee_publications', 'id')) \gexec
+```
+
+Подготовьте внутри API-контейнера защищённые файлы libpq `pg_service.conf` и
+`pgpass` из окружения/секрет-хранилища владельца: service
+`miem_publication_import`, БД `miem_workers`, пользователь
+`miem_publication_import`. Используйте внутренний PostgreSQL endpoint Compose,
+порт 5432; не публикуйте порт БД. Поле password хранится только в `pgpass`,
+доступном UID 65534 с mode 600. Не копируйте `.env` приложения. Этот шаг
+зависит от секрет-хранилища владельца; универсальной команды получения секрета нет.
+
+Установите файлы (команды root; каталог контейнера не должен быть volume с
+непроверенными файлами). `Статьи.csv` расположен рядом с каталогом репозитория:
+
+```sh
+docker exec miem_workers-api-1 mkdir -p /opt/miem-import /tmp/miem-import
+docker cp scripts/import_publications.py miem_workers-api-1:/opt/miem-import/import_publications.py
+docker cp ../Статьи.csv miem_workers-api-1:/opt/miem-import/articles.csv
+docker exec miem_workers-api-1 chown -R root:root /opt/miem-import
+docker exec miem_workers-api-1 chmod 755 /opt/miem-import
+docker exec miem_workers-api-1 chmod 644 /opt/miem-import/import_publications.py /opt/miem-import/articles.csv
+docker exec miem_workers-api-1 chown 65534:65534 /opt/miem-import/pg_service.conf /opt/miem-import/pgpass /tmp/miem-import
+docker exec miem_workers-api-1 chmod 600 /opt/miem-import/pg_service.conf /opt/miem-import/pgpass
+docker exec miem_workers-api-1 chmod 700 /tmp/miem-import
+docker exec miem_workers-api-1 sha256sum /opt/miem-import/articles.csv
+install -m 755 -o root -g root scripts/import-publications-run /usr/local/sbin/miem-publication-import
+install -m 755 -o root -g root scripts/import-publications-ssh /usr/local/sbin/miem-publication-import-ssh
+useradd --create-home --shell /bin/sh miem-pub-import
+install -d -m 700 -o miem-pub-import -g miem-pub-import /home/miem-pub-import/.ssh
+```
+
+Проверьте SHA256 CSV по указанному выше значению и script по проверенной копии PR.
+Секретные файлы должны существовать перед командами chown/chmod. Root-владелец
+каталога не позволяет UID 65534 заменить программу/CSV; этот UID может писать
+только отчёт и свои файлы конфигурации. Не предоставляйте SSH-пользователю группу
+docker или общий sudo. Каталоги и executables выше должны быть root-owned и
+недоступны для записи этому пользователю. Убедитесь, что абсолютный путь Docker
+на сервере — `/usr/bin/docker`.
+
+Создайте `/etc/sudoers.d/miem-publication-import` с mode 440:
+
+```sudoers
+miem-pub-import ALL=(root) NOPASSWD: /usr/local/sbin/miem-publication-import dry-run, /usr/local/sbin/miem-publication-import apply
+```
+
+```sh
+visudo -cf /etc/sudoers.d/miem-publication-import
+```
+
+В `/home/miem-pub-import/.ssh/authorized_keys` добавьте **только публичный** ключ
+владельца с ограничениями (замените `<PUBLIC_KEY>` самим публичным ключом):
+
+```text
+restrict,command="/usr/local/sbin/miem-publication-import-ssh" <PUBLIC_KEY>
+```
+
+```sh
+chown root:root /home/miem-pub-import/.ssh/authorized_keys
+chmod 644 /home/miem-pub-import/.ssh/authorized_keys
+chown root:root /home/miem-pub-import /home/miem-pub-import/.ssh
+chmod 755 /home/miem-pub-import
+chmod 755 /home/miem-pub-import/.ssh
+```
+
+Root-владение home/.ssh предотвращает замену ограниченного authorized_keys.
+Убедитесь, что SSH не разрешает этому пользователю другие ключи, пароль,
+keyboard-interactive, пользовательские environment/rc или общий shell.
+При необходимости задайте в sshd `Match User miem-pub-import` с
+`AuthenticationMethods publickey`, `PasswordAuthentication no`,
+`KbdInteractiveAuthentication no`, `DisableForwarding yes`, `PermitTTY no`,
+`PermitUserRC no` и указанным ForceCommand. Проверка/reload SSH зависит от
+конфигурации сервера; перед reload сохраните действующую сессию владельца.
+
+С клиентской машины параметры хоста/ключа берутся из окружения владельца:
+
+```sh
+ssh -i "$IMPORT_SSH_KEY" -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=yes "miem-pub-import@$PROD_HOST" 'import-publications dry-run'
+```
+
+После проверки сводки, выдачи write-grants и отдельного решения владельца:
+
+```sh
+ssh -i "$IMPORT_SSH_KEY" -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=yes "miem-pub-import@$PROD_HOST" 'import-publications apply'
+ssh -i "$IMPORT_SSH_KEY" -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=yes "miem-pub-import@$PROD_HOST" 'import-publications dry-run'
+```
+
+Повторный dry-run должен показать ноль insert/update. При concurrent changes
+сводки могут различаться; без production-проверки фактические количества неизвестны.
+Владелец забирает отчёты без предоставления SCP/general shell временному ключу:
+
+```sh
+docker cp miem_workers-api-1:/tmp/miem-import/apply.json ./publication-import-apply.json
+docker cp miem_workers-api-1:/tmp/miem-import/dry-run.json ./publication-import-dry-run.json
+```
+
+## Отзыв доступа
+
+Владелец прекращает новые SSH-входы: перемещает временный authorized_keys в свой
+закрытый архив и удаляет только этот sudoers-файл. Не удаляйте данные БД или volumes.
+
+```sh
+mv /home/miem-pub-import/.ssh/authorized_keys /var/backups/miem-publication-import/authorized_keys.disabled
+rm /etc/sudoers.d/miem-publication-import
+```
+
+В psql владельцем:
+
+```sql
+ALTER ROLE miem_publication_import NOLOGIN;
+SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE usename = 'miem_publication_import' AND pid <> pg_backend_pid();
+REVOKE ALL PRIVILEGES ON DATABASE miem_workers FROM miem_publication_import;
+REVOKE ALL PRIVILEGES ON SCHEMA public FROM miem_publication_import;
+REVOKE SELECT (id, full_name) ON public.employees FROM miem_publication_import;
+REVOKE SELECT (id, employee_id, title, year, doi_url, raw_data),
+       INSERT (employee_id, title, year, doi_url, raw_data, source_hash, created_at, updated_at),
+       UPDATE (title, year, doi_url, raw_data, updated_at)
+       ON public.employee_publications FROM miem_publication_import;
+SELECT format('REVOKE ALL PRIVILEGES ON SEQUENCE %s FROM miem_publication_import', pg_get_serial_sequence('public.employee_publications', 'id')) \gexec
+DROP ROLE miem_publication_import;
+```
+
+После сохранения отчётов и backup удалите только временные секреты подключения:
+
+```sh
+docker exec miem_workers-api-1 rm /opt/miem-import/pgpass /opt/miem-import/pg_service.conf
+```
+
+Эти действия не используют `DROP OWNED` и не удаляют публикации. SQL роли не
+является миграцией схемы приложения. Если DROP ROLE не проходит, проверьте
+оставшиеся grants; не применяйте CASCADE/удаление данных.

infra/install-postgres-backup-cron.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+set -eu
+
+temporary=$(mktemp)
+app_dir=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)
+trap 'rm -f -- "$temporary"' EXIT HUP INT TERM
+
+crontab -l 2>/dev/null | grep -v 'postgres-backup.sh' > "$temporary" || true
+printf '%s\n' "30 3 * * 0 flock -w 3600 /tmp/miem-postgres-backup.lock $app_dir/infra/postgres-backup.sh || logger -t miem-postgres-backup -- 'backup skipped or failed'" >> "$temporary"
+crontab "$temporary"

infra/postgres-backup.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+set -eu
+umask 077
+
+app_dir=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)
+backup_dir="$HOME/backups/postgres"
+backup="$backup_dir/postgres-latest.sql"
+temporary="$backup.tmp.$$"
+
+log() { logger -t miem-postgres-backup -- "$*" 2>/dev/null || true; }
+
+mkdir -p "$backup_dir"
+trap 'status=$?; rm -f -- "$temporary"; [ "$status" -eq 0 ] || log "backup failed"; exit "$status"' EXIT HUP INT TERM
+cd "$app_dir"
+
+docker compose exec -T postgres sh -c 'pg_dumpall -U "$POSTGRES_USER"' > "$temporary"
+test -s "$temporary"
+mv "$temporary" "$backup"
+find "$HOME/backups/miem_workers" -maxdepth 1 -type f -name 'miem_workers-*.sql' -delete 2>/dev/null || true
+log "created $backup"

migrations/001_init.sql

@@ -13,6 +13,7 @@ CREATE TABLE IF NOT EXISTS crawl_runs (
   finished_at TIMESTAMPTZ,
   found_count INTEGER NOT NULL DEFAULT 0,
   parsed_count INTEGER NOT NULL DEFAULT 0,
+  skipped_count INTEGER NOT NULL DEFAULT 0,
   new_count INTEGER NOT NULL DEFAULT 0,
   error_count INTEGER NOT NULL DEFAULT 0,
   dismissed_count INTEGER NOT NULL DEFAULT 0,
@@ -32,6 +33,7 @@ CREATE TABLE IF NOT EXISTS employees (
   dismissed_at TIMESTAMPTZ,
   parser_version VARCHAR(32),
   current_data JSONB,
+  has_academic_degree BOOLEAN NOT NULL DEFAULT FALSE,
   current_checksum VARCHAR(64),
   created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
   updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
@@ -39,6 +41,7 @@ CREATE TABLE IF NOT EXISTS employees (
 
 CREATE INDEX IF NOT EXISTS ix_employees_full_name ON employees (full_name);
 CREATE INDEX IF NOT EXISTS ix_employees_status ON employees (status);
+CREATE INDEX IF NOT EXISTS ix_employees_has_academic_degree ON employees (has_academic_degree);
 
 CREATE TABLE IF NOT EXISTS employee_snapshots (
   id SERIAL PRIMARY KEY,
@@ -73,3 +76,22 @@ CREATE TABLE IF NOT EXISTS profile_tabs (
 );
 
 CREATE INDEX IF NOT EXISTS ix_profile_tabs_employee_id ON profile_tabs (employee_id);
+
+CREATE TABLE IF NOT EXISTS parse_resource_cache (
+  id SERIAL PRIMARY KEY,
+  profile_key VARCHAR(255) NOT NULL,
+  resource_key VARCHAR(255) NOT NULL,
+  method VARCHAR(16) NOT NULL,
+  url TEXT NOT NULL,
+  request_fingerprint VARCHAR(64) NOT NULL,
+  etag TEXT,
+  last_modified TEXT,
+  body_hash VARCHAR(64) NOT NULL,
+  body_snapshot BYTEA NOT NULL,
+  parser_version VARCHAR(32),
+  fetched_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  CONSTRAINT uq_parse_resource_cache_resource UNIQUE (profile_key, resource_key, request_fingerprint)
+);
+
+CREATE INDEX IF NOT EXISTS ix_parse_resource_cache_profile_key
+  ON parse_resource_cache (profile_key);

migrations/005_parse_resource_cache.sql

@@ -0,0 +1,21 @@
+ALTER TABLE crawl_runs
+ADD COLUMN IF NOT EXISTS skipped_count INTEGER NOT NULL DEFAULT 0;
+
+CREATE TABLE IF NOT EXISTS parse_resource_cache (
+  id SERIAL PRIMARY KEY,
+  profile_key VARCHAR(255) NOT NULL,
+  resource_key VARCHAR(255) NOT NULL,
+  method VARCHAR(16) NOT NULL,
+  url TEXT NOT NULL,
+  request_fingerprint VARCHAR(64) NOT NULL,
+  etag TEXT,
+  last_modified TEXT,
+  body_hash VARCHAR(64) NOT NULL,
+  body_snapshot BYTEA NOT NULL,
+  parser_version VARCHAR(32),
+  fetched_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  CONSTRAINT uq_parse_resource_cache_resource UNIQUE (profile_key, resource_key, request_fingerprint)
+);
+
+CREATE INDEX IF NOT EXISTS ix_parse_resource_cache_profile_key
+  ON parse_resource_cache (profile_key);

migrations/006_employee_publications.sql

@@ -0,0 +1,39 @@
+CREATE TABLE IF NOT EXISTS employee_publications (
+  id SERIAL PRIMARY KEY,
+  employee_id INTEGER NOT NULL REFERENCES employees(id) ON DELETE CASCADE,
+  publication_id VARCHAR(64),
+  title TEXT NOT NULL,
+  year INTEGER,
+  publication_type VARCHAR(64),
+  language VARCHAR(16),
+  status INTEGER,
+  url TEXT,
+  doi_url TEXT,
+  other_url TEXT,
+  document_url TEXT,
+  citation_text TEXT,
+  annotation JSONB,
+  description JSONB,
+  authors JSONB,
+  raw_data JSONB,
+  source_hash VARCHAR(64) NOT NULL,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  CONSTRAINT uq_employee_publications_employee_publication UNIQUE (employee_id, publication_id),
+  CONSTRAINT uq_employee_publications_employee_source_hash UNIQUE (employee_id, source_hash)
+);
+
+CREATE INDEX IF NOT EXISTS ix_employee_publications_employee_id
+  ON employee_publications (employee_id);
+
+CREATE INDEX IF NOT EXISTS ix_employee_publications_publication_id
+  ON employee_publications (publication_id);
+
+CREATE INDEX IF NOT EXISTS ix_employee_publications_doi_url
+  ON employee_publications (doi_url);
+
+CREATE INDEX IF NOT EXISTS ix_employee_publications_year
+  ON employee_publications (year);
+
+CREATE INDEX IF NOT EXISTS ix_employee_publications_publication_type
+  ON employee_publications (publication_type);

migrations/007_employee_news_links.sql

@@ -0,0 +1,27 @@
+CREATE TABLE IF NOT EXISTS employee_news_links (
+  id SERIAL PRIMARY KEY,
+  employee_id INTEGER NOT NULL REFERENCES employees(id) ON DELETE CASCADE,
+  title TEXT NOT NULL,
+  url TEXT,
+  summary TEXT,
+  published_at TIMESTAMPTZ,
+  published_year INTEGER,
+  source_hash VARCHAR(64) NOT NULL,
+  raw_data JSONB,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  CONSTRAINT uq_employee_news_links_employee_url UNIQUE (employee_id, url),
+  CONSTRAINT uq_employee_news_links_employee_source_hash UNIQUE (employee_id, source_hash)
+);
+
+CREATE INDEX IF NOT EXISTS ix_employee_news_links_employee_id
+  ON employee_news_links (employee_id);
+
+CREATE INDEX IF NOT EXISTS ix_employee_news_links_url
+  ON employee_news_links (url);
+
+CREATE INDEX IF NOT EXISTS ix_employee_news_links_published_at
+  ON employee_news_links (published_at);
+
+CREATE INDEX IF NOT EXISTS ix_employee_news_links_published_year
+  ON employee_news_links (published_year);

migrations/008_profile_verification.sql

@@ -0,0 +1,17 @@
+ALTER TABLE employees
+ADD COLUMN IF NOT EXISTS profile_unavailable_streak INTEGER NOT NULL DEFAULT 0;
+
+ALTER TABLE employees
+ADD COLUMN IF NOT EXISTS last_profile_check_at TIMESTAMPTZ;
+
+CREATE TABLE IF NOT EXISTS employee_profile_urls (
+  id SERIAL PRIMARY KEY,
+  employee_id INTEGER NOT NULL REFERENCES employees(id) ON DELETE CASCADE,
+  url TEXT NOT NULL,
+  first_seen_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  last_seen_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  CONSTRAINT uq_employee_profile_urls_employee_url UNIQUE (employee_id, url)
+);
+
+CREATE INDEX IF NOT EXISTS ix_employee_profile_urls_employee_id
+  ON employee_profile_urls (employee_id);

migrations/009_academic_degree_filter.sql

@@ -0,0 +1,11 @@
+ALTER TABLE employees
+ADD COLUMN IF NOT EXISTS has_academic_degree BOOLEAN NOT NULL DEFAULT FALSE;
+
+UPDATE employees
+SET has_academic_degree = COALESCE(
+  current_data::text ~* '(кандидат|доктор).{0,80}наук|ph\.?[[:space:]]*d\.?',
+  FALSE
+);
+
+CREATE INDEX IF NOT EXISTS ix_employees_has_academic_degree
+  ON employees (has_academic_degree);

pyproject.toml

@@ -1,28 +1,28 @@
-[project]
-name = "miem-workers"
-version = "0.5.0"
-description = "MIEM employees parser, admin API, and MCP server"
-requires-python = ">=3.11"
-dependencies = [
-  "apscheduler>=3.10.4",
-  "beautifulsoup4>=4.12.3",
-  "fastapi>=0.115.0",
-  "httpx>=0.27.0",
-  "jinja2>=3.1.4",
-  "lxml>=5.2.0",
-  "psycopg[binary]>=3.2.0",
-  "pydantic-settings>=2.4.0",
-  "python-multipart>=0.0.9",
-  "requests>=2.32.0",
-  "sqlalchemy>=2.0.32",
-  "uvicorn[standard]>=0.30.0",
-]
-
-[project.optional-dependencies]
-dev = [
-  "pytest>=8.3.0",
-]
-
-[tool.pytest.ini_options]
-testpaths = ["tests"]
-pythonpath = ["."]
+[project]
+name = "miem-workers"
+version = "0.10.4"
+description = "MIEM employees parser, admin API, and web admin"
+requires-python = ">=3.11"
+dependencies = [
+  "apscheduler>=3.10.4",
+  "beautifulsoup4>=4.12.3",
+  "fastapi>=0.115.0",
+  "httpx>=0.27.0",
+  "jinja2>=3.1.4",
+  "lxml>=5.2.0",
+  "psycopg[binary]>=3.2.0",
+  "pydantic-settings>=2.4.0",
+  "python-multipart>=0.0.9",
+  "requests>=2.32.0",
+  "sqlalchemy>=2.0.32",
+  "uvicorn[standard]>=0.30.0",
+]
+
+[project.optional-dependencies]
+dev = [
+  "pytest>=8.3.0",
+]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["."]

scripts/import-publications-run

@@ -0,0 +1,25 @@
+#!/bin/sh
+# Install root-owned; caller cannot choose container, executable, input or env.
+set -eu
+case "${1:-}" in
+    dry-run) mode=dry-run ;;
+    apply) mode=apply ;;
+    *) exit 1 ;;
+esac
+[ "$#" -eq 1 ] || exit 1
+if [ "$mode" = apply ]; then
+    exec /usr/bin/docker exec --user 65534:65534 \
+        -e PGSERVICEFILE=/opt/miem-import/pg_service.conf \
+        -e PGSERVICE=miem_publication_import \
+        -e PGPASSFILE=/opt/miem-import/pgpass \
+        -e PYTHONNOUSERSITE=1 \
+        miem_workers-api-1 python -I /opt/miem-import/import_publications.py \
+        /opt/miem-import/articles.csv --apply --report /tmp/miem-import/apply.json
+fi
+exec /usr/bin/docker exec --user 65534:65534 \
+    -e PGSERVICEFILE=/opt/miem-import/pg_service.conf \
+    -e PGSERVICE=miem_publication_import \
+    -e PGPASSFILE=/opt/miem-import/pgpass \
+    -e PYTHONNOUSERSITE=1 \
+    miem_workers-api-1 python -I /opt/miem-import/import_publications.py \
+    /opt/miem-import/articles.csv --report /tmp/miem-import/dry-run.json

scripts/import-publications-ssh

@@ -0,0 +1,8 @@
+#!/bin/sh
+# Forced SSH command: no arguments, paths, forwarding or shell from the client.
+set -eu
+case "${SSH_ORIGINAL_COMMAND:-}" in
+    'import-publications dry-run') exec sudo -n /usr/local/sbin/miem-publication-import dry-run ;;
+    'import-publications apply') exec sudo -n /usr/local/sbin/miem-publication-import apply ;;
+    *) printf '%s\n' 'Allowed: import-publications dry-run | import-publications apply' >&2; exit 1 ;;
+esac

scripts/import_publications.py

@@ -0,0 +1,192 @@
+"""Owner-run CSV enrichment; no app startup, .env loading, DDL or deletes."""
+
+import argparse
+import csv
+import hashlib
+import json
+import re
+import unicodedata
+from collections import Counter, defaultdict
+from pathlib import Path
+
+
+def normalized(value):
+    return " ".join(unicodedata.normalize("NFKC", value or "").casefold().replace("ё", "е").split())
+
+
+def doi(value):
+    return re.sub(r"^(?:https?://(?:dx\.)?doi\.org/|doi:\s*)", "", (value or "").strip(), flags=re.I).lower()
+
+
+def read_rows(path):
+    with path.open(encoding="utf-8-sig", newline="") as source:
+        reader = csv.DictReader(source, delimiter=";")
+        required = {"Название публикации", "Год", "DOI", "Авторы НИУ ВШЭ"}
+        if not required <= set(reader.fieldnames or []):
+            raise ValueError("Missing required CSV headers")
+        rows = []
+        for number, row in enumerate(reader, 2):
+            if None in row or any(v is None for v in row.values()):
+                raise ValueError(f"Malformed CSV record {number}")
+            rows.append((number, {k.strip(): v.strip() for k, v in row.items()}))
+        return rows
+
+
+def plan(rows, employees, publications):
+    names = defaultdict(list)
+    by_employee = defaultdict(list)
+    for employee in employees:
+        if normalized(employee["full_name"]):
+            names[normalized(employee["full_name"])].append(employee["id"])
+    for publication in publications:
+        by_employee[publication["employee_id"]].append(dict(publication))
+    actions, audit, seen = [], [], set()
+    for number, row in rows:
+        title, identifier = row["Название публикации"], doi(row["DOI"])
+        author_names = sorted(set(normalized(a) for a in row["Авторы НИУ ВШЭ"].split(";")) - {""})
+        year = int(row["Год"]) if row["Год"].isdigit() else None
+        if not title or not year or not 1000 <= year <= 9999 or not author_names or (identifier and not re.fullmatch(r"10\.\d{4,9}/\S+", identifier)):
+            audit.append({"row": number, "employee_id": None, "result": "skip", "reason": "invalid_fields"})
+            continue
+        for author in author_names:
+            ids = names.get(author, [])
+            entry = {"row": number, "employee_id": ids[0] if len(ids) == 1 else None}
+            def skip(reason):
+                audit.append({**entry, "result": "skip", "reason": reason})
+            if len(ids) != 1:
+                skip("employee_not_found" if not ids else "ambiguous_employee")
+                continue
+            employee_id = ids[0]
+            candidates = by_employee[employee_id]
+            exact = [p for p in candidates if identifier and doi(p.get("doi_url")) == identifier]
+            title_matches = [p for p in candidates if normalized(p["title"]) == normalized(title) and p.get("year") in (None, year)]
+            matches = {p["id"]: p for p in exact + title_matches}
+            if len(matches) > 1:
+                skip("ambiguous_publication")
+                continue
+            existing = next(iter(matches.values()), None)
+            if existing and identifier and doi(existing.get("doi_url")) and doi(existing["doi_url"]) != identifier:
+                skip("doi_conflict")
+                continue
+            if existing and existing.get("year") and existing["year"] != year:
+                skip("year_conflict")
+                continue
+            key = (employee_id, identifier or (normalized(title), year))
+            duplicate = key in seen
+            seen.add(key)
+            metadata = {k: v for k, v in row.items() if v}
+            raw = existing.get("raw_data") if existing else None
+            if raw is not None and not isinstance(raw, dict):
+                skip("unsupported_raw_data")
+                continue
+            raw = dict(raw or {})
+            old_metadata = raw.get("csv_enrichment", {})
+            if not isinstance(old_metadata, dict):
+                skip("unsupported_csv_metadata")
+                continue
+            conflicts = sorted(k for k, v in metadata.items() if old_metadata.get(k) and old_metadata[k] != v)
+            if conflicts:
+                entry["conflict_fields"] = conflicts
+            merged = dict(old_metadata)
+            for k, v in metadata.items():
+                if not merged.get(k):
+                    merged[k] = v
+            raw["csv_enrichment"] = merged
+            values = {"raw_data": raw}
+            if identifier and (not existing or not existing.get("doi_url")):
+                values["doi_url"] = "https://doi.org/" + identifier
+            if not existing or not existing.get("year"):
+                values["year"] = year
+            if not existing or not existing.get("title"):
+                values["title"] = title
+            if existing:
+                values = {k: v for k, v in values.items() if existing.get(k) != v}
+                if not values:
+                    skip("metadata_conflict_first_value_kept" if conflicts else "csv_duplicate" if duplicate else "already_present")
+                    continue
+                if isinstance(existing["id"], str):
+                    actions[int(existing["id"].split("-")[1])]["values"].update(values)
+                    existing.update(values)
+                    skip("metadata_conflict_first_value_kept" if conflicts else "merged_csv_duplicate")
+                    continue
+                action = {"result": "update", "id": existing["id"], "values": values}
+                existing.update(values)
+            else:
+                source_hash = hashlib.sha256(json.dumps(key, ensure_ascii=False, sort_keys=True).encode()).hexdigest()
+                values.update(employee_id=employee_id, source_hash=source_hash)
+                action = {"result": "insert", "values": values}
+                by_employee[employee_id].append({**values, "id": f"planned-{len(actions)}"})
+            actions.append(action)
+            audit.append({**entry, "result": action["result"], "reason": "metadata_conflict_first_value_kept" if conflicts else ""})
+    return actions, audit
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("file", type=Path)
+    parser.add_argument("--apply", action="store_true", help="Commit; default is read-only dry-run")
+    parser.add_argument("--report", type=Path, required=True)
+    args = parser.parse_args()
+    pending_report = args.report.with_suffix(args.report.suffix + ".pending")
+    if args.file.resolve() in (args.report.resolve(), pending_report.resolve()):
+        raise ValueError("Report must not overwrite input")
+    source_digest = hashlib.sha256(args.file.read_bytes()).hexdigest()
+    rows = read_rows(args.file)
+    # libpq environment/service only; never load application settings or .env.
+    import psycopg
+    from psycopg.rows import dict_row
+    from psycopg.types.json import Jsonb
+    from psycopg import sql
+
+    # Check report access before any possible commit; pending file is not a success report.
+    with pending_report.open("x", encoding="utf-8"):
+        pass
+    with psycopg.connect("", row_factory=dict_row) as connection:
+        connection.execute("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE" + ("" if args.apply else " READ ONLY"))
+        connection.execute("SET LOCAL search_path = pg_catalog, public")
+        connection.execute("SET LOCAL statement_timeout = '60s'")
+        connection.execute("SET LOCAL lock_timeout = '5s'")
+        if args.apply:
+            locked = connection.execute("SELECT pg_try_advisory_xact_lock(716203841)").fetchone()
+            if not next(iter(locked.values())):
+                raise RuntimeError("Another import is running")
+        employees = connection.execute("SELECT id, full_name FROM public.employees").fetchall()
+        publications = connection.execute("SELECT id, employee_id, title, year, doi_url, raw_data FROM public.employee_publications").fetchall()
+        actions, audit = plan(rows, employees, publications)
+        if args.apply:
+            for action in actions:
+                values = dict(action["values"])
+                if "raw_data" in values:
+                    values["raw_data"] = Jsonb(values["raw_data"])
+                columns = list(values)
+                if action["result"] == "insert":
+                    query = sql.SQL("INSERT INTO public.employee_publications ({}, created_at, updated_at) VALUES ({}, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)").format(
+                        sql.SQL(", ").join(map(sql.Identifier, columns)), sql.SQL(", ").join(sql.Placeholder() for _ in columns))
+                    connection.execute(query, list(values.values()))
+                else:
+                    query = sql.SQL("UPDATE public.employee_publications SET {}, updated_at = CURRENT_TIMESTAMP WHERE id = %s").format(
+                        sql.SQL(", ").join(sql.SQL("{} = %s").format(sql.Identifier(k)) for k in columns))
+                    result = connection.execute(query, [*values.values(), action["id"]])
+                    if result.rowcount != 1:
+                        raise RuntimeError("Publication changed concurrently")
+        if hashlib.sha256(args.file.read_bytes()).hexdigest() != source_digest:
+            raise RuntimeError("Input changed during import")
+        summary = {"mode": "apply" if args.apply else "dry-run", "csv_rows": len(rows),
+                   "sha256": source_digest,
+                   "counts": {key: sum(item["result"] == key for item in audit) for key in ("insert", "update", "skip")},
+                   "metadata_conflicts": sum(bool(item.get("conflict_fields")) for item in audit),
+                   "skip_reasons": dict(Counter(item["reason"] for item in audit if item["result"] == "skip"))}
+        pending_report.write_text(json.dumps({"summary": summary, "records": audit}, ensure_ascii=False, indent=2), encoding="utf-8")
+        if not args.apply:
+            connection.rollback()
+    pending_report.replace(args.report)
+    print(json.dumps(summary, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as error:
+        # Driver errors may contain connection credentials or personal data.
+        print(f"Import failed ({type(error).__name__}); transaction not partially applied. Check configuration and permissions.")
+        raise SystemExit(1)

tests/browser_admin.py

@@ -0,0 +1,143 @@
+"""Run from the repository root: python tests/browser_admin.py.
+
+Requires Playwright and Chromium. Uses in-memory data and intercepted requests;
+never contacts the production service. Optional BROWSER_EXECUTABLE selects an
+already installed Chromium instead of the Playwright download.
+"""
+import os
+from pathlib import Path
+import sys
+from tempfile import mkdtemp
+from urllib.parse import urlsplit
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from fastapi.testclient import TestClient
+from playwright.sync_api import expect, sync_playwright
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.pool import StaticPool
+
+from app.config import Settings, get_settings
+from app.db import Base, get_db
+from app.main import app
+from app.models import Employee
+from app.security import SESSION_COOKIE, sign_session
+
+
+def main():
+    engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
+    Base.metadata.create_all(engine)
+    factory = sessionmaker(bind=engine)
+    with factory() as db:
+        db.add_all(Employee(profile_key=f"staff:{i}", canonical_url=f"https://www.hse.ru/staff/{i}",
+                            full_name=f"Сотрудник {i:02d}", current_data={"positions": ["Преподаватель"]})
+                   for i in range(30))
+        db.commit()
+
+    def database():
+        with factory() as db:
+            yield db
+
+    settings = Settings(_env_file=None, session_secret="browser-test-secret")
+    app.dependency_overrides[get_db] = database
+    app.dependency_overrides[get_settings] = lambda: settings
+    client = TestClient(app)
+    client.cookies.set(SESSION_COOKIE, sign_session(settings.admin_username, settings))
+    output = Path(mkdtemp(prefix="miem-admin-browser-"))
+    pending = []
+
+    def serve(route):
+        url = urlsplit(route.request.url)
+        if url.path == "/api/crawl-runs/latest":
+            pending.append(route)
+            return
+        response = client.get(url.path + ("?" + url.query if url.query else ""))
+        route.fulfill(status=response.status_code, body=response.content,
+                      content_type=response.headers.get("content-type", "text/plain"))
+
+    try:
+        with sync_playwright() as playwright:
+            browser = playwright.chromium.launch(executable_path=os.getenv("BROWSER_EXECUTABLE"))
+            page = browser.new_page(viewport={"width": 1440, "height": 1000})
+            page.route("**/*", serve)
+            errors = []
+            page.on("pageerror", lambda error: errors.append(str(error)))
+            page.goto("http://miem.test/admin/directory?limit=25&offset=25")
+            expect(page.get_by_role("status")).to_have_text("Показаны 26–30 из 30")
+            assert page.locator("th:visible").count() == 5
+            assert page.locator(".directory__table-wrap").evaluate("e => e.scrollWidth <= e.clientWidth")
+            skip_link = page.get_by_role("link", name="Перейти к содержимому")
+            assert skip_link.evaluate("e => getComputedStyle(e).left === '-999px'")
+            page.keyboard.press("Tab")
+            expect(skip_link).to_be_focused()
+            assert skip_link.evaluate("e => getComputedStyle(e).left === '8px'")
+            page.keyboard.press("Tab")
+            assert skip_link.evaluate("e => getComputedStyle(e).left === '-999px'")
+
+            for width in (1440, 1024, 921, 920, 768, 621, 620, 390, 320):
+                page.set_viewport_size({"width": width, "height": 844})
+                assert page.locator(".directory__filters").evaluate("e => e.scrollWidth <= e.clientWidth")
+                assert page.locator(".directory__input").evaluate_all("""controls => controls.every(control => {
+                  const field = control.closest('.directory__field').getBoundingClientRect();
+                  const rect = control.getBoundingClientRect();
+                  return rect.left >= field.left - 1 && rect.right <= field.right + 1;
+                })""")
+            page.set_viewport_size({"width": 1440, "height": 1000})
+            page.get_by_label("ФИО или ссылка").fill("Сотрудник 00")
+            page.get_by_role("button", name="Применить фильтры").click()
+            expect(page.get_by_role("status")).to_have_text("Показаны 1–1 из 1")
+            assert "offset=" not in page.url
+            trigger = page.get_by_role("button", name="Настроить колонки")
+            trigger.focus()
+            page.keyboard.press("Enter")
+            dialog = page.get_by_role("dialog", name="Отображаемые колонки")
+            expect(dialog).to_be_visible()
+            for _ in range(22):
+                page.keyboard.press("Tab")
+                # Native dialogs may let Tab reach browser chrome, never background controls.
+                assert dialog.evaluate("e => document.activeElement === document.body || e.contains(document.activeElement)")
+            page.keyboard.press("Escape")
+            expect(trigger).to_be_focused()
+            trigger.click()
+            page.get_by_role("button", name="Контакты", exact=True).click()
+            page.keyboard.press("Escape")
+            page.reload()
+            expect(page.locator('th[data-column="email"]')).to_be_visible()
+            page.evaluate("localStorage.clear()")
+            page.goto("http://miem.test/admin/directory")
+            page.screenshot(path=str(output / "desktop.png"), full_page=True)
+            page.set_viewport_size({"width": 390, "height": 844})
+            assert page.evaluate("document.documentElement.scrollWidth <= innerWidth")
+            assert page.locator('th[data-column="full_name"]').evaluate("e => getComputedStyle(e).position") == "sticky"
+            page.screenshot(path=str(output / "mobile.png"), full_page=True)
+            trigger.click()
+            expect(dialog).to_be_visible()
+            assert dialog.evaluate("e => e.getBoundingClientRect().right <= innerWidth")
+            page.screenshot(path=str(output / "mobile-dialog.png"), full_page=True)
+            page.keyboard.press("Escape")
+            page.get_by_label("ФИО или ссылка").fill("Нет совпадений")
+            page.get_by_role("button", name="Применить фильтры").click()
+            expect(page.get_by_text("Сотрудники не найдены", exact=True)).to_be_visible()
+            expect(page.get_by_role("link", name="Сбросить фильтры", exact=True)).to_be_visible()
+            page.goto("http://miem.test/admin")
+            page.wait_for_function("document.querySelector('[data-progress-panel]').getAttribute('aria-busy') === 'true'")
+            pending.pop(0).fulfill(status=503, body="unavailable")
+            expect(page.locator("[data-progress-error]")).to_be_visible()
+            expect(page.get_by_text("Показаны последние полученные данные.", exact=False)).to_be_visible()
+            page.get_by_role("button", name="Повторить").click()
+            page.wait_for_timeout(100)
+            pending.pop(0).fulfill(json={"running": None, "latest": None})
+            expect(page.locator("[data-progress-error]")).to_be_hidden()
+            expect(page.get_by_text("Запусков пока нет. Запустите парсинг.", exact=True)).to_be_visible()
+            assert not errors, errors
+            browser.close()
+    finally:
+        app.dependency_overrides.clear()
+        client.close()
+        engine.dispose()
+    print(f"Browser checks passed; screenshots: {output}")
+
+
+if __name__ == "__main__":
+    main()

tests/postgres/Dockerfile

@@ -0,0 +1,7 @@
+FROM python:3.12-slim
+WORKDIR /app
+COPY requirements.txt .
+RUN python -m pip install --no-cache-dir -r requirements.txt
+COPY app ./app
+COPY scripts/import_publications.py ./scripts/import_publications.py
+COPY tests/conftest.py tests/test_publication_import_postgres.py ./tests/

tests/postgres/Dockerfile.dockerignore

@@ -0,0 +1,18 @@
+**
+!requirements.txt
+!app/
+!app/**
+!scripts/
+!scripts/import_publications.py
+!tests/
+!tests/conftest.py
+!tests/test_publication_import_postgres.py
+!tests/postgres/
+!tests/postgres/Dockerfile
+!tests/postgres/Dockerfile.dockerignore
+app/data/
+**/__pycache__/
+**/.env*
+**/*credentials*
+**/*.key
+**/*.pem

tests/postgres/Dockerfile.postgres

@@ -0,0 +1,2 @@
+FROM postgres:16-alpine
+COPY init.sql /docker-entrypoint-initdb.d/10-test.sql

tests/postgres/Dockerfile.postgres.dockerignore

@@ -0,0 +1,4 @@
+**
+!init.sql
+!Dockerfile.postgres
+!Dockerfile.postgres.dockerignore

tests/postgres/compose.vps.yml

@@ -0,0 +1,69 @@
+name: miem-publication-integration-vps
+
+services:
+  postgres-test:
+    image: miem-publication-integration-postgres:pr55
+    pull_policy: never
+    environment:
+      POSTGRES_USER: test_admin
+      POSTGRES_PASSWORD: isolated_test_admin
+      POSTGRES_DB: miem_import_test
+    command: ["postgres", "-c", "shared_buffers=64MB", "-c", "max_connections=12"]
+    cpus: 0.5
+    mem_limit: 512m
+    memswap_limit: 512m
+    pids_limit: 128
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "test_admin", "-d", "miem_import_test"]
+      interval: 2s
+      timeout: 5s
+      retries: 30
+    logging:
+      driver: json-file
+      options:
+        max-size: "5m"
+        max-file: "1"
+    networks: [isolated]
+
+  tests:
+    image: miem-publication-integration-runner:pr55
+    pull_policy: never
+    user: "65534:65534"
+    read_only: true
+    cap_drop: [ALL]
+    security_opt: ["no-new-privileges:true"]
+    cpus: 0.5
+    mem_limit: 512m
+    memswap_limit: 512m
+    pids_limit: 128
+    tmpfs:
+      - /tmp:size=128m,mode=1777
+    environment:
+      DATABASE_URL: postgresql+psycopg://test_admin:isolated_test_admin@postgres-test:5432/miem_import_test
+      MIEM_POSTGRES_INTEGRATION: "1"
+      PYTHONDONTWRITEBYTECODE: "1"
+      PYTEST_ADDOPTS: "-p no:cacheprovider"
+    command: ["python", "-m", "pytest", "-q", "tests/test_publication_import_postgres.py"]
+    depends_on:
+      postgres-test:
+        condition: service_healthy
+    logging:
+      driver: json-file
+      options:
+        max-size: "5m"
+        max-file: "1"
+    networks: [isolated]
+
+networks:
+  isolated:
+    internal: true
+
+volumes:
+  pgdata:
+    driver: local
+    driver_opts:
+      type: tmpfs
+      device: tmpfs
+      o: size=256m

tests/postgres/compose.yml

@@ -0,0 +1,36 @@
+name: miem-publication-integration
+
+services:
+  postgres-test:
+    image: postgres:16-alpine
+    environment:
+      POSTGRES_USER: test_admin
+      POSTGRES_PASSWORD: isolated_test_admin
+      POSTGRES_DB: miem_import_test
+    tmpfs:
+      - /var/lib/postgresql/data
+    volumes:
+      - ./init.sql:/docker-entrypoint-initdb.d/10-test.sql:ro
+    healthcheck:
+      test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "test_admin", "-d", "miem_import_test"]
+      interval: 2s
+      timeout: 5s
+      retries: 30
+    networks: [isolated]
+
+  tests:
+    build:
+      context: ../..
+      dockerfile: tests/postgres/Dockerfile
+    environment:
+      DATABASE_URL: postgresql+psycopg://test_admin:isolated_test_admin@postgres-test:5432/miem_import_test
+      MIEM_POSTGRES_INTEGRATION: "1"
+    command: ["python", "-m", "pytest", "-q", "tests/test_publication_import_postgres.py"]
+    depends_on:
+      postgres-test:
+        condition: service_healthy
+    networks: [isolated]
+
+networks:
+  isolated:
+    internal: true

tests/postgres/init.sql

@@ -0,0 +1,6 @@
+-- Synthetic credentials for this ephemeral, private test cluster only.
+CREATE DATABASE miem_crawl_test OWNER test_admin;
+CREATE ROLE import_test LOGIN PASSWORD 'isolated_test_import'
+    NOSUPERUSER NOCREATEDB NOCREATEROLE NOINHERIT NOREPLICATION NOBYPASSRLS
+    CONNECTION LIMIT 1;
+GRANT CONNECT ON DATABASE miem_import_test, miem_crawl_test TO import_test;

tests/test_academic_degrees.py

@@ -0,0 +1,26 @@
+from app.services.academic_degrees import academic_degrees
+
+
+def test_academic_degrees_extracts_all_degree_names_without_surrounding_text():
+    data = {
+        "sections": [
+            {
+                "title": "Образование",
+                "year_entries": [
+                    {"text": "2008 — кандидат технических наук, доцент"},
+                    {"text": "2018 — Доктор физико-математических наук"},
+                ],
+            },
+            {
+                "title": "Учёные степени",
+                "table": {"rows": [{"cells": ["Ph.D.", "Доктор физико-математических наук"]}]},
+            },
+            {"title": "Публикации", "items": ["Доктор медицинских наук выступил автором статьи"]},
+        ]
+    }
+
+    assert academic_degrees(data) == [
+        "кандидат технических наук",
+        "Доктор физико-математических наук",
+        "Ph.D.",
+    ]

tests/test_admin_data.py

@@ -1,6 +1,6 @@
 from datetime import datetime, timezone
 
-from app.models import CrawlError, CrawlRun, CrawlRunEmployeeChange, Employee
+from app.models import CrawlError, CrawlRun, CrawlRunEmployeeChange, Employee, EmployeeNewsLink
 from app.services.admin_data import (
     employee_detail_payload,
     employee_display_payload,
@@ -35,6 +35,7 @@ def test_employee_display_payload_extracts_common_fields(db_session):
             "sections": [
                 {"type": "publications", "publications": [{"title": "Paper"}]},
                 {"type": "courses_by_year", "courses": [{"title": "Course"}]},
+                {"type": "news", "news_links": [{"title": "News", "url": "https://example.test/news"}]},
             ],
         },
     )
@@ -46,9 +47,55 @@ def test_employee_display_payload_extracts_common_fields(db_session):
     assert payload["email_text"] == "person@hse.ru"
     assert payload["publications_count"] == 1
     assert payload["courses_count"] == 1
+    assert payload["news_count"] == 1
     assert payload["first_seen_display"] != "Не указано"
 
 
+def test_list_employees_page_filters_and_displays_academic_degrees(db_session):
+    db_session.add_all(
+        [
+            Employee(
+                profile_key="staff:degree",
+                canonical_url="https://www.hse.ru/staff/degree",
+                full_name="Doctor",
+                status="active",
+                first_seen_at=datetime.now(timezone.utc),
+                last_seen_at=datetime.now(timezone.utc),
+                has_academic_degree=True,
+                current_data={
+                    "sections": [
+                        {
+                            "title": "Образование и учёные степени",
+                            "year_entries": [{"year": 2020, "text": "Доктор технических наук"}],
+                        }
+                    ]
+                },
+            ),
+            Employee(
+                profile_key="staff:no-degree",
+                canonical_url="https://www.hse.ru/staff/no-degree",
+                full_name="Master",
+                status="active",
+                first_seen_at=datetime.now(timezone.utc),
+                last_seen_at=datetime.now(timezone.utc),
+                has_academic_degree=False,
+                current_data={"sections": [{"title": "Образование", "items": ["Магистратура"]}]},
+            ),
+        ]
+    )
+    db_session.commit()
+
+    page = list_employees_page(db_session, has_academic_degree=True)
+
+    assert page["total"] == 1
+    assert page["employees"][0]["full_name"] == "Doctor"
+    assert page["employees"][0]["academic_degree_text"] == "Доктор технических наук"
+
+    page = list_employees_page(db_session, has_academic_degree=False)
+
+    assert page["employees"][0]["full_name"] == "Master"
+
+
 def test_employee_detail_payload_normalizes_human_readable_sections(db_session):
     employee = Employee(
         profile_key="staff:person",
@@ -104,6 +151,19 @@ def test_employee_detail_payload_normalizes_human_readable_sections(db_session):
                     "type": "generic",
                     "raw_text": "Fallback text",
                 },
+                {
+                    "title": "В новостях",
+                    "type": "news",
+                    "news_links": [
+                        {
+                            "title": "News title",
+                            "url": "https://example.test/news",
+                            "summary": "News summary",
+                            "published_at": "2026-04-28T00:00:00+00:00",
+                            "published_year": 2026,
+                        }
+                    ],
+                },
             ],
         },
     )
@@ -118,6 +178,41 @@ def test_employee_detail_payload_normalizes_human_readable_sections(db_session):
     assert payload["sections"][2]["courses"][0]["title"] == "Course"
     assert payload["sections"][3]["theses"][0]["student"] == "Student Name"
     assert payload["sections"][4]["paragraphs"] == ["Fallback text"]
+    assert payload["sections"][5]["news_links"][0]["title"] == "News title"
+    assert payload["news_links"][0]["published_display"] == "28.04.2026"
+
+
+def test_employee_payload_prefers_stored_news_links(db_session):
+    employee = Employee(
+        profile_key="staff:news",
+        canonical_url="https://www.hse.ru/staff/news",
+        full_name="News Person",
+        status="active",
+        first_seen_at=datetime.now(timezone.utc),
+        last_seen_at=datetime.now(timezone.utc),
+        current_data={"sections": [{"type": "news", "news_links": [{"title": "Old news"}]}]},
+    )
+    db_session.add(employee)
+    db_session.commit()
+    db_session.add(
+        EmployeeNewsLink(
+            employee_id=employee.id,
+            title="Stored news",
+            url="https://example.test/stored",
+            summary="Stored summary",
+            published_at=datetime(2026, 4, 28, tzinfo=timezone.utc),
+            published_year=2026,
+            source_hash="b" * 64,
+        )
+    )
+    db_session.commit()
+
+    display = employee_display_payload(employee)
+    detail = employee_detail_payload(employee)
+
+    assert display["news_count"] == 1
+    assert detail["news_links"][0]["title"] == "Stored news"
+    assert detail["news_links"][0]["published_display"] == "28.04.2026"
 
 
 def test_employee_payloads_tolerate_malformed_current_data(db_session):
@@ -173,6 +268,18 @@ def test_list_employees_page_filters_sorts_and_paginates(db_session):
     assert page["limit"] == 50
 
 
+def test_catalog_recovers_from_out_of_range_offset(db_session):
+    db_session.add(Employee(profile_key="staff:page", canonical_url="https://www.hse.ru/staff/page", full_name="Page Person"))
+    db_session.commit()
+
+    page = list_employees_page(db_session, q="Page", offset=100, limit=25)
+    assert (page["offset"], page["page"], page["total"]) == (0, 1, 1)
+    assert len(page["employees"]) == 1
+    empty = list_employees_page(db_session, q="missing", offset=100)
+    assert empty["offset"] == 0
+    assert empty["employees"] == []
+
+
 def test_stats_payload_uses_latest_run_new_count(db_session):
     db_session.add(
         Employee(
@@ -200,13 +307,14 @@ def test_run_payload_calculates_progress():
         status="running",
         found_count=10,
         parsed_count=4,
+        skipped_count=2,
         error_count=1,
     )
 
     payload = run_payload(run)
 
-    assert payload["processed_count"] == 5
-    assert payload["progress_percent"] == 50.0
+    assert payload["processed_count"] == 7
+    assert payload["progress_percent"] == 70.0
     assert payload["status_display"] == "Выполняется"
 
 

tests/test_admin_templates.py

@@ -1,93 +1,128 @@
-from pathlib import Path
-
-
-def test_base_navigation_is_russian_and_has_no_legacy_employees_link():
-    template = Path("app/templates/base.html").read_text(encoding="utf-8")
-
-    assert "Обзор" in template
-    assert "Сотрудники" in template
-    assert "Запуски" in template
-    assert "Выйти" in template
-    assert '<a class="admin__brand-link" href="/admin">MIEM Employees</a>' in template
-    assert ">Employees<" not in template
-    assert "/admin/employees" not in template
-
-
-def test_directory_template_is_russian_and_uses_display_dates():
-    template = Path("app/templates/directory.html").read_text(encoding="utf-8")
-
-    assert "Сотрудники" in template
-    assert "Колонки" in template
-    assert "Применить" in template
-    assert "На странице: {{ value }}" in template
-    assert "{% for value in [25, 50, 100] %}" in template
-    assert "Найдено:" in template
-    assert "employee.first_seen_display" in template
-    assert "employee.last_seen_display" in template
-    assert "employee.dismissed_display" in template
-    assert "Directory" not in template
-    assert "employees found" not in template
-
-
-def test_admin_employees_route_redirects_to_directory():
-    source = Path("app/admin.py").read_text(encoding="utf-8")
-
-    assert 'RedirectResponse("/admin/directory", status_code=303)' in source
-
-
-def test_dashboard_limits_latest_runs_to_five():
-    source = Path("app/admin.py").read_text(encoding="utf-8")
-
-    assert "order_by(desc(CrawlRun.started_at)).limit(5)" in source
-    assert "order_by(desc(CrawlRun.started_at)).limit(10)" not in source
-
-
-def test_runs_template_links_to_run_detail():
-    template = Path("app/templates/runs.html").read_text(encoding="utf-8")
-
-    assert 'onclick="window.location.href=\'/admin/runs/{{ run.id }}\'"' in template
-    assert "onkeydown=\"if (event.key === 'Enter' || event.key === ' ')" in template
-    assert 'role="link"' in template
-    assert 'tabindex="0"' in template
-    assert 'data-row-href="/admin/runs/{{ run.id }}"' not in template
-    assert '<a class="admin__link" href="/admin/runs/{{ run.id }}">' not in template
-
-
-def test_run_detail_template_extends_base_and_shows_change_groups():
-    template = Path("app/templates/run_detail.html").read_text(encoding="utf-8")
-
-    assert '{% extends "base.html" %}' in template
-    assert 'id="new-employees"' in template
-    assert "Новые сотрудники" in template
-    assert "Потеряшки" in template
-    assert "Уволенные" in template
-    assert "Детализация сотрудников для этого запуска недоступна" in template
-
-
+from pathlib import Path
+
+
+def test_base_navigation_is_russian_and_has_no_legacy_employees_link():
+    template = Path("app/templates/base.html").read_text(encoding="utf-8")
+
+    assert "Обзор" in template
+    assert "Сотрудники" in template
+    assert "Запуски" in template
+    assert "Выйти" in template
+    assert '<a class="admin__brand-link" href="/admin">MIEM Employees</a>' in template
+    assert ">Employees<" not in template
+    assert "/admin/employees" not in template
+
+
+def test_directory_template_is_russian_and_uses_display_dates():
+    template = Path("app/templates/directory.html").read_text(encoding="utf-8")
+
+    assert "Сотрудники" in template
+    assert "колонки" in template
+    assert "Применить фильтры" in template
+    assert "Сотрудников на странице" in template
+    assert "{% for value in [25, 50, 100] %}" in template
+    assert "Показаны" in template
+    assert "Новости" in template
+    assert "Есть учёная степень" in template
+    assert 'data-column="academic_degree"' in template
+    assert "employee.news_count" in template
+    assert "employee.first_seen_display" in template
+    assert "employee.last_seen_display" in template
+    assert "employee.dismissed_display" in template
+    assert "verification_required" in template
+    assert '<label class="directory__field">Статус' in template
+    assert 'id="columns-dialog"' in template
+    assert 'aria-controls="columns-dialog"' in template
+    assert 'aria-labelledby="columns-dialog-title"' in template
+    assert "Сбросить фильтры" in template
+    assert 'data-columns-preset="review"' in template
+    assert '<dialog class="columns-modal"' in template
+    assert 'id="columns-dialog"' in template
+    assert 'aria-controls="columns-dialog"' in template
+    assert "Сбросить фильтры" in template
+    assert "Directory" not in template
+    assert "employees found" not in template
+
+
+def test_admin_employees_route_redirects_to_directory():
+    source = Path("app/admin.py").read_text(encoding="utf-8")
+
+    assert 'RedirectResponse("/admin/directory", status_code=303)' in source
+
+
+def test_dashboard_limits_latest_runs_to_five():
+    source = Path("app/admin.py").read_text(encoding="utf-8")
+
+    assert "order_by(desc(CrawlRun.started_at)).limit(5)" in source
+    assert "order_by(desc(CrawlRun.started_at)).limit(10)" not in source
+
+
+def test_runs_template_links_to_run_detail():
+    template = Path("app/templates/runs.html").read_text(encoding="utf-8")
+
+    assert '<a class="admin__link" href="/admin/runs/{{ run.id }}">' in template
+    assert "onclick=" not in template
+    assert 'role="link"' not in template
+
+
+def test_run_detail_template_extends_base_and_shows_change_groups():
+    template = Path("app/templates/run_detail.html").read_text(encoding="utf-8")
+
+    assert '{% extends "base.html" %}' in template
+    assert 'id="new-employees"' in template
+    assert "Новые сотрудники" in template
+    assert "Потеряшки" in template
+    assert "Требуют проверки" in template
+    assert "Уволенные" in template
+    assert "Детализация сотрудников для этого запуска недоступна" in template
+
+
 def test_dashboard_metric_cards_link_to_admin_targets():
-    template = Path("app/templates/dashboard.html").read_text(encoding="utf-8")
-
-    assert 'href="/admin/directory"' in template
-    assert 'href="/admin/directory?status=active"' in template
-    assert '/admin/runs/{{ latest_run.id }}#new-employees' in template
-    assert 'href="/admin/directory?status=dismissed"' in template
+    template = Path("app/templates/dashboard.html").read_text(encoding="utf-8")
+
+    assert 'href="/admin/directory"' in template
+    assert 'href="/admin/directory?status=active"' in template
+    assert 'href="/admin/directory?status=verification_required"' in template
+    assert '/admin/runs/{{ latest_run.id }}#new-employees' in template
+    assert 'href="/admin/directory?status=dismissed"' in template
     assert 'href="/admin/runs"' in template
 
 
-def test_dashboard_latest_run_rows_link_to_run_detail():
+def test_dashboard_has_dismissed_status_refresh_action():
     template = Path("app/templates/dashboard.html").read_text(encoding="utf-8")
 
-    assert 'onclick="window.location.href=\'/admin/runs/{{ run.id }}\'"' in template
-    assert "onkeydown=\"if (event.key === 'Enter' || event.key === ' ')" in template
-    assert 'role="link"' in template
-    assert 'tabindex="0"' in template
-    assert 'data-row-href="/admin/runs/{{ run.id }}"' not in template
-    assert '<a class="admin__link" href="/admin/runs/{{ run.id }}">' not in template
+    assert 'action="/admin/dismissed/refresh"' in template
+    assert "Проверить уволенных" in template
 
 
-def test_admin_js_supports_keyboard_activation_for_clickable_rows():
+def test_progress_template_has_accessible_progressbar():
+    for name in ("dashboard.html", "runs.html"):
+        template = Path(f"app/templates/{name}").read_text(encoding="utf-8")
+        assert 'role="progressbar"' in template
+        assert 'aria-valuemin="0"' in template
+        assert 'aria-valuemax="100"' in template
+        assert "Прогресс парсинга" in template
+
+
+def test_dashboard_latest_run_rows_link_to_run_detail():
+    template = Path("app/templates/dashboard.html").read_text(encoding="utf-8")
+
+    assert '<a class="admin__link" href="/admin/runs/{{ run.id }}">' in template
+    assert "onclick=" not in template
+    assert 'role="link"' not in template
+
+
+def test_admin_js_uses_native_dialog_for_column_settings():
     source = Path("app/static/admin.js").read_text(encoding="utf-8")
 
-    assert 'addEventListener("keydown"' in source
-    assert '"Enter"' in source
-    assert '" "' in source
+    assert "showModal()" in source
+    assert "modal.close()" in source
+    assert 'setAttribute("aria-expanded", "true")' in source
+    assert "data-progress-retry" in source
+
+
+def test_base_template_has_skip_link_and_current_navigation_state():
+    template = Path("app/templates/base.html").read_text(encoding="utf-8")
+
+    assert 'href="#main-content"' in template
+    assert 'aria-current="page"' in template

tests/test_api.py

@@ -0,0 +1,154 @@
+from datetime import datetime, timezone
+from types import SimpleNamespace
+
+from fastapi.testclient import TestClient
+from sqlalchemy import create_engine, select
+from sqlalchemy.exc import OperationalError
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.pool import StaticPool
+
+from app.config import Settings, get_settings
+from app.db import Base, get_db
+from app.main import app
+from app.models import CrawlRun, CrawlRunEmployeeChange, Employee
+from app.security import SESSION_COOKIE, sign_session
+
+
+def test_health_returns_versions():
+    response = TestClient(app).get("/api/health")
+
+    assert response.status_code == 200
+    assert response.json()["backend_version"] == "0.10.4"
+
+
+def test_api_employees_and_stats_require_admin_session():
+    engine = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False}, poolclass=StaticPool)
+    Base.metadata.create_all(engine)
+    session_factory = sessionmaker(bind=engine)
+    db = session_factory()
+    employee = Employee(
+        profile_key="staff:alpha",
+        profile_type="staff",
+        profile_id="alpha",
+        canonical_url="https://www.hse.ru/staff/alpha",
+        full_name="Alpha Person",
+        status="active",
+        first_seen_at=datetime.now(timezone.utc),
+        last_seen_at=datetime.now(timezone.utc),
+        current_data={"contacts": {"emails": ["alpha@hse.ru"]}, "sections": []},
+    )
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="completed", new_count=1)
+    db.add_all([employee, run])
+    db.commit()
+    db.add(CrawlRunEmployeeChange(
+        crawl_run_id=run.id,
+        employee_id=employee.id,
+        profile_key=employee.profile_key,
+        profile_url=employee.canonical_url,
+        full_name=employee.full_name,
+        change_type="new",
+        profile_available=True,
+        message="added",
+    ))
+    db.commit()
+    run_id = run.id
+    db.close()
+
+    settings = Settings(admin_username="admin", admin_password="password", session_secret="session-secret")
+
+    def override_db():
+        session = session_factory()
+        try:
+            yield session
+        finally:
+            session.close()
+
+    app.dependency_overrides[get_db] = override_db
+    app.dependency_overrides[get_settings] = lambda: settings
+    client = TestClient(app)
+    client.cookies.set(SESSION_COOKIE, sign_session("admin", settings))
+
+    employees = client.get("/api/employees", params={"q": "Alpha", "has_email": True})
+    stats = client.get("/api/stats")
+    run_details = client.get(f"/api/crawl-runs/{run_id}")
+
+    assert employees.status_code == 200
+    assert employees.json()["total"] == 1
+    assert stats.status_code == 200
+    assert stats.json()["new_in_last_run"] == 1
+    assert run_details.status_code == 200
+    assert run_details.json()["changes"]["new"][0]["full_name"] == "Alpha Person"
+    directory = client.get("/admin/directory?q=Alpha&offset=100")
+    assert directory.status_code == 200
+    assert "Показаны 1–1 из 1" in directory.text
+    assert 'name="offset"' not in directory.text
+    assert "Alpha Person" in directory.text
+    empty = client.get("/admin/directory?q=missing&offset=100")
+    assert "Показаны 0–0 из 0" in empty.text
+    assert "Сбросить фильтры" in empty.text
+    app.dependency_overrides.clear()
+
+
+def test_admin_refresh_employee_route_updates_only_requested_employee(monkeypatch):
+    engine = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False}, poolclass=StaticPool)
+    Base.metadata.create_all(engine)
+    session_factory = sessionmaker(bind=engine)
+    db = session_factory()
+    db.add(Employee(
+        profile_key="org_person:133709486",
+        profile_type="org_person",
+        profile_id="133709486",
+        canonical_url="https://www.hse.ru/org/persons/133709486",
+        full_name="Будков Юрий Алексеевич",
+        status="active",
+    ))
+    db.commit()
+    employee_id = db.scalar(select(Employee.id))
+    db.close()
+
+    settings = Settings(admin_username="admin", admin_password="password", session_secret="session-secret")
+
+    def override_db():
+        session = session_factory()
+        try:
+            yield session
+        finally:
+            session.close()
+
+    calls = []
+
+    def fake_refresh_employee(db, refreshed_employee, route_settings):
+        calls.append((refreshed_employee.id, route_settings))
+        return SimpleNamespace(status="completed")
+
+    app.dependency_overrides[get_db] = override_db
+    app.dependency_overrides[get_settings] = lambda: settings
+    monkeypatch.setattr("app.admin.refresh_employee", fake_refresh_employee)
+    client = TestClient(app)
+    client.cookies.set(SESSION_COOKIE, sign_session("admin", settings))
+
+    response = client.post(f"/admin/employees/{employee_id}/refresh", follow_redirects=False)
+
+    assert response.status_code == 303
+    assert response.headers["location"] == f"/admin/employees/{employee_id}?refresh_status=success"
+    assert calls == [(employee_id, settings)]
+    app.dependency_overrides.clear()
+
+
+def test_directory_database_error_has_retry_and_preserves_filters(db_session, monkeypatch):
+    settings = Settings(_env_file=None, session_secret="test-session-secret")
+    app.dependency_overrides[get_db] = lambda: db_session
+    app.dependency_overrides[get_settings] = lambda: settings
+    def unavailable(*args, **kwargs):
+        raise OperationalError("SELECT", {}, Exception("private database details"))
+    monkeypatch.setattr("app.admin.list_employees_page", unavailable)
+    try:
+        client = TestClient(app)
+        client.cookies.set(SESSION_COOKIE, sign_session(settings.admin_username, settings))
+        response = client.get("/admin/directory?q=Alice")
+        assert response.status_code == 503
+        assert 'href="/admin/directory?q=Alice"' in response.text
+        assert "Повторить загрузку" in response.text
+        assert "private database details" not in response.text
+    finally:
+        app.dependency_overrides.clear()

tests/test_api_mcp.py

@@ -1,430 +0,0 @@
-import json
-from datetime import datetime, timezone
-from types import SimpleNamespace
-
-from fastapi.testclient import TestClient
-from sqlalchemy import create_engine, select
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.pool import StaticPool
-
-from app.config import Settings, get_settings
-from app.db import Base, get_db
-from app.main import app
-from app.models import CrawlRun, CrawlRunEmployeeChange, Employee
-from app.security import SESSION_COOKIE, sign_session
-
-
-def test_health_returns_versions():
-    client = TestClient(app)
-
-    response = client.get("/api/health")
-
-    assert response.status_code == 200
-    assert response.json()["backend_version"] == "0.5.0"
-
-
-def test_mcp_lists_tools_without_auth_and_ignores_auth_header():
-    engine = create_engine(
-        "sqlite:///:memory:",
-        connect_args={"check_same_thread": False},
-        poolclass=StaticPool,
-    )
-    Base.metadata.create_all(engine)
-    Session = sessionmaker(bind=engine)
-
-    def override_db():
-        session = Session()
-        try:
-            yield session
-        finally:
-            session.close()
-
-    app.dependency_overrides[get_db] = override_db
-    client = TestClient(app)
-
-    without_auth = client.post("/mcp", json={"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}})
-    with_auth = client.post(
-        "/mcp",
-        headers={"Authorization": "Bearer anything"},
-        json={"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}},
-    )
-
-    assert without_auth.status_code == 200
-    assert with_auth.status_code == 200
-    tool_names = {tool["name"] for tool in without_auth.json()["result"]["tools"]}
-    assert "search_employees" in tool_names
-    assert "get_service_info" in tool_names
-    assert "sync_employees" in tool_names
-    assert any(tool["name"] == "get_crawl_run_details" for tool in without_auth.json()["result"]["tools"])
-    assert with_auth.json()["result"]["tools"] == without_auth.json()["result"]["tools"]
-
-    app.dependency_overrides.clear()
-
-
-def test_mcp_search_employees_returns_matching_employee():
-    engine = create_engine(
-        "sqlite:///:memory:",
-        connect_args={"check_same_thread": False},
-        poolclass=StaticPool,
-    )
-    Base.metadata.create_all(engine)
-    Session = sessionmaker(bind=engine)
-    session = Session()
-    session.add(
-        Employee(
-            profile_key="staff:avsergeev",
-            profile_type="staff",
-            profile_id="avsergeev",
-            canonical_url="https://www.hse.ru/staff/avsergeev",
-            full_name="Сергеев Алексей Викторович",
-            status="active",
-            first_seen_at=datetime.now(timezone.utc),
-            last_seen_at=datetime.now(timezone.utc),
-            current_data={"sections": []},
-        )
-    )
-    session.commit()
-    session.close()
-
-    def override_db():
-        db = Session()
-        try:
-            yield db
-        finally:
-            db.close()
-
-    app.dependency_overrides[get_db] = override_db
-    client = TestClient(app)
-
-    response = client.post(
-        "/mcp",
-        json={
-            "jsonrpc": "2.0",
-            "id": 1,
-            "method": "tools/call",
-            "params": {"name": "search_employees", "arguments": {"query": "Сергеев"}},
-        },
-    )
-
-    assert response.status_code == 200
-    assert "Сергеев Алексей Викторович" in response.json()["result"]["content"][0]["text"]
-
-    app.dependency_overrides.clear()
-
-
-def test_mcp_service_info_returns_tools_and_dataset_hash():
-    engine = create_engine(
-        "sqlite:///:memory:",
-        connect_args={"check_same_thread": False},
-        poolclass=StaticPool,
-    )
-    Base.metadata.create_all(engine)
-    Session = sessionmaker(bind=engine)
-    session = Session()
-    session.add(
-        Employee(
-            profile_key="staff:alpha",
-            profile_type="staff",
-            profile_id="alpha",
-            canonical_url="https://www.hse.ru/staff/alpha",
-            full_name="Alpha Person",
-            status="active",
-            current_checksum="a" * 64,
-            current_data={"sections": []},
-        )
-    )
-    session.commit()
-    session.close()
-
-    def override_db():
-        db = Session()
-        try:
-            yield db
-        finally:
-            db.close()
-
-    app.dependency_overrides[get_db] = override_db
-    client = TestClient(app)
-
-    response = client.post(
-        "/mcp",
-        json={"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": "get_service_info", "arguments": {}}},
-    )
-
-    assert response.status_code == 200
-    payload = json.loads(response.json()["result"]["content"][0]["text"])
-    assert payload["service_name"] == "miem-employees"
-    assert payload["backend_version"] == "0.5.0"
-    assert payload["dataset"]["hash"]
-    assert any(tool["name"] == "sync_employees" for tool in payload["tools"])
-
-    app.dependency_overrides.clear()
-
-
-def test_mcp_sync_employees_full_empty_and_unknown_hash_modes():
-    engine = create_engine(
-        "sqlite:///:memory:",
-        connect_args={"check_same_thread": False},
-        poolclass=StaticPool,
-    )
-    Base.metadata.create_all(engine)
-    Session = sessionmaker(bind=engine)
-    session = Session()
-    session.add(
-        Employee(
-            profile_key="staff:alpha",
-            profile_type="staff",
-            profile_id="alpha",
-            canonical_url="https://www.hse.ru/staff/alpha",
-            full_name="Alpha Person",
-            status="active",
-            current_checksum="a" * 64,
-            current_data={"sections": [{"type": "paragraphs"}]},
-        )
-    )
-    session.commit()
-    session.close()
-
-    def override_db():
-        db = Session()
-        try:
-            yield db
-        finally:
-            db.close()
-
-    app.dependency_overrides[get_db] = override_db
-    client = TestClient(app)
-
-    full_response = client.post(
-        "/mcp",
-        json={"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": "sync_employees", "arguments": {}}},
-    )
-    full_payload = json.loads(full_response.json()["result"]["content"][0]["text"])
-    current_hash = full_payload["to_hash"]
-
-    empty_response = client.post(
-        "/mcp",
-        json={
-            "jsonrpc": "2.0",
-            "id": 2,
-            "method": "tools/call",
-            "params": {"name": "sync_employees", "arguments": {"client_hash": current_hash}},
-        },
-    )
-    empty_payload = json.loads(empty_response.json()["result"]["content"][0]["text"])
-
-    unknown_response = client.post(
-        "/mcp",
-        json={
-            "jsonrpc": "2.0",
-            "id": 3,
-            "method": "tools/call",
-            "params": {"name": "sync_employees", "arguments": {"client_hash": "missing"}},
-        },
-    )
-    unknown_payload = json.loads(unknown_response.json()["result"]["content"][0]["text"])
-
-    assert full_payload["mode"] == "full"
-    assert full_payload["items"][0]["data"] == {"sections": [{"type": "paragraphs"}]}
-    assert empty_payload["mode"] == "delta"
-    assert empty_payload["changes"] == {"added": [], "updated": [], "dismissed": [], "removed": []}
-    assert unknown_payload["mode"] == "full"
-    assert unknown_payload["reason"] == "unknown_client_hash"
-
-    app.dependency_overrides.clear()
-
-
-def test_mcp_get_crawl_run_details_returns_changes():
-    engine = create_engine(
-        "sqlite:///:memory:",
-        connect_args={"check_same_thread": False},
-        poolclass=StaticPool,
-    )
-    Base.metadata.create_all(engine)
-    Session = sessionmaker(bind=engine)
-    session = Session()
-    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="completed", new_count=1)
-    employee = Employee(
-        profile_key="staff:new",
-        profile_type="staff",
-        profile_id="new",
-        canonical_url="https://www.hse.ru/staff/new",
-        full_name="New Person",
-        status="active",
-        first_seen_at=datetime.now(timezone.utc),
-        last_seen_at=datetime.now(timezone.utc),
-    )
-    session.add_all([run, employee])
-    session.commit()
-    session.add(
-        CrawlRunEmployeeChange(
-            crawl_run_id=run.id,
-            employee_id=employee.id,
-            profile_key=employee.profile_key,
-            profile_url=employee.canonical_url,
-            full_name=employee.full_name,
-            change_type="new",
-            profile_available=True,
-            message="added",
-        )
-    )
-    session.commit()
-    run_id = run.id
-    session.close()
-
-    def override_db():
-        db = Session()
-        try:
-            yield db
-        finally:
-            db.close()
-
-    app.dependency_overrides[get_db] = override_db
-    client = TestClient(app)
-
-    response = client.post(
-        "/mcp",
-        json={
-            "jsonrpc": "2.0",
-            "id": 1,
-            "method": "tools/call",
-            "params": {"name": "get_crawl_run_details", "arguments": {"run_id": run_id}},
-        },
-    )
-
-    assert response.status_code == 200
-    text = response.json()["result"]["content"][0]["text"]
-    assert "New Person" in text
-    assert "changes_detail_available" in text
-
-    app.dependency_overrides.clear()
-
-
-def test_mcp_protected_resource_metadata_route_is_removed():
-    client = TestClient(app)
-
-    response = client.get("/.well-known/oauth-protected-resource")
-
-    assert response.status_code == 404
-
-
-def test_api_employees_and_stats_require_admin_session():
-    engine = create_engine(
-        "sqlite:///:memory:",
-        connect_args={"check_same_thread": False},
-        poolclass=StaticPool,
-    )
-    Base.metadata.create_all(engine)
-    Session = sessionmaker(bind=engine)
-    db = Session()
-    db.add(
-        Employee(
-            profile_key="staff:alpha",
-            profile_type="staff",
-            profile_id="alpha",
-            canonical_url="https://www.hse.ru/staff/alpha",
-            full_name="Alpha Person",
-            status="active",
-            first_seen_at=datetime.now(timezone.utc),
-            last_seen_at=datetime.now(timezone.utc),
-            current_data={"contacts": {"emails": ["alpha@hse.ru"]}, "sections": []},
-        )
-    )
-    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="completed", new_count=1)
-    db.add(run)
-    db.commit()
-    db.add(
-        CrawlRunEmployeeChange(
-            crawl_run_id=run.id,
-            employee_id=1,
-            profile_key="staff:alpha",
-            profile_url="https://www.hse.ru/staff/alpha",
-            full_name="Alpha Person",
-            change_type="new",
-            profile_available=True,
-            message="added",
-        )
-    )
-    db.commit()
-    run_id = run.id
-    db.close()
-
-    settings = Settings(admin_username="admin", admin_password="password", session_secret="session-secret")
-
-    def override_db():
-        session = Session()
-        try:
-            yield session
-        finally:
-            session.close()
-
-    app.dependency_overrides[get_db] = override_db
-    app.dependency_overrides[get_settings] = lambda: settings
-    client = TestClient(app)
-    client.cookies.set(SESSION_COOKIE, sign_session("admin", settings))
-
-    employees = client.get("/api/employees", params={"q": "Alpha", "has_email": True})
-    stats = client.get("/api/stats")
-    run_details = client.get(f"/api/crawl-runs/{run_id}")
-
-    assert employees.status_code == 200
-    assert employees.json()["total"] == 1
-    assert stats.status_code == 200
-    assert stats.json()["new_in_last_run"] == 1
-    assert run_details.status_code == 200
-    assert run_details.json()["changes"]["new"][0]["full_name"] == "Alpha Person"
-
-    app.dependency_overrides.clear()
-
-
-def test_admin_refresh_employee_route_updates_only_requested_employee(monkeypatch):
-    engine = create_engine(
-        "sqlite:///:memory:",
-        connect_args={"check_same_thread": False},
-        poolclass=StaticPool,
-    )
-    Base.metadata.create_all(engine)
-    Session = sessionmaker(bind=engine)
-    db = Session()
-    db.add(
-        Employee(
-            profile_key="org_person:133709486",
-            profile_type="org_person",
-            profile_id="133709486",
-            canonical_url="https://www.hse.ru/org/persons/133709486",
-            full_name="Будков Юрий Алексеевич",
-            status="active",
-        )
-    )
-    db.commit()
-    employee_id = db.scalar(select(Employee.id))
-    db.close()
-
-    settings = Settings(admin_username="admin", admin_password="password", session_secret="session-secret")
-
-    def override_db():
-        session = Session()
-        try:
-            yield session
-        finally:
-            session.close()
-
-    calls = []
-
-    def fake_refresh_employee(db, refreshed_employee, route_settings):
-        calls.append((refreshed_employee.id, route_settings))
-        return SimpleNamespace(status="completed")
-
-    app.dependency_overrides[get_db] = override_db
-    app.dependency_overrides[get_settings] = lambda: settings
-    monkeypatch.setattr("app.admin.refresh_employee", fake_refresh_employee)
-    client = TestClient(app)
-    client.cookies.set(SESSION_COOKIE, sign_session("admin", settings))
-
-    response = client.post(f"/admin/employees/{employee_id}/refresh", follow_redirects=False)
-
-    assert response.status_code == 303
-    assert response.headers["location"] == f"/admin/employees/{employee_id}?refresh_status=success"
-    assert calls == [(employee_id, settings)]
-
-    app.dependency_overrides.clear()

tests/test_config.py

@@ -11,3 +11,12 @@ def test_numeric_crawl_limit_is_parsed():
     settings = Settings(crawl_limit="25")
 
     assert settings.crawl_limit == 25
+
+
+def test_dismissal_safety_settings_have_conservative_defaults(monkeypatch):
+    monkeypatch.delenv("DISMISSAL_CONFIRMATION_RUNS", raising=False)
+    monkeypatch.delenv("MAX_AUTO_DISMISSALS_PER_RUN", raising=False)
+    settings = Settings(_env_file=None)
+
+    assert settings.dismissal_confirmation_runs == 3
+    assert settings.max_auto_dismissals_per_run == 25

tests/test_council.py

@@ -0,0 +1,76 @@
+import csv
+from datetime import date
+
+from fastapi.testclient import TestClient
+
+from app.config import Settings, get_settings
+from app.db import get_db
+from app.main import app
+from app.models import Employee
+from app.security import SESSION_COOKIE, sign_session
+from app.services import council
+
+
+def test_council_counts_unique_articles_and_only_active_doctors(db_session, monkeypatch, tmp_path):
+    fields = ["Название публикации", "Год", "DOI", "EID (Scopus)", "UT (WoS)",
+              "Авторы НИУ ВШЭ", "Квартиль scopus", "Квартиль wos"]
+    export = tmp_path / "articles.csv"
+    with export.open("w", encoding="utf-8-sig", newline="") as output:
+        writer = csv.DictWriter(output, fieldnames=fields, delimiter=";")
+        writer.writeheader()
+        for number in range(10):
+            writer.writerow({"Название публикации": f"Article {number}", "Год": 2024,
+                             "DOI": f"10.1234/{number}", "EID (Scopus)": f"eid-{number}",
+                             "Авторы НИУ ВШЭ": "Иванов Иван Иванович; Петров Пётр Петрович",
+                             "Квартиль scopus": "Q1"})
+        writer.writerow({"Название публикации": "Article 0", "Год": 2024, "DOI": "10.1234/0",
+                         "UT (WoS)": "ut-0", "Авторы НИУ ВШЭ": "Иванов Иван Иванович",
+                         "Квартиль wos": "Q2"})
+    monkeypatch.setattr(council, "EXPORT", export)
+    council._articles_by_author.cache_clear()
+    db_session.add_all([
+        Employee(profile_key="doctor", canonical_url="https://example.org/1", full_name="Иванов Иван Иванович",
+                 status="active", current_data={"sections": [{"title": "Учёные степени",
+                 "items": ["Доктор технических наук"]}]}),
+        Employee(profile_key="candidate", canonical_url="https://example.org/2", full_name="Петров Пётр Петрович",
+                 status="active", current_data={"sections": [{"title": "Учёные степени",
+                 "items": ["Кандидат технических наук"]}]}),
+    ])
+    db_session.commit()
+    result = council.council_candidates(db_session, date(2026, 9, 30))
+    assert result["total"] == 1
+    assert result["candidates"][0]["count"] == 10
+    assert len(result["candidates"][0]["articles"][0]["sources"]) == 2
+    assert council.period(date(2024, 2, 29))[0] == date(2019, 2, 28)
+    settings = Settings(_env_file=None, session_secret="test-session-secret")
+    monkeypatch.setattr("app.admin.council_candidates", lambda db: council.council_candidates(db, date(2026, 9, 30)))
+    app.dependency_overrides[get_db] = lambda: db_session
+    app.dependency_overrides[get_settings] = lambda: settings
+    try:
+        client = TestClient(app)
+        client.cookies.set(SESSION_COOKIE, sign_session(settings.admin_username, settings))
+        response = client.get("/admin/council")
+        assert response.status_code == 200
+        assert "Иванов Иван Иванович" in response.text
+        assert "Article 0" in response.text
+        assert "публикаций по выгрузке" in response.text
+    finally:
+        app.dependency_overrides.clear()
+    council._articles_by_author.cache_clear()
+
+
+def test_council_page_requires_admin_and_explains_preliminary_result(db_session, monkeypatch):
+    monkeypatch.setattr(council, "_articles_by_author", lambda: {})
+    settings = Settings(_env_file=None, session_secret="test-session-secret")
+    app.dependency_overrides[get_db] = lambda: db_session
+    app.dependency_overrides[get_settings] = lambda: settings
+    try:
+        client = TestClient(app)
+        assert client.get("/admin/council", follow_redirects=False).status_code in (302, 303, 401, 403)
+        client.cookies.set(SESSION_COOKIE, sign_session(settings.admin_username, settings))
+        response = client.get("/admin/council")
+        assert response.status_code == 200
+        assert "Предварительный список" in response.text
+        assert "К1 или RSCI могут отсутствовать" in response.text
+    finally:
+        app.dependency_overrides.clear()

tests/test_crawler.py

@@ -1,113 +1,690 @@
-from datetime import datetime, timezone
-
-from app.models import CrawlRun, CrawlRunEmployeeChange, Employee
-from app.services.crawler import _mark_dismissed, _upsert_employee
+import gzip
+import pytest
+from datetime import datetime, timezone
+
+from app.models import (
+    CrawlError,
+    CrawlRun,
+    CrawlRunEmployeeChange,
+    Employee,
+    EmployeeNewsLink,
+    EmployeePublication,
+    EmployeeSnapshot,
+    ParseResourceCache,
+)
+from app.config import Settings
+from app.services.crawler import _checksum, _mark_dismissed, _upsert_employee, refresh_dismissed_status
+from app.services.resource_cache import ResourceCache
+
+
+class FakeResponse:
+    def __init__(self, status_code):
+        self.status_code = status_code
+
+
+class FakeSession:
+    def __init__(self, statuses):
+        self.statuses = statuses
+
+    def get(self, url, **_kwargs):
+        return FakeResponse(self.statuses[url])
+
+
+class ConditionalResponse:
+    def __init__(self, status_code, text="", headers=None):
+        self.status_code = status_code
+        self._text = text
+        self.headers = headers or {}
+        self.text_read = False
+
+    @property
+    def text(self):
+        self.text_read = True
+        return self._text
+
+    def raise_for_status(self):
+        return None
+
+
+class ConditionalSession:
+    def __init__(self):
+        self.requests = []
+        self.not_modified_response = ConditionalResponse(304)
+
+    def get(self, url, **kwargs):
+        self.requests.append((url, kwargs))
+        if kwargs["headers"].get("If-None-Match") == '"cached"':
+            return self.not_modified_response
+        return ConditionalResponse(200, "fresh", {"ETag": '"fresh"'})
 
 
-class FakeResponse:
-    def __init__(self, status_code):
-        self.status_code = status_code
-
-
-class FakeSession:
-    def __init__(self, statuses):
-        self.statuses = statuses
-
-    def get(self, url, **_kwargs):
-        return FakeResponse(self.statuses[url])
-
-
-def test_mark_dismissed_records_missing_source_when_profile_is_available(db_session):
-    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
-    db_session.add(run)
-    db_session.add(
-        Employee(
-            profile_key="staff:kept",
-            canonical_url="https://www.hse.ru/staff/kept",
-            status="active",
-            first_seen_at=datetime.now(timezone.utc),
-            last_seen_at=datetime.now(timezone.utc),
-        )
+def test_refresh_dismissed_status_reactivates_only_profiles_in_source(monkeypatch, db_session):
+    now = datetime.now(timezone.utc)
+    found = Employee(
+        profile_key="staff:returned",
+        canonical_url="https://www.hse.ru/staff/returned",
+        status="dismissed",
+        dismissed_at=now,
+        first_seen_at=now,
+        last_seen_at=now,
     )
-    db_session.add(
-        Employee(
-            profile_key="staff:missing",
-            canonical_url="https://www.hse.ru/staff/missing",
-            status="active",
-            first_seen_at=datetime.now(timezone.utc),
-            last_seen_at=datetime.now(timezone.utc),
-        )
-    )
-    db_session.commit()
-
-    dismissed = _mark_dismissed(
-        db_session,
-        run,
-        {"staff:kept"},
-        FakeSession({"https://www.hse.ru/staff/missing": 200}),
-        30,
-    )
-
-    assert dismissed == 0
-    assert db_session.query(Employee).filter_by(profile_key="staff:kept").one().status == "active"
-    missing = db_session.query(Employee).filter_by(profile_key="staff:missing").one()
-    assert missing.status == "active"
-    assert missing.dismissed_at is None
-    change = db_session.query(CrawlRunEmployeeChange).one()
-    assert change.change_type == "missing_from_source"
-    assert change.profile_available is True
-
-
-def test_mark_dismissed_marks_missing_employee_when_profile_is_unavailable(db_session):
-    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
-    employee = Employee(
+    still_dismissed = Employee(
         profile_key="staff:gone",
         canonical_url="https://www.hse.ru/staff/gone",
-        status="active",
-        first_seen_at=datetime.now(timezone.utc),
-        last_seen_at=datetime.now(timezone.utc),
+        status="dismissed",
+        dismissed_at=now,
+        first_seen_at=now,
+        last_seen_at=now,
     )
-    db_session.add_all([run, employee])
+    db_session.add_all([found, still_dismissed])
     db_session.commit()
-
-    dismissed = _mark_dismissed(
-        db_session,
-        run,
-        set(),
-        FakeSession({"https://www.hse.ru/staff/gone": 404}),
-        30,
+    monkeypatch.setattr(
+        "app.services.crawler.collect_profile_links",
+        lambda *_args, **_kwargs: ["https://www.hse.ru/staff/returned"],
     )
 
-    assert dismissed == 1
-    assert employee.status == "dismissed"
-    assert employee.dismissed_at is not None
-    change = db_session.query(CrawlRunEmployeeChange).one()
-    assert change.change_type == "dismissed"
-    assert change.profile_available is False
+    run = refresh_dismissed_status(db_session, Settings())
+
+    assert run.status == "completed"
+    assert run.parsed_count == 1
+    assert run.skipped_count == 1
+    assert found.status == "active"
+    assert found.dismissed_at is None
+    assert still_dismissed.status == "dismissed"
+
+
+def test_mark_dismissed_records_missing_source_when_profile_is_available(db_session):
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    db_session.add(run)
+    db_session.add(
+        Employee(
+            profile_key="staff:kept",
+            canonical_url="https://www.hse.ru/staff/kept",
+            status="active",
+            first_seen_at=datetime.now(timezone.utc),
+            last_seen_at=datetime.now(timezone.utc),
+        )
+    )
+    db_session.add(
+        Employee(
+            profile_key="staff:missing",
+            canonical_url="https://www.hse.ru/staff/missing",
+            status="active",
+            first_seen_at=datetime.now(timezone.utc),
+            last_seen_at=datetime.now(timezone.utc),
+        )
+    )
+    db_session.commit()
+
+    dismissed = _mark_dismissed(
+        db_session,
+        run,
+        {"staff:kept"},
+        FakeSession({"https://www.hse.ru/staff/missing": 200}),
+        30,
+    )
+
+    assert dismissed == 0
+    assert db_session.query(Employee).filter_by(profile_key="staff:kept").one().status == "active"
+    missing = db_session.query(Employee).filter_by(profile_key="staff:missing").one()
+    assert missing.status == "active"
+    assert missing.dismissed_at is None
+    change = db_session.query(CrawlRunEmployeeChange).one()
+    assert change.change_type == "missing_from_source"
+    assert change.profile_available is True
+
+
+def test_mark_dismissed_requires_consecutive_unavailable_checks(db_session):
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    employee = Employee(
+        profile_key="staff:gone",
+        canonical_url="https://www.hse.ru/staff/gone",
+        status="active",
+        first_seen_at=datetime.now(timezone.utc),
+        last_seen_at=datetime.now(timezone.utc),
+    )
+    db_session.add_all([run, employee])
+    db_session.commit()
+
+    first_check = _mark_dismissed(
+        db_session,
+        run,
+        set(),
+        FakeSession({"https://www.hse.ru/staff/gone": 404}),
+        30,
+        confirmation_runs=2,
+    )
+
+    assert first_check == 0
+    assert employee.status == "verification_required"
+    assert employee.dismissed_at is None
+    assert employee.profile_unavailable_streak == 1
+    assert db_session.query(CrawlRunEmployeeChange).one().change_type == "verification_required"
+
+    second_check = _mark_dismissed(
+        db_session,
+        run,
+        set(),
+        FakeSession({"https://www.hse.ru/staff/gone": 404}),
+        30,
+        confirmation_runs=2,
+    )
+
+    assert second_check == 1
+    assert employee.status == "dismissed"
+    assert employee.dismissed_at is not None
+    assert employee.profile_unavailable_streak == 2
+    change = db_session.query(CrawlRunEmployeeChange).order_by(CrawlRunEmployeeChange.id).all()[-1]
+    assert change.change_type == "dismissed"
+    assert change.profile_available is False
+
+
+def test_mark_dismissed_does_not_dismiss_on_server_error(db_session):
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    employee = Employee(
+        profile_key="staff:temporary-error",
+        canonical_url="https://www.hse.ru/staff/temporary-error",
+        status="active",
+        first_seen_at=datetime.now(timezone.utc),
+        last_seen_at=datetime.now(timezone.utc),
+    )
+    db_session.add_all([run, employee])
+    db_session.commit()
+
+    dismissed = _mark_dismissed(
+        db_session,
+        run,
+        set(),
+        FakeSession({"https://www.hse.ru/staff/temporary-error": 503}),
+        30,
+        confirmation_runs=1,
+    )
+
+    assert dismissed == 0
+    assert employee.status == "active"
+    assert employee.profile_unavailable_streak == 0
+    assert db_session.query(CrawlError).one().error_type == "ProfileAvailabilityCheckError"
+
+
+def test_available_profile_resets_verification_streak(db_session):
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    employee = Employee(
+        profile_key="staff:restored",
+        canonical_url="https://www.hse.ru/staff/restored",
+        status="active",
+        first_seen_at=datetime.now(timezone.utc),
+        last_seen_at=datetime.now(timezone.utc),
+    )
+    db_session.add_all([run, employee])
+    db_session.commit()
+
+    _mark_dismissed(
+        db_session,
+        run,
+        set(),
+        FakeSession({"https://www.hse.ru/staff/restored": 404}),
+        30,
+        confirmation_runs=3,
+    )
+    _mark_dismissed(
+        db_session,
+        run,
+        set(),
+        FakeSession({"https://www.hse.ru/staff/restored": 200}),
+        30,
+        confirmation_runs=3,
+    )
+
+    assert employee.status == "active"
+    assert employee.profile_unavailable_streak == 0
+
+
+def test_mark_dismissed_blocks_mass_auto_dismissals(db_session):
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    employees = [
+        Employee(
+            profile_key=f"staff:gone-{index}",
+            canonical_url=f"https://www.hse.ru/staff/gone-{index}",
+            status="active",
+            first_seen_at=datetime.now(timezone.utc),
+            last_seen_at=datetime.now(timezone.utc),
+        )
+        for index in range(2)
+    ]
+    db_session.add_all([run, *employees])
+    db_session.commit()
+
+    dismissed = _mark_dismissed(
+        db_session,
+        run,
+        set(),
+        FakeSession({employee.canonical_url: 404 for employee in employees}),
+        30,
+        confirmation_runs=1,
+        max_auto_dismissals=1,
+    )
+
+    assert dismissed == 0
+    assert {employee.status for employee in employees} == {"verification_required"}
+    assert "приостановлено" in run.message
+
+
+def test_upsert_employee_increments_new_count_and_records_change_for_new_employee(db_session):
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    db_session.add(run)
+    db_session.commit()
+
+    employee, _ = _upsert_employee(
+        db_session,
+        run,
+        {
+            "source_url": "https://www.hse.ru/staff/newperson",
+            "profile_type": "staff",
+            "profile_id": "newperson",
+            "full_name": "New Person",
+            "tabs": [],
+            "sections": [
+                {
+                    "title": "Образование и учёные степени",
+                    "year_entries": [{"text": "Кандидат технических наук"}],
+                }
+            ],
+            "parser_version": "0.2.0",
+            "_html": "<html></html>",
+        },
+    )
+    db_session.commit()
+
+    assert run.new_count == 1
+    assert employee.has_academic_degree is True
+    change = db_session.query(CrawlRunEmployeeChange).one()
+    assert change.change_type == "new"
+    assert change.full_name == "New Person"
+
+
+def test_upsert_employee_reconciles_profile_moved_to_new_url(db_session):
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    employee = Employee(
+        profile_key="staff:abelov",
+        canonical_url="https://www.hse.ru/staff/abelov",
+        full_name="Белов Александр Владимирович",
+        status="active",
+        first_seen_at=datetime.now(timezone.utc),
+        last_seen_at=datetime.now(timezone.utc),
+    )
+    db_session.add_all([run, employee])
+    db_session.commit()
+    employee_id = employee.id
+
+    updated, changed = _upsert_employee(
+        db_session,
+        run,
+        {
+            "source_url": "https://www.hse.ru/org/persons/47634735",
+            "profile_type": "org_person",
+            "profile_id": "47634735",
+            "full_name": "Белов Александр Владимирович",
+            "tabs": [],
+            "sections": [],
+            "parser_version": "0.7.0",
+            "_html": "<html></html>",
+        },
+    )
+    db_session.commit()
+
+    assert changed is True
+    assert updated.id == employee_id
+    assert updated.profile_key == "org_person:47634735"
+    assert updated.canonical_url == "https://www.hse.ru/org/persons/47634735"
+    assert updated.status == "active"
+    assert run.new_count == 0
+    assert db_session.query(Employee).count() == 1
+    assert {item.url for item in updated.profile_urls} == {
+        "https://www.hse.ru/staff/abelov",
+        "https://www.hse.ru/org/persons/47634735",
+    }
+
+
+def test_resource_cache_uses_etag_and_reuses_cached_body_on_304(db_session):
+    db_session.add(
+        ParseResourceCache(
+            profile_key="staff:cached",
+            resource_key="main-html",
+            method="GET",
+            url="https://www.hse.ru/staff/cached",
+            request_fingerprint="020d59db7b358d9023d0f185bcbf5a9c085d3cf2bf91d92d48eee9147e8d0f01",
+            etag='"cached"',
+            body_hash="cached-hash",
+            body_snapshot=gzip.compress("cached body".encode("utf-8")),
+            parser_version="0.6.0",
+        )
+    )
+    db_session.commit()
+    session = ConditionalSession()
+
+    result = ResourceCache(db_session).fetch_text(
+        session,
+        profile_key="staff:cached",
+        resource_key="main-html",
+        method="GET",
+        url="https://www.hse.ru/staff/cached",
+        headers={"User-Agent": "test"},
+        timeout=10,
+    )
+
+    assert session.requests[0][1]["headers"]["If-None-Match"] == '"cached"'
+    assert result.text == "cached body"
+    assert result.from_cache is True
+    assert session.not_modified_response.text_read is False
+
+
+def test_upsert_employee_skips_snapshot_when_checksum_is_unchanged(db_session):
+    first_run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    second_run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    db_session.add_all([first_run, second_run])
+    db_session.commit()
+
+    _, first_changed = _upsert_employee(db_session, first_run, _parsed_employee("same"))
+    _, second_changed = _upsert_employee(db_session, second_run, _parsed_employee("same"))
+    db_session.commit()
+
+    assert first_changed is True
+    assert second_changed is False
+    assert db_session.query(EmployeeSnapshot).count() == 1
+
+
+def test_upsert_employee_saves_publications_and_reuses_existing_rows(db_session):
+    first_run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    second_run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    db_session.add_all([first_run, second_run])
+    db_session.commit()
+
+    parsed = _parsed_employee("published")
+    parsed["sections"] = [
+        {
+            "type": "publications",
+            "publications": [
+                {
+                    "id": "888959076",
+                    "publication_id": "888959076",
+                    "title": "Detailed Publication",
+                    "year": 2023,
+                    "publication_type": "ARTICLE",
+                    "language": "ru",
+                    "status": 1,
+                    "url": "https://publications.hse.ru/view/888959076",
+                    "doi_url": "https://doi.org/10.1/test",
+                    "citation_text": "Detailed citation",
+                    "annotation": {"ru": "Аннотация"},
+                    "description": {"main": "Detailed citation"},
+                    "authors": [{"id": "1", "title_ru": "Автор"}],
+                    "raw_data": {"id": "888959076", "title": "Detailed Publication"},
+                }
+            ],
+        }
+    ]
+
+    employee, _ = _upsert_employee(db_session, first_run, parsed)
+    db_session.commit()
+    _upsert_employee(db_session, second_run, _parsed_employee_with_publication("published"))
+    db_session.commit()
+
+    publications = db_session.query(EmployeePublication).filter_by(employee_id=employee.id).all()
+    assert len(publications) == 1
+    assert publications[0].doi_url == "https://doi.org/10.1/test"
+    assert publications[0].authors == [{"id": "1", "title_ru": "Автор"}]
+
+
+def test_upsert_employee_records_publication_errors_without_failing_employee(monkeypatch, db_session):
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    db_session.add(run)
+    db_session.commit()
+
+    def broken_sync(*_args, **_kwargs):
+        raise RuntimeError("boom")
+
+    monkeypatch.setattr("app.services.crawler._sync_employee_publications", broken_sync)
+
+    employee, changed = _upsert_employee(db_session, run, _parsed_employee_with_publication("error-safe"))
+    db_session.commit()
+
+    assert changed is True
+    assert employee.full_name == "Same Person"
+    assert db_session.query(Employee).filter_by(profile_key="staff:error-safe").one()
+    error = db_session.query(CrawlError).one()
+    assert "публикации" in error.message.lower()
+
+
+def test_upsert_employee_saves_news_links_and_reuses_existing_rows(db_session):
+    first_run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    second_run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    db_session.add_all([first_run, second_run])
+    db_session.commit()
+
+    employee, _ = _upsert_employee(db_session, first_run, _parsed_employee_with_news("news-person"))
+    db_session.commit()
+    _upsert_employee(db_session, second_run, _parsed_employee_with_news("news-person"))
+    db_session.commit()
+
+    news_links = db_session.query(EmployeeNewsLink).filter_by(employee_id=employee.id).all()
+    assert len(news_links) == 1
+    assert news_links[0].title == "News Title"
+    assert news_links[0].url == "https://www.hse.ru/news/1.html"
+    assert news_links[0].published_year == 2026
+
+
+def test_upsert_employee_records_news_errors_without_failing_employee(monkeypatch, db_session):
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    db_session.add(run)
+    db_session.commit()
+
+    def broken_sync(*_args, **_kwargs):
+        raise RuntimeError("boom")
+
+    monkeypatch.setattr("app.services.crawler._sync_employee_news_links", broken_sync)
+
+    employee, changed = _upsert_employee(db_session, run, _parsed_employee_with_news("news-error-safe"))
+    db_session.commit()
+
+    assert changed is True
+    assert employee.full_name == "Same Person"
+    assert db_session.query(Employee).filter_by(profile_key="staff:news-error-safe").one()
+    error = db_session.query(CrawlError).one()
+    assert "новости" in error.message.lower()
+
+
+def test_checksum_changes_when_widget_data_changes():
+    base = _parsed_employee("widgets")
+    changed = _parsed_employee("widgets")
+    changed["sections"] = [
+        {
+            "type": "publications",
+            "publications": [{"id": "1", "title": "New publication"}],
+        }
+    ]
+
+    assert _checksum(base) != _checksum(changed)
+
+
+def test_checksum_ignores_date_dependent_experience_text():
+    first = _parsed_employee("experience")
+    second = _parsed_employee("experience")
+    first["sections"] = [{"raw_text": "Стаж работы в НИУ ВШЭ: 5 лет"}]
+    second["sections"] = [{"raw_text": "Стаж работы в НИУ ВШЭ: 6 лет"}]
+
+    assert _checksum(first) == _checksum(second)
+
+
+def _parsed_employee(profile_id: str) -> dict:
+    return {
+        "source_url": f"https://www.hse.ru/staff/{profile_id}",
+        "profile_type": "staff",
+        "profile_id": profile_id,
+        "full_name": "Same Person",
+        "tabs": [],
+        "sections": [],
+        "parser_version": "0.6.0",
+        "_html": "<html></html>",
+    }
+
+
+def _parsed_employee_with_publication(profile_id: str) -> dict:
+    parsed = _parsed_employee(profile_id)
+    parsed["sections"] = [
+        {
+            "type": "publications",
+            "publications": [
+                {
+                    "id": "888959076",
+                    "publication_id": "888959076",
+                    "title": "Detailed Publication",
+                    "year": 2023,
+                    "publication_type": "ARTICLE",
+                    "language": "ru",
+                    "status": 1,
+                    "url": "https://publications.hse.ru/view/888959076",
+                    "doi_url": "https://doi.org/10.1/test",
+                    "citation_text": "Detailed citation",
+                    "annotation": {"ru": "Аннотация"},
+                    "description": {"main": "Detailed citation"},
+                    "authors": [{"id": "1", "title_ru": "Автор"}],
+                    "raw_data": {"id": "888959076", "title": "Detailed Publication"},
+                }
+            ],
+        }
+    ]
+    return parsed
+
+
+def _parsed_employee_with_news(profile_id: str) -> dict:
+    parsed = _parsed_employee(profile_id)
+    parsed["sections"] = [
+        {
+            "type": "news",
+            "news_links": [
+                {
+                    "title": "News Title",
+                    "url": "https://www.hse.ru/news/1.html",
+                    "summary": "News summary",
+                    "published_at": "2026-04-28T00:00:00+00:00",
+                    "published_year": 2026,
+                    "raw_data": {"title": "News Title", "url": "https://www.hse.ru/news/1.html"},
+                }
+            ],
+        }
+    ]
+    return parsed
 
 
-def test_upsert_employee_increments_new_count_and_records_change_for_new_employee(db_session):
+def test_csv_enrichment_survives_repeated_crawl_without_duplicate(db_session):
+    from scripts.import_publications import plan
+    from app.services.crawler import _sync_employee_publications
+
     run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
     db_session.add(run)
     db_session.commit()
-
-    _upsert_employee(
-        db_session,
-        run,
-        {
-            "source_url": "https://www.hse.ru/staff/newperson",
-            "profile_type": "staff",
-            "profile_id": "newperson",
-            "full_name": "New Person",
-            "tabs": [],
-            "sections": [],
-            "parser_version": "0.2.0",
-            "_html": "<html></html>",
-        },
-    )
+    employee, _ = _upsert_employee(db_session, run, _parsed_employee_with_publication("csv-preserved"))
     db_session.commit()
+    existing = db_session.query(EmployeePublication).filter_by(employee_id=employee.id).one()
+    rows = [(number, {"Название публикации": title, "Год": str(year), "DOI": identifier,
+                      "Авторы НИУ ВШЭ": employee.full_name, "Квартиль scopus": "Q1"})
+            for number, title, year, identifier in (
+                (2, existing.title, existing.year, ""),
+                (3, "CSV only", 2024, "10.1234/csv-only"),
+                (4, "Later HSE", 2024, "10.1234/later"))]
+    actions, _ = plan(rows, [{"id": employee.id, "full_name": employee.full_name}],
+                      [{field: getattr(existing, field) for field in
+                        ("id", "employee_id", "title", "year", "doi_url", "raw_data")}])
+    for action in actions:
+        if action["result"] == "insert":
+            db_session.add(EmployeePublication(**action["values"]))
+        else:
+            for field, value in action["values"].items():
+                setattr(existing, field, value)
+    db_session.add(EmployeePublication(employee_id=employee.id, title="Obsolete HSE",
+                                       source_hash="obsolete", raw_data={"id": "obsolete"}))
+    db_session.commit()
+    original_id = existing.id
+    imported_ids = {p.title: p.id for p in db_session.query(EmployeePublication).all()}
 
-    assert run.new_count == 1
-    change = db_session.query(CrawlRunEmployeeChange).one()
-    assert change.change_type == "new"
-    assert change.full_name == "New Person"
+    parsed = _parsed_employee_with_publication("csv-preserved")
+    publication = parsed["sections"][0]["publications"][0]
+    publication.pop("year")
+    publication.pop("doi_url")
+    publication["raw_data"]["fresh"] = True
+    for _ in range(2):
+        _sync_employee_publications(db_session, employee, parsed)
+        db_session.commit()
+    remaining = db_session.query(EmployeePublication).filter_by(employee_id=employee.id).all()
+    assert len(remaining) == 3
+    assert all(p.raw_data["csv_enrichment"]["Квартиль scopus"] == "Q1" for p in remaining)
+    refreshed = db_session.get(EmployeePublication, original_id)
+    assert refreshed.raw_data["fresh"] is True
+    assert refreshed.year == 2023 and refreshed.doi_url == "https://doi.org/10.1/test"
+    assert db_session.get(EmployeePublication, imported_ids["CSV only"]) is not None
+
+    parsed["sections"][0]["publications"].append({
+        "publication_id": "later-id", "title": "Later HSE", "year": 2024,
+        "doi_url": "http://dx.doi.org/10.1234/LATER", "raw_data": {"id": "later-id"}})
+    for _ in range(2):
+        _sync_employee_publications(db_session, employee, parsed)
+        db_session.commit()
+    assert db_session.query(EmployeePublication).filter_by(employee_id=employee.id).count() == 3
+    adopted = db_session.get(EmployeePublication, imported_ids["Later HSE"])
+    assert adopted.publication_id == "later-id"
+    assert adopted.raw_data["csv_enrichment"]["Квартиль scopus"] == "Q1"
+
+
+def test_ambiguous_csv_match_rolls_back_publication_sync(db_session):
+    from app.services.crawler import _try_sync_employee_publications
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    db_session.add(run)
+    db_session.commit()
+    employee, _ = _upsert_employee(db_session, run, _parsed_employee("csv-ambiguous"))
+    for index in range(2):
+        db_session.add(EmployeePublication(employee_id=employee.id, title="Same", year=2024,
+            doi_url="https://doi.org/10.1234/same", source_hash=f"csv-{index}",
+            raw_data={"csv_enrichment": {"Год": "2024"}}))
+    db_session.commit()
+    parsed = {"sections": [{"type": "publications", "publications": [
+        {"publication_id": "new", "title": "New"},
+        {"publication_id": "same", "title": "Same", "doi_url": "https://doi.org/10.1234/same", "year": 2024}]}]}
+    _try_sync_employee_publications(db_session, run, employee, parsed)
+    db_session.commit()
+    assert db_session.query(EmployeePublication).filter_by(employee_id=employee.id).count() == 2
+    assert db_session.query(CrawlError).filter_by(crawl_run_id=run.id, error_type="ValueError").count() == 1
+
+
+@pytest.mark.parametrize("bound_id", [None, "hse-A"])
+@pytest.mark.parametrize("incoming_ids", [("hse-A", "hse-B"), ("hse-B", "hse-A")])
+def test_two_hse_ids_cannot_overwrite_one_csv_record(db_session, bound_id, incoming_ids):
+    from app.services.crawler import _try_sync_employee_publications
+    from app.services.admin_data import run_detail_payload
+    run = CrawlRun(source_url="https://miem.hse.ru/persons", status="running")
+    db_session.add(run)
+    db_session.commit()
+    employee, _ = _upsert_employee(db_session, run, _parsed_employee("csv-hse-id-conflict"))
+    original_raw = {"csv_enrichment": {"Год": "2024"}, "original": "keep"}
+    original_authors = [{"title_ru": "Synthetic original author"}]
+    record = EmployeePublication(employee_id=employee.id, title="Same", year=2024,
+        publication_id=bound_id, source_hash="csv-original", raw_data=original_raw,
+        authors=original_authors)
+    db_session.add(record)
+    db_session.commit()
+    original_id = record.id
+    parsed = {"sections": [{"type": "publications", "publications": [
+        {"publication_id": identifier, "title": "Same", "year": 2024,
+         "authors": [{"id": identifier}], "raw_data": {"id": identifier}}
+        for identifier in incoming_ids]}]}
+    _try_sync_employee_publications(db_session, run, employee, parsed)
+    db_session.commit()
+    db_session.expire_all()
+    kept = db_session.get(EmployeePublication, original_id)
+    assert db_session.query(EmployeePublication).filter_by(employee_id=employee.id).count() == 1
+    assert kept.publication_id == bound_id and kept.source_hash == "csv-original"
+    assert kept.raw_data == original_raw and kept.authors == original_authors
+    errors = run_detail_payload(db_session, run)["errors"]
+    assert len(errors) == 1
+    assert errors[0]["error_type"] == "ValueError"
+    assert "Ambiguous HSE IDs" in errors[0]["message"]
+    assert "hse-A" in errors[0]["message"] and "hse-B" in errors[0]["message"]

tests/test_dataset_versions.py

@@ -1,88 +0,0 @@
-from datetime import datetime, timezone
-
-from app.models import Employee
-from app.services.dataset_versions import get_or_create_current_version, sync_employees_payload
-
-
-def _employee(profile_key: str, checksum: str, *, status: str = "active") -> Employee:
-    return Employee(
-        profile_key=profile_key,
-        profile_type=profile_key.split(":", 1)[0],
-        profile_id=profile_key.split(":", 1)[1],
-        canonical_url=f"https://www.hse.ru/{profile_key}",
-        full_name=profile_key,
-        status=status,
-        first_seen_at=datetime.now(timezone.utc),
-        last_seen_at=datetime.now(timezone.utc),
-        current_data={"profile_key": profile_key},
-        current_checksum=checksum,
-    )
-
-
-def test_dataset_version_hash_is_stable_for_same_employee_state(db_session):
-    db_session.add(_employee("staff:alpha", "a" * 64))
-    db_session.commit()
-
-    first = get_or_create_current_version(db_session)
-    db_session.commit()
-    second = get_or_create_current_version(db_session)
-
-    assert second.id == first.id
-    assert second.hash == first.hash
-    assert second.employee_count == 1
-
-
-def test_dataset_version_hash_changes_when_employee_checksum_changes(db_session):
-    employee = _employee("staff:alpha", "a" * 64)
-    db_session.add(employee)
-    db_session.commit()
-    first = get_or_create_current_version(db_session)
-    db_session.commit()
-
-    employee.current_checksum = "b" * 64
-    db_session.commit()
-    second = get_or_create_current_version(db_session)
-
-    assert second.hash != first.hash
-    assert second.previous_hash == first.hash
-
-
-def test_sync_employees_diff_spans_multiple_intermediate_versions(db_session):
-    alpha = _employee("staff:alpha", "a" * 64)
-    db_session.add(alpha)
-    db_session.commit()
-    first = get_or_create_current_version(db_session)
-    db_session.commit()
-
-    beta = _employee("staff:beta", "b" * 64)
-    db_session.add(beta)
-    db_session.commit()
-    get_or_create_current_version(db_session)
-    db_session.commit()
-
-    alpha.current_checksum = "c" * 64
-    alpha.current_data = {"profile_key": "staff:alpha", "changed": True}
-    db_session.commit()
-
-    payload = sync_employees_payload(db_session, client_hash=first.hash, include_data=False)
-
-    assert payload["mode"] == "delta"
-    assert [item["profile_key"] for item in payload["changes"]["added"]] == ["staff:beta"]
-    assert [item["profile_key"] for item in payload["changes"]["updated"]] == ["staff:alpha"]
-    assert payload["changes"]["dismissed"] == []
-    assert payload["changes"]["removed"] == []
-
-
-def test_sync_employees_reports_dismissed_as_tombstone(db_session):
-    alpha = _employee("staff:alpha", "a" * 64)
-    db_session.add(alpha)
-    db_session.commit()
-    first = get_or_create_current_version(db_session)
-    db_session.commit()
-
-    alpha.status = "dismissed"
-    db_session.commit()
-    payload = sync_employees_payload(db_session, client_hash=first.hash, include_data=False)
-
-    assert payload["changes"]["dismissed"][0]["profile_key"] == "staff:alpha"
-    assert payload["changes"]["dismissed"][0]["status"] == "dismissed"

tests/test_db_schema.py

@@ -0,0 +1,144 @@
+from sqlalchemy import create_engine, inspect, text
+
+from app.db import _ensure_runtime_schema
+
+
+def test_runtime_schema_adds_skipped_count_to_existing_crawl_runs_table(monkeypatch):
+    engine = create_engine("sqlite:///:memory:")
+    with engine.begin() as connection:
+        connection.execute(
+            text(
+                """
+                CREATE TABLE crawl_runs (
+                  id INTEGER PRIMARY KEY,
+                  source_url TEXT NOT NULL,
+                  status VARCHAR(32) NOT NULL DEFAULT 'running',
+                  found_count INTEGER NOT NULL DEFAULT 0,
+                  parsed_count INTEGER NOT NULL DEFAULT 0
+                )
+                """
+            )
+        )
+    monkeypatch.setattr("app.db.engine", engine)
+
+    _ensure_runtime_schema()
+
+    columns = {column["name"] for column in inspect(engine).get_columns("crawl_runs")}
+    assert "skipped_count" in columns
+
+
+def test_runtime_schema_creates_employee_publications_table_when_employees_exist(monkeypatch):
+    engine = create_engine("sqlite:///:memory:")
+    with engine.begin() as connection:
+        connection.execute(
+            text(
+                """
+                CREATE TABLE employees (
+                  id INTEGER PRIMARY KEY,
+                  profile_key VARCHAR(255) NOT NULL UNIQUE,
+                  canonical_url TEXT NOT NULL,
+                  status VARCHAR(32) NOT NULL DEFAULT 'active',
+                  first_seen_at DATETIME NOT NULL,
+                  last_seen_at DATETIME NOT NULL,
+                  created_at DATETIME NOT NULL,
+                  updated_at DATETIME NOT NULL
+                )
+                """
+            )
+        )
+        connection.execute(
+            text(
+                """
+                CREATE TABLE crawl_runs (
+                  id INTEGER PRIMARY KEY,
+                  source_url TEXT NOT NULL,
+                  status VARCHAR(32) NOT NULL DEFAULT 'running',
+                  found_count INTEGER NOT NULL DEFAULT 0,
+                  parsed_count INTEGER NOT NULL DEFAULT 0,
+                  skipped_count INTEGER NOT NULL DEFAULT 0
+                )
+                """
+            )
+        )
+    monkeypatch.setattr("app.db.engine", engine)
+
+    _ensure_runtime_schema()
+    _ensure_runtime_schema()
+
+    inspector = inspect(engine)
+    assert "employee_publications" in inspector.get_table_names()
+    columns = {column["name"] for column in inspector.get_columns("employee_publications")}
+    assert {"employee_id", "publication_id", "doi_url", "authors", "raw_data", "source_hash"}.issubset(columns)
+
+
+def test_runtime_schema_creates_employee_news_links_table_when_employees_exist(monkeypatch):
+    engine = create_engine("sqlite:///:memory:")
+    with engine.begin() as connection:
+        connection.execute(
+            text(
+                """
+                CREATE TABLE employees (
+                  id INTEGER PRIMARY KEY,
+                  profile_key VARCHAR(255) NOT NULL UNIQUE,
+                  canonical_url TEXT NOT NULL,
+                  status VARCHAR(32) NOT NULL DEFAULT 'active',
+                  first_seen_at DATETIME NOT NULL,
+                  last_seen_at DATETIME NOT NULL,
+                  created_at DATETIME NOT NULL,
+                  updated_at DATETIME NOT NULL
+                )
+                """
+            )
+        )
+        connection.execute(
+            text(
+                """
+                CREATE TABLE crawl_runs (
+                  id INTEGER PRIMARY KEY,
+                  source_url TEXT NOT NULL,
+                  status VARCHAR(32) NOT NULL DEFAULT 'running',
+                  found_count INTEGER NOT NULL DEFAULT 0,
+                  parsed_count INTEGER NOT NULL DEFAULT 0,
+                  skipped_count INTEGER NOT NULL DEFAULT 0
+                )
+                """
+            )
+        )
+    monkeypatch.setattr("app.db.engine", engine)
+
+    _ensure_runtime_schema()
+    _ensure_runtime_schema()
+
+    inspector = inspect(engine)
+    assert "employee_news_links" in inspector.get_table_names()
+    columns = {column["name"] for column in inspector.get_columns("employee_news_links")}
+    assert {"employee_id", "title", "url", "summary", "published_at", "published_year", "source_hash", "raw_data"}.issubset(columns)
+
+
+def test_runtime_schema_adds_profile_verification_fields(monkeypatch):
+    engine = create_engine("sqlite:///:memory:")
+    with engine.begin() as connection:
+        connection.execute(
+            text(
+                """
+                CREATE TABLE employees (
+                  id INTEGER PRIMARY KEY,
+                  profile_key VARCHAR(255) NOT NULL UNIQUE,
+                  canonical_url TEXT NOT NULL,
+                  status VARCHAR(32) NOT NULL DEFAULT 'active',
+                  first_seen_at DATETIME NOT NULL,
+                  last_seen_at DATETIME NOT NULL,
+                  created_at DATETIME NOT NULL,
+                  updated_at DATETIME NOT NULL
+                )
+                """
+            )
+        )
+    monkeypatch.setattr("app.db.engine", engine)
+
+    _ensure_runtime_schema()
+
+    inspector = inspect(engine)
+    columns = {column["name"] for column in inspector.get_columns("employees")}
+    assert {"profile_unavailable_streak", "last_profile_check_at", "has_academic_degree"}.issubset(columns)
+    assert "employee_profile_urls" in inspector.get_table_names()

tests/test_employee_detail_template.py

@@ -13,6 +13,9 @@ def test_employee_detail_template_is_human_readable():
     assert "section.list_items" in template
     assert "Основная информация" in template
     assert "Контакты" in template
+    assert "В новостях" in template
+    assert "employee_view.news_links" in template
+    assert "news.summary" in template
     assert "Разделы профиля" in template
     assert "graduation_theses" in template
     assert "Год защиты" in template

tests/test_parser.py

@@ -34,7 +34,21 @@ class FakeSession:
                             "type": "ARTICLE",
                             "title": "Дублирование пакетов",
                             "year": 2023,
+                            "language": {"name": "ru"},
+                            "status": 1,
+                            "authorsByType": {
+                                "author": [
+                                    {
+                                        "id": "568398853",
+                                        "href": "/org/persons/568398853",
+                                        "title": {"ru": "Левицкий И. А.", "en": ""},
+                                        "reverseTitle": {"ru": "И. А. Левицкий", "en": ""},
+                                    }
+                                ]
+                            },
                             "description": {"short": {"ru": "Информационные процессы. 2023."}},
+                            "annotation": {"ru": "<p>Русская аннотация</p>"},
+                            "documents": {"DOI": {"href": "https://doi.org/10.1/test"}},
                         }
                     ],
                 },
@@ -153,6 +167,9 @@ def test_enrich_sections_from_hse_widgets_loads_publications_and_vkr():
 
     assert publications["publications_count"] == 1
     assert publications["publications"][0]["url"] == "https://publications.hse.ru/view/888959076"
+    assert publications["publications"][0]["doi_url"] == "https://doi.org/10.1/test"
+    assert publications["publications"][0]["annotation"] == {"ru": "Русская аннотация"}
+    assert publications["publications"][0]["authors"][0]["is_current_employee"] is True
     assert theses["theses_count"] == 1
     assert theses["theses"][0]["student"] == "Лесняк Владислав Евгеньевич"
     assert theses["theses"][0]["project_url"] == "https://www.hse.ru/edu/vkr/1045750164"
@@ -215,3 +232,45 @@ def test_news_heading_with_publications_word_does_not_absorb_widget_publications
     assert len(publications) == 1
     assert publications[0]["title"] == "Публикации и исследования"
     assert publications[0]["publications_count"] == 1
+
+
+def test_extract_sections_parses_employee_news_links():
+    soup = BeautifulSoup(
+        """
+        <div class="b-person-data posts hidden printable" data-tab="press_links_news" tab-node="press_links_news">
+          <div class="post f8">
+            <div class="post__extra">
+              <div class="post-meta">
+                <div class="post-meta__date">
+                  <div class="post-meta__day">28</div>
+                  <div class="post-meta__month">апр.</div>
+                  <div class="post-meta__year">2026</div>
+                </div>
+              </div>
+            </div>
+            <div class="post__content">
+              <h2 class="first_child"><a class="link" href="/news/edu/1153850518.html">Как финал ВсОШ формирует кадры</a></h2>
+              <div class="post__text"><p class="with-indent">Краткое описание новости.</p></div>
+            </div>
+          </div>
+          <div class="post f8">
+            <div class="post__content">
+              <h2><a href="https://miem.hse.ru/news/1123589375.html">Партнер магистратуры</a></h2>
+            </div>
+          </div>
+        </div>
+        """,
+        "html.parser",
+    )
+
+    sections = extract_sections(soup, "https://www.hse.ru/staff/avsergeev")
+
+    assert len(sections) == 1
+    news = sections[0]
+    assert news["type"] == "news"
+    assert news["news_count"] == 2
+    assert news["news_links"][0]["title"] == "Как финал ВсОШ формирует кадры"
+    assert news["news_links"][0]["url"] == "https://www.hse.ru/news/edu/1153850518.html"
+    assert news["news_links"][0]["summary"] == "Краткое описание новости."
+    assert news["news_links"][0]["published_at"] == "2026-04-28T00:00:00+00:00"
+    assert news["news_links"][0]["published_year"] == 2026

tests/test_publication_import.py

@@ -0,0 +1,233 @@
+"""Offline checks: synthetic identities only, no settings or database access."""
+from scripts.import_publications import plan, read_rows
+from pathlib import Path
+
+
+def row(title="Test", doi="10.1234/test", year="2024", authors="Test Person"):
+    return {"Название публикации": title, "DOI": doi, "Год": year,
+            "Авторы НИУ ВШЭ": authors}
+
+
+EMPLOYEES = [{"id": 1, "full_name": "Test Person"}]
+
+
+def test_repeat_and_no_new_employees():
+    rows = [(2, row()), (3, row()), (4, row(authors="Missing Person"))]
+    actions, audit = plan(rows, EMPLOYEES, [])
+    assert len(actions) == 1 and actions[0]["result"] == "insert"
+    assert [a["reason"] for a in audit] == ["", "csv_duplicate", "employee_not_found"]
+    stored = [{"id": 11, **actions[0]["values"]}]
+    repeat, _ = plan(rows, EMPLOYEES, stored)
+    assert repeat == []
+
+
+def test_ambiguity_and_conflicts():
+    assert not plan([(2, row())], EMPLOYEES * 2, [])[0]
+    stored = [{"id": 1, "employee_id": 1, "title": "Test", "year": 2024,
+               "doi_url": "https://doi.org/10.1234/other", "raw_data": {}}]
+    assert plan([(2, row())], EMPLOYEES, stored)[1][0]["reason"] == "doi_conflict"
+    stored[0]["doi_url"] = "https://doi.org/10.1234/test"
+    assert plan([(2, row(year="2025"))], EMPLOYEES, stored)[1][0]["reason"] == "year_conflict"
+    assert plan([(2, row())], EMPLOYEES, stored * 2)[0]  # same id is one identity
+    stored.append({**stored[0], "id": 2})
+    assert plan([(2, row())], EMPLOYEES, stored)[1][0]["reason"] == "ambiguous_publication"
+
+
+def test_fill_only_preserves_metadata_and_empty_values():
+    stored = [{"id": 1, "employee_id": 1, "title": "Test", "year": 2024,
+               "doi_url": None, "raw_data": {"hse": "keep", "csv_enrichment": {"extra": "keep"}}}]
+    actions, _ = plan([(2, {**row(), "extra": "", "new": "added"})], EMPLOYEES, stored)
+    values = actions[0]["values"]
+    assert set(values) == {"doi_url", "raw_data"}
+    assert values["raw_data"]["hse"] == "keep"
+    assert values["raw_data"]["csv_enrichment"]["extra"] == "keep"
+    assert values["raw_data"]["csv_enrichment"]["new"] == "added"
+    assert "new" not in stored[0]["raw_data"]["csv_enrichment"]
+
+
+def test_missing_doi_then_doi_does_not_update_synthetic_id():
+    actions, _ = plan([(2, row(doi="")), (3, row())], EMPLOYEES, [])
+    assert len(actions) == 1 and actions[0]["result"] == "insert"
+    assert actions[0]["values"]["doi_url"] == "https://doi.org/10.1234/test"
+
+
+def test_invalid_rows_and_owner_file():
+    assert not plan([(2, row(year="oops")), (3, row(doi="bad"))], EMPLOYEES, [])[0]
+    path = Path(__file__).resolve().parents[2] / "Статьи.csv"
+    if path.exists():
+        rows = read_rows(path)
+        assert len(rows) == 1797
+        actions, audit = plan(rows, [], [])
+        assert not actions and audit and all(a["result"] == "skip" for a in audit)
+
+
+def test_cli_read_only_and_atomic_failure(tmp_path, monkeypatch):
+    import sys
+    import psycopg
+    from scripts import import_publications
+
+    source = tmp_path / "input.csv"
+    source.write_text("Название публикации;Год;DOI;Авторы НИУ ВШЭ\nFirst;2024;10.1234/first;Test Person\nSecond;2024;10.1234/second;Test Person\n", encoding="utf-8")
+
+    class Result:
+        rowcount = 1
+
+        def __init__(self, rows):
+            self.rows = rows
+
+        def fetchall(self):
+            return self.rows
+
+        def fetchone(self):
+            return self.rows[0]
+
+    class Connection:
+        def __init__(self, fail=False):
+            self.queries, self.writes = [], []
+            self.fail, self.rolled_back, self.committed = fail, False, False
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, kind, *_):
+            if kind:
+                self.rollback()
+            elif not self.rolled_back:
+                self.committed = True
+
+        def rollback(self):
+            self.rolled_back = True
+            self.writes.clear()
+
+        def execute(self, query, params=None):
+            query = query if isinstance(query, str) else query.as_string()
+            self.queries.append(query)
+            if query.startswith("SELECT id, full_name"):
+                return Result(EMPLOYEES)
+            if query.startswith("SELECT id, employee_id"):
+                return Result([])
+            if "pg_try_advisory" in query:
+                return Result([{"locked": True}])
+            if query.startswith("INSERT"):
+                self.writes.append(params)
+                if self.fail and len(self.writes) == 2:
+                    raise RuntimeError("Synthetic second-write failure")
+            return Result([])
+
+    report = tmp_path / "dry.json"
+    dry = Connection()
+    monkeypatch.setattr(psycopg, "connect", lambda *a, **k: dry)
+    monkeypatch.setattr(sys, "argv", ["import_publications", str(source), "--report", str(report)])
+    import_publications.main()
+    assert dry.rolled_back and not dry.committed and not dry.writes
+    assert not any(q.startswith(("INSERT", "UPDATE", "DELETE")) for q in dry.queries)
+    assert "READ ONLY" in dry.queries[0]
+    assert report.exists()
+
+    failed = Connection(fail=True)
+    monkeypatch.setattr(psycopg, "connect", lambda *a, **k: failed)
+    report = tmp_path / "apply.json"
+    monkeypatch.setattr(sys, "argv", ["import_publications", str(source), "--report", str(report), "--apply"])
+    import pytest
+    with pytest.raises(RuntimeError):
+        import_publications.main()
+    assert failed.rolled_back and not failed.committed and not failed.writes
+    assert not report.exists()
+
+    success = Connection()
+    monkeypatch.setattr(psycopg, "connect", lambda *a, **k: success)
+    monkeypatch.setattr(sys, "argv", ["import_publications", str(source), "--report", str(tmp_path / "success.json"), "--apply"])
+    import_publications.main()
+    assert success.committed and len(success.writes) == 2
+    assert not any(q.startswith(("DELETE", "ALTER", "CREATE")) for q in success.queries)
+
+
+def test_report_cannot_overwrite_source(tmp_path, monkeypatch):
+    import sys
+    import pytest
+    from scripts import import_publications
+    source = tmp_path / "input.csv"
+    source.write_text("unchanged", encoding="utf-8")
+    monkeypatch.setattr(sys, "argv", ["import_publications", str(source), "--report", str(source)])
+    with pytest.raises(ValueError):
+        import_publications.main()
+    assert source.read_text() == "unchanged"
+
+
+def test_duplicate_rows_fill_late_metadata_and_report_conflicts():
+    rows = [(2, {**row(), "EID (Scopus)": "", "Квартиль scopus": ""}),
+            (3, {**row(), "EID (Scopus)": "new-id", "Квартиль scopus": "Q1"}),
+            (4, {**row(), "EID (Scopus)": "different-id", "Квартиль scopus": "Q2", "UT (WoS)": "late-wos"})]
+    actions, audit = plan(rows, EMPLOYEES, [])
+    assert len(actions) == 1 and actions[0]["result"] == "insert"
+    metadata = actions[0]["values"]["raw_data"]["csv_enrichment"]
+    assert metadata["EID (Scopus)"] == "new-id" and metadata["Квартиль scopus"] == "Q1"
+    assert metadata["UT (WoS)"] == "late-wos"
+    assert audit[1]["reason"] == "merged_csv_duplicate"
+    assert audit[2]["conflict_fields"] == ["EID (Scopus)", "Квартиль scopus"]
+    assert audit[2]["result"] == "skip"
+    repeated, repeated_audit = plan(rows, EMPLOYEES, [{"id": 11, **actions[0]["values"]}])
+    assert repeated == []
+    assert repeated_audit[2]["reason"] == "metadata_conflict_first_value_kept"
+    assert "different-id" not in str(audit)
+
+
+def test_cli_apply_year_only_preserves_existing_json(tmp_path, monkeypatch):
+    import json
+    import sys
+    import psycopg
+    from scripts import import_publications
+    source = tmp_path / "input.csv"
+    fields = row()
+    source.write_text(";".join(fields) + "\n" + ";".join(fields.values()) + "\n", encoding="utf-8")
+    original_raw = {"hse": "keep", "csv_enrichment": fields.copy()}
+    stored = {"id": 11, "employee_id": 1, "title": "Test", "year": None,
+              "doi_url": "https://doi.org/10.1234/test", "raw_data": original_raw}
+    actions, _ = plan([(2, fields)], EMPLOYEES, [stored])
+    assert actions[0]["values"] == {"year": 2024}
+
+    class Result:
+        rowcount = 1
+
+        def __init__(self, rows):
+            self.rows = rows
+
+        def fetchall(self):
+            return self.rows
+
+        def fetchone(self):
+            return self.rows[0]
+
+    class Connection:
+        committed = False
+        updates = []
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, error, *_):
+            self.committed = error is None
+
+        def execute(self, query, params=None):
+            query = query if isinstance(query, str) else query.as_string()
+            if query.startswith("SELECT id, full_name"):
+                return Result(EMPLOYEES)
+            if query.startswith("SELECT id, employee_id"):
+                return Result([stored])
+            if "pg_try_advisory" in query:
+                return Result([{"locked": True}])
+            if query.startswith("UPDATE"):
+                self.updates.append((query, params))
+            return Result([])
+
+    connection = Connection()
+    report = tmp_path / "report.json"
+    monkeypatch.setattr(psycopg, "connect", lambda *a, **k: connection)
+    monkeypatch.setattr(sys, "argv", ["import_publications", str(source), "--report", str(report), "--apply"])
+    import_publications.main()
+    assert connection.committed and len(connection.updates) == 1
+    query, params = connection.updates[0]
+    assert '"year" = %s' in query and "raw_data" not in query
+    assert params == [2024, 11]
+    assert stored["raw_data"] == original_raw and stored["year"] is None
+    assert json.loads(report.read_text())["summary"]["counts"] == {"insert": 0, "update": 1, "skip": 0}

tests/test_publication_import_postgres.py

@@ -0,0 +1,280 @@
+"""Real PostgreSQL checks; opt in only inside tests/postgres/compose.yml."""
+
+import copy
+import csv
+import json
+import os
+import subprocess
+import sys
+from contextlib import contextmanager
+from pathlib import Path
+
+import pytest
+import requests
+from sqlalchemy import create_engine, select, text
+from sqlalchemy.orm import Session
+
+from app.config import Settings
+from app.db import Base
+from app.models import CrawlError, CrawlRun, Employee, EmployeePublication
+from app.services.crawler import _try_sync_employee_publications, run_crawl
+
+
+ADMIN_URL = "postgresql+psycopg://test_admin:isolated_test_admin@postgres-test:5432/miem_import_test"
+pytestmark = pytest.mark.skipif(
+    os.environ.get("MIEM_POSTGRES_INTEGRATION") != "1", reason="Requires isolated PostgreSQL Compose project"
+)
+
+
+@pytest.fixture(autouse=True)
+def isolated_environment(monkeypatch):
+    assert os.environ.get("DATABASE_URL") == ADMIN_URL, "Use the dedicated test Compose project"
+
+    def forbidden_request(*args, **kwargs):
+        raise AssertionError("External HTTP requests are forbidden in PostgreSQL integration tests")
+
+    monkeypatch.setattr(requests.sessions.Session, "request", forbidden_request)
+
+
+@contextmanager
+def database(name):
+    assert name in {"miem_import_test", "miem_crawl_test"}
+    engine = create_engine(ADMIN_URL.replace("/miem_import_test", "/" + name))
+    try:
+        with engine.connect() as connection:
+            assert connection.scalar(text("SELECT current_database()")) == name
+        Base.metadata.create_all(engine)
+        with engine.begin() as connection:
+            connection.execute(text("GRANT USAGE ON SCHEMA public TO import_test"))
+            connection.execute(text("GRANT SELECT (id, full_name) ON public.employees TO import_test"))
+            connection.execute(text("GRANT SELECT (id, employee_id, title, year, doi_url, raw_data) ON public.employee_publications TO import_test"))
+            connection.execute(text("GRANT INSERT (employee_id, title, year, doi_url, raw_data, source_hash, created_at, updated_at) ON public.employee_publications TO import_test"))
+            connection.execute(text("GRANT UPDATE (title, year, doi_url, raw_data, updated_at) ON public.employee_publications TO import_test"))
+            # Created from the actual model: not an assumed sequence name.
+            sequence = connection.scalar(text("SELECT pg_get_serial_sequence('public.employee_publications', 'id')"))
+            assert sequence == "public.employee_publications_id_seq"
+            connection.execute(text("GRANT USAGE ON SEQUENCE public.employee_publications_id_seq TO import_test"))
+            permissions = connection.execute(text("""SELECT
+                has_schema_privilege('import_test', 'public', 'CREATE'),
+                has_table_privilege('import_test', 'public.employees', 'INSERT'),
+                has_table_privilege('import_test', 'public.employee_publications', 'DELETE'),
+                has_sequence_privilege('import_test', 'public.employee_publications_id_seq', 'UPDATE')
+            """)).one()
+            assert not any(permissions)
+        with Session(engine) as session:
+            yield session, name
+    finally:
+        # Only the dedicated ephemeral test DB; never DROP DATABASE or roles.
+        Base.metadata.drop_all(engine)
+        engine.dispose()
+
+
+@pytest.fixture
+def pg_import_db():
+    with database("miem_import_test") as db:
+        yield db
+
+
+@pytest.fixture
+def pg_crawl_db():
+    with database("miem_crawl_test") as db:
+        yield db
+
+
+def employee(session, name="Synthetic One", key="one"):
+    record = Employee(full_name=name, profile_key="staff:" + key,
+                      canonical_url="https://synthetic.invalid/staff/" + key)
+    session.add(record)
+    session.commit()
+    return record
+
+
+def article(title="Synthetic article", identifier="10.1234/test", year="2024", author="Synthetic One", **metadata):
+    return {"Название публикации": title, "Год": year, "DOI": identifier,
+            "Авторы НИУ ВШЭ": author, **metadata}
+
+
+def write_csv(path, rows):
+    columns = list(dict.fromkeys(key for row in rows for key in row))
+    with path.open("w", encoding="utf-8-sig", newline="") as output:
+        writer = csv.DictWriter(output, fieldnames=columns, delimiter=";")
+        writer.writeheader()
+        writer.writerows(rows)
+
+
+def cli(tmp_path, db, rows, *, apply=False, expected_code=0):
+    _, name = db
+    source = tmp_path / "articles.csv"
+    report = tmp_path / "report.json"
+    write_csv(source, rows)
+    command = [sys.executable, str(Path(__file__).resolve().parents[1] / "scripts/import_publications.py"),
+               str(source), "--report", str(report)]
+    if apply:
+        command.append("--apply")
+    # Explicit whitelist: no production env, .env, pg_service or ~/.pgpass.
+    environment = {"PATH": os.environ["PATH"], "LANG": "C.UTF-8", "PGHOST": "postgres-test",
+                   "PGPORT": "5432", "PGDATABASE": name, "PGUSER": "import_test",
+                   "PGPASSWORD": "isolated_test_import", "PGPASSFILE": "/dev/null",
+                   "PGSERVICEFILE": "/dev/null", "PGCONNECT_TIMEOUT": "5"}
+    result = subprocess.run(command, env=environment, cwd=tmp_path, capture_output=True, text=True, timeout=60)
+    assert result.returncode == expected_code, result.stdout + result.stderr
+    assert "Synthetic One" not in result.stdout and "isolated_test_import" not in result.stdout
+    return json.loads(report.read_text()) if report.exists() else None
+
+
+def snapshot(session):
+    session.expire_all()
+    tables = {table.name: session.execute(select(table).order_by(*table.primary_key.columns)).all()
+              for table in Base.metadata.sorted_tables}
+    sequences = session.execute(text("SELECT sequencename, last_value FROM pg_sequences WHERE schemaname = 'public' ORDER BY sequencename")).all()
+    session.commit()
+    return tables, sequences
+
+
+def test_postgres_dry_run_changes_neither_rows_nor_sequences(pg_import_db, tmp_path):
+    session, _ = pg_import_db
+    employee(session)
+    before = snapshot(session)
+    report = cli(tmp_path, pg_import_db, [article()])
+    assert report["summary"]["counts"] == {"insert": 1, "update": 0, "skip": 0}
+    assert snapshot(session) == before
+
+
+def test_postgres_apply_and_reimport(pg_import_db, tmp_path):
+    session, _ = pg_import_db
+    person = employee(session)
+    existing = EmployeePublication(employee_id=person.id, title="Existing", year=2024,
+                                   source_hash="existing", raw_data={"hse": "keep"})
+    session.add(existing)
+    session.commit()
+    rows = [article(title="Existing"), article(title="New", identifier="10.1234/new"),
+            article(title="Unknown author", identifier="10.1234/unknown", author="Synthetic Missing")]
+    report = cli(tmp_path, pg_import_db, rows, apply=True)
+    assert report["summary"]["counts"] == {"insert": 1, "update": 1, "skip": 1}
+    session.expire_all()
+    assert session.query(Employee).count() == 1
+    assert session.query(EmployeePublication).count() == 2
+    assert existing.doi_url == "https://doi.org/10.1234/test" and existing.raw_data["hse"] == "keep"
+    before = snapshot(session)
+    repeated = cli(tmp_path, pg_import_db, rows, apply=True)
+    assert repeated["summary"]["counts"] == {"insert": 0, "update": 0, "skip": 3}
+    assert snapshot(session) == before
+
+
+def test_postgres_apply_failure_rolls_back_update_and_insert(pg_import_db, tmp_path):
+    session, _ = pg_import_db
+    person = employee(session)
+    existing = EmployeePublication(employee_id=person.id, title="Existing", year=2024,
+                                   source_hash="existing", raw_data={"hse": "keep"})
+    session.add(existing)
+    session.commit()
+    # Test-only constraint forces a real server error after update + successful INSERT.
+    session.execute(text("ALTER TABLE public.employee_publications ADD CONSTRAINT test_failure CHECK (title <> 'Reject')"))
+    session.commit()
+    before, _ = snapshot(session)
+    rows = [article(title="Existing"), article(title="Accepted", identifier="10.1234/accepted"),
+            article(title="Reject", identifier="10.1234/reject")]
+    assert cli(tmp_path, pg_import_db, rows, apply=True, expected_code=1) is None
+    after, _ = snapshot(session)
+    assert after == before  # nextval may advance on rollback; rows must not change.
+    assert (tmp_path / "report.json.pending").exists()
+
+
+def test_postgres_year_only_update_preserves_json(pg_import_db, tmp_path):
+    session, _ = pg_import_db
+    person = employee(session)
+    row = article()
+    raw = {"hse": {"keep": True}, "csv_enrichment": row.copy()}
+    record = EmployeePublication(employee_id=person.id, title=row["Название публикации"],
+                                 year=None, doi_url="https://doi.org/10.1234/test",
+                                 source_hash="existing", raw_data=raw)
+    session.add(record)
+    session.commit()
+    report = cli(tmp_path, pg_import_db, [row], apply=True)
+    session.expire_all()
+    assert report["summary"]["counts"]["update"] == 1
+    assert record.year == 2024 and record.raw_data == raw
+
+
+def test_postgres_late_csv_metadata_and_conflicts(pg_import_db, tmp_path):
+    session, _ = pg_import_db
+    employee(session)
+    rows = [article(), article(**{"EID (Scopus)": "late-id", "Квартиль scopus": "Q1"}),
+            article(**{"EID (Scopus)": "conflict-id", "UT (WoS)": "late-wos"})]
+    report = cli(tmp_path, pg_import_db, rows, apply=True)
+    session.expire_all()
+    assert session.query(EmployeePublication).count() == 1
+    metadata = session.query(EmployeePublication).one().raw_data["csv_enrichment"]
+    assert metadata["EID (Scopus)"] == "late-id" and metadata["Квартиль scopus"] == "Q1"
+    assert metadata["UT (WoS)"] == "late-wos"
+    assert report["summary"]["metadata_conflicts"] == 1
+    assert report["records"][2]["conflict_fields"] == ["EID (Scopus)"]
+    before = snapshot(session)
+    repeated = cli(tmp_path, pg_import_db, rows, apply=True)
+    assert repeated["summary"]["counts"]["insert"] == repeated["summary"]["counts"]["update"] == 0
+    assert snapshot(session) == before
+
+
+def test_postgres_crawler_preserves_import_and_adopts_hse_id(pg_crawl_db, tmp_path):
+    session, _ = pg_crawl_db
+    person = employee(session)
+    cli(tmp_path, pg_crawl_db, [article(), article(title="CSV only", identifier="10.1234/only")], apply=True)
+    session.expire_all()
+    ids = {p.title: p.id for p in session.query(EmployeePublication).all()}
+    run = CrawlRun(source_url="https://synthetic.invalid/persons", status="running")
+    session.add(run)
+    session.commit()
+    parsed = {"sections": [{"type": "publications", "publications": [
+        {"publication_id": "hse-one", "title": "Synthetic article", "year": 2024,
+         "doi_url": "https://doi.org/10.1234/test", "raw_data": {"id": "hse-one"}}]}]}
+    for _ in range(2):
+        _try_sync_employee_publications(session, run, person, copy.deepcopy(parsed))
+        session.commit()
+        session.expire_all()
+    assert session.query(EmployeePublication).count() == 2
+    kept = session.get(EmployeePublication, ids["Synthetic article"])
+    assert kept.publication_id == "hse-one" and "csv_enrichment" in kept.raw_data
+    assert session.get(EmployeePublication, ids["CSV only"]).raw_data["csv_enrichment"]
+    assert session.query(CrawlError).count() == 0
+
+
+@pytest.mark.parametrize("bound_id", [None, "hse-A"])
+@pytest.mark.parametrize("incoming_ids", [("hse-A", "hse-B"), ("hse-B", "hse-A")])
+def test_postgres_hse_id_conflict_does_not_block_other_employee(pg_crawl_db, tmp_path, monkeypatch, bound_id, incoming_ids):
+    session, _ = pg_crawl_db
+    first = employee(session)
+    second = employee(session, "Synthetic Two", "two")
+    cli(tmp_path, pg_crawl_db, [article(identifier="")], apply=True)
+    session.expire_all()
+    original = session.query(EmployeePublication).filter_by(employee_id=first.id).one()
+    original.publication_id = bound_id
+    original.authors = [{"id": "original-author"}]
+    session.commit()
+    fields = ("publication_id", "source_hash", "raw_data", "authors", "updated_at")
+    before = {field: copy.deepcopy(getattr(original, field)) for field in fields}
+    session.commit()
+
+    def profile(person, publications):
+        return {"source_url": person.canonical_url, "profile_type": "staff",
+                "profile_id": person.profile_key.split(":")[1], "full_name": person.full_name,
+                "parser_version": "integration-test", "tabs": [],
+                "sections": [{"type": "publications", "publications": publications}]}
+
+    profiles = {
+        first.canonical_url: profile(first, [{"publication_id": key, "title": "Synthetic article", "year": 2024,
+                                             "authors": [{"id": key}], "raw_data": {"id": key}} for key in incoming_ids]),
+        second.canonical_url: profile(second, [{"publication_id": "second-article", "title": "Other employee article",
+                                               "year": 2024, "raw_data": {"id": "second-article"}}]),
+    }
+    monkeypatch.setattr("app.services.crawler.collect_profile_links", lambda *a, **k: list(profiles))
+    monkeypatch.setattr("app.services.crawler.parse_person_profile", lambda _, url, *a, **k: copy.deepcopy(profiles[url]))
+    settings = Settings(_env_file=None, database_url=ADMIN_URL.replace("miem_import_test", "miem_crawl_test"),
+                        source_url="https://synthetic.invalid/persons", request_delay_seconds=0)
+    run = run_crawl(session, settings)
+    session.expire_all()
+    assert run.status == "completed" and run.parsed_count == 2
+    kept = session.query(EmployeePublication).filter_by(employee_id=first.id).one()
+    assert {field: getattr(kept, field) for field in fields} == before
+    assert session.query(EmployeePublication).filter_by(employee_id=second.id).one().publication_id == "second-article"
+    error = session.query(CrawlError).filter_by(crawl_run_id=run.id).one()
+    assert "Ambiguous HSE IDs" in error.message