Compare commits
30 Commits
fix/api-on
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| f6a01500d7 | |||
| ec7aec310a | |||
| 0f91ed0d8f | |||
| 1e9cc50a73 | |||
| 10d2cf0101 | |||
| 44949996bc | |||
| baad49b976 | |||
| ceca2017cb | |||
| 83e726efe5 | |||
| 7d6d8759a5 | |||
| 5d89ae3891 | |||
| 880ac6c53e | |||
| 2d19eef9c2 | |||
| a1d60984e3 | |||
| d5f41e015f | |||
| e1ba77c337 | |||
| a989c5c53c | |||
| a82930027d | |||
| 78493b9248 | |||
| 5d3fb68c2f | |||
| dc060985a3 | |||
| 1d8a854a12 | |||
| d07711f665 | |||
| cf810f9cad | |||
| e142138c1b | |||
| c031fb9f4a | |||
| 70a4719bab | |||
| 3bdb8b1d90 | |||
| f3fb714126 | |||
| 3ed1dd8832 |
@@ -10,6 +10,8 @@ CRAWL_LIMIT=
|
|||||||
REQUEST_TIMEOUT=30
|
REQUEST_TIMEOUT=30
|
||||||
REQUEST_DELAY_SECONDS=1
|
REQUEST_DELAY_SECONDS=1
|
||||||
PARSER_USE_PLAYWRIGHT=false
|
PARSER_USE_PLAYWRIGHT=false
|
||||||
|
DISMISSAL_CONFIRMATION_RUNS=3
|
||||||
|
MAX_AUTO_DISMISSALS_PER_RUN=25
|
||||||
|
|
||||||
ADMIN_USERNAME=admin
|
ADMIN_USERNAME=admin
|
||||||
ADMIN_PASSWORD=change-me
|
ADMIN_PASSWORD=change-me
|
||||||
|
|||||||
88
.gitea/workflows/ci.yml
Normal file
88
.gitea/workflows/ci.yml
Normal file
@@ -0,0 +1,88 @@
|
|||||||
|
name: CI
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
pull_request:
|
||||||
|
branches: [main]
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
test:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
env:
|
||||||
|
DATABASE_URL: sqlite:///./ci.db
|
||||||
|
PYTHONUNBUFFERED: "1"
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: "3.12"
|
||||||
|
cache: pip
|
||||||
|
- run: python -m pip install --upgrade pip
|
||||||
|
- run: python -m pip install -r requirements.txt
|
||||||
|
- run: python -m pytest -q
|
||||||
|
- run: node --check app/static/admin.js
|
||||||
|
|
||||||
|
deploy:
|
||||||
|
if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || github.event_name == 'workflow_dispatch'
|
||||||
|
needs: test
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
concurrency:
|
||||||
|
group: production-deploy
|
||||||
|
cancel-in-progress: false
|
||||||
|
env:
|
||||||
|
PROD_PATH: /srv/miem_workers
|
||||||
|
RELEASE_NAME: miem-workers-release-${{ github.run_id }}.tar
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- name: Prepare release
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
tar --exclude=.git --exclude=.env -cf "$RUNNER_TEMP/$RELEASE_NAME" .
|
||||||
|
printf '%s\n' "$PROD_SSH_KEY" > "$RUNNER_TEMP/deploy_key"
|
||||||
|
printf '%s\n' "$PROD_KNOWN_HOSTS" > "$RUNNER_TEMP/known_hosts"
|
||||||
|
chmod 600 "$RUNNER_TEMP/deploy_key" "$RUNNER_TEMP/known_hosts"
|
||||||
|
env:
|
||||||
|
PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }}
|
||||||
|
PROD_KNOWN_HOSTS: ${{ secrets.PROD_KNOWN_HOSTS }}
|
||||||
|
- name: Upload release
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
scp -i "$RUNNER_TEMP/deploy_key" \
|
||||||
|
-o BatchMode=yes \
|
||||||
|
-o UserKnownHostsFile="$RUNNER_TEMP/known_hosts" \
|
||||||
|
"$RUNNER_TEMP/$RELEASE_NAME" \
|
||||||
|
"$PROD_USER@$PROD_HOST:$RELEASE_NAME"
|
||||||
|
env:
|
||||||
|
PROD_HOST: ${{ secrets.PROD_HOST }}
|
||||||
|
PROD_USER: ${{ secrets.PROD_USER }}
|
||||||
|
- name: Deploy and verify
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
ssh -i "$RUNNER_TEMP/deploy_key" \
|
||||||
|
-o BatchMode=yes \
|
||||||
|
-o UserKnownHostsFile="$RUNNER_TEMP/known_hosts" \
|
||||||
|
"$PROD_USER@$PROD_HOST" \
|
||||||
|
"PROD_PATH='$PROD_PATH' RELEASE_NAME='$RELEASE_NAME' bash -s" <<'REMOTE'
|
||||||
|
set -euo pipefail
|
||||||
|
cd "$PROD_PATH"
|
||||||
|
release="$HOME/$RELEASE_NAME"
|
||||||
|
mkdir -p "$HOME/backups/miem_workers"
|
||||||
|
backup="$HOME/backups/miem_workers/miem_workers-$(date +%Y%m%d-%H%M%S).sql"
|
||||||
|
docker compose exec -T postgres sh -c 'pg_dump -U "$POSTGRES_USER" "$POSTGRES_DB"' > "$backup"
|
||||||
|
docker run --rm \
|
||||||
|
-v "$release:/src/release.tar:ro" \
|
||||||
|
-v "$PROD_PATH":/dst \
|
||||||
|
alpine:3.20 sh -c 'tar -xf /src/release.tar -C /dst'
|
||||||
|
docker compose up -d --build
|
||||||
|
curl --fail --silent --show-error http://127.0.0.1:8000/api/health
|
||||||
|
docker compose ps
|
||||||
|
rm -f "$release"
|
||||||
|
REMOTE
|
||||||
|
env:
|
||||||
|
PROD_HOST: ${{ secrets.PROD_HOST }}
|
||||||
|
PROD_USER: ${{ secrets.PROD_USER }}
|
||||||
36
CHANGELOG.md
36
CHANGELOG.md
@@ -1,5 +1,41 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## 0.8.6
|
||||||
|
|
||||||
|
- Исправлена загрузка release-архива production CD в домашний каталог deploy-пользователя.
|
||||||
|
|
||||||
|
## 0.8.5
|
||||||
|
|
||||||
|
- Исправлена адаптивная компоновка фильтров справочника и скрытие skip-link до клавиатурного фокуса.
|
||||||
|
|
||||||
|
## 0.8.4
|
||||||
|
|
||||||
|
- Подключены к конфигурации ограничения массового автоматического увольнения, используемые плановым обходом.
|
||||||
|
|
||||||
|
## 0.8.3
|
||||||
|
|
||||||
|
- Удалены устаревшие пояснение о недоступном endpoint и проверка его отсутствия.
|
||||||
|
|
||||||
|
## 0.8.2
|
||||||
|
|
||||||
|
- Добавлен production CD workflow с backup PostgreSQL и проверкой healthcheck после обновления.
|
||||||
|
|
||||||
|
## 0.8.1
|
||||||
|
|
||||||
|
- Улучшены доступность, фильтры, состояния и адаптивность административного каталога.
|
||||||
|
|
||||||
|
## 0.7.8
|
||||||
|
|
||||||
|
- В колонке учёной степени отображаются все найденные степени без лишнего текста.
|
||||||
|
|
||||||
|
## 0.7.7
|
||||||
|
|
||||||
|
- Удалена неиспользуемая интеграция обмена данными и связанная документация и тесты.
|
||||||
|
|
||||||
|
## 0.7.6
|
||||||
|
|
||||||
|
- Возвращён еженедельный автоматический запуск обхода сотрудников.
|
||||||
|
|
||||||
## 0.7.5
|
## 0.7.5
|
||||||
|
|
||||||
- Production Compose запускает только API и PostgreSQL.
|
- Production Compose запускает только API и PostgreSQL.
|
||||||
|
|||||||
@@ -1,671 +0,0 @@
|
|||||||
# MCP: описание работы, структуры и тулзов
|
|
||||||
|
|
||||||
Документ описывает MCP endpoint сервиса `miem-employees` по текущей реализации в `app/mcp.py`.
|
|
||||||
|
|
||||||
## Где находится MCP
|
|
||||||
|
|
||||||
- FastAPI router: `app.mcp.router`
|
|
||||||
- Подключение к приложению: `app/main.py`
|
|
||||||
- HTTP endpoint: `POST /mcp`
|
|
||||||
- Локально при обычном запуске API: `http://localhost:8000/mcp`
|
|
||||||
- В Docker Compose endpoint обслуживает `api`: `http://localhost:8000/mcp`
|
|
||||||
- Авторизация на уровне приложения: отсутствует. Заголовок `Authorization` не проверяется и не влияет на ответ.
|
|
||||||
|
|
||||||
Если доступ к MCP нужно ограничить, это должно делаться внешним контуром: bind на localhost, VPN, firewall, reverse proxy или отдельная сетевая политика.
|
|
||||||
|
|
||||||
## Протокол
|
|
||||||
|
|
||||||
Endpoint принимает JSON-RPC 2.0 over HTTP.
|
|
||||||
|
|
||||||
Общий формат запроса:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"jsonrpc": "2.0",
|
|
||||||
"id": 1,
|
|
||||||
"method": "tools/list",
|
|
||||||
"params": {}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Общий формат успешного ответа:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"jsonrpc": "2.0",
|
|
||||||
"id": 1,
|
|
||||||
"result": {}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Общий формат ошибки:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"jsonrpc": "2.0",
|
|
||||||
"id": 1,
|
|
||||||
"error": {
|
|
||||||
"code": -32601,
|
|
||||||
"message": "Method not found"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Поддерживаемая версия MCP-протокола:
|
|
||||||
|
|
||||||
```text
|
|
||||||
2024-11-05
|
|
||||||
```
|
|
||||||
|
|
||||||
Имя сервиса:
|
|
||||||
|
|
||||||
```text
|
|
||||||
miem-employees
|
|
||||||
```
|
|
||||||
|
|
||||||
Версия сервера берется из `app.version.BACKEND_VERSION`.
|
|
||||||
|
|
||||||
## Поддерживаемые JSON-RPC методы
|
|
||||||
|
|
||||||
### initialize
|
|
||||||
|
|
||||||
Возвращает метаданные MCP-сервера и capabilities.
|
|
||||||
|
|
||||||
Запрос:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"jsonrpc": "2.0",
|
|
||||||
"id": 1,
|
|
||||||
"method": "initialize",
|
|
||||||
"params": {}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Ответ:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"jsonrpc": "2.0",
|
|
||||||
"id": 1,
|
|
||||||
"result": {
|
|
||||||
"protocolVersion": "2024-11-05",
|
|
||||||
"serverInfo": {
|
|
||||||
"name": "miem-employees",
|
|
||||||
"version": "0.7.0"
|
|
||||||
},
|
|
||||||
"capabilities": {
|
|
||||||
"tools": {}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### tools/list
|
|
||||||
|
|
||||||
Возвращает список доступных tools с JSON Schema для аргументов.
|
|
||||||
|
|
||||||
Запрос:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"jsonrpc": "2.0",
|
|
||||||
"id": 1,
|
|
||||||
"method": "tools/list",
|
|
||||||
"params": {}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Ответ содержит массив `result.tools`.
|
|
||||||
|
|
||||||
### tools/call
|
|
||||||
|
|
||||||
Вызывает один tool по имени.
|
|
||||||
|
|
||||||
Запрос:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"jsonrpc": "2.0",
|
|
||||||
"id": 1,
|
|
||||||
"method": "tools/call",
|
|
||||||
"params": {
|
|
||||||
"name": "search_employees",
|
|
||||||
"arguments": {
|
|
||||||
"query": "Сергеев",
|
|
||||||
"limit": 20
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Ответ tool всегда заворачивается в MCP content-массив:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"jsonrpc": "2.0",
|
|
||||||
"id": 1,
|
|
||||||
"result": {
|
|
||||||
"content": [
|
|
||||||
{
|
|
||||||
"type": "text",
|
|
||||||
"text": "{\"items\":[]}"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Поле `text` содержит сериализованный JSON с `ensure_ascii=false`. Клиент должен распарсить это поле как JSON, если ему нужна структурированная нагрузка.
|
|
||||||
|
|
||||||
## Ошибки
|
|
||||||
|
|
||||||
- Неизвестный JSON-RPC метод: `code = -32601`, `message = "Method not found"`.
|
|
||||||
- Исключения при обработке tool: `code = -32000`, `message` содержит текст исключения.
|
|
||||||
- Если сущность не найдена внутри отдельных tools, HTTP и JSON-RPC ответ остаются успешными, а полезная нагрузка содержит `{"error": "not_found"}`.
|
|
||||||
|
|
||||||
## Источники данных
|
|
||||||
|
|
||||||
MCP читает данные из основной базы через SQLAlchemy session из `app.db.get_db`.
|
|
||||||
|
|
||||||
Основные таблицы и модели:
|
|
||||||
|
|
||||||
- `employees`: текущая карточка сотрудника, статус, профиль, `current_data`, checksum.
|
|
||||||
- `employee_publications`: нормализованные публикации сотрудников с авторами, DOI, аннотацией, описанием, citation text и raw JSON из HSE Publications.
|
|
||||||
- `employee_news_links`: нормализованные ссылки на новости из блока профиля «В новостях» с заголовком, URL, кратким описанием, датой, годом публикации и raw JSON карточки.
|
|
||||||
- `crawl_runs`: история запусков парсинга.
|
|
||||||
- `crawl_run_employee_changes`: детальные изменения сотрудников в рамках запуска.
|
|
||||||
- `crawl_errors`: ошибки парсинга в рамках запуска.
|
|
||||||
- `dataset_versions`: версии полного набора сотрудников.
|
|
||||||
- `dataset_version_items`: состав конкретной версии набора сотрудников.
|
|
||||||
|
|
||||||
## Общая структура employee payload
|
|
||||||
|
|
||||||
Краткая карточка сотрудника:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"profile_key": "staff:avsergeev",
|
|
||||||
"profile_id": "avsergeev",
|
|
||||||
"full_name": "Сергеев Алексей Викторович",
|
|
||||||
"status": "active",
|
|
||||||
"canonical_url": "https://www.hse.ru/staff/avsergeev",
|
|
||||||
"last_seen_at": "2026-05-14T10:00:00+00:00",
|
|
||||||
"dismissed_at": null
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
В sync payload дополнительно отдается `checksum`.
|
|
||||||
|
|
||||||
Полная карточка дополнительно содержит:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"data": {
|
|
||||||
"contacts": {},
|
|
||||||
"sections": []
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
`data` соответствует распарсенному JSON профиля сотрудника. Внутри `sections` могут быть секции с публикациями, курсами, ВКР, новостями, таблицами, ссылками и произвольными текстовыми блоками.
|
|
||||||
|
|
||||||
Пример секции новостей внутри `data.sections`:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"title": "В новостях",
|
|
||||||
"slug": "v_novostyah",
|
|
||||||
"type": "news",
|
|
||||||
"news_count": 1,
|
|
||||||
"news_links": [
|
|
||||||
{
|
|
||||||
"title": "Название новости",
|
|
||||||
"url": "https://www.hse.ru/news/edu/1153850518.html",
|
|
||||||
"summary": "Краткое описание новости.",
|
|
||||||
"published_at": "2026-04-28T00:00:00+00:00",
|
|
||||||
"published_year": 2026
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Для новостей отдельного MCP tool сейчас нет: они доступны через `get_employee(...).data.sections` или через полную синхронизацию `sync_employees(include_data=true)`.
|
|
||||||
|
|
||||||
## Tools
|
|
||||||
|
|
||||||
### get_service_info
|
|
||||||
|
|
||||||
Назначение: вернуть метаданные сервиса, список tools и текущую версию набора сотрудников.
|
|
||||||
|
|
||||||
Аргументы: отсутствуют.
|
|
||||||
|
|
||||||
Возвращает:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"service_name": "miem-employees",
|
|
||||||
"backend_version": "0.7.0",
|
|
||||||
"protocolVersion": "2024-11-05",
|
|
||||||
"tools": [],
|
|
||||||
"dataset": {
|
|
||||||
"hash": "sha256",
|
|
||||||
"previous_hash": "sha256 или null",
|
|
||||||
"created_at": "2026-05-14T10:00:00+00:00",
|
|
||||||
"crawl_run_id": 123,
|
|
||||||
"employee_count": 100,
|
|
||||||
"active_count": 95,
|
|
||||||
"dismissed_count": 5
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Особенность: перед ответом сервис создает актуальную `dataset_version`, если текущий набор сотрудников еще не имеет версии.
|
|
||||||
|
|
||||||
### sync_employees
|
|
||||||
|
|
||||||
Назначение: синхронизировать клиентский кэш сотрудников по hash набора данных.
|
|
||||||
|
|
||||||
Аргументы:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"client_hash": "sha256 или null",
|
|
||||||
"include_data": true
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
- `client_hash`: hash версии, которая уже есть у клиента. Если не передан, отдается полный snapshot.
|
|
||||||
- `include_data`: управляет включением полного `data` в карточки сотрудников. По умолчанию `true`.
|
|
||||||
|
|
||||||
Полный ответ без `client_hash`:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"mode": "full",
|
|
||||||
"from_hash": null,
|
|
||||||
"to_hash": "current-sha256",
|
|
||||||
"dataset": {},
|
|
||||||
"items": []
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Если клиентский hash совпадает с текущим:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"mode": "delta",
|
|
||||||
"from_hash": "current-sha256",
|
|
||||||
"to_hash": "current-sha256",
|
|
||||||
"dataset": {},
|
|
||||||
"changes": {
|
|
||||||
"added": [],
|
|
||||||
"updated": [],
|
|
||||||
"dismissed": [],
|
|
||||||
"removed": []
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Если `client_hash` неизвестен серверу:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"mode": "full",
|
|
||||||
"from_hash": "missing",
|
|
||||||
"to_hash": "current-sha256",
|
|
||||||
"dataset": {},
|
|
||||||
"items": [],
|
|
||||||
"reason": "unknown_client_hash"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Если `client_hash` найден и отличается от текущего:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"mode": "delta",
|
|
||||||
"from_hash": "old-sha256",
|
|
||||||
"to_hash": "current-sha256",
|
|
||||||
"dataset": {},
|
|
||||||
"changes": {
|
|
||||||
"added": [],
|
|
||||||
"updated": [],
|
|
||||||
"dismissed": [],
|
|
||||||
"removed": []
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Логика delta:
|
|
||||||
|
|
||||||
- `added`: сотрудник появился в новой версии.
|
|
||||||
- `updated`: изменился checksum или статус, и сотрудник активен.
|
|
||||||
- `dismissed`: сотрудник есть в новой версии, но получил статус `dismissed`.
|
|
||||||
- `removed`: `profile_key` был в старой версии, но отсутствует в новой.
|
|
||||||
|
|
||||||
Hash набора считается по отсортированному списку `{profile_key, status, checksum}`.
|
|
||||||
|
|
||||||
### search_employees
|
|
||||||
|
|
||||||
Назначение: найти сотрудников по ФИО или canonical URL.
|
|
||||||
|
|
||||||
Аргументы:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"query": "Сергеев",
|
|
||||||
"status": "active",
|
|
||||||
"limit": 20
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
- `query`: обязательный по schema, но в коде пустая строка означает поиск без текстового фильтра.
|
|
||||||
- `status`: опционально, только `active` или `dismissed`.
|
|
||||||
- `limit`: максимум 100, по умолчанию 20.
|
|
||||||
|
|
||||||
Возвращает массив кратких employee payload без `data`:
|
|
||||||
|
|
||||||
```json
|
|
||||||
[
|
|
||||||
{
|
|
||||||
"profile_key": "staff:avsergeev",
|
|
||||||
"profile_id": "avsergeev",
|
|
||||||
"full_name": "Сергеев Алексей Викторович",
|
|
||||||
"status": "active",
|
|
||||||
"canonical_url": "https://www.hse.ru/staff/avsergeev",
|
|
||||||
"last_seen_at": "2026-05-14T10:00:00+00:00",
|
|
||||||
"dismissed_at": null
|
|
||||||
}
|
|
||||||
]
|
|
||||||
```
|
|
||||||
|
|
||||||
### get_employee
|
|
||||||
|
|
||||||
Назначение: получить одну карточку сотрудника.
|
|
||||||
|
|
||||||
Аргументы:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"profile_id_or_url": "avsergeev"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Поиск выполняется по:
|
|
||||||
|
|
||||||
- `profile_key`
|
|
||||||
- `profile_id`
|
|
||||||
- точному `canonical_url`
|
|
||||||
- частичному совпадению `canonical_url`
|
|
||||||
|
|
||||||
Возвращает полный employee payload с `data`.
|
|
||||||
|
|
||||||
Если сотрудник не найден:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"error": "not_found"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### list_employee_publications
|
|
||||||
|
|
||||||
Назначение: вернуть публикации сотрудника. Если есть нормализованные строки в `employee_publications`, tool возвращает детальные публикационные данные: авторов, DOI, аннотацию, описание, citation text, год, тип, язык, статус и ссылки. Если детальная таблица еще не заполнена, tool использует старый fallback из `employees.current_data.sections[].publications`.
|
|
||||||
|
|
||||||
Аргументы:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"profile_id_or_url": "avsergeev"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Поиск сотрудника выполняется так же, как в `get_employee`: по `profile_key`, `profile_id`, точному или частичному `canonical_url`.
|
|
||||||
|
|
||||||
Порядок источников:
|
|
||||||
|
|
||||||
- сначала `employee_publications`, отсортированные по году, названию и внутреннему id;
|
|
||||||
- если записей нет, секции `current_data.sections` с `type = "publications"` и массивами `publications`.
|
|
||||||
|
|
||||||
Ответ:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"employee": {
|
|
||||||
"profile_key": "org_person:803294906",
|
|
||||||
"profile_id": "803294906",
|
|
||||||
"full_name": "Борисов Сергей Петрович",
|
|
||||||
"status": "active",
|
|
||||||
"canonical_url": "https://www.hse.ru/org/persons/803294906",
|
|
||||||
"last_seen_at": "2026-05-14T10:00:00+00:00",
|
|
||||||
"dismissed_at": null
|
|
||||||
},
|
|
||||||
"items": [
|
|
||||||
{
|
|
||||||
"id": "888959076",
|
|
||||||
"publication_id": "888959076",
|
|
||||||
"title": "Название публикации",
|
|
||||||
"text": "Краткое описание или citation",
|
|
||||||
"url": "https://publications.hse.ru/view/888959076",
|
|
||||||
"year": 2023,
|
|
||||||
"type": "ARTICLE",
|
|
||||||
"publication_type": "ARTICLE",
|
|
||||||
"language": "ru",
|
|
||||||
"status": 1,
|
|
||||||
"doi_url": "https://doi.org/10.53921/18195822_2023_23_4_624",
|
|
||||||
"other_url": "https://example.test",
|
|
||||||
"document_url": "https://example.test/file.pdf",
|
|
||||||
"citation_text": "Авторы. Название публикации // Журнал. 2023.",
|
|
||||||
"annotation": {
|
|
||||||
"ru": "Аннотация",
|
|
||||||
"en": "Abstract"
|
|
||||||
},
|
|
||||||
"description": {
|
|
||||||
"main": "Авторы. Название публикации // Журнал. 2023."
|
|
||||||
},
|
|
||||||
"authors": [
|
|
||||||
{
|
|
||||||
"id": "803294906",
|
|
||||||
"href": "https://www.hse.ru/org/persons/803294906",
|
|
||||||
"title_ru": "Борисов С. П.",
|
|
||||||
"title_en": "",
|
|
||||||
"reverse_title_ru": "С. П. Борисов",
|
|
||||||
"reverse_title_en": "",
|
|
||||||
"alt_name": "S. P. Borisov",
|
|
||||||
"other_name": null,
|
|
||||||
"is_current_employee": true
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
В fallback-режиме из `current_data` старые элементы могут содержать только базовые поля `title`, `text`, `url` и `id`.
|
|
||||||
|
|
||||||
Если сотрудник не найден:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"items": []
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Если сотрудник найден, но публикаций нет:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"employee": {},
|
|
||||||
"items": []
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### list_employee_courses
|
|
||||||
|
|
||||||
Назначение: вернуть курсы преподавания сотрудника из распарсенных секций профиля.
|
|
||||||
|
|
||||||
Аргументы:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"profile_id_or_url": "avsergeev"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Сервис ищет секции `current_data.sections` с `type = "courses_by_year"` и объединяет массивы `courses`.
|
|
||||||
|
|
||||||
Ответ:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"employee": {},
|
|
||||||
"items": [
|
|
||||||
{
|
|
||||||
"title": "Название курса",
|
|
||||||
"url": "https://..."
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Если сотрудник или данные профиля отсутствуют:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"items": []
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### get_crawl_status
|
|
||||||
|
|
||||||
Назначение: вернуть последний запуск парсинга.
|
|
||||||
|
|
||||||
Аргументы: отсутствуют.
|
|
||||||
|
|
||||||
Ответ:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"id": 123,
|
|
||||||
"status": "completed",
|
|
||||||
"source_url": "https://miem.hse.ru/persons",
|
|
||||||
"started_at": "2026-05-14T10:00:00+00:00",
|
|
||||||
"finished_at": "2026-05-14T10:10:00+00:00",
|
|
||||||
"found_count": 100,
|
|
||||||
"parsed_count": 98,
|
|
||||||
"error_count": 2,
|
|
||||||
"dismissed_count": 1
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Если запусков еще не было:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"status": "never_run"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### get_crawl_run_details
|
|
||||||
|
|
||||||
Назначение: вернуть детальную информацию по конкретному запуску парсинга: summary, изменения сотрудников и ошибки.
|
|
||||||
|
|
||||||
Аргументы:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"run_id": 123
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Ответ:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"id": 123,
|
|
||||||
"source_url": "https://miem.hse.ru/persons",
|
|
||||||
"status": "completed",
|
|
||||||
"status_display": "Завершен",
|
|
||||||
"started_at": "2026-05-14T10:00:00+00:00",
|
|
||||||
"finished_at": "2026-05-14T10:10:00+00:00",
|
|
||||||
"started_display": "14.05.2026 13:00",
|
|
||||||
"finished_display": "14.05.2026 13:10",
|
|
||||||
"found_count": 100,
|
|
||||||
"parsed_count": 98,
|
|
||||||
"new_count": 3,
|
|
||||||
"error_count": 2,
|
|
||||||
"dismissed_count": 1,
|
|
||||||
"processed_count": 100,
|
|
||||||
"progress_percent": 100.0,
|
|
||||||
"message": null,
|
|
||||||
"changes_detail_available": true,
|
|
||||||
"changes": {
|
|
||||||
"new": [],
|
|
||||||
"missing_from_source": [],
|
|
||||||
"dismissed": []
|
|
||||||
},
|
|
||||||
"errors": []
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Если запуск не найден:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"error": "not_found"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Примеры curl
|
|
||||||
|
|
||||||
Список tools:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl http://localhost:8000/mcp \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}'
|
|
||||||
```
|
|
||||||
|
|
||||||
Поиск сотрудника:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl http://localhost:8000/mcp \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"search_employees","arguments":{"query":"Сергеев","limit":5}}}'
|
|
||||||
```
|
|
||||||
|
|
||||||
Полная синхронизация:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl http://localhost:8000/mcp \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"sync_employees","arguments":{"include_data":false}}}'
|
|
||||||
```
|
|
||||||
|
|
||||||
Delta-синхронизация:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl http://localhost:8000/mcp \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{"jsonrpc":"2.0","id":4,"method":"tools/call","params":{"name":"sync_employees","arguments":{"client_hash":"known-sha256","include_data":true}}}'
|
|
||||||
```
|
|
||||||
|
|
||||||
## Как MCP используется клиентом
|
|
||||||
|
|
||||||
1. Клиент вызывает `initialize` и проверяет `protocolVersion`.
|
|
||||||
2. Клиент вызывает `tools/list`, чтобы получить актуальный список tools и input schemas.
|
|
||||||
3. Для поиска и точечных запросов клиент вызывает `tools/call` с `search_employees`, `get_employee`, `list_employee_publications`, `list_employee_courses`, `get_crawl_status` или `get_crawl_run_details`.
|
|
||||||
4. Для локального кэша клиент вызывает `get_service_info` или `sync_employees`.
|
|
||||||
5. Клиент хранит последний `dataset.hash`.
|
|
||||||
6. При следующей синхронизации клиент передает hash как `client_hash`.
|
|
||||||
7. Сервер возвращает пустую delta, delta с изменениями или полный snapshot, если hash неизвестен.
|
|
||||||
|
|
||||||
## Важные особенности реализации
|
|
||||||
|
|
||||||
- MCP endpoint read-only: tools не запускают парсинг и не меняют сотрудников напрямую.
|
|
||||||
- `get_service_info` и `sync_employees` могут создать новую запись `dataset_versions`, если состояние сотрудников изменилось и новой версии еще нет.
|
|
||||||
- Все tool payloads возвращаются как JSON-строка внутри `content[0].text`.
|
|
||||||
- `search_employees` ищет через `ilike` по `full_name` и `canonical_url`.
|
|
||||||
- `get_employee` допускает частичный URL, поэтому строка `133709486` может найти `https://www.hse.ru/org/persons/133709486`.
|
|
||||||
- Временные значения сериализуются через `isoformat()`, display-поля для админских payload формируются в часовом поясе `Europe/Moscow`.
|
|
||||||
83
README.md
83
README.md
@@ -1,10 +1,11 @@
|
|||||||
# MIEM Employees Server
|
# MIEM Employees Server
|
||||||
|
|
||||||
Сервис собирает сотрудников МИЭМ с сайта ВШЭ, хранит карточки и историю обновлений в Postgres, показывает минимальную админку и отдает read-only MCP endpoint для ИИ-агентов.
|
Сервис собирает сотрудников МИЭМ с сайта ВШЭ, хранит карточки и историю обновлений в Postgres и показывает минимальную админку.
|
||||||
|
|
||||||
## Архитектура
|
## Архитектура
|
||||||
|
|
||||||
- `api`: FastAPI, REST API, HTML-админка, MCP endpoint и healthcheck.
|
- `api`: FastAPI, REST API, HTML-админка и healthcheck.
|
||||||
|
- `worker`: weekly scheduler, который запускает парсинг по `CRAWL_CRON`.
|
||||||
- `postgres`: основная БД.
|
- `postgres`: основная БД.
|
||||||
|
|
||||||
Парсер использует фиксированный источник сотрудников, по умолчанию `https://miem.hse.ru/persons`. Для каждой карточки сохраняются ФИО, должности, год начала работы, контакты, идентификаторы, вкладки профиля, секции, публикации, курсы, ВКР, новости, JSON-снапшот и сжатый HTML-снапшот. Детальные публикации дополнительно нормализуются в отдельную таблицу `employee_publications`, а новости из блока «В новостях» — в `employee_news_links`. Ссылки обходятся только из меню профиля самого сотрудника (`person-menu`), например `#sci`, `#teaching`, `#main`.
|
Парсер использует фиксированный источник сотрудников, по умолчанию `https://miem.hse.ru/persons`. Для каждой карточки сохраняются ФИО, должности, год начала работы, контакты, идентификаторы, вкладки профиля, секции, публикации, курсы, ВКР, новости, JSON-снапшот и сжатый HTML-снапшот. Детальные публикации дополнительно нормализуются в отдельную таблицу `employee_publications`, а новости из блока «В новостях» — в `employee_news_links`. Ссылки обходятся только из меню профиля самого сотрудника (`person-menu`), например `#sci`, `#teaching`, `#main`.
|
||||||
@@ -20,7 +21,8 @@ cp .env.example .env
|
|||||||
Основные настройки:
|
Основные настройки:
|
||||||
|
|
||||||
- `DATABASE_URL`: строка подключения SQLAlchemy.
|
- `DATABASE_URL`: строка подключения SQLAlchemy.
|
||||||
- `SOURCE_URL`: список сотрудников МИЭМ.
|
- `SOURCE_URL`: список сотрудников МИЭМ.
|
||||||
|
- `CRAWL_CRON`: расписание в формате crontab, по умолчанию `0 3 * * 1`.
|
||||||
- `CRAWL_LIMIT`: опциональный лимит профилей для тестового запуска.
|
- `CRAWL_LIMIT`: опциональный лимит профилей для тестового запуска.
|
||||||
- `ADMIN_USERNAME`, `ADMIN_PASSWORD`: логин и пароль админки.
|
- `ADMIN_USERNAME`, `ADMIN_PASSWORD`: логин и пароль админки.
|
||||||
- `SESSION_SECRET`: секрет подписи cookie.
|
- `SESSION_SECRET`: секрет подписи cookie.
|
||||||
@@ -41,9 +43,15 @@ uvicorn app.main:app --reload
|
|||||||
|
|
||||||
В админке доступны:
|
В админке доступны:
|
||||||
|
|
||||||
- `Dashboard`: общая статистика, последний добавленный сотрудник, прогресс текущего/последнего парсинга и ручной запуск.
|
- «Обзор»: статистика, последний добавленный сотрудник, прогресс парсинга и ручной запуск.
|
||||||
- `Directory`: настраиваемая таблица сотрудников с фильтрами, сортировкой, пагинацией и выбором колонок.
|
- «Сотрудники»: поиск, фильтры, сортировка, пагинация и выбор колонок. По умолчанию показаны ФИО, статус, должности, дата последнего обнаружения и внешний профиль. В диалоге колонок доступны наборы «Проверка», «Контакты» и «Все поля»; ранее сохранённый выбор сохраняется.
|
||||||
- `Runs`: история запусков, ошибки и progress bar.
|
- «Запуски»: история обходов, ошибки и доступный индикатор прогресса.
|
||||||
|
|
||||||
|
Все фильтры применяются кнопкой «Применить фильтры» с переходом на первую страницу. «Сбросить» очищает условия; при устаревшем номере страницы каталог возвращает первую. Подписи полей показывают выбранные условия, над таблицей указан диапазон результатов. Пояснения дат и статусов находятся под фильтрами.
|
||||||
|
|
||||||
|
Каталог поддерживает навигацию с клавиатуры и системный диалог колонок с Escape и возвратом фокуса. На узких экранах таблица прокручивается внутри страницы, колонка ФИО закреплена. Используется системный шрифт без внешних загрузок.
|
||||||
|
|
||||||
|
Каталог формируется сервером: отдельная клиентская загрузка списка не нужна, переход показывает браузер. Пустой результат предлагает сбросить фильтры, пустая база — запустить парсинг. При ошибке базы каталог возвращает HTTP 503 и предлагает повторить загрузку с теми же фильтрами. При обновлении прогресса показывается состояние загрузки; при ошибке сохраняются последние значения с предупреждением об их актуальности и кнопкой «Повторить». Автоматические попытки продолжаются каждые 4 секунды, запрос ограничен 15 секундами; параллельные запросы не запускаются.
|
||||||
|
|
||||||
## Docker Compose
|
## Docker Compose
|
||||||
|
|
||||||
@@ -54,7 +62,9 @@ docker compose up -d --build --remove-orphans
|
|||||||
По умолчанию:
|
По умолчанию:
|
||||||
|
|
||||||
- API и админка: `http://localhost:8000`
|
- API и админка: `http://localhost:8000`
|
||||||
- Postgres: `localhost:5432`
|
- PostgreSQL: `postgres:5432` внутри сети Compose; порт на хост не опубликован.
|
||||||
|
|
||||||
|
Compose запускает `api`, `worker` и `postgres`. API привязан к localhost; для внешнего доступа нужен настроенный reverse proxy. REST API данных требует сессию администратора.
|
||||||
|
|
||||||
Таблицы создаются приложением при старте. При обновлении существующей базы приложение также добавляет недостающие runtime-колонки, например `crawl_runs.skipped_count`. SQL-миграции для ручного применения лежат в `migrations/`.
|
Таблицы создаются приложением при старте. При обновлении существующей базы приложение также добавляет недостающие runtime-колонки, например `crawl_runs.skipped_count`. SQL-миграции для ручного применения лежат в `migrations/`.
|
||||||
|
|
||||||
@@ -69,8 +79,6 @@ docker compose up -d --build --remove-orphans
|
|||||||
|
|
||||||
`employee_publications` содержит `publication_id`, название, год, тип публикации, язык, статус, ссылку на карточку HSE Publications, DOI, внешние/document-ссылки, citation text, аннотацию, описание, авторов, raw JSON ответа `searchPubs` и `source_hash` для безопасного повторного upsert. Уникальность поддерживается по `(employee_id, publication_id)` и `(employee_id, source_hash)`, поэтому повторный crawl не должен создавать дубликаты.
|
`employee_publications` содержит `publication_id`, название, год, тип публикации, язык, статус, ссылку на карточку HSE Publications, DOI, внешние/document-ссылки, citation text, аннотацию, описание, авторов, raw JSON ответа `searchPubs` и `source_hash` для безопасного повторного upsert. Уникальность поддерживается по `(employee_id, publication_id)` и `(employee_id, source_hash)`, поэтому повторный crawl не должен создавать дубликаты.
|
||||||
|
|
||||||
`list_employee_publications` сначала читает `employee_publications`; если детальных строк еще нет, возвращает старые публикации из `current_data`.
|
|
||||||
|
|
||||||
Новости сотрудников также хранятся в двух видах:
|
Новости сотрудников также хранятся в двух видах:
|
||||||
|
|
||||||
- краткий список остается внутри `employees.current_data.sections[].news_links`;
|
- краткий список остается внутри `employees.current_data.sections[].news_links`;
|
||||||
@@ -80,7 +88,7 @@ docker compose up -d --build --remove-orphans
|
|||||||
|
|
||||||
## Парсинг
|
## Парсинг
|
||||||
|
|
||||||
Ручной запуск доступен в админке на `Dashboard` и странице `Runs` или через REST:
|
Worker запускает обход по `CRAWL_CRON`. Ручной запуск также доступен в админке на `Dashboard` и странице `Runs` или через REST:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl -X POST http://localhost:8000/api/crawl-runs --cookie "miem_admin_session=..."
|
curl -X POST http://localhost:8000/api/crawl-runs --cookie "miem_admin_session=..."
|
||||||
@@ -105,41 +113,26 @@ curl -X POST http://localhost:8000/api/crawl-runs --cookie "miem_admin_session=.
|
|||||||
|
|
||||||
Во время выполнения парсинга `found_count`, `parsed_count`, `skipped_count` и `error_count` обновляются в базе. Админка опрашивает `/api/crawl-runs/latest` и показывает прогресс как `(parsed_count + skipped_count + error_count) / found_count`.
|
Во время выполнения парсинга `found_count`, `parsed_count`, `skipped_count` и `error_count` обновляются в базе. Админка опрашивает `/api/crawl-runs/latest` и показывает прогресс как `(parsed_count + skipped_count + error_count) / found_count`.
|
||||||
|
|
||||||
## MCP
|
## Обслуживание
|
||||||
|
|
||||||
Endpoint: `POST /mcp`, без авторизации на уровне приложения.
|
|
||||||
|
|
||||||
Поддерживаемые tools:
|
|
||||||
|
|
||||||
- `get_service_info()`
|
|
||||||
- `sync_employees(client_hash?, include_data?)`
|
|
||||||
- `search_employees(query, status?, limit?)`
|
|
||||||
- `get_employee(profile_id_or_url)`
|
|
||||||
- `list_employee_publications(profile_id_or_url)` — публикации сотрудника; при наличии данных из `employee_publications` возвращает авторов, DOI, аннотацию, описание, citation text, год, тип, язык, статус и ссылку HSE Publications.
|
|
||||||
- `list_employee_courses(profile_id_or_url)`
|
|
||||||
- `get_crawl_status()`
|
|
||||||
- `get_crawl_run_details(run_id)`
|
|
||||||
|
|
||||||
`get_service_info` возвращает метаданные сервиса, список tools и текущую версию набора сотрудников. `sync_employees` отдает полный snapshot или delta по `client_hash`; checksum набора строится по сотрудникам, их статусам и текущим checksums. Ответы tools возвращаются как JSON-строка внутри MCP `content[0].text`.
|
|
||||||
|
|
||||||
Новости сотрудника отдельной MCP tool не имеют: они доступны в `get_employee(...).data.sections` и `sync_employees(include_data=true)` как секция `type = "news"` с массивом `news_links`.
|
|
||||||
|
|
||||||
Пример локального запроса списка tools:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl http://localhost:8000/mcp \
|
docker compose logs -f api
|
||||||
-H "Content-Type: application/json" \
|
docker compose logs -f worker
|
||||||
-d '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}'
|
|
||||||
```
|
|
||||||
|
|
||||||
Если MCP нужно ограничить, делайте это на сетевом уровне: localhost binding, VPN, firewall, reverse proxy или другой внешний контур доступа.
|
|
||||||
|
|
||||||
## Обслуживание
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker compose logs -f api
|
|
||||||
docker compose exec postgres pg_dump -U miem miem_workers > backup.sql
|
docker compose exec postgres pg_dump -U miem miem_workers > backup.sql
|
||||||
docker compose down
|
docker compose down
|
||||||
```
|
```
|
||||||
|
|
||||||
Версия сервиса: `0.7.5`. Админка всегда показывает версии backend и frontend в footer.
|
Production deploy выполняется job `deploy` workflow `CI` после успешного `test` на push в `main` или вручную. Для него нужны Actions secrets `PROD_HOST`, `PROD_USER`, `PROD_SSH_KEY` и `PROD_KNOWN_HOSTS`; `.env` и ключи в репозиторий не добавляются. Перед обновлением workflow сохраняет PostgreSQL backup в домашний каталог deploy-пользователя, пересобирает `api` и `worker` и проверяет `/api/health`. Одновременно выполняется только один production deploy.
|
||||||
|
|
||||||
|
## Проверки
|
||||||
|
|
||||||
|
После установки зависимостей из `requirements.txt` выполните:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python -m pytest -q
|
||||||
|
node --check app/static/admin.js
|
||||||
|
```
|
||||||
|
|
||||||
|
Тесты с данными используют временную SQLite; отдельные API smoke-тесты запускают приложение с его текущей конфигурацией. Браузерная проверка: `pip install playwright`, `python -m playwright install chromium`, затем `python tests/browser_admin.py`. Она проверяет реальный рендеринг страниц, клавиатуру, диалог, фильтры и восстановление прогресса после ошибки. Скриншоты сохраняются во временную папку; путь выводится в конце.
|
||||||
|
|
||||||
|
Версия сервиса: `0.8.6`. Админка всегда показывает версии backend и frontend в footer.
|
||||||
|
|||||||
33
app/admin.py
33
app/admin.py
@@ -2,6 +2,7 @@ from fastapi import APIRouter, BackgroundTasks, Depends, Form, Request
|
|||||||
from fastapi.responses import HTMLResponse, RedirectResponse
|
from fastapi.responses import HTMLResponse, RedirectResponse
|
||||||
from fastapi.templating import Jinja2Templates
|
from fastapi.templating import Jinja2Templates
|
||||||
from sqlalchemy import desc, func, select
|
from sqlalchemy import desc, func, select
|
||||||
|
from sqlalchemy.exc import SQLAlchemyError
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.config import Settings, get_settings
|
from app.config import Settings, get_settings
|
||||||
@@ -93,19 +94,23 @@ def directory(
|
|||||||
parsed_started_to = _parse_date(started_to)
|
parsed_started_to = _parse_date(started_to)
|
||||||
parsed_has_email = None if has_email in (None, "") else has_email == "true"
|
parsed_has_email = None if has_email in (None, "") else has_email == "true"
|
||||||
parsed_has_academic_degree = None if has_academic_degree in (None, "") else has_academic_degree == "true"
|
parsed_has_academic_degree = None if has_academic_degree in (None, "") else has_academic_degree == "true"
|
||||||
page = list_employees_page(
|
try:
|
||||||
db,
|
page = list_employees_page(
|
||||||
status=status,
|
db,
|
||||||
q=q,
|
status=status,
|
||||||
started_from=parsed_started_from,
|
q=q,
|
||||||
started_to=parsed_started_to,
|
started_from=parsed_started_from,
|
||||||
has_email=parsed_has_email,
|
started_to=parsed_started_to,
|
||||||
has_academic_degree=parsed_has_academic_degree,
|
has_email=parsed_has_email,
|
||||||
sort=sort,
|
has_academic_degree=parsed_has_academic_degree,
|
||||||
direction=direction,
|
sort=sort,
|
||||||
limit=limit,
|
direction=direction,
|
||||||
offset=offset,
|
limit=limit,
|
||||||
)
|
offset=offset,
|
||||||
|
)
|
||||||
|
except SQLAlchemyError:
|
||||||
|
db.rollback()
|
||||||
|
return _render(request, "directory_error.html", {}, status_code=503)
|
||||||
return _render(
|
return _render(
|
||||||
request,
|
request,
|
||||||
"directory.html",
|
"directory.html",
|
||||||
@@ -121,7 +126,7 @@ def directory(
|
|||||||
"sort": sort,
|
"sort": sort,
|
||||||
"direction": direction,
|
"direction": direction,
|
||||||
"limit": page["limit"],
|
"limit": page["limit"],
|
||||||
"offset": offset,
|
"offset": page["offset"],
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -13,6 +13,8 @@ class Settings(BaseSettings):
|
|||||||
request_timeout: int = 30
|
request_timeout: int = 30
|
||||||
request_delay_seconds: float = 1.0
|
request_delay_seconds: float = 1.0
|
||||||
parser_use_playwright: bool = False
|
parser_use_playwright: bool = False
|
||||||
|
dismissal_confirmation_runs: int = Field(default=3, ge=1)
|
||||||
|
max_auto_dismissals_per_run: int = Field(default=25, ge=1)
|
||||||
|
|
||||||
admin_username: str = "admin"
|
admin_username: str = "admin"
|
||||||
admin_password: str = "admin"
|
admin_password: str = "admin"
|
||||||
|
|||||||
@@ -4,14 +4,12 @@ from fastapi.staticfiles import StaticFiles
|
|||||||
from app.admin import router as admin_router
|
from app.admin import router as admin_router
|
||||||
from app.api import router as api_router
|
from app.api import router as api_router
|
||||||
from app.db import init_db
|
from app.db import init_db
|
||||||
from app.mcp import router as mcp_router
|
|
||||||
from app.version import BACKEND_VERSION
|
from app.version import BACKEND_VERSION
|
||||||
|
|
||||||
app = FastAPI(title="MIEM Employees", version=BACKEND_VERSION)
|
app = FastAPI(title="MIEM Employees", version=BACKEND_VERSION)
|
||||||
app.mount("/static", StaticFiles(directory="app/static"), name="static")
|
app.mount("/static", StaticFiles(directory="app/static"), name="static")
|
||||||
app.include_router(api_router)
|
app.include_router(api_router)
|
||||||
app.include_router(admin_router)
|
app.include_router(admin_router)
|
||||||
app.include_router(mcp_router)
|
|
||||||
|
|
||||||
|
|
||||||
@app.on_event("startup")
|
@app.on_event("startup")
|
||||||
|
|||||||
262
app/mcp.py
262
app/mcp.py
@@ -1,262 +0,0 @@
|
|||||||
import json
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, Request
|
|
||||||
from sqlalchemy import desc, or_, select
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from app.db import get_db
|
|
||||||
from app.models import CrawlRun, Employee, EmployeePublication
|
|
||||||
from app.services.admin_data import run_detail_payload
|
|
||||||
from app.services.dataset_versions import service_info_payload, sync_employees_payload
|
|
||||||
from app.version import BACKEND_VERSION
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/mcp")
|
|
||||||
PROTOCOL_VERSION = "2024-11-05"
|
|
||||||
SERVICE_NAME = "miem-employees"
|
|
||||||
|
|
||||||
|
|
||||||
TOOLS = [
|
|
||||||
{
|
|
||||||
"name": "get_service_info",
|
|
||||||
"description": "Return service metadata, supported tools, and current dataset version.",
|
|
||||||
"inputSchema": {"type": "object", "properties": {}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "sync_employees",
|
|
||||||
"description": "Synchronize employees by dataset hash. Returns a full snapshot or a delta from client_hash.",
|
|
||||||
"inputSchema": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"client_hash": {"type": "string"},
|
|
||||||
"include_data": {"type": "boolean", "default": True},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "search_employees",
|
|
||||||
"description": "Search MIEM employees by name or profile URL.",
|
|
||||||
"inputSchema": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"query": {"type": "string"},
|
|
||||||
"status": {"type": "string", "enum": ["active", "dismissed"]},
|
|
||||||
"limit": {"type": "integer", "default": 20},
|
|
||||||
},
|
|
||||||
"required": ["query"],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "get_employee",
|
|
||||||
"description": "Get one employee by profile id, profile key, or canonical URL.",
|
|
||||||
"inputSchema": {"type": "object", "properties": {"profile_id_or_url": {"type": "string"}}, "required": ["profile_id_or_url"]},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "list_employee_publications",
|
|
||||||
"description": (
|
|
||||||
"List employee publications with detailed fields when available: authors, DOI URL, annotation, "
|
|
||||||
"description, citation text, year, publication type, language, status, and HSE Publications URL."
|
|
||||||
),
|
|
||||||
"inputSchema": {"type": "object", "properties": {"profile_id_or_url": {"type": "string"}}, "required": ["profile_id_or_url"]},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "list_employee_courses",
|
|
||||||
"description": "List teaching courses parsed from an employee profile.",
|
|
||||||
"inputSchema": {"type": "object", "properties": {"profile_id_or_url": {"type": "string"}}, "required": ["profile_id_or_url"]},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "get_crawl_status",
|
|
||||||
"description": "Return the latest crawl run status.",
|
|
||||||
"inputSchema": {"type": "object", "properties": {}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "get_crawl_run_details",
|
|
||||||
"description": "Return detailed employee changes and errors for one crawl run.",
|
|
||||||
"inputSchema": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {"run_id": {"type": "integer"}},
|
|
||||||
"required": ["run_id"],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("")
|
|
||||||
async def mcp_http(
|
|
||||||
request: Request,
|
|
||||||
db: Session = Depends(get_db),
|
|
||||||
) -> dict:
|
|
||||||
payload = await request.json()
|
|
||||||
method = payload.get("method")
|
|
||||||
request_id = payload.get("id")
|
|
||||||
params = payload.get("params") or {}
|
|
||||||
|
|
||||||
try:
|
|
||||||
if method == "initialize":
|
|
||||||
result = {
|
|
||||||
"protocolVersion": PROTOCOL_VERSION,
|
|
||||||
"serverInfo": {"name": SERVICE_NAME, "version": BACKEND_VERSION},
|
|
||||||
"capabilities": {"tools": {}},
|
|
||||||
}
|
|
||||||
elif method == "tools/list":
|
|
||||||
result = {"tools": TOOLS}
|
|
||||||
elif method == "tools/call":
|
|
||||||
result = _call_tool(db, params.get("name"), params.get("arguments") or {})
|
|
||||||
else:
|
|
||||||
return {"jsonrpc": "2.0", "id": request_id, "error": {"code": -32601, "message": "Method not found"}}
|
|
||||||
return {"jsonrpc": "2.0", "id": request_id, "result": result}
|
|
||||||
except Exception as exc:
|
|
||||||
return {"jsonrpc": "2.0", "id": request_id, "error": {"code": -32000, "message": str(exc)}}
|
|
||||||
|
|
||||||
|
|
||||||
def _call_tool(db: Session, name: str, arguments: dict) -> dict:
|
|
||||||
if name == "get_service_info":
|
|
||||||
return _tool_response(
|
|
||||||
service_info_payload(
|
|
||||||
db,
|
|
||||||
tools=TOOLS,
|
|
||||||
service_name=SERVICE_NAME,
|
|
||||||
backend_version=BACKEND_VERSION,
|
|
||||||
protocol_version=PROTOCOL_VERSION,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
if name == "sync_employees":
|
|
||||||
return _tool_response(
|
|
||||||
sync_employees_payload(
|
|
||||||
db,
|
|
||||||
client_hash=arguments.get("client_hash"),
|
|
||||||
include_data=bool(arguments.get("include_data", True)),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
if name == "search_employees":
|
|
||||||
return _tool_response(_search_employees(db, arguments))
|
|
||||||
if name == "get_employee":
|
|
||||||
employee = _find_employee(db, arguments["profile_id_or_url"])
|
|
||||||
return _tool_response(_employee_payload(employee) if employee else {"error": "not_found"})
|
|
||||||
if name == "list_employee_publications":
|
|
||||||
employee = _find_employee(db, arguments["profile_id_or_url"])
|
|
||||||
return _tool_response(_collect_section_items(employee, "publications"))
|
|
||||||
if name == "list_employee_courses":
|
|
||||||
employee = _find_employee(db, arguments["profile_id_or_url"])
|
|
||||||
return _tool_response(_collect_section_items(employee, "courses_by_year"))
|
|
||||||
if name == "get_crawl_status":
|
|
||||||
run = db.scalar(select(CrawlRun).order_by(desc(CrawlRun.started_at)).limit(1))
|
|
||||||
return _tool_response(_run_payload(run) if run else {"status": "never_run"})
|
|
||||||
if name == "get_crawl_run_details":
|
|
||||||
run = db.get(CrawlRun, int(arguments["run_id"]))
|
|
||||||
return _tool_response(run_detail_payload(db, run) if run else {"error": "not_found"})
|
|
||||||
raise ValueError(f"Unknown tool: {name}")
|
|
||||||
|
|
||||||
|
|
||||||
def _search_employees(db: Session, arguments: dict) -> list[dict]:
|
|
||||||
query = arguments.get("query", "")
|
|
||||||
limit = min(int(arguments.get("limit") or 20), 100)
|
|
||||||
stmt = select(Employee)
|
|
||||||
if arguments.get("status"):
|
|
||||||
stmt = stmt.where(Employee.status == arguments["status"])
|
|
||||||
if query:
|
|
||||||
pattern = f"%{query}%"
|
|
||||||
stmt = stmt.where(or_(Employee.full_name.ilike(pattern), Employee.canonical_url.ilike(pattern)))
|
|
||||||
employees = db.scalars(stmt.order_by(Employee.full_name).limit(limit)).all()
|
|
||||||
return [_employee_payload(employee, include_data=False) for employee in employees]
|
|
||||||
|
|
||||||
|
|
||||||
def _find_employee(db: Session, value: str) -> Employee | None:
|
|
||||||
pattern = value.strip()
|
|
||||||
stmt = select(Employee).where(
|
|
||||||
or_(
|
|
||||||
Employee.profile_key == pattern,
|
|
||||||
Employee.profile_id == pattern,
|
|
||||||
Employee.canonical_url == pattern,
|
|
||||||
Employee.canonical_url.ilike(f"%{pattern}%"),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return db.scalar(stmt.limit(1))
|
|
||||||
|
|
||||||
|
|
||||||
def _collect_section_items(employee: Employee | None, section_type: str) -> dict:
|
|
||||||
if not employee:
|
|
||||||
return {"items": []}
|
|
||||||
if section_type == "publications":
|
|
||||||
publications = _stored_publications(employee)
|
|
||||||
if publications:
|
|
||||||
return {"employee": _employee_payload(employee, include_data=False), "items": publications}
|
|
||||||
if not employee.current_data:
|
|
||||||
return {"employee": _employee_payload(employee, include_data=False), "items": []}
|
|
||||||
items = []
|
|
||||||
for section in employee.current_data.get("sections") or []:
|
|
||||||
if section.get("type") != section_type:
|
|
||||||
continue
|
|
||||||
if section_type == "publications":
|
|
||||||
items.extend(section.get("publications") or [])
|
|
||||||
elif section_type == "courses_by_year":
|
|
||||||
items.extend(section.get("courses") or [])
|
|
||||||
return {"employee": _employee_payload(employee, include_data=False), "items": items}
|
|
||||||
|
|
||||||
|
|
||||||
def _stored_publications(employee: Employee) -> list[dict]:
|
|
||||||
return [_publication_payload(publication) for publication in sorted(employee.publications, key=_publication_sort_key)]
|
|
||||||
|
|
||||||
|
|
||||||
def _publication_sort_key(publication: EmployeePublication) -> tuple:
|
|
||||||
return (publication.year or 0, publication.title or "", publication.id)
|
|
||||||
|
|
||||||
|
|
||||||
def _publication_payload(publication: EmployeePublication) -> dict:
|
|
||||||
text = publication.citation_text or publication.title
|
|
||||||
payload = {
|
|
||||||
"id": publication.publication_id,
|
|
||||||
"publication_id": publication.publication_id,
|
|
||||||
"title": publication.title,
|
|
||||||
"text": text,
|
|
||||||
"url": publication.url,
|
|
||||||
}
|
|
||||||
optional = {
|
|
||||||
"year": publication.year,
|
|
||||||
"type": publication.publication_type,
|
|
||||||
"publication_type": publication.publication_type,
|
|
||||||
"language": publication.language,
|
|
||||||
"status": publication.status,
|
|
||||||
"doi_url": publication.doi_url,
|
|
||||||
"other_url": publication.other_url,
|
|
||||||
"document_url": publication.document_url,
|
|
||||||
"citation_text": publication.citation_text,
|
|
||||||
"annotation": publication.annotation,
|
|
||||||
"description": publication.description,
|
|
||||||
"authors": publication.authors,
|
|
||||||
}
|
|
||||||
payload.update({key: value for key, value in optional.items() if value not in (None, [], {})})
|
|
||||||
return payload
|
|
||||||
|
|
||||||
|
|
||||||
def _employee_payload(employee: Employee, include_data: bool = True) -> dict:
|
|
||||||
payload = {
|
|
||||||
"profile_key": employee.profile_key,
|
|
||||||
"profile_id": employee.profile_id,
|
|
||||||
"full_name": employee.full_name,
|
|
||||||
"status": employee.status,
|
|
||||||
"canonical_url": employee.canonical_url,
|
|
||||||
"last_seen_at": employee.last_seen_at.isoformat() if employee.last_seen_at else None,
|
|
||||||
"dismissed_at": employee.dismissed_at.isoformat() if employee.dismissed_at else None,
|
|
||||||
}
|
|
||||||
if include_data:
|
|
||||||
payload["data"] = employee.current_data
|
|
||||||
return payload
|
|
||||||
|
|
||||||
|
|
||||||
def _run_payload(run: CrawlRun) -> dict:
|
|
||||||
return {
|
|
||||||
"id": run.id,
|
|
||||||
"status": run.status,
|
|
||||||
"source_url": run.source_url,
|
|
||||||
"started_at": run.started_at.isoformat() if run.started_at else None,
|
|
||||||
"finished_at": run.finished_at.isoformat() if run.finished_at else None,
|
|
||||||
"found_count": run.found_count,
|
|
||||||
"parsed_count": run.parsed_count,
|
|
||||||
"skipped_count": run.skipped_count,
|
|
||||||
"error_count": run.error_count,
|
|
||||||
"dismissed_count": run.dismissed_count,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _tool_response(data: object) -> dict:
|
|
||||||
return {"content": [{"type": "text", "text": json.dumps(data, ensure_ascii=False, default=str)}]}
|
|
||||||
@@ -162,7 +162,6 @@ class CrawlRun(Base):
|
|||||||
message: Mapped[str | None] = mapped_column(Text)
|
message: Mapped[str | None] = mapped_column(Text)
|
||||||
|
|
||||||
employee_changes: Mapped[list["CrawlRunEmployeeChange"]] = relationship(back_populates="crawl_run")
|
employee_changes: Mapped[list["CrawlRunEmployeeChange"]] = relationship(back_populates="crawl_run")
|
||||||
dataset_versions: Mapped[list["DatasetVersion"]] = relationship(back_populates="crawl_run")
|
|
||||||
|
|
||||||
|
|
||||||
class CrawlRunEmployeeChange(Base):
|
class CrawlRunEmployeeChange(Base):
|
||||||
@@ -242,42 +241,3 @@ class ParseResourceCache(Base):
|
|||||||
body_snapshot: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
|
body_snapshot: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
|
||||||
parser_version: Mapped[str | None] = mapped_column(String(32))
|
parser_version: Mapped[str | None] = mapped_column(String(32))
|
||||||
fetched_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)
|
fetched_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)
|
||||||
|
|
||||||
|
|
||||||
class DatasetVersion(Base):
|
|
||||||
__tablename__ = "dataset_versions"
|
|
||||||
__table_args__ = (
|
|
||||||
UniqueConstraint("hash", name="uq_dataset_versions_hash"),
|
|
||||||
Index("ix_dataset_versions_created_at", "created_at"),
|
|
||||||
)
|
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
|
||||||
hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
||||||
previous_hash: Mapped[str | None] = mapped_column(String(64))
|
|
||||||
crawl_run_id: Mapped[int | None] = mapped_column(ForeignKey("crawl_runs.id"))
|
|
||||||
employee_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
|
||||||
active_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
|
||||||
dismissed_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)
|
|
||||||
|
|
||||||
crawl_run: Mapped[CrawlRun | None] = relationship(back_populates="dataset_versions")
|
|
||||||
items: Mapped[list["DatasetVersionItem"]] = relationship(back_populates="dataset_version", cascade="all, delete-orphan")
|
|
||||||
|
|
||||||
|
|
||||||
class DatasetVersionItem(Base):
|
|
||||||
__tablename__ = "dataset_version_items"
|
|
||||||
__table_args__ = (
|
|
||||||
UniqueConstraint("dataset_version_id", "profile_key", name="uq_dataset_version_items_version_profile"),
|
|
||||||
Index("ix_dataset_version_items_hash", "dataset_version_id"),
|
|
||||||
Index("ix_dataset_version_items_profile_key", "profile_key"),
|
|
||||||
)
|
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
|
||||||
dataset_version_id: Mapped[int] = mapped_column(ForeignKey("dataset_versions.id"), nullable=False)
|
|
||||||
profile_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
|
||||||
employee_id: Mapped[int | None] = mapped_column(ForeignKey("employees.id"))
|
|
||||||
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
||||||
checksum: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
||||||
|
|
||||||
dataset_version: Mapped[DatasetVersion] = relationship(back_populates="items")
|
|
||||||
employee: Mapped[Employee | None] = relationship()
|
|
||||||
|
|||||||
@@ -2,13 +2,16 @@ import re
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
_PATTERN = re.compile(r"\b(?:кандидат|доктор)\s+[\w\s-]{0,80}?\s+наук\b|\bph\.?\s*d\.?\b", re.IGNORECASE)
|
_PATTERN = re.compile(r"\b(?:кандидат|доктор)(?:\s+[\w-]+){0,12}\s+наук\b|\bph\.?\s*d\.?(?!\w)", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
def academic_degrees(data: dict[str, Any] | None) -> list[str]:
|
def academic_degrees(data: dict[str, Any] | None) -> list[str]:
|
||||||
degrees = []
|
degrees = []
|
||||||
|
seen = set()
|
||||||
for section in (data or {}).get("sections") or []:
|
for section in (data or {}).get("sections") or []:
|
||||||
if not isinstance(section, dict) or not re.search(r"уч[её]н.*степен|academic degree", str(section.get("title") or ""), re.IGNORECASE):
|
if not isinstance(section, dict) or not re.search(
|
||||||
|
r"образован|степен|academic degree|education", str(section.get("title") or ""), re.IGNORECASE
|
||||||
|
):
|
||||||
continue
|
continue
|
||||||
values = [
|
values = [
|
||||||
*(entry.get("text") for entry in section.get("year_entries") or [] if isinstance(entry, dict)),
|
*(entry.get("text") for entry in section.get("year_entries") or [] if isinstance(entry, dict)),
|
||||||
@@ -16,8 +19,15 @@ def academic_degrees(data: dict[str, Any] | None) -> list[str]:
|
|||||||
*(section.get("items") or []),
|
*(section.get("items") or []),
|
||||||
section.get("raw_text"),
|
section.get("raw_text"),
|
||||||
]
|
]
|
||||||
|
table = section.get("table") or {}
|
||||||
|
for row in table.get("rows") or []:
|
||||||
|
if isinstance(row, dict):
|
||||||
|
values.extend(row.get("cells") or [])
|
||||||
for value in values:
|
for value in values:
|
||||||
text = str(value or "").strip()
|
text = str(value or "").strip()
|
||||||
if text and _PATTERN.search(text) and text not in degrees:
|
for match in _PATTERN.finditer(text):
|
||||||
degrees.append(text)
|
degree = match.group(0).strip()
|
||||||
|
if degree.casefold() not in seen:
|
||||||
|
seen.add(degree.casefold())
|
||||||
|
degrees.append(degree)
|
||||||
return degrees
|
return degrees
|
||||||
|
|||||||
@@ -132,6 +132,8 @@ def list_employees_page(
|
|||||||
has_academic_degree=has_academic_degree,
|
has_academic_degree=has_academic_degree,
|
||||||
)
|
)
|
||||||
total = db.scalar(select(func.count()).select_from(base_stmt.subquery())) or 0
|
total = db.scalar(select(func.count()).select_from(base_stmt.subquery())) or 0
|
||||||
|
if offset >= total:
|
||||||
|
offset = 0
|
||||||
sort_column = EMPLOYEE_SORTS.get(sort, Employee.full_name)
|
sort_column = EMPLOYEE_SORTS.get(sort, Employee.full_name)
|
||||||
order = desc(sort_column) if direction == "desc" else sort_column
|
order = desc(sort_column) if direction == "desc" else sort_column
|
||||||
employees = db.scalars(base_stmt.order_by(order).limit(limit).offset(offset)).all()
|
employees = db.scalars(base_stmt.order_by(order).limit(limit).offset(offset)).all()
|
||||||
|
|||||||
@@ -26,7 +26,6 @@ from app.parser.collector import collect_profile_links
|
|||||||
from app.parser.profile import parse_person_profile
|
from app.parser.profile import parse_person_profile
|
||||||
from app.parser.profile_url import profile_key
|
from app.parser.profile_url import profile_key
|
||||||
from app.services.academic_degrees import academic_degrees
|
from app.services.academic_degrees import academic_degrees
|
||||||
from app.services.dataset_versions import get_or_create_current_version
|
|
||||||
from app.services.resource_cache import ResourceCache
|
from app.services.resource_cache import ResourceCache
|
||||||
|
|
||||||
HEADERS = {
|
HEADERS = {
|
||||||
@@ -102,7 +101,6 @@ def run_crawl(db: Session, settings: Settings) -> CrawlRun:
|
|||||||
max_auto_dismissals=settings.max_auto_dismissals_per_run,
|
max_auto_dismissals=settings.max_auto_dismissals_per_run,
|
||||||
)
|
)
|
||||||
run.status = "completed"
|
run.status = "completed"
|
||||||
get_or_create_current_version(db, crawl_run_id=run.id)
|
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
run.status = "failed"
|
run.status = "failed"
|
||||||
run.message = str(exc)
|
run.message = str(exc)
|
||||||
@@ -146,7 +144,6 @@ def refresh_dismissed_status(db: Session, settings: Settings) -> CrawlRun:
|
|||||||
)
|
)
|
||||||
run.parsed_count += 1
|
run.parsed_count += 1
|
||||||
run.status = "completed"
|
run.status = "completed"
|
||||||
get_or_create_current_version(db, crawl_run_id=run.id)
|
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
run.status = "failed"
|
run.status = "failed"
|
||||||
run.error_count = 1
|
run.error_count = 1
|
||||||
@@ -194,7 +191,6 @@ def refresh_employee(db: Session, employee: Employee, settings: Settings) -> Cra
|
|||||||
else:
|
else:
|
||||||
run.skipped_count = 1
|
run.skipped_count = 1
|
||||||
run.status = "completed"
|
run.status = "completed"
|
||||||
get_or_create_current_version(db, crawl_run_id=run.id)
|
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
run.status = "failed"
|
run.status = "failed"
|
||||||
run.error_count = 1
|
run.error_count = 1
|
||||||
|
|||||||
@@ -1,227 +0,0 @@
|
|||||||
import hashlib
|
|
||||||
import json
|
|
||||||
from dataclasses import dataclass
|
|
||||||
|
|
||||||
from sqlalchemy import desc, select
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from app.models import DatasetVersion, DatasetVersionItem, Employee
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class EmployeeMarker:
|
|
||||||
profile_key: str
|
|
||||||
employee_id: int | None
|
|
||||||
status: str
|
|
||||||
checksum: str
|
|
||||||
|
|
||||||
|
|
||||||
def get_or_create_current_version(db: Session, *, crawl_run_id: int | None = None) -> DatasetVersion:
|
|
||||||
employees = db.scalars(select(Employee).order_by(Employee.profile_key)).all()
|
|
||||||
markers = [_employee_marker(employee) for employee in employees]
|
|
||||||
dataset_hash = _dataset_hash(markers)
|
|
||||||
latest = get_latest_version(db)
|
|
||||||
if latest and latest.hash == dataset_hash:
|
|
||||||
return latest
|
|
||||||
|
|
||||||
active_count = sum(1 for marker in markers if marker.status == "active")
|
|
||||||
dismissed_count = sum(1 for marker in markers if marker.status == "dismissed")
|
|
||||||
version = DatasetVersion(
|
|
||||||
hash=dataset_hash,
|
|
||||||
previous_hash=latest.hash if latest else None,
|
|
||||||
crawl_run_id=crawl_run_id,
|
|
||||||
employee_count=len(markers),
|
|
||||||
active_count=active_count,
|
|
||||||
dismissed_count=dismissed_count,
|
|
||||||
)
|
|
||||||
db.add(version)
|
|
||||||
db.flush()
|
|
||||||
for marker in markers:
|
|
||||||
db.add(
|
|
||||||
DatasetVersionItem(
|
|
||||||
dataset_version_id=version.id,
|
|
||||||
profile_key=marker.profile_key,
|
|
||||||
employee_id=marker.employee_id,
|
|
||||||
status=marker.status,
|
|
||||||
checksum=marker.checksum,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
db.flush()
|
|
||||||
return version
|
|
||||||
|
|
||||||
|
|
||||||
def get_latest_version(db: Session) -> DatasetVersion | None:
|
|
||||||
return db.scalar(select(DatasetVersion).order_by(desc(DatasetVersion.created_at), desc(DatasetVersion.id)).limit(1))
|
|
||||||
|
|
||||||
|
|
||||||
def get_version_by_hash(db: Session, dataset_hash: str | None) -> DatasetVersion | None:
|
|
||||||
if not dataset_hash:
|
|
||||||
return None
|
|
||||||
return db.scalar(select(DatasetVersion).where(DatasetVersion.hash == dataset_hash).limit(1))
|
|
||||||
|
|
||||||
|
|
||||||
def service_info_payload(db: Session, *, tools: list[dict], service_name: str, backend_version: str, protocol_version: str) -> dict:
|
|
||||||
version = get_or_create_current_version(db)
|
|
||||||
db.commit()
|
|
||||||
return {
|
|
||||||
"service_name": service_name,
|
|
||||||
"backend_version": backend_version,
|
|
||||||
"protocolVersion": protocol_version,
|
|
||||||
"tools": tools,
|
|
||||||
"dataset": _version_payload(version),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def sync_employees_payload(db: Session, *, client_hash: str | None = None, include_data: bool = True) -> dict:
|
|
||||||
current = get_or_create_current_version(db)
|
|
||||||
db.commit()
|
|
||||||
if not client_hash:
|
|
||||||
return _full_sync_payload(db, current, include_data=include_data, reason=None)
|
|
||||||
if client_hash == current.hash:
|
|
||||||
return {
|
|
||||||
"mode": "delta",
|
|
||||||
"from_hash": client_hash,
|
|
||||||
"to_hash": current.hash,
|
|
||||||
"dataset": _version_payload(current),
|
|
||||||
"changes": {"added": [], "updated": [], "dismissed": [], "removed": []},
|
|
||||||
}
|
|
||||||
|
|
||||||
previous = get_version_by_hash(db, client_hash)
|
|
||||||
if not previous:
|
|
||||||
return _full_sync_payload(db, current, include_data=include_data, reason="unknown_client_hash", from_hash=client_hash)
|
|
||||||
|
|
||||||
return _delta_sync_payload(db, previous, current, include_data=include_data)
|
|
||||||
|
|
||||||
|
|
||||||
def _full_sync_payload(
|
|
||||||
db: Session,
|
|
||||||
current: DatasetVersion,
|
|
||||||
*,
|
|
||||||
include_data: bool,
|
|
||||||
reason: str | None,
|
|
||||||
from_hash: str | None = None,
|
|
||||||
) -> dict:
|
|
||||||
employees = db.scalars(select(Employee).order_by(Employee.profile_key)).all()
|
|
||||||
payload = {
|
|
||||||
"mode": "full",
|
|
||||||
"from_hash": from_hash,
|
|
||||||
"to_hash": current.hash,
|
|
||||||
"dataset": _version_payload(current),
|
|
||||||
"items": [_employee_payload(employee, include_data=include_data) for employee in employees],
|
|
||||||
}
|
|
||||||
if reason:
|
|
||||||
payload["reason"] = reason
|
|
||||||
return payload
|
|
||||||
|
|
||||||
|
|
||||||
def _delta_sync_payload(db: Session, previous: DatasetVersion, current: DatasetVersion, *, include_data: bool) -> dict:
|
|
||||||
previous_items = _items_by_profile_key(previous)
|
|
||||||
current_items = _items_by_profile_key(current)
|
|
||||||
employees = {employee.profile_key: employee for employee in db.scalars(select(Employee)).all()}
|
|
||||||
added = []
|
|
||||||
updated = []
|
|
||||||
dismissed = []
|
|
||||||
removed = []
|
|
||||||
|
|
||||||
for profile_key, current_item in sorted(current_items.items()):
|
|
||||||
previous_item = previous_items.get(profile_key)
|
|
||||||
employee = employees.get(profile_key)
|
|
||||||
if not previous_item:
|
|
||||||
if employee:
|
|
||||||
added.append(_employee_payload(employee, include_data=include_data))
|
|
||||||
continue
|
|
||||||
if previous_item.checksum == current_item.checksum and previous_item.status == current_item.status:
|
|
||||||
continue
|
|
||||||
if current_item.status == "dismissed":
|
|
||||||
dismissed.append(_tombstone(profile_key, current_item.status, employee))
|
|
||||||
elif employee:
|
|
||||||
updated.append(_employee_payload(employee, include_data=include_data))
|
|
||||||
|
|
||||||
for profile_key, previous_item in sorted(previous_items.items()):
|
|
||||||
if profile_key not in current_items:
|
|
||||||
removed.append(_tombstone(profile_key, "removed", employees.get(profile_key), checksum=previous_item.checksum))
|
|
||||||
|
|
||||||
return {
|
|
||||||
"mode": "delta",
|
|
||||||
"from_hash": previous.hash,
|
|
||||||
"to_hash": current.hash,
|
|
||||||
"dataset": _version_payload(current),
|
|
||||||
"changes": {
|
|
||||||
"added": added,
|
|
||||||
"updated": updated,
|
|
||||||
"dismissed": dismissed,
|
|
||||||
"removed": removed,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _items_by_profile_key(version: DatasetVersion) -> dict[str, DatasetVersionItem]:
|
|
||||||
return {item.profile_key: item for item in version.items}
|
|
||||||
|
|
||||||
|
|
||||||
def _version_payload(version: DatasetVersion) -> dict:
|
|
||||||
return {
|
|
||||||
"hash": version.hash,
|
|
||||||
"previous_hash": version.previous_hash,
|
|
||||||
"created_at": version.created_at.isoformat() if version.created_at else None,
|
|
||||||
"crawl_run_id": version.crawl_run_id,
|
|
||||||
"employee_count": version.employee_count,
|
|
||||||
"active_count": version.active_count,
|
|
||||||
"dismissed_count": version.dismissed_count,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _employee_marker(employee: Employee) -> EmployeeMarker:
|
|
||||||
return EmployeeMarker(
|
|
||||||
profile_key=employee.profile_key,
|
|
||||||
employee_id=employee.id,
|
|
||||||
status=employee.status,
|
|
||||||
checksum=employee.current_checksum or _payload_hash(employee.current_data or {}),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _dataset_hash(markers: list[EmployeeMarker]) -> str:
|
|
||||||
payload = [
|
|
||||||
{"profile_key": marker.profile_key, "status": marker.status, "checksum": marker.checksum}
|
|
||||||
for marker in sorted(markers, key=lambda item: item.profile_key)
|
|
||||||
]
|
|
||||||
return _payload_hash(payload)
|
|
||||||
|
|
||||||
|
|
||||||
def _payload_hash(value: object) -> str:
|
|
||||||
payload = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str)
|
|
||||||
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
|
||||||
|
|
||||||
|
|
||||||
def _employee_payload(employee: Employee, *, include_data: bool) -> dict:
|
|
||||||
payload = {
|
|
||||||
"profile_key": employee.profile_key,
|
|
||||||
"profile_id": employee.profile_id,
|
|
||||||
"full_name": employee.full_name,
|
|
||||||
"status": employee.status,
|
|
||||||
"canonical_url": employee.canonical_url,
|
|
||||||
"last_seen_at": employee.last_seen_at.isoformat() if employee.last_seen_at else None,
|
|
||||||
"dismissed_at": employee.dismissed_at.isoformat() if employee.dismissed_at else None,
|
|
||||||
"checksum": employee.current_checksum or _payload_hash(employee.current_data or {}),
|
|
||||||
}
|
|
||||||
if include_data:
|
|
||||||
payload["data"] = employee.current_data
|
|
||||||
return payload
|
|
||||||
|
|
||||||
|
|
||||||
def _tombstone(profile_key: str, status: str, employee: Employee | None, *, checksum: str | None = None) -> dict:
|
|
||||||
payload = {
|
|
||||||
"profile_key": profile_key,
|
|
||||||
"status": status,
|
|
||||||
"checksum": checksum or (employee.current_checksum if employee else None),
|
|
||||||
}
|
|
||||||
if employee:
|
|
||||||
payload.update(
|
|
||||||
{
|
|
||||||
"profile_id": employee.profile_id,
|
|
||||||
"full_name": employee.full_name,
|
|
||||||
"canonical_url": employee.canonical_url,
|
|
||||||
"dismissed_at": employee.dismissed_at.isoformat() if employee.dismissed_at else None,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
return payload
|
|
||||||
@@ -5,9 +5,23 @@
|
|||||||
flex-direction: column;
|
flex-direction: column;
|
||||||
color: #1f2937;
|
color: #1f2937;
|
||||||
background: #f6f7f9;
|
background: #f6f7f9;
|
||||||
font-family: Arial, sans-serif;
|
font-family: system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.admin__skip-link {
|
||||||
|
position: absolute;
|
||||||
|
top: 8px;
|
||||||
|
left: -999px;
|
||||||
|
z-index: 100;
|
||||||
|
padding: 8px 12px;
|
||||||
|
color: #ffffff;
|
||||||
|
background: #0f766e;
|
||||||
|
}
|
||||||
|
|
||||||
|
.admin__skip-link:focus-visible {
|
||||||
|
left: 8px;
|
||||||
|
}
|
||||||
|
|
||||||
.admin__header {
|
.admin__header {
|
||||||
display: flex;
|
display: flex;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
@@ -16,6 +30,7 @@
|
|||||||
padding: 18px 32px;
|
padding: 18px 32px;
|
||||||
background: #ffffff;
|
background: #ffffff;
|
||||||
border-bottom: 1px solid #d9dee7;
|
border-bottom: 1px solid #d9dee7;
|
||||||
|
flex-wrap: wrap;
|
||||||
}
|
}
|
||||||
|
|
||||||
.admin__brand {
|
.admin__brand {
|
||||||
@@ -40,6 +55,16 @@
|
|||||||
font-weight: 700;
|
font-weight: 700;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.admin__link[aria-current="page"] {
|
||||||
|
text-decoration: underline;
|
||||||
|
text-underline-offset: 4px;
|
||||||
|
}
|
||||||
|
|
||||||
|
:is(a, button, input, select, summary):focus-visible {
|
||||||
|
outline: 3px solid #0f766e;
|
||||||
|
outline-offset: 2px;
|
||||||
|
}
|
||||||
|
|
||||||
.admin__main {
|
.admin__main {
|
||||||
flex: 1;
|
flex: 1;
|
||||||
width: min(1180px, calc(100% - 32px));
|
width: min(1180px, calc(100% - 32px));
|
||||||
@@ -107,10 +132,6 @@
|
|||||||
border-collapse: collapse;
|
border-collapse: collapse;
|
||||||
}
|
}
|
||||||
|
|
||||||
.table__row {
|
|
||||||
cursor: pointer;
|
|
||||||
}
|
|
||||||
|
|
||||||
.table__row:hover {
|
.table__row:hover {
|
||||||
background: #f0fdfa;
|
background: #f0fdfa;
|
||||||
}
|
}
|
||||||
@@ -161,8 +182,9 @@
|
|||||||
border-radius: 6px;
|
border-radius: 6px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.button {
|
.button {
|
||||||
padding: 10px 14px;
|
min-height: 40px;
|
||||||
|
padding: 10px 14px;
|
||||||
border: 0;
|
border: 0;
|
||||||
border-radius: 6px;
|
border-radius: 6px;
|
||||||
color: #ffffff;
|
color: #ffffff;
|
||||||
@@ -178,6 +200,19 @@
|
|||||||
|
|
||||||
.button--compact {
|
.button--compact {
|
||||||
padding: 8px 12px;
|
padding: 8px 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.button:hover {
|
||||||
|
filter: brightness(0.9);
|
||||||
|
}
|
||||||
|
|
||||||
|
.button:active {
|
||||||
|
filter: brightness(0.8);
|
||||||
|
}
|
||||||
|
|
||||||
|
.button:disabled {
|
||||||
|
cursor: wait;
|
||||||
|
opacity: 0.65;
|
||||||
}
|
}
|
||||||
|
|
||||||
.code {
|
.code {
|
||||||
@@ -426,25 +461,25 @@
|
|||||||
gap: 12px;
|
gap: 12px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.progress-panel__header {
|
.progress-panel__header {
|
||||||
display: flex;
|
display: flex;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
justify-content: space-between;
|
justify-content: space-between;
|
||||||
gap: 16px;
|
gap: 16px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.progress-panel__actions {
|
.progress-panel__actions {
|
||||||
display: flex;
|
display: flex;
|
||||||
flex-wrap: wrap;
|
flex-wrap: wrap;
|
||||||
gap: 8px;
|
gap: 8px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.button--secondary {
|
.button--secondary {
|
||||||
color: #0f766e;
|
color: #0f766e;
|
||||||
background: #ccfbf1;
|
background: #ccfbf1;
|
||||||
}
|
}
|
||||||
|
|
||||||
.progress-panel__body {
|
.progress-panel__body {
|
||||||
display: grid;
|
display: grid;
|
||||||
gap: 10px;
|
gap: 10px;
|
||||||
}
|
}
|
||||||
@@ -467,6 +502,12 @@
|
|||||||
color: #6b7280;
|
color: #6b7280;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.progress-panel__error {
|
||||||
|
margin: 0;
|
||||||
|
color: #991b1b;
|
||||||
|
font-weight: 700;
|
||||||
|
}
|
||||||
|
|
||||||
.progress-bar {
|
.progress-bar {
|
||||||
height: 12px;
|
height: 12px;
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
@@ -478,7 +519,6 @@
|
|||||||
height: 100%;
|
height: 100%;
|
||||||
width: 0;
|
width: 0;
|
||||||
background: #0f766e;
|
background: #0f766e;
|
||||||
transition: width 0.25s ease;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
.directory {
|
.directory {
|
||||||
@@ -503,9 +543,9 @@
|
|||||||
color: #6b7280;
|
color: #6b7280;
|
||||||
}
|
}
|
||||||
|
|
||||||
.directory__filters {
|
.directory__filters {
|
||||||
display: grid;
|
display: grid;
|
||||||
grid-template-columns: minmax(220px, 1.7fr) repeat(6, minmax(120px, 1fr));
|
grid-template-columns: minmax(0, 1.1fr) minmax(0, 2fr) minmax(0, 1.25fr) minmax(0, 2fr);
|
||||||
gap: 10px;
|
gap: 10px;
|
||||||
padding: 16px;
|
padding: 16px;
|
||||||
background: #ffffff;
|
background: #ffffff;
|
||||||
@@ -513,13 +553,58 @@
|
|||||||
border-radius: 8px;
|
border-radius: 8px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.directory__input {
|
.directory__filter-group {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: 1fr;
|
||||||
|
align-content: start;
|
||||||
|
gap: 8px;
|
||||||
|
min-width: 0;
|
||||||
|
margin: 0;
|
||||||
|
padding: 0;
|
||||||
|
border: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.directory__filter-group--search {
|
||||||
|
grid-template-columns: 1fr;
|
||||||
|
}
|
||||||
|
|
||||||
|
.directory__filter-legend {
|
||||||
|
grid-column: 1 / -1;
|
||||||
|
margin-bottom: 2px;
|
||||||
|
color: #374151;
|
||||||
|
font-size: 12px;
|
||||||
|
font-weight: 700;
|
||||||
|
}
|
||||||
|
|
||||||
|
.directory__field {
|
||||||
|
display: grid;
|
||||||
|
gap: 5px;
|
||||||
|
min-width: 0;
|
||||||
|
color: #4b5563;
|
||||||
|
font-size: 12px;
|
||||||
|
font-weight: 700;
|
||||||
|
}
|
||||||
|
|
||||||
|
.directory__input {
|
||||||
|
box-sizing: border-box;
|
||||||
|
width: 100%;
|
||||||
|
font: inherit;
|
||||||
|
font-size: 16px;
|
||||||
|
font-weight: 400;
|
||||||
min-width: 0;
|
min-width: 0;
|
||||||
padding: 10px 12px;
|
padding: 10px 12px;
|
||||||
border: 1px solid #cbd5e1;
|
border: 1px solid #cbd5e1;
|
||||||
border-radius: 6px;
|
border-radius: 6px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.directory__filter-actions {
|
||||||
|
display: flex;
|
||||||
|
align-items: end;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
gap: 8px;
|
||||||
|
grid-column: 1 / -1;
|
||||||
|
}
|
||||||
|
|
||||||
.directory__table-wrap {
|
.directory__table-wrap {
|
||||||
overflow-x: auto;
|
overflow-x: auto;
|
||||||
background: #ffffff;
|
background: #ffffff;
|
||||||
@@ -539,9 +624,18 @@
|
|||||||
font-weight: 700;
|
font-weight: 700;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.directory__help {
|
||||||
|
color: #4b5563;
|
||||||
|
line-height: 1.5;
|
||||||
|
}
|
||||||
|
|
||||||
|
.directory__help summary {
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
|
||||||
.directory-table {
|
.directory-table {
|
||||||
width: 100%;
|
width: 100%;
|
||||||
min-width: 1120px;
|
min-width: 640px;
|
||||||
border-collapse: collapse;
|
border-collapse: collapse;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -556,14 +650,19 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
.directory-table__cell {
|
.directory-table__cell {
|
||||||
|
overflow-wrap: anywhere;
|
||||||
max-width: 280px;
|
max-width: 280px;
|
||||||
padding: 12px 10px;
|
padding: 12px 10px;
|
||||||
border-bottom: 1px solid #e5e7eb;
|
border-bottom: 1px solid #e5e7eb;
|
||||||
vertical-align: top;
|
vertical-align: top;
|
||||||
|
}
|
||||||
|
|
||||||
|
.directory-table .badge {
|
||||||
|
white-space: nowrap;
|
||||||
}
|
}
|
||||||
|
|
||||||
.directory-table__row {
|
.directory-table__row {
|
||||||
cursor: pointer;
|
cursor: default;
|
||||||
}
|
}
|
||||||
|
|
||||||
.directory-table__row:hover {
|
.directory-table__row:hover {
|
||||||
@@ -576,33 +675,32 @@
|
|||||||
text-align: center;
|
text-align: center;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.directory-table__empty span,
|
||||||
|
.directory-table__empty a {
|
||||||
|
display: block;
|
||||||
|
margin-top: 8px;
|
||||||
|
}
|
||||||
|
|
||||||
.directory-table__cell--hidden,
|
.directory-table__cell--hidden,
|
||||||
.directory-table__head--hidden {
|
.directory-table__head--hidden {
|
||||||
display: none;
|
display: none;
|
||||||
}
|
}
|
||||||
|
|
||||||
.columns-modal {
|
.columns-modal {
|
||||||
position: fixed;
|
width: min(620px, calc(100% - 40px));
|
||||||
inset: 0;
|
max-height: min(720px, calc(100vh - 40px));
|
||||||
z-index: 50;
|
margin: auto;
|
||||||
display: grid;
|
padding: 0;
|
||||||
place-items: center;
|
border: 0;
|
||||||
padding: 20px;
|
background: transparent;
|
||||||
}
|
}
|
||||||
|
|
||||||
.columns-modal[hidden] {
|
.columns-modal::backdrop {
|
||||||
display: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
.columns-modal__backdrop {
|
|
||||||
position: absolute;
|
|
||||||
inset: 0;
|
|
||||||
background: rgba(17, 24, 39, 0.54);
|
background: rgba(17, 24, 39, 0.54);
|
||||||
}
|
}
|
||||||
|
|
||||||
.columns-modal__panel {
|
.columns-modal__panel {
|
||||||
position: relative;
|
position: relative;
|
||||||
width: min(620px, 100%);
|
|
||||||
max-height: min(720px, calc(100vh - 40px));
|
max-height: min(720px, calc(100vh - 40px));
|
||||||
overflow: auto;
|
overflow: auto;
|
||||||
padding: 20px;
|
padding: 20px;
|
||||||
@@ -640,16 +738,37 @@
|
|||||||
border-radius: 6px;
|
border-radius: 6px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.columns-modal__checkbox {
|
.columns-modal__checkbox {
|
||||||
width: 16px;
|
width: 16px;
|
||||||
height: 16px;
|
height: 16px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.columns-modal__presets {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
gap: 8px;
|
||||||
|
margin-top: 18px;
|
||||||
|
padding: 0;
|
||||||
|
border: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.columns-modal__preset-label {
|
||||||
|
color: #4b5563;
|
||||||
|
font-size: 13px;
|
||||||
|
font-weight: 700;
|
||||||
|
}
|
||||||
|
|
||||||
@media (max-width: 920px) {
|
@media (max-width: 920px) {
|
||||||
.directory__filters {
|
.directory__filters {
|
||||||
grid-template-columns: 1fr 1fr;
|
grid-template-columns: 1fr 1fr;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.directory__filter-group--search,
|
||||||
|
.directory__filter-actions {
|
||||||
|
grid-column: 1 / -1;
|
||||||
|
}
|
||||||
|
|
||||||
.progress-panel__header,
|
.progress-panel__header,
|
||||||
.directory__header,
|
.directory__header,
|
||||||
.employee-card__header {
|
.employee-card__header {
|
||||||
@@ -659,7 +778,40 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
@media (max-width: 620px) {
|
@media (max-width: 620px) {
|
||||||
|
.directory-table [data-column="full_name"] {
|
||||||
|
position: sticky;
|
||||||
|
left: 0;
|
||||||
|
z-index: 1;
|
||||||
|
min-width: 130px;
|
||||||
|
max-width: 160px;
|
||||||
|
background: #ffffff;
|
||||||
|
}
|
||||||
|
.admin__header {
|
||||||
|
align-items: flex-start;
|
||||||
|
padding: 16px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.admin__nav {
|
||||||
|
width: 100%;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.admin__main {
|
||||||
|
width: min(100% - 24px, 1180px);
|
||||||
|
margin: 20px auto;
|
||||||
|
}
|
||||||
|
|
||||||
.directory__filters {
|
.directory__filters {
|
||||||
grid-template-columns: 1fr;
|
grid-template-columns: 1fr;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.directory__filter-group,
|
||||||
|
.directory__filter-group--search,
|
||||||
|
.directory__filter-actions {
|
||||||
|
grid-column: 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
.directory__field {
|
||||||
|
font-size: 13px;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,14 +3,15 @@
|
|||||||
"full_name",
|
"full_name",
|
||||||
"status",
|
"status",
|
||||||
"positions",
|
"positions",
|
||||||
"hse_start_year",
|
|
||||||
"email",
|
|
||||||
"academic_degree",
|
|
||||||
"last_seen_at",
|
"last_seen_at",
|
||||||
"dismissed_at",
|
|
||||||
"profile",
|
"profile",
|
||||||
];
|
];
|
||||||
const storageKey = "miem.directory.columns";
|
const storageKey = "miem.directory.columns";
|
||||||
|
const columnPresets = {
|
||||||
|
review: ["full_name", "status", "academic_degree", "last_seen_at", "profile"],
|
||||||
|
contacts: ["full_name", "status", "email", "phone", "address", "profile"],
|
||||||
|
full: ["full_name", "status", "positions", "hse_start_year", "email", "phone", "address", "academic_degree", "publications_count", "courses_count", "news_count", "first_seen_at", "last_seen_at", "dismissed_at", "profile"],
|
||||||
|
};
|
||||||
|
|
||||||
function readColumns() {
|
function readColumns() {
|
||||||
try {
|
try {
|
||||||
@@ -40,18 +41,33 @@
|
|||||||
if (!document.querySelector("[data-directory-table]")) return;
|
if (!document.querySelector("[data-directory-table]")) return;
|
||||||
let columns = readColumns();
|
let columns = readColumns();
|
||||||
const modal = document.querySelector("[data-columns-modal]");
|
const modal = document.querySelector("[data-columns-modal]");
|
||||||
|
const trigger = document.querySelector("[data-columns-open]");
|
||||||
applyColumns(columns);
|
applyColumns(columns);
|
||||||
|
|
||||||
document.querySelectorAll("[data-columns-open]").forEach((button) => {
|
trigger.addEventListener("click", () => {
|
||||||
button.addEventListener("click", () => {
|
modal.showModal();
|
||||||
modal.hidden = false;
|
trigger.setAttribute("aria-expanded", "true");
|
||||||
});
|
modal.querySelector("input")?.focus();
|
||||||
});
|
});
|
||||||
document.querySelectorAll("[data-columns-close]").forEach((button) => {
|
document.querySelectorAll("[data-columns-close]").forEach((button) => {
|
||||||
button.addEventListener("click", () => {
|
button.addEventListener("click", () => {
|
||||||
modal.hidden = true;
|
modal.close();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
document.querySelectorAll("[data-columns-preset]").forEach((button) => {
|
||||||
|
button.addEventListener("click", () => {
|
||||||
|
columns = columnPresets[button.dataset.columnsPreset];
|
||||||
|
writeColumns(columns);
|
||||||
|
applyColumns(columns);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
modal.addEventListener("click", (event) => {
|
||||||
|
if (event.target === modal) modal.close();
|
||||||
|
});
|
||||||
|
modal.addEventListener("close", () => {
|
||||||
|
trigger.setAttribute("aria-expanded", "false");
|
||||||
|
trigger.focus();
|
||||||
|
});
|
||||||
document.querySelectorAll("[data-column-toggle]").forEach((checkbox) => {
|
document.querySelectorAll("[data-column-toggle]").forEach((checkbox) => {
|
||||||
checkbox.addEventListener("change", () => {
|
checkbox.addEventListener("change", () => {
|
||||||
columns = Array.from(document.querySelectorAll("[data-column-toggle]:checked")).map((item) => item.value);
|
columns = Array.from(document.querySelectorAll("[data-column-toggle]:checked")).map((item) => item.value);
|
||||||
@@ -62,28 +78,14 @@
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function setupClickableRows() {
|
|
||||||
const openRow = (row) => {
|
|
||||||
window.location.href = row.dataset.rowHref;
|
|
||||||
};
|
|
||||||
|
|
||||||
document.querySelectorAll("[data-row-href]").forEach((row) => {
|
|
||||||
row.addEventListener("click", (event) => {
|
|
||||||
if (event.target.closest("a, button, input, select, label")) return;
|
|
||||||
openRow(row);
|
|
||||||
});
|
|
||||||
row.addEventListener("keydown", (event) => {
|
|
||||||
if (!["Enter", " "].includes(event.key)) return;
|
|
||||||
if (event.target.closest("a, button, input, select, label")) return;
|
|
||||||
event.preventDefault();
|
|
||||||
openRow(row);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function setupProgress() {
|
function setupProgress() {
|
||||||
const panel = document.querySelector("[data-progress-panel]");
|
const panel = document.querySelector("[data-progress-panel]");
|
||||||
if (!panel) return;
|
if (!panel) return;
|
||||||
|
const state = document.createElement("p");
|
||||||
|
state.className = "progress-panel__empty";
|
||||||
|
state.setAttribute("role", "status");
|
||||||
|
panel.append(state);
|
||||||
|
let pending = false;
|
||||||
|
|
||||||
const update = (run) => {
|
const update = (run) => {
|
||||||
if (!run) return;
|
if (!run) return;
|
||||||
@@ -94,6 +96,7 @@
|
|||||||
const errors = document.querySelector("[data-progress-errors]");
|
const errors = document.querySelector("[data-progress-errors]");
|
||||||
const fill = document.querySelector("[data-progress-fill]");
|
const fill = document.querySelector("[data-progress-fill]");
|
||||||
const percent = document.querySelector("[data-progress-percent]");
|
const percent = document.querySelector("[data-progress-percent]");
|
||||||
|
const error = document.querySelector("[data-progress-error]");
|
||||||
if (status) status.textContent = run.status_display || run.status;
|
if (status) status.textContent = run.status_display || run.status;
|
||||||
if (processed) processed.textContent = run.processed_count;
|
if (processed) processed.textContent = run.processed_count;
|
||||||
if (found) found.textContent = run.found_count;
|
if (found) found.textContent = run.found_count;
|
||||||
@@ -101,21 +104,44 @@
|
|||||||
if (errors) errors.textContent = run.error_count;
|
if (errors) errors.textContent = run.error_count;
|
||||||
if (fill) fill.style.width = `${run.progress_percent}%`;
|
if (fill) fill.style.width = `${run.progress_percent}%`;
|
||||||
if (percent) percent.textContent = run.progress_percent;
|
if (percent) percent.textContent = run.progress_percent;
|
||||||
|
if (fill) {
|
||||||
|
fill.parentElement.setAttribute("aria-valuenow", run.progress_percent);
|
||||||
|
fill.parentElement.setAttribute("aria-valuetext", `${run.progress_percent}%`);
|
||||||
|
}
|
||||||
|
if (error) error.hidden = true;
|
||||||
};
|
};
|
||||||
|
|
||||||
const poll = async () => {
|
const poll = async () => {
|
||||||
|
if (pending) return true;
|
||||||
|
pending = true;
|
||||||
|
state.textContent = "Обновляем прогресс…";
|
||||||
|
panel.setAttribute("aria-busy", "true");
|
||||||
try {
|
try {
|
||||||
const response = await fetch("/api/crawl-runs/latest", { credentials: "same-origin" });
|
const response = await fetch("/api/crawl-runs/latest", {
|
||||||
if (!response.ok) return false;
|
credentials: "same-origin",
|
||||||
|
signal: AbortSignal.timeout(15000),
|
||||||
|
});
|
||||||
|
if (!response.ok) throw new Error("progress request failed");
|
||||||
const data = await response.json();
|
const data = await response.json();
|
||||||
const run = data.running || data.latest;
|
const run = data.running || data.latest;
|
||||||
update(run);
|
update(run);
|
||||||
|
state.textContent = run ? "Прогресс обновлён" : "Запусков пока нет. Запустите парсинг.";
|
||||||
|
const error = document.querySelector("[data-progress-error]");
|
||||||
|
if (error) error.hidden = true;
|
||||||
return Boolean(data.running);
|
return Boolean(data.running);
|
||||||
} catch (_error) {
|
} catch (_error) {
|
||||||
return false;
|
const error = document.querySelector("[data-progress-error]");
|
||||||
|
if (error) error.hidden = false;
|
||||||
|
state.textContent = "Показаны последние полученные данные. Прогресс может быть устаревшим.";
|
||||||
|
return true;
|
||||||
|
} finally {
|
||||||
|
pending = false;
|
||||||
|
panel.setAttribute("aria-busy", "false");
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
document.querySelectorAll("[data-progress-retry]").forEach((button) => button.addEventListener("click", poll));
|
||||||
|
|
||||||
const interval = window.setInterval(async () => {
|
const interval = window.setInterval(async () => {
|
||||||
const keepGoing = await poll();
|
const keepGoing = await poll();
|
||||||
if (!keepGoing) window.clearInterval(interval);
|
if (!keepGoing) window.clearInterval(interval);
|
||||||
@@ -123,6 +149,5 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
setupColumns();
|
setupColumns();
|
||||||
setupClickableRows();
|
|
||||||
setupProgress();
|
setupProgress();
|
||||||
})();
|
})();
|
||||||
|
|||||||
@@ -7,18 +7,19 @@
|
|||||||
<link rel="stylesheet" href="/static/admin.css">
|
<link rel="stylesheet" href="/static/admin.css">
|
||||||
</head>
|
</head>
|
||||||
<body class="admin">
|
<body class="admin">
|
||||||
|
<a class="admin__skip-link" href="#main-content">Перейти к содержимому</a>
|
||||||
<header class="admin__header">
|
<header class="admin__header">
|
||||||
<h1 class="admin__brand"><a class="admin__brand-link" href="/admin">MIEM Employees</a></h1>
|
<h1 class="admin__brand"><a class="admin__brand-link" href="/admin">MIEM Employees</a></h1>
|
||||||
<nav class="admin__nav">
|
<nav class="admin__nav">
|
||||||
<a class="admin__link" href="/admin">Обзор</a>
|
<a class="admin__link" href="/admin"{% if request.url.path == "/admin" %} aria-current="page"{% endif %}>Обзор</a>
|
||||||
<a class="admin__link" href="/admin/directory">Сотрудники</a>
|
<a class="admin__link" href="/admin/directory"{% if request.url.path == "/admin/directory" %} aria-current="page"{% endif %}>Сотрудники</a>
|
||||||
<a class="admin__link" href="/admin/runs">Запуски</a>
|
<a class="admin__link" href="/admin/runs"{% if request.url.path.startswith("/admin/runs") %} aria-current="page"{% endif %}>Запуски</a>
|
||||||
<form method="post" action="/admin/logout">
|
<form method="post" action="/admin/logout">
|
||||||
<button class="button button--ghost" type="submit">Выйти</button>
|
<button class="button button--ghost" type="submit">Выйти</button>
|
||||||
</form>
|
</form>
|
||||||
</nav>
|
</nav>
|
||||||
</header>
|
</header>
|
||||||
<main class="admin__main">
|
<main class="admin__main" id="main-content">
|
||||||
{% block content %}{% endblock %}
|
{% block content %}{% endblock %}
|
||||||
</main>
|
</main>
|
||||||
<footer class="admin__footer">
|
<footer class="admin__footer">
|
||||||
|
|||||||
@@ -28,15 +28,15 @@
|
|||||||
</section>
|
</section>
|
||||||
<section class="panel progress-panel" data-progress-panel>
|
<section class="panel progress-panel" data-progress-panel>
|
||||||
<div class="progress-panel__header">
|
<div class="progress-panel__header">
|
||||||
<h2 class="panel__title">Прогресс парсинга</h2>
|
<h2 class="panel__title">Прогресс парсинга</h2>
|
||||||
<div class="progress-panel__actions">
|
<div class="progress-panel__actions">
|
||||||
<form method="post" action="/admin/crawl-now">
|
<form method="post" action="/admin/crawl-now">
|
||||||
<button class="button" type="submit">Запустить парсинг</button>
|
<button class="button" type="submit">Запустить парсинг</button>
|
||||||
</form>
|
</form>
|
||||||
<form method="post" action="/admin/dismissed/refresh">
|
<form method="post" action="/admin/dismissed/refresh">
|
||||||
<button class="button button--secondary" type="submit">Проверить уволенных</button>
|
<button class="button button--secondary" type="submit">Проверить уволенных</button>
|
||||||
</form>
|
</form>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{% set run = counts.current_running_run or latest_run %}
|
{% set run = counts.current_running_run or latest_run %}
|
||||||
<div class="progress-panel__body" data-progress-body>
|
<div class="progress-panel__body" data-progress-body>
|
||||||
@@ -46,10 +46,11 @@
|
|||||||
<span>без изменений: <span data-progress-skipped>{{ run.skipped_count if run else 0 }}</span></span>
|
<span>без изменений: <span data-progress-skipped>{{ run.skipped_count if run else 0 }}</span></span>
|
||||||
<span>ошибок: <span data-progress-errors>{{ run.error_count if run else 0 }}</span></span>
|
<span>ошибок: <span data-progress-errors>{{ run.error_count if run else 0 }}</span></span>
|
||||||
</div>
|
</div>
|
||||||
<div class="progress-bar" aria-label="Parsing progress">
|
<div class="progress-bar" role="progressbar" aria-label="Прогресс парсинга" aria-valuemin="0" aria-valuemax="100" aria-valuenow="{{ run.progress_percent if run else 0 }}" aria-valuetext="{{ run.progress_percent if run else 0 }}%">
|
||||||
<div class="progress-bar__fill" data-progress-fill style="width: {{ run.progress_percent if run else 0 }}%"></div>
|
<div class="progress-bar__fill" data-progress-fill style="width: {{ run.progress_percent if run else 0 }}%"></div>
|
||||||
</div>
|
</div>
|
||||||
<div class="progress-panel__percent"><span data-progress-percent>{{ run.progress_percent if run else 0 }}</span>%</div>
|
<div class="progress-panel__percent"><span data-progress-percent>{{ run.progress_percent if run else 0 }}</span>%</div>
|
||||||
|
<p class="progress-panel__error" data-progress-error role="status" hidden>Не удалось обновить прогресс. <button class="button button--ghost button--compact" type="button" data-progress-retry>Повторить</button></p>
|
||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
<section class="panel">
|
<section class="panel">
|
||||||
@@ -58,7 +59,7 @@
|
|||||||
<thead><tr><th class="table__head">ID</th><th class="table__head">Статус</th><th class="table__head">Обработано</th><th class="table__head">Без изменений</th><th class="table__head">Ошибки</th><th class="table__head">Старт</th></tr></thead>
|
<thead><tr><th class="table__head">ID</th><th class="table__head">Статус</th><th class="table__head">Обработано</th><th class="table__head">Без изменений</th><th class="table__head">Ошибки</th><th class="table__head">Старт</th></tr></thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
{% for run in runs %}
|
{% for run in runs %}
|
||||||
<tr class="table__row" onclick="window.location.href='/admin/runs/{{ run.id }}'" onkeydown="if (event.key === 'Enter' || event.key === ' ') { event.preventDefault(); window.location.href='/admin/runs/{{ run.id }}'; }" role="link" tabindex="0"><td class="table__cell">{{ run.id }}</td><td class="table__cell">{{ run.status_display }}</td><td class="table__cell">{{ run.parsed_count }}</td><td class="table__cell">{{ run.skipped_count }}</td><td class="table__cell">{{ run.error_count }}</td><td class="table__cell">{{ run.started_display }}</td></tr>
|
<tr class="table__row"><td class="table__cell"><a class="admin__link" href="/admin/runs/{{ run.id }}">{{ run.id }}</a></td><td class="table__cell">{{ run.status_display }}</td><td class="table__cell">{{ run.parsed_count }}</td><td class="table__cell">{{ run.skipped_count }}</td><td class="table__cell">{{ run.error_count }}</td><td class="table__cell">{{ run.started_display }}</td></tr>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
|
|||||||
@@ -5,49 +5,70 @@
|
|||||||
<div class="directory__header">
|
<div class="directory__header">
|
||||||
<div>
|
<div>
|
||||||
<h2 class="directory__title">Сотрудники</h2>
|
<h2 class="directory__title">Сотрудники</h2>
|
||||||
<p class="directory__summary">Найдено: {{ page.total }}</p>
|
{% set range_start = page.offset + 1 if page.total else 0 %}
|
||||||
|
{% set range_end = [page.offset + page.employees|length, page.total]|min %}
|
||||||
|
<p class="directory__summary" role="status">Показаны {{ range_start }}–{{ range_end }} из {{ page.total }}</p>
|
||||||
</div>
|
</div>
|
||||||
<button class="button" type="button" data-columns-open>Колонки</button>
|
<button class="button" type="button" data-columns-open aria-controls="columns-dialog" aria-expanded="false">Настроить колонки</button>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<form class="directory__filters" method="get" action="/admin/directory">
|
<form class="directory__filters" method="get" action="/admin/directory">
|
||||||
<input class="directory__input" name="q" value="{{ filters.q }}" placeholder="ФИО или ссылка">
|
<fieldset class="directory__filter-group directory__filter-group--search">
|
||||||
<select class="directory__input" name="status">
|
<legend class="directory__filter-legend">Поиск</legend>
|
||||||
|
<label class="directory__field">ФИО или ссылка<input class="directory__input" name="q" value="{{ filters.q }}" placeholder="Например, Иванов или hse.ru"></label>
|
||||||
|
</fieldset>
|
||||||
|
<fieldset class="directory__filter-group">
|
||||||
|
<legend class="directory__filter-legend">Статус и данные</legend>
|
||||||
|
<label class="directory__field">Статус<select class="directory__input" name="status">
|
||||||
<option value="" {% if not filters.status %}selected{% endif %}>Все статусы</option>
|
<option value="" {% if not filters.status %}selected{% endif %}>Все статусы</option>
|
||||||
<option value="active" {% if filters.status == "active" %}selected{% endif %}>Работает</option>
|
<option value="active" {% if filters.status == "active" %}selected{% endif %}>Работает</option>
|
||||||
<option value="verification_required" {% if filters.status == "verification_required" %}selected{% endif %}>Требует проверки</option>
|
<option value="verification_required" {% if filters.status == "verification_required" %}selected{% endif %}>Требует проверки</option>
|
||||||
<option value="dismissed" {% if filters.status == "dismissed" %}selected{% endif %}>Уволен</option>
|
<option value="dismissed" {% if filters.status == "dismissed" %}selected{% endif %}>Уволен</option>
|
||||||
</select>
|
</select></label>
|
||||||
<select class="directory__input" name="has_email">
|
<label class="directory__field">Email<select class="directory__input" name="has_email">
|
||||||
<option value="" {% if not filters.has_email %}selected{% endif %}>Любой email</option>
|
<option value="" {% if not filters.has_email %}selected{% endif %}>Любой email</option>
|
||||||
<option value="true" {% if filters.has_email == "true" %}selected{% endif %}>Есть email</option>
|
<option value="true" {% if filters.has_email == "true" %}selected{% endif %}>Есть email</option>
|
||||||
<option value="false" {% if filters.has_email == "false" %}selected{% endif %}>Нет email</option>
|
<option value="false" {% if filters.has_email == "false" %}selected{% endif %}>Нет email</option>
|
||||||
</select>
|
</select></label>
|
||||||
<select class="directory__input" name="has_academic_degree" aria-label="Учёная степень">
|
<label class="directory__field">Учёная степень<select class="directory__input" name="has_academic_degree">
|
||||||
<option value="" {% if not filters.has_academic_degree %}selected{% endif %}>Любая учёная степень</option>
|
<option value="" {% if not filters.has_academic_degree %}selected{% endif %}>Любая учёная степень</option>
|
||||||
<option value="true" {% if filters.has_academic_degree == "true" %}selected{% endif %}>Есть учёная степень</option>
|
<option value="true" {% if filters.has_academic_degree == "true" %}selected{% endif %}>Есть учёная степень</option>
|
||||||
<option value="false" {% if filters.has_academic_degree == "false" %}selected{% endif %}>Нет учёной степени</option>
|
<option value="false" {% if filters.has_academic_degree == "false" %}selected{% endif %}>Нет учёной степени</option>
|
||||||
</select>
|
</select></label>
|
||||||
<input class="directory__input" type="date" name="started_from" value="{{ filters.started_from }}" aria-label="Впервые найден с">
|
</fieldset>
|
||||||
<input class="directory__input" type="date" name="started_to" value="{{ filters.started_to }}" aria-label="Впервые найден по">
|
<fieldset class="directory__filter-group">
|
||||||
<select class="directory__input" name="sort">
|
<legend class="directory__filter-legend">Период обнаружения</legend>
|
||||||
|
<label class="directory__field">От<input class="directory__input" type="date" name="started_from" value="{{ filters.started_from }}"></label>
|
||||||
|
<label class="directory__field">До<input class="directory__input" type="date" name="started_to" value="{{ filters.started_to }}"></label>
|
||||||
|
</fieldset>
|
||||||
|
<fieldset class="directory__filter-group">
|
||||||
|
<legend class="directory__filter-legend">Сортировка и страница</legend>
|
||||||
|
<label class="directory__field">Сортировать по<select class="directory__input" name="sort">
|
||||||
{% for value, label in [("full_name", "ФИО"), ("status", "Статус"), ("hse_start_year", "Год начала"), ("first_seen_at", "Впервые найден"), ("last_seen_at", "Последний раз найден"), ("dismissed_at", "Дата увольнения")] %}
|
{% for value, label in [("full_name", "ФИО"), ("status", "Статус"), ("hse_start_year", "Год начала"), ("first_seen_at", "Впервые найден"), ("last_seen_at", "Последний раз найден"), ("dismissed_at", "Дата увольнения")] %}
|
||||||
<option value="{{ value }}" {% if filters.sort == value %}selected{% endif %}>Сортировка: {{ label }}</option>
|
<option value="{{ value }}" {% if filters.sort == value %}selected{% endif %}>{{ label }}</option>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
</select>
|
</select></label>
|
||||||
<select class="directory__input" name="direction">
|
<label class="directory__field">Направление<select class="directory__input" name="direction">
|
||||||
<option value="asc" {% if filters.direction == "asc" %}selected{% endif %}>По возрастанию</option>
|
<option value="asc" {% if filters.direction == "asc" %}selected{% endif %}>По возрастанию</option>
|
||||||
<option value="desc" {% if filters.direction == "desc" %}selected{% endif %}>По убыванию</option>
|
<option value="desc" {% if filters.direction == "desc" %}selected{% endif %}>По убыванию</option>
|
||||||
</select>
|
</select></label>
|
||||||
<select class="directory__input" name="limit" onchange="this.form.offset.value = 0; this.form.submit()">
|
<label class="directory__field">Сотрудников на странице<select class="directory__input" name="limit">
|
||||||
{% for value in [25, 50, 100] %}
|
{% for value in [25, 50, 100] %}
|
||||||
<option value="{{ value }}" {% if filters.limit == value %}selected{% endif %}>На странице: {{ value }}</option>
|
<option value="{{ value }}" {% if filters.limit == value %}selected{% endif %}>{{ value }}</option>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
</select>
|
</select></label>
|
||||||
<input type="hidden" name="offset" value="{{ filters.offset }}">
|
</fieldset>
|
||||||
<button class="button" type="submit">Применить</button>
|
<div class="directory__filter-actions">
|
||||||
|
<button class="button" type="submit">Применить фильтры</button>
|
||||||
|
<a class="button button--ghost" href="/admin/directory">Сбросить</a>
|
||||||
|
</div>
|
||||||
</form>
|
</form>
|
||||||
|
|
||||||
|
<details class="directory__help">
|
||||||
|
<summary>Что означают даты и статусы</summary>
|
||||||
|
<p>«Год начала» — начало работы в ВШЭ. «Впервые найден» и «Последний раз найден» — даты обнаружения сотрудника в списке источника, а не даты приёма и увольнения. «Требует проверки» означает, что профиль недоступен и увольнение ещё не подтверждено.</p>
|
||||||
|
</details>
|
||||||
|
|
||||||
<div class="directory__table-wrap">
|
<div class="directory__table-wrap">
|
||||||
<table class="directory-table" data-directory-table>
|
<table class="directory-table" data-directory-table>
|
||||||
<thead>
|
<thead>
|
||||||
@@ -71,9 +92,9 @@
|
|||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
{% for employee in page.employees %}
|
{% for employee in page.employees %}
|
||||||
<tr class="directory-table__row" data-row-href="/admin/employees/{{ employee.id }}">
|
<tr class="directory-table__row">
|
||||||
<td class="directory-table__cell" data-column="full_name">{{ employee.full_name or "Без имени" }}</td>
|
<td class="directory-table__cell" data-column="full_name"><a class="admin__link" href="/admin/employees/{{ employee.id }}">{{ employee.full_name or "Без имени" }}</a></td>
|
||||||
<td class="directory-table__cell" data-column="status"><span class="badge {% if employee.status == "dismissed" %}badge--dismissed{% endif %}">{{ employee.status_display }}</span></td>
|
<td class="directory-table__cell" data-column="status"><span class="badge {% if employee.status == "dismissed" %}badge--dismissed{% elif employee.status == "verification_required" %}badge--verification{% endif %}">{{ employee.status_display }}</span></td>
|
||||||
<td class="directory-table__cell" data-column="positions">{{ employee.positions_text }}</td>
|
<td class="directory-table__cell" data-column="positions">{{ employee.positions_text }}</td>
|
||||||
<td class="directory-table__cell" data-column="hse_start_year">{{ employee.hse_start_year or "" }}</td>
|
<td class="directory-table__cell" data-column="hse_start_year">{{ employee.hse_start_year or "" }}</td>
|
||||||
<td class="directory-table__cell" data-column="email">{{ employee.email_text }}</td>
|
<td class="directory-table__cell" data-column="email">{{ employee.email_text }}</td>
|
||||||
@@ -86,10 +107,18 @@
|
|||||||
<td class="directory-table__cell" data-column="first_seen_at">{{ employee.first_seen_display }}</td>
|
<td class="directory-table__cell" data-column="first_seen_at">{{ employee.first_seen_display }}</td>
|
||||||
<td class="directory-table__cell" data-column="last_seen_at">{{ employee.last_seen_display }}</td>
|
<td class="directory-table__cell" data-column="last_seen_at">{{ employee.last_seen_display }}</td>
|
||||||
<td class="directory-table__cell" data-column="dismissed_at">{{ employee.dismissed_display }}</td>
|
<td class="directory-table__cell" data-column="dismissed_at">{{ employee.dismissed_display }}</td>
|
||||||
<td class="directory-table__cell" data-column="profile"><a class="admin__link" href="{{ employee.canonical_url }}">Открыть</a></td>
|
<td class="directory-table__cell" data-column="profile"><a class="admin__link" href="{{ employee.canonical_url }}" target="_blank" rel="noopener">Внешний профиль</a></td>
|
||||||
</tr>
|
</tr>
|
||||||
{% else %}
|
{% else %}
|
||||||
<tr><td class="directory-table__empty" colspan="15">По этим фильтрам сотрудники не найдены.</td></tr>
|
<tr><td class="directory-table__empty" colspan="15">
|
||||||
|
<strong>Сотрудники не найдены</strong>
|
||||||
|
{% if filters.q or filters.status or filters.has_email or filters.has_academic_degree or filters.started_from or filters.started_to %}
|
||||||
|
<span>Попробуйте изменить запрос или сбросить фильтры.</span>
|
||||||
|
<a class="admin__link" href="/admin/directory">Сбросить фильтры</a>
|
||||||
|
{% else %}
|
||||||
|
<span>Запустите парсинг, чтобы загрузить сотрудников.</span>
|
||||||
|
{% endif %}
|
||||||
|
</td></tr>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
@@ -108,11 +137,10 @@
|
|||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
<div class="columns-modal" data-columns-modal hidden>
|
<dialog class="columns-modal" id="columns-dialog" data-columns-modal aria-labelledby="columns-dialog-title">
|
||||||
<div class="columns-modal__backdrop" data-columns-close></div>
|
<section class="columns-modal__panel">
|
||||||
<section class="columns-modal__panel" aria-label="Настройка колонок">
|
|
||||||
<div class="columns-modal__header">
|
<div class="columns-modal__header">
|
||||||
<h3 class="columns-modal__title">Отображаемые колонки</h3>
|
<h3 class="columns-modal__title" id="columns-dialog-title">Отображаемые колонки</h3>
|
||||||
<button class="button button--ghost" type="button" data-columns-close>Закрыть</button>
|
<button class="button button--ghost" type="button" data-columns-close>Закрыть</button>
|
||||||
</div>
|
</div>
|
||||||
<div class="columns-modal__grid">
|
<div class="columns-modal__grid">
|
||||||
@@ -120,8 +148,14 @@
|
|||||||
<label class="columns-modal__option"><input class="columns-modal__checkbox" type="checkbox" value="{{ key }}" data-column-toggle> {{ label }}</label>
|
<label class="columns-modal__option"><input class="columns-modal__checkbox" type="checkbox" value="{{ key }}" data-column-toggle> {{ label }}</label>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
</div>
|
</div>
|
||||||
|
<fieldset class="columns-modal__presets">
|
||||||
|
<legend class="columns-modal__preset-label">Быстрый набор</legend>
|
||||||
|
<button class="button button--ghost button--compact" type="button" data-columns-preset="review">Проверка</button>
|
||||||
|
<button class="button button--ghost button--compact" type="button" data-columns-preset="contacts">Контакты</button>
|
||||||
|
<button class="button button--ghost button--compact" type="button" data-columns-preset="full">Все поля</button>
|
||||||
|
</fieldset>
|
||||||
</section>
|
</section>
|
||||||
</div>
|
</dialog>
|
||||||
{% endblock %}
|
{% endblock %}
|
||||||
{% block scripts %}
|
{% block scripts %}
|
||||||
<script src="/static/admin.js"></script>
|
<script src="/static/admin.js"></script>
|
||||||
|
|||||||
9
app/templates/directory_error.html
Normal file
9
app/templates/directory_error.html
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
{% extends "base.html" %}
|
||||||
|
{% block title %}Каталог недоступен · MIEM Employees{% endblock %}
|
||||||
|
{% block content %}
|
||||||
|
<section class="panel">
|
||||||
|
<h2 class="panel__title">Не удалось загрузить сотрудников</h2>
|
||||||
|
<p>База данных временно недоступна. Повторите запрос; выбранные фильтры сохранятся.</p>
|
||||||
|
<a class="button" href="{{ request.url.path }}{% if request.url.query %}?{{ request.url.query }}{% endif %}">Повторить загрузку</a>
|
||||||
|
</section>
|
||||||
|
{% endblock %}
|
||||||
@@ -17,10 +17,11 @@
|
|||||||
<span>без изменений: <span data-progress-skipped>{{ run.skipped_count }}</span></span>
|
<span>без изменений: <span data-progress-skipped>{{ run.skipped_count }}</span></span>
|
||||||
<span>ошибок: <span data-progress-errors>{{ run.error_count }}</span></span>
|
<span>ошибок: <span data-progress-errors>{{ run.error_count }}</span></span>
|
||||||
</div>
|
</div>
|
||||||
<div class="progress-bar" aria-label="Parsing progress">
|
<div class="progress-bar" role="progressbar" aria-label="Прогресс парсинга" aria-valuemin="0" aria-valuemax="100" aria-valuenow="{{ percent }}" aria-valuetext="{{ percent }}%">
|
||||||
<div class="progress-bar__fill" data-progress-fill style="width: {{ percent }}%"></div>
|
<div class="progress-bar__fill" data-progress-fill style="width: {{ percent }}%"></div>
|
||||||
</div>
|
</div>
|
||||||
<div class="progress-panel__percent"><span data-progress-percent>{{ percent }}</span>%</div>
|
<div class="progress-panel__percent"><span data-progress-percent>{{ percent }}</span>%</div>
|
||||||
|
<p class="progress-panel__error" data-progress-error role="status" hidden>Не удалось обновить прогресс. <button class="button button--ghost button--compact" type="button" data-progress-retry>Повторить</button></p>
|
||||||
</div>
|
</div>
|
||||||
{% else %}
|
{% else %}
|
||||||
<div class="progress-panel" data-progress-panel>
|
<div class="progress-panel" data-progress-panel>
|
||||||
@@ -30,17 +31,18 @@
|
|||||||
<span>без изменений: <span data-progress-skipped>0</span></span>
|
<span>без изменений: <span data-progress-skipped>0</span></span>
|
||||||
<span>ошибок: <span data-progress-errors>0</span></span>
|
<span>ошибок: <span data-progress-errors>0</span></span>
|
||||||
</div>
|
</div>
|
||||||
<div class="progress-bar" aria-label="Parsing progress">
|
<div class="progress-bar" role="progressbar" aria-label="Прогресс парсинга" aria-valuemin="0" aria-valuemax="100" aria-valuenow="0" aria-valuetext="0%">
|
||||||
<div class="progress-bar__fill" data-progress-fill style="width: 0%"></div>
|
<div class="progress-bar__fill" data-progress-fill style="width: 0%"></div>
|
||||||
</div>
|
</div>
|
||||||
<div class="progress-panel__percent"><span data-progress-percent>0</span>%</div>
|
<div class="progress-panel__percent"><span data-progress-percent>0</span>%</div>
|
||||||
|
<p class="progress-panel__error" data-progress-error role="status" hidden>Не удалось обновить прогресс. <button class="button button--ghost button--compact" type="button" data-progress-retry>Повторить</button></p>
|
||||||
</div>
|
</div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
<table class="table">
|
<table class="table">
|
||||||
<thead><tr><th class="table__head">ID</th><th class="table__head">Статус</th><th class="table__head">Найдено</th><th class="table__head">Обработано</th><th class="table__head">Без изменений</th><th class="table__head">Новые</th><th class="table__head">Ошибки</th><th class="table__head">Уволены</th><th class="table__head">Старт</th></tr></thead>
|
<thead><tr><th class="table__head">ID</th><th class="table__head">Статус</th><th class="table__head">Найдено</th><th class="table__head">Обработано</th><th class="table__head">Без изменений</th><th class="table__head">Новые</th><th class="table__head">Ошибки</th><th class="table__head">Уволены</th><th class="table__head">Старт</th></tr></thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
{% for run in runs %}
|
{% for run in runs %}
|
||||||
<tr class="table__row" onclick="window.location.href='/admin/runs/{{ run.id }}'" onkeydown="if (event.key === 'Enter' || event.key === ' ') { event.preventDefault(); window.location.href='/admin/runs/{{ run.id }}'; }" role="link" tabindex="0"><td class="table__cell">{{ run.id }}</td><td class="table__cell">{{ run.status_display }}</td><td class="table__cell">{{ run.found_count }}</td><td class="table__cell">{{ run.parsed_count }}</td><td class="table__cell">{{ run.skipped_count }}</td><td class="table__cell">{{ run.new_count }}</td><td class="table__cell">{{ run.error_count }}</td><td class="table__cell">{{ run.dismissed_count }}</td><td class="table__cell">{{ run.started_display }}</td></tr>
|
<tr class="table__row"><td class="table__cell"><a class="admin__link" href="/admin/runs/{{ run.id }}">{{ run.id }}</a></td><td class="table__cell">{{ run.status_display }}</td><td class="table__cell">{{ run.found_count }}</td><td class="table__cell">{{ run.parsed_count }}</td><td class="table__cell">{{ run.skipped_count }}</td><td class="table__cell">{{ run.new_count }}</td><td class="table__cell">{{ run.error_count }}</td><td class="table__cell">{{ run.dismissed_count }}</td><td class="table__cell">{{ run.started_display }}</td></tr>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
|
|||||||
@@ -1,3 +1,3 @@
|
|||||||
APP_VERSION = "0.7.5"
|
APP_VERSION = "0.8.6"
|
||||||
FRONTEND_VERSION = "0.7.5"
|
FRONTEND_VERSION = "0.8.6"
|
||||||
BACKEND_VERSION = "0.7.5"
|
BACKEND_VERSION = "0.8.6"
|
||||||
|
|||||||
@@ -25,5 +25,15 @@ services:
|
|||||||
postgres:
|
postgres:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
|
|
||||||
|
worker:
|
||||||
|
build: .
|
||||||
|
command: python -m app.worker
|
||||||
|
env_file: .env
|
||||||
|
environment:
|
||||||
|
DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-miem}:${POSTGRES_PASSWORD:-miem_password}@postgres:5432/${POSTGRES_DB:-miem_workers}
|
||||||
|
depends_on:
|
||||||
|
postgres:
|
||||||
|
condition: service_healthy
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
postgres_data:
|
postgres_data:
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "miem-workers"
|
name = "miem-workers"
|
||||||
version = "0.7.5"
|
version = "0.8.6"
|
||||||
description = "MIEM employees parser, admin API, and MCP server"
|
description = "MIEM employees parser, admin API, and web admin"
|
||||||
requires-python = ">=3.11"
|
requires-python = ">=3.11"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"apscheduler>=3.10.4",
|
"apscheduler>=3.10.4",
|
||||||
|
|||||||
143
tests/browser_admin.py
Normal file
143
tests/browser_admin.py
Normal file
@@ -0,0 +1,143 @@
|
|||||||
|
"""Run from the repository root: python tests/browser_admin.py.
|
||||||
|
|
||||||
|
Requires Playwright and Chromium. Uses in-memory data and intercepted requests;
|
||||||
|
never contacts the production service. Optional BROWSER_EXECUTABLE selects an
|
||||||
|
already installed Chromium instead of the Playwright download.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
from tempfile import mkdtemp
|
||||||
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||||
|
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
from playwright.sync_api import expect, sync_playwright
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
from sqlalchemy.pool import StaticPool
|
||||||
|
|
||||||
|
from app.config import Settings, get_settings
|
||||||
|
from app.db import Base, get_db
|
||||||
|
from app.main import app
|
||||||
|
from app.models import Employee
|
||||||
|
from app.security import SESSION_COOKIE, sign_session
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
|
||||||
|
Base.metadata.create_all(engine)
|
||||||
|
factory = sessionmaker(bind=engine)
|
||||||
|
with factory() as db:
|
||||||
|
db.add_all(Employee(profile_key=f"staff:{i}", canonical_url=f"https://www.hse.ru/staff/{i}",
|
||||||
|
full_name=f"Сотрудник {i:02d}", current_data={"positions": ["Преподаватель"]})
|
||||||
|
for i in range(30))
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
def database():
|
||||||
|
with factory() as db:
|
||||||
|
yield db
|
||||||
|
|
||||||
|
settings = Settings(_env_file=None, session_secret="browser-test-secret")
|
||||||
|
app.dependency_overrides[get_db] = database
|
||||||
|
app.dependency_overrides[get_settings] = lambda: settings
|
||||||
|
client = TestClient(app)
|
||||||
|
client.cookies.set(SESSION_COOKIE, sign_session(settings.admin_username, settings))
|
||||||
|
output = Path(mkdtemp(prefix="miem-admin-browser-"))
|
||||||
|
pending = []
|
||||||
|
|
||||||
|
def serve(route):
|
||||||
|
url = urlsplit(route.request.url)
|
||||||
|
if url.path == "/api/crawl-runs/latest":
|
||||||
|
pending.append(route)
|
||||||
|
return
|
||||||
|
response = client.get(url.path + ("?" + url.query if url.query else ""))
|
||||||
|
route.fulfill(status=response.status_code, body=response.content,
|
||||||
|
content_type=response.headers.get("content-type", "text/plain"))
|
||||||
|
|
||||||
|
try:
|
||||||
|
with sync_playwright() as playwright:
|
||||||
|
browser = playwright.chromium.launch(executable_path=os.getenv("BROWSER_EXECUTABLE"))
|
||||||
|
page = browser.new_page(viewport={"width": 1440, "height": 1000})
|
||||||
|
page.route("**/*", serve)
|
||||||
|
errors = []
|
||||||
|
page.on("pageerror", lambda error: errors.append(str(error)))
|
||||||
|
page.goto("http://miem.test/admin/directory?limit=25&offset=25")
|
||||||
|
expect(page.get_by_role("status")).to_have_text("Показаны 26–30 из 30")
|
||||||
|
assert page.locator("th:visible").count() == 5
|
||||||
|
assert page.locator(".directory__table-wrap").evaluate("e => e.scrollWidth <= e.clientWidth")
|
||||||
|
skip_link = page.get_by_role("link", name="Перейти к содержимому")
|
||||||
|
assert skip_link.evaluate("e => getComputedStyle(e).left === '-999px'")
|
||||||
|
page.keyboard.press("Tab")
|
||||||
|
expect(skip_link).to_be_focused()
|
||||||
|
assert skip_link.evaluate("e => getComputedStyle(e).left === '8px'")
|
||||||
|
page.keyboard.press("Tab")
|
||||||
|
assert skip_link.evaluate("e => getComputedStyle(e).left === '-999px'")
|
||||||
|
|
||||||
|
for width in (1440, 1024, 921, 920, 768, 621, 620, 390, 320):
|
||||||
|
page.set_viewport_size({"width": width, "height": 844})
|
||||||
|
assert page.locator(".directory__filters").evaluate("e => e.scrollWidth <= e.clientWidth")
|
||||||
|
assert page.locator(".directory__input").evaluate_all("""controls => controls.every(control => {
|
||||||
|
const field = control.closest('.directory__field').getBoundingClientRect();
|
||||||
|
const rect = control.getBoundingClientRect();
|
||||||
|
return rect.left >= field.left - 1 && rect.right <= field.right + 1;
|
||||||
|
})""")
|
||||||
|
page.set_viewport_size({"width": 1440, "height": 1000})
|
||||||
|
page.get_by_label("ФИО или ссылка").fill("Сотрудник 00")
|
||||||
|
page.get_by_role("button", name="Применить фильтры").click()
|
||||||
|
expect(page.get_by_role("status")).to_have_text("Показаны 1–1 из 1")
|
||||||
|
assert "offset=" not in page.url
|
||||||
|
trigger = page.get_by_role("button", name="Настроить колонки")
|
||||||
|
trigger.focus()
|
||||||
|
page.keyboard.press("Enter")
|
||||||
|
dialog = page.get_by_role("dialog", name="Отображаемые колонки")
|
||||||
|
expect(dialog).to_be_visible()
|
||||||
|
for _ in range(22):
|
||||||
|
page.keyboard.press("Tab")
|
||||||
|
# Native dialogs may let Tab reach browser chrome, never background controls.
|
||||||
|
assert dialog.evaluate("e => document.activeElement === document.body || e.contains(document.activeElement)")
|
||||||
|
page.keyboard.press("Escape")
|
||||||
|
expect(trigger).to_be_focused()
|
||||||
|
trigger.click()
|
||||||
|
page.get_by_role("button", name="Контакты", exact=True).click()
|
||||||
|
page.keyboard.press("Escape")
|
||||||
|
page.reload()
|
||||||
|
expect(page.locator('th[data-column="email"]')).to_be_visible()
|
||||||
|
page.evaluate("localStorage.clear()")
|
||||||
|
page.goto("http://miem.test/admin/directory")
|
||||||
|
page.screenshot(path=str(output / "desktop.png"), full_page=True)
|
||||||
|
page.set_viewport_size({"width": 390, "height": 844})
|
||||||
|
assert page.evaluate("document.documentElement.scrollWidth <= innerWidth")
|
||||||
|
assert page.locator('th[data-column="full_name"]').evaluate("e => getComputedStyle(e).position") == "sticky"
|
||||||
|
page.screenshot(path=str(output / "mobile.png"), full_page=True)
|
||||||
|
trigger.click()
|
||||||
|
expect(dialog).to_be_visible()
|
||||||
|
assert dialog.evaluate("e => e.getBoundingClientRect().right <= innerWidth")
|
||||||
|
page.screenshot(path=str(output / "mobile-dialog.png"), full_page=True)
|
||||||
|
page.keyboard.press("Escape")
|
||||||
|
page.get_by_label("ФИО или ссылка").fill("Нет совпадений")
|
||||||
|
page.get_by_role("button", name="Применить фильтры").click()
|
||||||
|
expect(page.get_by_text("Сотрудники не найдены", exact=True)).to_be_visible()
|
||||||
|
expect(page.get_by_role("link", name="Сбросить фильтры", exact=True)).to_be_visible()
|
||||||
|
page.goto("http://miem.test/admin")
|
||||||
|
page.wait_for_function("document.querySelector('[data-progress-panel]').getAttribute('aria-busy') === 'true'")
|
||||||
|
pending.pop(0).fulfill(status=503, body="unavailable")
|
||||||
|
expect(page.locator("[data-progress-error]")).to_be_visible()
|
||||||
|
expect(page.get_by_text("Показаны последние полученные данные.", exact=False)).to_be_visible()
|
||||||
|
page.get_by_role("button", name="Повторить").click()
|
||||||
|
page.wait_for_timeout(100)
|
||||||
|
pending.pop(0).fulfill(json={"running": None, "latest": None})
|
||||||
|
expect(page.locator("[data-progress-error]")).to_be_hidden()
|
||||||
|
expect(page.get_by_text("Запусков пока нет. Запустите парсинг.", exact=True)).to_be_visible()
|
||||||
|
assert not errors, errors
|
||||||
|
browser.close()
|
||||||
|
finally:
|
||||||
|
app.dependency_overrides.clear()
|
||||||
|
client.close()
|
||||||
|
engine.dispose()
|
||||||
|
print(f"Browser checks passed; screenshots: {output}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
26
tests/test_academic_degrees.py
Normal file
26
tests/test_academic_degrees.py
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
from app.services.academic_degrees import academic_degrees
|
||||||
|
|
||||||
|
|
||||||
|
def test_academic_degrees_extracts_all_degree_names_without_surrounding_text():
|
||||||
|
data = {
|
||||||
|
"sections": [
|
||||||
|
{
|
||||||
|
"title": "Образование",
|
||||||
|
"year_entries": [
|
||||||
|
{"text": "2008 — кандидат технических наук, доцент"},
|
||||||
|
{"text": "2018 — Доктор физико-математических наук"},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "Учёные степени",
|
||||||
|
"table": {"rows": [{"cells": ["Ph.D.", "Доктор физико-математических наук"]}]},
|
||||||
|
},
|
||||||
|
{"title": "Публикации", "items": ["Доктор медицинских наук выступил автором статьи"]},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
assert academic_degrees(data) == [
|
||||||
|
"кандидат технических наук",
|
||||||
|
"Доктор физико-математических наук",
|
||||||
|
"Ph.D.",
|
||||||
|
]
|
||||||
@@ -268,6 +268,18 @@ def test_list_employees_page_filters_sorts_and_paginates(db_session):
|
|||||||
assert page["limit"] == 50
|
assert page["limit"] == 50
|
||||||
|
|
||||||
|
|
||||||
|
def test_catalog_recovers_from_out_of_range_offset(db_session):
|
||||||
|
db_session.add(Employee(profile_key="staff:page", canonical_url="https://www.hse.ru/staff/page", full_name="Page Person"))
|
||||||
|
db_session.commit()
|
||||||
|
|
||||||
|
page = list_employees_page(db_session, q="Page", offset=100, limit=25)
|
||||||
|
assert (page["offset"], page["page"], page["total"]) == (0, 1, 1)
|
||||||
|
assert len(page["employees"]) == 1
|
||||||
|
empty = list_employees_page(db_session, q="missing", offset=100)
|
||||||
|
assert empty["offset"] == 0
|
||||||
|
assert empty["employees"] == []
|
||||||
|
|
||||||
|
|
||||||
def test_stats_payload_uses_latest_run_new_count(db_session):
|
def test_stats_payload_uses_latest_run_new_count(db_session):
|
||||||
db_session.add(
|
db_session.add(
|
||||||
Employee(
|
Employee(
|
||||||
|
|||||||
@@ -17,11 +17,11 @@ def test_directory_template_is_russian_and_uses_display_dates():
|
|||||||
template = Path("app/templates/directory.html").read_text(encoding="utf-8")
|
template = Path("app/templates/directory.html").read_text(encoding="utf-8")
|
||||||
|
|
||||||
assert "Сотрудники" in template
|
assert "Сотрудники" in template
|
||||||
assert "Колонки" in template
|
assert "колонки" in template
|
||||||
assert "Применить" in template
|
assert "Применить фильтры" in template
|
||||||
assert "На странице: {{ value }}" in template
|
assert "Сотрудников на странице" in template
|
||||||
assert "{% for value in [25, 50, 100] %}" in template
|
assert "{% for value in [25, 50, 100] %}" in template
|
||||||
assert "Найдено:" in template
|
assert "Показаны" in template
|
||||||
assert "Новости" in template
|
assert "Новости" in template
|
||||||
assert "Есть учёная степень" in template
|
assert "Есть учёная степень" in template
|
||||||
assert 'data-column="academic_degree"' in template
|
assert 'data-column="academic_degree"' in template
|
||||||
@@ -29,7 +29,17 @@ def test_directory_template_is_russian_and_uses_display_dates():
|
|||||||
assert "employee.first_seen_display" in template
|
assert "employee.first_seen_display" in template
|
||||||
assert "employee.last_seen_display" in template
|
assert "employee.last_seen_display" in template
|
||||||
assert "employee.dismissed_display" in template
|
assert "employee.dismissed_display" in template
|
||||||
assert "verification_required" in template
|
assert "verification_required" in template
|
||||||
|
assert '<label class="directory__field">Статус' in template
|
||||||
|
assert 'id="columns-dialog"' in template
|
||||||
|
assert 'aria-controls="columns-dialog"' in template
|
||||||
|
assert 'aria-labelledby="columns-dialog-title"' in template
|
||||||
|
assert "Сбросить фильтры" in template
|
||||||
|
assert 'data-columns-preset="review"' in template
|
||||||
|
assert '<dialog class="columns-modal"' in template
|
||||||
|
assert 'id="columns-dialog"' in template
|
||||||
|
assert 'aria-controls="columns-dialog"' in template
|
||||||
|
assert "Сбросить фильтры" in template
|
||||||
assert "Directory" not in template
|
assert "Directory" not in template
|
||||||
assert "employees found" not in template
|
assert "employees found" not in template
|
||||||
|
|
||||||
@@ -50,12 +60,9 @@ def test_dashboard_limits_latest_runs_to_five():
|
|||||||
def test_runs_template_links_to_run_detail():
|
def test_runs_template_links_to_run_detail():
|
||||||
template = Path("app/templates/runs.html").read_text(encoding="utf-8")
|
template = Path("app/templates/runs.html").read_text(encoding="utf-8")
|
||||||
|
|
||||||
assert 'onclick="window.location.href=\'/admin/runs/{{ run.id }}\'"' in template
|
assert '<a class="admin__link" href="/admin/runs/{{ run.id }}">' in template
|
||||||
assert "onkeydown=\"if (event.key === 'Enter' || event.key === ' ')" in template
|
assert "onclick=" not in template
|
||||||
assert 'role="link"' in template
|
assert 'role="link"' not in template
|
||||||
assert 'tabindex="0"' in template
|
|
||||||
assert 'data-row-href="/admin/runs/{{ run.id }}"' not in template
|
|
||||||
assert '<a class="admin__link" href="/admin/runs/{{ run.id }}">' not in template
|
|
||||||
|
|
||||||
|
|
||||||
def test_run_detail_template_extends_base_and_shows_change_groups():
|
def test_run_detail_template_extends_base_and_shows_change_groups():
|
||||||
@@ -86,22 +93,36 @@ def test_dashboard_has_dismissed_status_refresh_action():
|
|||||||
|
|
||||||
assert 'action="/admin/dismissed/refresh"' in template
|
assert 'action="/admin/dismissed/refresh"' in template
|
||||||
assert "Проверить уволенных" in template
|
assert "Проверить уволенных" in template
|
||||||
|
|
||||||
|
|
||||||
|
def test_progress_template_has_accessible_progressbar():
|
||||||
|
for name in ("dashboard.html", "runs.html"):
|
||||||
|
template = Path(f"app/templates/{name}").read_text(encoding="utf-8")
|
||||||
|
assert 'role="progressbar"' in template
|
||||||
|
assert 'aria-valuemin="0"' in template
|
||||||
|
assert 'aria-valuemax="100"' in template
|
||||||
|
assert "Прогресс парсинга" in template
|
||||||
|
|
||||||
|
|
||||||
def test_dashboard_latest_run_rows_link_to_run_detail():
|
def test_dashboard_latest_run_rows_link_to_run_detail():
|
||||||
template = Path("app/templates/dashboard.html").read_text(encoding="utf-8")
|
template = Path("app/templates/dashboard.html").read_text(encoding="utf-8")
|
||||||
|
|
||||||
assert 'onclick="window.location.href=\'/admin/runs/{{ run.id }}\'"' in template
|
assert '<a class="admin__link" href="/admin/runs/{{ run.id }}">' in template
|
||||||
assert "onkeydown=\"if (event.key === 'Enter' || event.key === ' ')" in template
|
assert "onclick=" not in template
|
||||||
assert 'role="link"' in template
|
assert 'role="link"' not in template
|
||||||
assert 'tabindex="0"' in template
|
|
||||||
assert 'data-row-href="/admin/runs/{{ run.id }}"' not in template
|
|
||||||
assert '<a class="admin__link" href="/admin/runs/{{ run.id }}">' not in template
|
|
||||||
|
|
||||||
|
|
||||||
def test_admin_js_supports_keyboard_activation_for_clickable_rows():
|
def test_admin_js_uses_native_dialog_for_column_settings():
|
||||||
source = Path("app/static/admin.js").read_text(encoding="utf-8")
|
source = Path("app/static/admin.js").read_text(encoding="utf-8")
|
||||||
|
|
||||||
assert 'addEventListener("keydown"' in source
|
assert "showModal()" in source
|
||||||
assert '"Enter"' in source
|
assert "modal.close()" in source
|
||||||
assert '" "' in source
|
assert 'setAttribute("aria-expanded", "true")' in source
|
||||||
|
assert "data-progress-retry" in source
|
||||||
|
|
||||||
|
|
||||||
|
def test_base_template_has_skip_link_and_current_navigation_state():
|
||||||
|
template = Path("app/templates/base.html").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
assert 'href="#main-content"' in template
|
||||||
|
assert 'aria-current="page"' in template
|
||||||
|
|||||||
154
tests/test_api.py
Normal file
154
tests/test_api.py
Normal file
@@ -0,0 +1,154 @@
|
|||||||
|
from datetime import datetime, timezone
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
from sqlalchemy import create_engine, select
|
||||||
|
from sqlalchemy.exc import OperationalError
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
from sqlalchemy.pool import StaticPool
|
||||||
|
|
||||||
|
from app.config import Settings, get_settings
|
||||||
|
from app.db import Base, get_db
|
||||||
|
from app.main import app
|
||||||
|
from app.models import CrawlRun, CrawlRunEmployeeChange, Employee
|
||||||
|
from app.security import SESSION_COOKIE, sign_session
|
||||||
|
|
||||||
|
|
||||||
|
def test_health_returns_versions():
|
||||||
|
response = TestClient(app).get("/api/health")
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.json()["backend_version"] == "0.8.6"
|
||||||
|
|
||||||
|
|
||||||
|
def test_api_employees_and_stats_require_admin_session():
|
||||||
|
engine = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False}, poolclass=StaticPool)
|
||||||
|
Base.metadata.create_all(engine)
|
||||||
|
session_factory = sessionmaker(bind=engine)
|
||||||
|
db = session_factory()
|
||||||
|
employee = Employee(
|
||||||
|
profile_key="staff:alpha",
|
||||||
|
profile_type="staff",
|
||||||
|
profile_id="alpha",
|
||||||
|
canonical_url="https://www.hse.ru/staff/alpha",
|
||||||
|
full_name="Alpha Person",
|
||||||
|
status="active",
|
||||||
|
first_seen_at=datetime.now(timezone.utc),
|
||||||
|
last_seen_at=datetime.now(timezone.utc),
|
||||||
|
current_data={"contacts": {"emails": ["alpha@hse.ru"]}, "sections": []},
|
||||||
|
)
|
||||||
|
run = CrawlRun(source_url="https://miem.hse.ru/persons", status="completed", new_count=1)
|
||||||
|
db.add_all([employee, run])
|
||||||
|
db.commit()
|
||||||
|
db.add(CrawlRunEmployeeChange(
|
||||||
|
crawl_run_id=run.id,
|
||||||
|
employee_id=employee.id,
|
||||||
|
profile_key=employee.profile_key,
|
||||||
|
profile_url=employee.canonical_url,
|
||||||
|
full_name=employee.full_name,
|
||||||
|
change_type="new",
|
||||||
|
profile_available=True,
|
||||||
|
message="added",
|
||||||
|
))
|
||||||
|
db.commit()
|
||||||
|
run_id = run.id
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
settings = Settings(admin_username="admin", admin_password="password", session_secret="session-secret")
|
||||||
|
|
||||||
|
def override_db():
|
||||||
|
session = session_factory()
|
||||||
|
try:
|
||||||
|
yield session
|
||||||
|
finally:
|
||||||
|
session.close()
|
||||||
|
|
||||||
|
app.dependency_overrides[get_db] = override_db
|
||||||
|
app.dependency_overrides[get_settings] = lambda: settings
|
||||||
|
client = TestClient(app)
|
||||||
|
client.cookies.set(SESSION_COOKIE, sign_session("admin", settings))
|
||||||
|
|
||||||
|
employees = client.get("/api/employees", params={"q": "Alpha", "has_email": True})
|
||||||
|
stats = client.get("/api/stats")
|
||||||
|
run_details = client.get(f"/api/crawl-runs/{run_id}")
|
||||||
|
|
||||||
|
assert employees.status_code == 200
|
||||||
|
assert employees.json()["total"] == 1
|
||||||
|
assert stats.status_code == 200
|
||||||
|
assert stats.json()["new_in_last_run"] == 1
|
||||||
|
assert run_details.status_code == 200
|
||||||
|
assert run_details.json()["changes"]["new"][0]["full_name"] == "Alpha Person"
|
||||||
|
directory = client.get("/admin/directory?q=Alpha&offset=100")
|
||||||
|
assert directory.status_code == 200
|
||||||
|
assert "Показаны 1–1 из 1" in directory.text
|
||||||
|
assert 'name="offset"' not in directory.text
|
||||||
|
assert "Alpha Person" in directory.text
|
||||||
|
empty = client.get("/admin/directory?q=missing&offset=100")
|
||||||
|
assert "Показаны 0–0 из 0" in empty.text
|
||||||
|
assert "Сбросить фильтры" in empty.text
|
||||||
|
app.dependency_overrides.clear()
|
||||||
|
|
||||||
|
|
||||||
|
def test_admin_refresh_employee_route_updates_only_requested_employee(monkeypatch):
|
||||||
|
engine = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False}, poolclass=StaticPool)
|
||||||
|
Base.metadata.create_all(engine)
|
||||||
|
session_factory = sessionmaker(bind=engine)
|
||||||
|
db = session_factory()
|
||||||
|
db.add(Employee(
|
||||||
|
profile_key="org_person:133709486",
|
||||||
|
profile_type="org_person",
|
||||||
|
profile_id="133709486",
|
||||||
|
canonical_url="https://www.hse.ru/org/persons/133709486",
|
||||||
|
full_name="Будков Юрий Алексеевич",
|
||||||
|
status="active",
|
||||||
|
))
|
||||||
|
db.commit()
|
||||||
|
employee_id = db.scalar(select(Employee.id))
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
settings = Settings(admin_username="admin", admin_password="password", session_secret="session-secret")
|
||||||
|
|
||||||
|
def override_db():
|
||||||
|
session = session_factory()
|
||||||
|
try:
|
||||||
|
yield session
|
||||||
|
finally:
|
||||||
|
session.close()
|
||||||
|
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def fake_refresh_employee(db, refreshed_employee, route_settings):
|
||||||
|
calls.append((refreshed_employee.id, route_settings))
|
||||||
|
return SimpleNamespace(status="completed")
|
||||||
|
|
||||||
|
app.dependency_overrides[get_db] = override_db
|
||||||
|
app.dependency_overrides[get_settings] = lambda: settings
|
||||||
|
monkeypatch.setattr("app.admin.refresh_employee", fake_refresh_employee)
|
||||||
|
client = TestClient(app)
|
||||||
|
client.cookies.set(SESSION_COOKIE, sign_session("admin", settings))
|
||||||
|
|
||||||
|
response = client.post(f"/admin/employees/{employee_id}/refresh", follow_redirects=False)
|
||||||
|
|
||||||
|
assert response.status_code == 303
|
||||||
|
assert response.headers["location"] == f"/admin/employees/{employee_id}?refresh_status=success"
|
||||||
|
assert calls == [(employee_id, settings)]
|
||||||
|
app.dependency_overrides.clear()
|
||||||
|
|
||||||
|
|
||||||
|
def test_directory_database_error_has_retry_and_preserves_filters(db_session, monkeypatch):
|
||||||
|
settings = Settings(_env_file=None, session_secret="test-session-secret")
|
||||||
|
app.dependency_overrides[get_db] = lambda: db_session
|
||||||
|
app.dependency_overrides[get_settings] = lambda: settings
|
||||||
|
def unavailable(*args, **kwargs):
|
||||||
|
raise OperationalError("SELECT", {}, Exception("private database details"))
|
||||||
|
monkeypatch.setattr("app.admin.list_employees_page", unavailable)
|
||||||
|
try:
|
||||||
|
client = TestClient(app)
|
||||||
|
client.cookies.set(SESSION_COOKIE, sign_session(settings.admin_username, settings))
|
||||||
|
response = client.get("/admin/directory?q=Alice")
|
||||||
|
assert response.status_code == 503
|
||||||
|
assert 'href="/admin/directory?q=Alice"' in response.text
|
||||||
|
assert "Повторить загрузку" in response.text
|
||||||
|
assert "private database details" not in response.text
|
||||||
|
finally:
|
||||||
|
app.dependency_overrides.clear()
|
||||||
@@ -1,532 +0,0 @@
|
|||||||
import json
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
from types import SimpleNamespace
|
|
||||||
|
|
||||||
from fastapi.testclient import TestClient
|
|
||||||
from sqlalchemy import create_engine, select
|
|
||||||
from sqlalchemy.orm import sessionmaker
|
|
||||||
from sqlalchemy.pool import StaticPool
|
|
||||||
|
|
||||||
from app.config import Settings, get_settings
|
|
||||||
from app.db import Base, get_db
|
|
||||||
from app.main import app
|
|
||||||
from app.models import CrawlRun, CrawlRunEmployeeChange, Employee, EmployeePublication
|
|
||||||
from app.security import SESSION_COOKIE, sign_session
|
|
||||||
|
|
||||||
|
|
||||||
def test_health_returns_versions():
|
|
||||||
client = TestClient(app)
|
|
||||||
|
|
||||||
response = client.get("/api/health")
|
|
||||||
|
|
||||||
assert response.status_code == 200
|
|
||||||
assert response.json()["backend_version"] == "0.7.3"
|
|
||||||
|
|
||||||
|
|
||||||
def test_mcp_lists_tools_without_auth_and_ignores_auth_header():
|
|
||||||
engine = create_engine(
|
|
||||||
"sqlite:///:memory:",
|
|
||||||
connect_args={"check_same_thread": False},
|
|
||||||
poolclass=StaticPool,
|
|
||||||
)
|
|
||||||
Base.metadata.create_all(engine)
|
|
||||||
Session = sessionmaker(bind=engine)
|
|
||||||
|
|
||||||
def override_db():
|
|
||||||
session = Session()
|
|
||||||
try:
|
|
||||||
yield session
|
|
||||||
finally:
|
|
||||||
session.close()
|
|
||||||
|
|
||||||
app.dependency_overrides[get_db] = override_db
|
|
||||||
client = TestClient(app)
|
|
||||||
|
|
||||||
without_auth = client.post("/mcp", json={"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}})
|
|
||||||
with_auth = client.post(
|
|
||||||
"/mcp",
|
|
||||||
headers={"Authorization": "Bearer anything"},
|
|
||||||
json={"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}},
|
|
||||||
)
|
|
||||||
|
|
||||||
assert without_auth.status_code == 200
|
|
||||||
assert with_auth.status_code == 200
|
|
||||||
tool_names = {tool["name"] for tool in without_auth.json()["result"]["tools"]}
|
|
||||||
assert "search_employees" in tool_names
|
|
||||||
assert "get_service_info" in tool_names
|
|
||||||
assert "sync_employees" in tool_names
|
|
||||||
assert any(tool["name"] == "get_crawl_run_details" for tool in without_auth.json()["result"]["tools"])
|
|
||||||
assert with_auth.json()["result"]["tools"] == without_auth.json()["result"]["tools"]
|
|
||||||
|
|
||||||
app.dependency_overrides.clear()
|
|
||||||
|
|
||||||
|
|
||||||
def test_mcp_search_employees_returns_matching_employee():
|
|
||||||
engine = create_engine(
|
|
||||||
"sqlite:///:memory:",
|
|
||||||
connect_args={"check_same_thread": False},
|
|
||||||
poolclass=StaticPool,
|
|
||||||
)
|
|
||||||
Base.metadata.create_all(engine)
|
|
||||||
Session = sessionmaker(bind=engine)
|
|
||||||
session = Session()
|
|
||||||
session.add(
|
|
||||||
Employee(
|
|
||||||
profile_key="staff:avsergeev",
|
|
||||||
profile_type="staff",
|
|
||||||
profile_id="avsergeev",
|
|
||||||
canonical_url="https://www.hse.ru/staff/avsergeev",
|
|
||||||
full_name="Сергеев Алексей Викторович",
|
|
||||||
status="active",
|
|
||||||
first_seen_at=datetime.now(timezone.utc),
|
|
||||||
last_seen_at=datetime.now(timezone.utc),
|
|
||||||
current_data={"sections": []},
|
|
||||||
)
|
|
||||||
)
|
|
||||||
session.commit()
|
|
||||||
session.close()
|
|
||||||
|
|
||||||
def override_db():
|
|
||||||
db = Session()
|
|
||||||
try:
|
|
||||||
yield db
|
|
||||||
finally:
|
|
||||||
db.close()
|
|
||||||
|
|
||||||
app.dependency_overrides[get_db] = override_db
|
|
||||||
client = TestClient(app)
|
|
||||||
|
|
||||||
response = client.post(
|
|
||||||
"/mcp",
|
|
||||||
json={
|
|
||||||
"jsonrpc": "2.0",
|
|
||||||
"id": 1,
|
|
||||||
"method": "tools/call",
|
|
||||||
"params": {"name": "search_employees", "arguments": {"query": "Сергеев"}},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
assert response.status_code == 200
|
|
||||||
assert "Сергеев Алексей Викторович" in response.json()["result"]["content"][0]["text"]
|
|
||||||
|
|
||||||
app.dependency_overrides.clear()
|
|
||||||
|
|
||||||
|
|
||||||
def test_mcp_service_info_returns_tools_and_dataset_hash():
|
|
||||||
engine = create_engine(
|
|
||||||
"sqlite:///:memory:",
|
|
||||||
connect_args={"check_same_thread": False},
|
|
||||||
poolclass=StaticPool,
|
|
||||||
)
|
|
||||||
Base.metadata.create_all(engine)
|
|
||||||
Session = sessionmaker(bind=engine)
|
|
||||||
session = Session()
|
|
||||||
session.add(
|
|
||||||
Employee(
|
|
||||||
profile_key="staff:alpha",
|
|
||||||
profile_type="staff",
|
|
||||||
profile_id="alpha",
|
|
||||||
canonical_url="https://www.hse.ru/staff/alpha",
|
|
||||||
full_name="Alpha Person",
|
|
||||||
status="active",
|
|
||||||
current_checksum="a" * 64,
|
|
||||||
current_data={"sections": []},
|
|
||||||
)
|
|
||||||
)
|
|
||||||
session.commit()
|
|
||||||
session.close()
|
|
||||||
|
|
||||||
def override_db():
|
|
||||||
db = Session()
|
|
||||||
try:
|
|
||||||
yield db
|
|
||||||
finally:
|
|
||||||
db.close()
|
|
||||||
|
|
||||||
app.dependency_overrides[get_db] = override_db
|
|
||||||
client = TestClient(app)
|
|
||||||
|
|
||||||
response = client.post(
|
|
||||||
"/mcp",
|
|
||||||
json={"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": "get_service_info", "arguments": {}}},
|
|
||||||
)
|
|
||||||
|
|
||||||
assert response.status_code == 200
|
|
||||||
payload = json.loads(response.json()["result"]["content"][0]["text"])
|
|
||||||
assert payload["service_name"] == "miem-employees"
|
|
||||||
assert payload["backend_version"] == "0.7.3"
|
|
||||||
assert payload["dataset"]["hash"]
|
|
||||||
assert any(tool["name"] == "sync_employees" for tool in payload["tools"])
|
|
||||||
|
|
||||||
app.dependency_overrides.clear()
|
|
||||||
|
|
||||||
|
|
||||||
def test_mcp_list_employee_publications_prefers_stored_publications_with_fallback():
|
|
||||||
engine = create_engine(
|
|
||||||
"sqlite:///:memory:",
|
|
||||||
connect_args={"check_same_thread": False},
|
|
||||||
poolclass=StaticPool,
|
|
||||||
)
|
|
||||||
Base.metadata.create_all(engine)
|
|
||||||
Session = sessionmaker(bind=engine)
|
|
||||||
session = Session()
|
|
||||||
stored_employee = Employee(
|
|
||||||
profile_key="staff:stored",
|
|
||||||
profile_type="staff",
|
|
||||||
profile_id="stored",
|
|
||||||
canonical_url="https://www.hse.ru/staff/stored",
|
|
||||||
full_name="Stored Person",
|
|
||||||
status="active",
|
|
||||||
current_data={
|
|
||||||
"sections": [
|
|
||||||
{
|
|
||||||
"type": "publications",
|
|
||||||
"publications": [{"title": "Old JSON Publication", "url": "https://example.test/old"}],
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
)
|
|
||||||
fallback_employee = Employee(
|
|
||||||
profile_key="staff:fallback",
|
|
||||||
profile_type="staff",
|
|
||||||
profile_id="fallback",
|
|
||||||
canonical_url="https://www.hse.ru/staff/fallback",
|
|
||||||
full_name="Fallback Person",
|
|
||||||
status="active",
|
|
||||||
current_data={
|
|
||||||
"sections": [
|
|
||||||
{
|
|
||||||
"type": "publications",
|
|
||||||
"publications": [{"title": "Fallback Publication", "url": "https://example.test/fallback"}],
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
)
|
|
||||||
session.add_all([stored_employee, fallback_employee])
|
|
||||||
session.commit()
|
|
||||||
session.add(
|
|
||||||
EmployeePublication(
|
|
||||||
employee_id=stored_employee.id,
|
|
||||||
publication_id="pub-1",
|
|
||||||
title="Stored Publication",
|
|
||||||
year=2024,
|
|
||||||
publication_type="ARTICLE",
|
|
||||||
url="https://publications.hse.ru/view/pub-1",
|
|
||||||
doi_url="https://doi.org/10.1/test",
|
|
||||||
citation_text="Stored Citation",
|
|
||||||
annotation={"ru": "Аннотация", "en": "Abstract"},
|
|
||||||
description={"main": "Stored Citation"},
|
|
||||||
authors=[{"id": "1", "title_ru": "Автор", "is_current_employee": True}],
|
|
||||||
source_hash="a" * 64,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
session.commit()
|
|
||||||
session.close()
|
|
||||||
|
|
||||||
def override_db():
|
|
||||||
db = Session()
|
|
||||||
try:
|
|
||||||
yield db
|
|
||||||
finally:
|
|
||||||
db.close()
|
|
||||||
|
|
||||||
app.dependency_overrides[get_db] = override_db
|
|
||||||
client = TestClient(app)
|
|
||||||
|
|
||||||
stored_response = client.post(
|
|
||||||
"/mcp",
|
|
||||||
json={
|
|
||||||
"jsonrpc": "2.0",
|
|
||||||
"id": 1,
|
|
||||||
"method": "tools/call",
|
|
||||||
"params": {"name": "list_employee_publications", "arguments": {"profile_id_or_url": "stored"}},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
fallback_response = client.post(
|
|
||||||
"/mcp",
|
|
||||||
json={
|
|
||||||
"jsonrpc": "2.0",
|
|
||||||
"id": 2,
|
|
||||||
"method": "tools/call",
|
|
||||||
"params": {"name": "list_employee_publications", "arguments": {"profile_id_or_url": "fallback"}},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
stored_payload = json.loads(stored_response.json()["result"]["content"][0]["text"])
|
|
||||||
fallback_payload = json.loads(fallback_response.json()["result"]["content"][0]["text"])
|
|
||||||
assert stored_payload["items"][0]["title"] == "Stored Publication"
|
|
||||||
assert stored_payload["items"][0]["doi_url"] == "https://doi.org/10.1/test"
|
|
||||||
assert stored_payload["items"][0]["annotation"] == {"ru": "Аннотация", "en": "Abstract"}
|
|
||||||
assert stored_payload["items"][0]["authors"] == [{"id": "1", "title_ru": "Автор", "is_current_employee": True}]
|
|
||||||
assert fallback_payload["items"][0]["title"] == "Fallback Publication"
|
|
||||||
|
|
||||||
app.dependency_overrides.clear()
|
|
||||||
|
|
||||||
|
|
||||||
def test_mcp_sync_employees_full_empty_and_unknown_hash_modes():
|
|
||||||
engine = create_engine(
|
|
||||||
"sqlite:///:memory:",
|
|
||||||
connect_args={"check_same_thread": False},
|
|
||||||
poolclass=StaticPool,
|
|
||||||
)
|
|
||||||
Base.metadata.create_all(engine)
|
|
||||||
Session = sessionmaker(bind=engine)
|
|
||||||
session = Session()
|
|
||||||
session.add(
|
|
||||||
Employee(
|
|
||||||
profile_key="staff:alpha",
|
|
||||||
profile_type="staff",
|
|
||||||
profile_id="alpha",
|
|
||||||
canonical_url="https://www.hse.ru/staff/alpha",
|
|
||||||
full_name="Alpha Person",
|
|
||||||
status="active",
|
|
||||||
current_checksum="a" * 64,
|
|
||||||
current_data={"sections": [{"type": "paragraphs"}]},
|
|
||||||
)
|
|
||||||
)
|
|
||||||
session.commit()
|
|
||||||
session.close()
|
|
||||||
|
|
||||||
def override_db():
|
|
||||||
db = Session()
|
|
||||||
try:
|
|
||||||
yield db
|
|
||||||
finally:
|
|
||||||
db.close()
|
|
||||||
|
|
||||||
app.dependency_overrides[get_db] = override_db
|
|
||||||
client = TestClient(app)
|
|
||||||
|
|
||||||
full_response = client.post(
|
|
||||||
"/mcp",
|
|
||||||
json={"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": "sync_employees", "arguments": {}}},
|
|
||||||
)
|
|
||||||
full_payload = json.loads(full_response.json()["result"]["content"][0]["text"])
|
|
||||||
current_hash = full_payload["to_hash"]
|
|
||||||
|
|
||||||
empty_response = client.post(
|
|
||||||
"/mcp",
|
|
||||||
json={
|
|
||||||
"jsonrpc": "2.0",
|
|
||||||
"id": 2,
|
|
||||||
"method": "tools/call",
|
|
||||||
"params": {"name": "sync_employees", "arguments": {"client_hash": current_hash}},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
empty_payload = json.loads(empty_response.json()["result"]["content"][0]["text"])
|
|
||||||
|
|
||||||
unknown_response = client.post(
|
|
||||||
"/mcp",
|
|
||||||
json={
|
|
||||||
"jsonrpc": "2.0",
|
|
||||||
"id": 3,
|
|
||||||
"method": "tools/call",
|
|
||||||
"params": {"name": "sync_employees", "arguments": {"client_hash": "missing"}},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
unknown_payload = json.loads(unknown_response.json()["result"]["content"][0]["text"])
|
|
||||||
|
|
||||||
assert full_payload["mode"] == "full"
|
|
||||||
assert full_payload["items"][0]["data"] == {"sections": [{"type": "paragraphs"}]}
|
|
||||||
assert empty_payload["mode"] == "delta"
|
|
||||||
assert empty_payload["changes"] == {"added": [], "updated": [], "dismissed": [], "removed": []}
|
|
||||||
assert unknown_payload["mode"] == "full"
|
|
||||||
assert unknown_payload["reason"] == "unknown_client_hash"
|
|
||||||
|
|
||||||
app.dependency_overrides.clear()
|
|
||||||
|
|
||||||
|
|
||||||
def test_mcp_get_crawl_run_details_returns_changes():
|
|
||||||
engine = create_engine(
|
|
||||||
"sqlite:///:memory:",
|
|
||||||
connect_args={"check_same_thread": False},
|
|
||||||
poolclass=StaticPool,
|
|
||||||
)
|
|
||||||
Base.metadata.create_all(engine)
|
|
||||||
Session = sessionmaker(bind=engine)
|
|
||||||
session = Session()
|
|
||||||
run = CrawlRun(source_url="https://miem.hse.ru/persons", status="completed", new_count=1)
|
|
||||||
employee = Employee(
|
|
||||||
profile_key="staff:new",
|
|
||||||
profile_type="staff",
|
|
||||||
profile_id="new",
|
|
||||||
canonical_url="https://www.hse.ru/staff/new",
|
|
||||||
full_name="New Person",
|
|
||||||
status="active",
|
|
||||||
first_seen_at=datetime.now(timezone.utc),
|
|
||||||
last_seen_at=datetime.now(timezone.utc),
|
|
||||||
)
|
|
||||||
session.add_all([run, employee])
|
|
||||||
session.commit()
|
|
||||||
session.add(
|
|
||||||
CrawlRunEmployeeChange(
|
|
||||||
crawl_run_id=run.id,
|
|
||||||
employee_id=employee.id,
|
|
||||||
profile_key=employee.profile_key,
|
|
||||||
profile_url=employee.canonical_url,
|
|
||||||
full_name=employee.full_name,
|
|
||||||
change_type="new",
|
|
||||||
profile_available=True,
|
|
||||||
message="added",
|
|
||||||
)
|
|
||||||
)
|
|
||||||
session.commit()
|
|
||||||
run_id = run.id
|
|
||||||
session.close()
|
|
||||||
|
|
||||||
def override_db():
|
|
||||||
db = Session()
|
|
||||||
try:
|
|
||||||
yield db
|
|
||||||
finally:
|
|
||||||
db.close()
|
|
||||||
|
|
||||||
app.dependency_overrides[get_db] = override_db
|
|
||||||
client = TestClient(app)
|
|
||||||
|
|
||||||
response = client.post(
|
|
||||||
"/mcp",
|
|
||||||
json={
|
|
||||||
"jsonrpc": "2.0",
|
|
||||||
"id": 1,
|
|
||||||
"method": "tools/call",
|
|
||||||
"params": {"name": "get_crawl_run_details", "arguments": {"run_id": run_id}},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
assert response.status_code == 200
|
|
||||||
text = response.json()["result"]["content"][0]["text"]
|
|
||||||
assert "New Person" in text
|
|
||||||
assert "changes_detail_available" in text
|
|
||||||
|
|
||||||
app.dependency_overrides.clear()
|
|
||||||
|
|
||||||
|
|
||||||
def test_mcp_protected_resource_metadata_route_is_removed():
|
|
||||||
client = TestClient(app)
|
|
||||||
|
|
||||||
response = client.get("/.well-known/oauth-protected-resource")
|
|
||||||
|
|
||||||
assert response.status_code == 404
|
|
||||||
|
|
||||||
|
|
||||||
def test_api_employees_and_stats_require_admin_session():
|
|
||||||
engine = create_engine(
|
|
||||||
"sqlite:///:memory:",
|
|
||||||
connect_args={"check_same_thread": False},
|
|
||||||
poolclass=StaticPool,
|
|
||||||
)
|
|
||||||
Base.metadata.create_all(engine)
|
|
||||||
Session = sessionmaker(bind=engine)
|
|
||||||
db = Session()
|
|
||||||
db.add(
|
|
||||||
Employee(
|
|
||||||
profile_key="staff:alpha",
|
|
||||||
profile_type="staff",
|
|
||||||
profile_id="alpha",
|
|
||||||
canonical_url="https://www.hse.ru/staff/alpha",
|
|
||||||
full_name="Alpha Person",
|
|
||||||
status="active",
|
|
||||||
first_seen_at=datetime.now(timezone.utc),
|
|
||||||
last_seen_at=datetime.now(timezone.utc),
|
|
||||||
current_data={"contacts": {"emails": ["alpha@hse.ru"]}, "sections": []},
|
|
||||||
)
|
|
||||||
)
|
|
||||||
run = CrawlRun(source_url="https://miem.hse.ru/persons", status="completed", new_count=1)
|
|
||||||
db.add(run)
|
|
||||||
db.commit()
|
|
||||||
db.add(
|
|
||||||
CrawlRunEmployeeChange(
|
|
||||||
crawl_run_id=run.id,
|
|
||||||
employee_id=1,
|
|
||||||
profile_key="staff:alpha",
|
|
||||||
profile_url="https://www.hse.ru/staff/alpha",
|
|
||||||
full_name="Alpha Person",
|
|
||||||
change_type="new",
|
|
||||||
profile_available=True,
|
|
||||||
message="added",
|
|
||||||
)
|
|
||||||
)
|
|
||||||
db.commit()
|
|
||||||
run_id = run.id
|
|
||||||
db.close()
|
|
||||||
|
|
||||||
settings = Settings(admin_username="admin", admin_password="password", session_secret="session-secret")
|
|
||||||
|
|
||||||
def override_db():
|
|
||||||
session = Session()
|
|
||||||
try:
|
|
||||||
yield session
|
|
||||||
finally:
|
|
||||||
session.close()
|
|
||||||
|
|
||||||
app.dependency_overrides[get_db] = override_db
|
|
||||||
app.dependency_overrides[get_settings] = lambda: settings
|
|
||||||
client = TestClient(app)
|
|
||||||
client.cookies.set(SESSION_COOKIE, sign_session("admin", settings))
|
|
||||||
|
|
||||||
employees = client.get("/api/employees", params={"q": "Alpha", "has_email": True})
|
|
||||||
stats = client.get("/api/stats")
|
|
||||||
run_details = client.get(f"/api/crawl-runs/{run_id}")
|
|
||||||
|
|
||||||
assert employees.status_code == 200
|
|
||||||
assert employees.json()["total"] == 1
|
|
||||||
assert stats.status_code == 200
|
|
||||||
assert stats.json()["new_in_last_run"] == 1
|
|
||||||
assert run_details.status_code == 200
|
|
||||||
assert run_details.json()["changes"]["new"][0]["full_name"] == "Alpha Person"
|
|
||||||
|
|
||||||
app.dependency_overrides.clear()
|
|
||||||
|
|
||||||
|
|
||||||
def test_admin_refresh_employee_route_updates_only_requested_employee(monkeypatch):
|
|
||||||
engine = create_engine(
|
|
||||||
"sqlite:///:memory:",
|
|
||||||
connect_args={"check_same_thread": False},
|
|
||||||
poolclass=StaticPool,
|
|
||||||
)
|
|
||||||
Base.metadata.create_all(engine)
|
|
||||||
Session = sessionmaker(bind=engine)
|
|
||||||
db = Session()
|
|
||||||
db.add(
|
|
||||||
Employee(
|
|
||||||
profile_key="org_person:133709486",
|
|
||||||
profile_type="org_person",
|
|
||||||
profile_id="133709486",
|
|
||||||
canonical_url="https://www.hse.ru/org/persons/133709486",
|
|
||||||
full_name="Будков Юрий Алексеевич",
|
|
||||||
status="active",
|
|
||||||
)
|
|
||||||
)
|
|
||||||
db.commit()
|
|
||||||
employee_id = db.scalar(select(Employee.id))
|
|
||||||
db.close()
|
|
||||||
|
|
||||||
settings = Settings(admin_username="admin", admin_password="password", session_secret="session-secret")
|
|
||||||
|
|
||||||
def override_db():
|
|
||||||
session = Session()
|
|
||||||
try:
|
|
||||||
yield session
|
|
||||||
finally:
|
|
||||||
session.close()
|
|
||||||
|
|
||||||
calls = []
|
|
||||||
|
|
||||||
def fake_refresh_employee(db, refreshed_employee, route_settings):
|
|
||||||
calls.append((refreshed_employee.id, route_settings))
|
|
||||||
return SimpleNamespace(status="completed")
|
|
||||||
|
|
||||||
app.dependency_overrides[get_db] = override_db
|
|
||||||
app.dependency_overrides[get_settings] = lambda: settings
|
|
||||||
monkeypatch.setattr("app.admin.refresh_employee", fake_refresh_employee)
|
|
||||||
client = TestClient(app)
|
|
||||||
client.cookies.set(SESSION_COOKIE, sign_session("admin", settings))
|
|
||||||
|
|
||||||
response = client.post(f"/admin/employees/{employee_id}/refresh", follow_redirects=False)
|
|
||||||
|
|
||||||
assert response.status_code == 303
|
|
||||||
assert response.headers["location"] == f"/admin/employees/{employee_id}?refresh_status=success"
|
|
||||||
assert calls == [(employee_id, settings)]
|
|
||||||
|
|
||||||
app.dependency_overrides.clear()
|
|
||||||
@@ -11,3 +11,12 @@ def test_numeric_crawl_limit_is_parsed():
|
|||||||
settings = Settings(crawl_limit="25")
|
settings = Settings(crawl_limit="25")
|
||||||
|
|
||||||
assert settings.crawl_limit == 25
|
assert settings.crawl_limit == 25
|
||||||
|
|
||||||
|
|
||||||
|
def test_dismissal_safety_settings_have_conservative_defaults(monkeypatch):
|
||||||
|
monkeypatch.delenv("DISMISSAL_CONFIRMATION_RUNS", raising=False)
|
||||||
|
monkeypatch.delenv("MAX_AUTO_DISMISSALS_PER_RUN", raising=False)
|
||||||
|
settings = Settings(_env_file=None)
|
||||||
|
|
||||||
|
assert settings.dismissal_confirmation_runs == 3
|
||||||
|
assert settings.max_auto_dismissals_per_run == 25
|
||||||
|
|||||||
@@ -1,88 +0,0 @@
|
|||||||
from datetime import datetime, timezone
|
|
||||||
|
|
||||||
from app.models import Employee
|
|
||||||
from app.services.dataset_versions import get_or_create_current_version, sync_employees_payload
|
|
||||||
|
|
||||||
|
|
||||||
def _employee(profile_key: str, checksum: str, *, status: str = "active") -> Employee:
|
|
||||||
return Employee(
|
|
||||||
profile_key=profile_key,
|
|
||||||
profile_type=profile_key.split(":", 1)[0],
|
|
||||||
profile_id=profile_key.split(":", 1)[1],
|
|
||||||
canonical_url=f"https://www.hse.ru/{profile_key}",
|
|
||||||
full_name=profile_key,
|
|
||||||
status=status,
|
|
||||||
first_seen_at=datetime.now(timezone.utc),
|
|
||||||
last_seen_at=datetime.now(timezone.utc),
|
|
||||||
current_data={"profile_key": profile_key},
|
|
||||||
current_checksum=checksum,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_dataset_version_hash_is_stable_for_same_employee_state(db_session):
|
|
||||||
db_session.add(_employee("staff:alpha", "a" * 64))
|
|
||||||
db_session.commit()
|
|
||||||
|
|
||||||
first = get_or_create_current_version(db_session)
|
|
||||||
db_session.commit()
|
|
||||||
second = get_or_create_current_version(db_session)
|
|
||||||
|
|
||||||
assert second.id == first.id
|
|
||||||
assert second.hash == first.hash
|
|
||||||
assert second.employee_count == 1
|
|
||||||
|
|
||||||
|
|
||||||
def test_dataset_version_hash_changes_when_employee_checksum_changes(db_session):
|
|
||||||
employee = _employee("staff:alpha", "a" * 64)
|
|
||||||
db_session.add(employee)
|
|
||||||
db_session.commit()
|
|
||||||
first = get_or_create_current_version(db_session)
|
|
||||||
db_session.commit()
|
|
||||||
|
|
||||||
employee.current_checksum = "b" * 64
|
|
||||||
db_session.commit()
|
|
||||||
second = get_or_create_current_version(db_session)
|
|
||||||
|
|
||||||
assert second.hash != first.hash
|
|
||||||
assert second.previous_hash == first.hash
|
|
||||||
|
|
||||||
|
|
||||||
def test_sync_employees_diff_spans_multiple_intermediate_versions(db_session):
|
|
||||||
alpha = _employee("staff:alpha", "a" * 64)
|
|
||||||
db_session.add(alpha)
|
|
||||||
db_session.commit()
|
|
||||||
first = get_or_create_current_version(db_session)
|
|
||||||
db_session.commit()
|
|
||||||
|
|
||||||
beta = _employee("staff:beta", "b" * 64)
|
|
||||||
db_session.add(beta)
|
|
||||||
db_session.commit()
|
|
||||||
get_or_create_current_version(db_session)
|
|
||||||
db_session.commit()
|
|
||||||
|
|
||||||
alpha.current_checksum = "c" * 64
|
|
||||||
alpha.current_data = {"profile_key": "staff:alpha", "changed": True}
|
|
||||||
db_session.commit()
|
|
||||||
|
|
||||||
payload = sync_employees_payload(db_session, client_hash=first.hash, include_data=False)
|
|
||||||
|
|
||||||
assert payload["mode"] == "delta"
|
|
||||||
assert [item["profile_key"] for item in payload["changes"]["added"]] == ["staff:beta"]
|
|
||||||
assert [item["profile_key"] for item in payload["changes"]["updated"]] == ["staff:alpha"]
|
|
||||||
assert payload["changes"]["dismissed"] == []
|
|
||||||
assert payload["changes"]["removed"] == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_sync_employees_reports_dismissed_as_tombstone(db_session):
|
|
||||||
alpha = _employee("staff:alpha", "a" * 64)
|
|
||||||
db_session.add(alpha)
|
|
||||||
db_session.commit()
|
|
||||||
first = get_or_create_current_version(db_session)
|
|
||||||
db_session.commit()
|
|
||||||
|
|
||||||
alpha.status = "dismissed"
|
|
||||||
db_session.commit()
|
|
||||||
payload = sync_employees_payload(db_session, client_hash=first.hash, include_data=False)
|
|
||||||
|
|
||||||
assert payload["changes"]["dismissed"][0]["profile_key"] == "staff:alpha"
|
|
||||||
assert payload["changes"]["dismissed"][0]["status"] == "dismissed"
|
|
||||||
Reference in New Issue
Block a user