From c031fb9f4ae54256d03edea2fa98dd6defadb23c Mon Sep 17 00:00:00 2001 From: admin Date: Thu, 27 Aug 2026 17:04:03 +0300 Subject: [PATCH 1/2] refactor: remove unused data integration --- CHANGELOG.md | 4 + MCP_DESCRIPTION.md | 671 ------------------------------------------ README.md | 35 +-- app/main.py | 2 - app/mcp.py | 262 ----------------- app/version.py | 6 +- pyproject.toml | 4 +- tests/test_api.py | 126 ++++++++ tests/test_api_mcp.py | 532 --------------------------------- 9 files changed, 138 insertions(+), 1504 deletions(-) delete mode 100644 MCP_DESCRIPTION.md delete mode 100644 app/mcp.py create mode 100644 tests/test_api.py delete mode 100644 tests/test_api_mcp.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 80086c6..4ea18dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.7.7 + +- Удалена неиспользуемая интеграция обмена данными и связанная документация и тесты. + ## 0.7.6 - Возвращён еженедельный автоматический запуск обхода сотрудников. diff --git a/MCP_DESCRIPTION.md b/MCP_DESCRIPTION.md deleted file mode 100644 index ebf4cf1..0000000 --- a/MCP_DESCRIPTION.md +++ /dev/null @@ -1,671 +0,0 @@ -# MCP: описание работы, структуры и тулзов - -Документ описывает MCP endpoint сервиса `miem-employees` по текущей реализации в `app/mcp.py`. - -## Где находится MCP - -- FastAPI router: `app.mcp.router` -- Подключение к приложению: `app/main.py` -- HTTP endpoint: `POST /mcp` -- Локально при обычном запуске API: `http://localhost:8000/mcp` -- В Docker Compose endpoint обслуживает `api`: `http://localhost:8000/mcp` -- Авторизация на уровне приложения: отсутствует. Заголовок `Authorization` не проверяется и не влияет на ответ. - -Если доступ к MCP нужно ограничить, это должно делаться внешним контуром: bind на localhost, VPN, firewall, reverse proxy или отдельная сетевая политика. - -## Протокол - -Endpoint принимает JSON-RPC 2.0 over HTTP. - -Общий формат запроса: - -```json -{ - "jsonrpc": "2.0", - "id": 1, - "method": "tools/list", - "params": {} -} -``` - -Общий формат успешного ответа: - -```json -{ - "jsonrpc": "2.0", - "id": 1, - "result": {} -} -``` - -Общий формат ошибки: - -```json -{ - "jsonrpc": "2.0", - "id": 1, - "error": { - "code": -32601, - "message": "Method not found" - } -} -``` - -Поддерживаемая версия MCP-протокола: - -```text -2024-11-05 -``` - -Имя сервиса: - -```text -miem-employees -``` - -Версия сервера берется из `app.version.BACKEND_VERSION`. - -## Поддерживаемые JSON-RPC методы - -### initialize - -Возвращает метаданные MCP-сервера и capabilities. - -Запрос: - -```json -{ - "jsonrpc": "2.0", - "id": 1, - "method": "initialize", - "params": {} -} -``` - -Ответ: - -```json -{ - "jsonrpc": "2.0", - "id": 1, - "result": { - "protocolVersion": "2024-11-05", - "serverInfo": { - "name": "miem-employees", - "version": "0.7.0" - }, - "capabilities": { - "tools": {} - } - } -} -``` - -### tools/list - -Возвращает список доступных tools с JSON Schema для аргументов. - -Запрос: - -```json -{ - "jsonrpc": "2.0", - "id": 1, - "method": "tools/list", - "params": {} -} -``` - -Ответ содержит массив `result.tools`. - -### tools/call - -Вызывает один tool по имени. - -Запрос: - -```json -{ - "jsonrpc": "2.0", - "id": 1, - "method": "tools/call", - "params": { - "name": "search_employees", - "arguments": { - "query": "Сергеев", - "limit": 20 - } - } -} -``` - -Ответ tool всегда заворачивается в MCP content-массив: - -```json -{ - "jsonrpc": "2.0", - "id": 1, - "result": { - "content": [ - { - "type": "text", - "text": "{\"items\":[]}" - } - ] - } -} -``` - -Поле `text` содержит сериализованный JSON с `ensure_ascii=false`. Клиент должен распарсить это поле как JSON, если ему нужна структурированная нагрузка. - -## Ошибки - -- Неизвестный JSON-RPC метод: `code = -32601`, `message = "Method not found"`. -- Исключения при обработке tool: `code = -32000`, `message` содержит текст исключения. -- Если сущность не найдена внутри отдельных tools, HTTP и JSON-RPC ответ остаются успешными, а полезная нагрузка содержит `{"error": "not_found"}`. - -## Источники данных - -MCP читает данные из основной базы через SQLAlchemy session из `app.db.get_db`. - -Основные таблицы и модели: - -- `employees`: текущая карточка сотрудника, статус, профиль, `current_data`, checksum. -- `employee_publications`: нормализованные публикации сотрудников с авторами, DOI, аннотацией, описанием, citation text и raw JSON из HSE Publications. -- `employee_news_links`: нормализованные ссылки на новости из блока профиля «В новостях» с заголовком, URL, кратким описанием, датой, годом публикации и raw JSON карточки. -- `crawl_runs`: история запусков парсинга. -- `crawl_run_employee_changes`: детальные изменения сотрудников в рамках запуска. -- `crawl_errors`: ошибки парсинга в рамках запуска. -- `dataset_versions`: версии полного набора сотрудников. -- `dataset_version_items`: состав конкретной версии набора сотрудников. - -## Общая структура employee payload - -Краткая карточка сотрудника: - -```json -{ - "profile_key": "staff:avsergeev", - "profile_id": "avsergeev", - "full_name": "Сергеев Алексей Викторович", - "status": "active", - "canonical_url": "https://www.hse.ru/staff/avsergeev", - "last_seen_at": "2026-05-14T10:00:00+00:00", - "dismissed_at": null -} -``` - -В sync payload дополнительно отдается `checksum`. - -Полная карточка дополнительно содержит: - -```json -{ - "data": { - "contacts": {}, - "sections": [] - } -} -``` - -`data` соответствует распарсенному JSON профиля сотрудника. Внутри `sections` могут быть секции с публикациями, курсами, ВКР, новостями, таблицами, ссылками и произвольными текстовыми блоками. - -Пример секции новостей внутри `data.sections`: - -```json -{ - "title": "В новостях", - "slug": "v_novostyah", - "type": "news", - "news_count": 1, - "news_links": [ - { - "title": "Название новости", - "url": "https://www.hse.ru/news/edu/1153850518.html", - "summary": "Краткое описание новости.", - "published_at": "2026-04-28T00:00:00+00:00", - "published_year": 2026 - } - ] -} -``` - -Для новостей отдельного MCP tool сейчас нет: они доступны через `get_employee(...).data.sections` или через полную синхронизацию `sync_employees(include_data=true)`. - -## Tools - -### get_service_info - -Назначение: вернуть метаданные сервиса, список tools и текущую версию набора сотрудников. - -Аргументы: отсутствуют. - -Возвращает: - -```json -{ - "service_name": "miem-employees", - "backend_version": "0.7.0", - "protocolVersion": "2024-11-05", - "tools": [], - "dataset": { - "hash": "sha256", - "previous_hash": "sha256 или null", - "created_at": "2026-05-14T10:00:00+00:00", - "crawl_run_id": 123, - "employee_count": 100, - "active_count": 95, - "dismissed_count": 5 - } -} -``` - -Особенность: перед ответом сервис создает актуальную `dataset_version`, если текущий набор сотрудников еще не имеет версии. - -### sync_employees - -Назначение: синхронизировать клиентский кэш сотрудников по hash набора данных. - -Аргументы: - -```json -{ - "client_hash": "sha256 или null", - "include_data": true -} -``` - -- `client_hash`: hash версии, которая уже есть у клиента. Если не передан, отдается полный snapshot. -- `include_data`: управляет включением полного `data` в карточки сотрудников. По умолчанию `true`. - -Полный ответ без `client_hash`: - -```json -{ - "mode": "full", - "from_hash": null, - "to_hash": "current-sha256", - "dataset": {}, - "items": [] -} -``` - -Если клиентский hash совпадает с текущим: - -```json -{ - "mode": "delta", - "from_hash": "current-sha256", - "to_hash": "current-sha256", - "dataset": {}, - "changes": { - "added": [], - "updated": [], - "dismissed": [], - "removed": [] - } -} -``` - -Если `client_hash` неизвестен серверу: - -```json -{ - "mode": "full", - "from_hash": "missing", - "to_hash": "current-sha256", - "dataset": {}, - "items": [], - "reason": "unknown_client_hash" -} -``` - -Если `client_hash` найден и отличается от текущего: - -```json -{ - "mode": "delta", - "from_hash": "old-sha256", - "to_hash": "current-sha256", - "dataset": {}, - "changes": { - "added": [], - "updated": [], - "dismissed": [], - "removed": [] - } -} -``` - -Логика delta: - -- `added`: сотрудник появился в новой версии. -- `updated`: изменился checksum или статус, и сотрудник активен. -- `dismissed`: сотрудник есть в новой версии, но получил статус `dismissed`. -- `removed`: `profile_key` был в старой версии, но отсутствует в новой. - -Hash набора считается по отсортированному списку `{profile_key, status, checksum}`. - -### search_employees - -Назначение: найти сотрудников по ФИО или canonical URL. - -Аргументы: - -```json -{ - "query": "Сергеев", - "status": "active", - "limit": 20 -} -``` - -- `query`: обязательный по schema, но в коде пустая строка означает поиск без текстового фильтра. -- `status`: опционально, только `active` или `dismissed`. -- `limit`: максимум 100, по умолчанию 20. - -Возвращает массив кратких employee payload без `data`: - -```json -[ - { - "profile_key": "staff:avsergeev", - "profile_id": "avsergeev", - "full_name": "Сергеев Алексей Викторович", - "status": "active", - "canonical_url": "https://www.hse.ru/staff/avsergeev", - "last_seen_at": "2026-05-14T10:00:00+00:00", - "dismissed_at": null - } -] -``` - -### get_employee - -Назначение: получить одну карточку сотрудника. - -Аргументы: - -```json -{ - "profile_id_or_url": "avsergeev" -} -``` - -Поиск выполняется по: - -- `profile_key` -- `profile_id` -- точному `canonical_url` -- частичному совпадению `canonical_url` - -Возвращает полный employee payload с `data`. - -Если сотрудник не найден: - -```json -{ - "error": "not_found" -} -``` - -### list_employee_publications - -Назначение: вернуть публикации сотрудника. Если есть нормализованные строки в `employee_publications`, tool возвращает детальные публикационные данные: авторов, DOI, аннотацию, описание, citation text, год, тип, язык, статус и ссылки. Если детальная таблица еще не заполнена, tool использует старый fallback из `employees.current_data.sections[].publications`. - -Аргументы: - -```json -{ - "profile_id_or_url": "avsergeev" -} -``` - -Поиск сотрудника выполняется так же, как в `get_employee`: по `profile_key`, `profile_id`, точному или частичному `canonical_url`. - -Порядок источников: - -- сначала `employee_publications`, отсортированные по году, названию и внутреннему id; -- если записей нет, секции `current_data.sections` с `type = "publications"` и массивами `publications`. - -Ответ: - -```json -{ - "employee": { - "profile_key": "org_person:803294906", - "profile_id": "803294906", - "full_name": "Борисов Сергей Петрович", - "status": "active", - "canonical_url": "https://www.hse.ru/org/persons/803294906", - "last_seen_at": "2026-05-14T10:00:00+00:00", - "dismissed_at": null - }, - "items": [ - { - "id": "888959076", - "publication_id": "888959076", - "title": "Название публикации", - "text": "Краткое описание или citation", - "url": "https://publications.hse.ru/view/888959076", - "year": 2023, - "type": "ARTICLE", - "publication_type": "ARTICLE", - "language": "ru", - "status": 1, - "doi_url": "https://doi.org/10.53921/18195822_2023_23_4_624", - "other_url": "https://example.test", - "document_url": "https://example.test/file.pdf", - "citation_text": "Авторы. Название публикации // Журнал. 2023.", - "annotation": { - "ru": "Аннотация", - "en": "Abstract" - }, - "description": { - "main": "Авторы. Название публикации // Журнал. 2023." - }, - "authors": [ - { - "id": "803294906", - "href": "https://www.hse.ru/org/persons/803294906", - "title_ru": "Борисов С. П.", - "title_en": "", - "reverse_title_ru": "С. П. Борисов", - "reverse_title_en": "", - "alt_name": "S. P. Borisov", - "other_name": null, - "is_current_employee": true - } - ] - } - ] -} -``` - -В fallback-режиме из `current_data` старые элементы могут содержать только базовые поля `title`, `text`, `url` и `id`. - -Если сотрудник не найден: - -```json -{ - "items": [] -} -``` - -Если сотрудник найден, но публикаций нет: - -```json -{ - "employee": {}, - "items": [] -} -``` - -### list_employee_courses - -Назначение: вернуть курсы преподавания сотрудника из распарсенных секций профиля. - -Аргументы: - -```json -{ - "profile_id_or_url": "avsergeev" -} -``` - -Сервис ищет секции `current_data.sections` с `type = "courses_by_year"` и объединяет массивы `courses`. - -Ответ: - -```json -{ - "employee": {}, - "items": [ - { - "title": "Название курса", - "url": "https://..." - } - ] -} -``` - -Если сотрудник или данные профиля отсутствуют: - -```json -{ - "items": [] -} -``` - -### get_crawl_status - -Назначение: вернуть последний запуск парсинга. - -Аргументы: отсутствуют. - -Ответ: - -```json -{ - "id": 123, - "status": "completed", - "source_url": "https://miem.hse.ru/persons", - "started_at": "2026-05-14T10:00:00+00:00", - "finished_at": "2026-05-14T10:10:00+00:00", - "found_count": 100, - "parsed_count": 98, - "error_count": 2, - "dismissed_count": 1 -} -``` - -Если запусков еще не было: - -```json -{ - "status": "never_run" -} -``` - -### get_crawl_run_details - -Назначение: вернуть детальную информацию по конкретному запуску парсинга: summary, изменения сотрудников и ошибки. - -Аргументы: - -```json -{ - "run_id": 123 -} -``` - -Ответ: - -```json -{ - "id": 123, - "source_url": "https://miem.hse.ru/persons", - "status": "completed", - "status_display": "Завершен", - "started_at": "2026-05-14T10:00:00+00:00", - "finished_at": "2026-05-14T10:10:00+00:00", - "started_display": "14.05.2026 13:00", - "finished_display": "14.05.2026 13:10", - "found_count": 100, - "parsed_count": 98, - "new_count": 3, - "error_count": 2, - "dismissed_count": 1, - "processed_count": 100, - "progress_percent": 100.0, - "message": null, - "changes_detail_available": true, - "changes": { - "new": [], - "missing_from_source": [], - "dismissed": [] - }, - "errors": [] -} -``` - -Если запуск не найден: - -```json -{ - "error": "not_found" -} -``` - -## Примеры curl - -Список tools: - -```bash -curl http://localhost:8000/mcp \ - -H "Content-Type: application/json" \ - -d '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}' -``` - -Поиск сотрудника: - -```bash -curl http://localhost:8000/mcp \ - -H "Content-Type: application/json" \ - -d '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"search_employees","arguments":{"query":"Сергеев","limit":5}}}' -``` - -Полная синхронизация: - -```bash -curl http://localhost:8000/mcp \ - -H "Content-Type: application/json" \ - -d '{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"sync_employees","arguments":{"include_data":false}}}' -``` - -Delta-синхронизация: - -```bash -curl http://localhost:8000/mcp \ - -H "Content-Type: application/json" \ - -d '{"jsonrpc":"2.0","id":4,"method":"tools/call","params":{"name":"sync_employees","arguments":{"client_hash":"known-sha256","include_data":true}}}' -``` - -## Как MCP используется клиентом - -1. Клиент вызывает `initialize` и проверяет `protocolVersion`. -2. Клиент вызывает `tools/list`, чтобы получить актуальный список tools и input schemas. -3. Для поиска и точечных запросов клиент вызывает `tools/call` с `search_employees`, `get_employee`, `list_employee_publications`, `list_employee_courses`, `get_crawl_status` или `get_crawl_run_details`. -4. Для локального кэша клиент вызывает `get_service_info` или `sync_employees`. -5. Клиент хранит последний `dataset.hash`. -6. При следующей синхронизации клиент передает hash как `client_hash`. -7. Сервер возвращает пустую delta, delta с изменениями или полный snapshot, если hash неизвестен. - -## Важные особенности реализации - -- MCP endpoint read-only: tools не запускают парсинг и не меняют сотрудников напрямую. -- `get_service_info` и `sync_employees` могут создать новую запись `dataset_versions`, если состояние сотрудников изменилось и новой версии еще нет. -- Все tool payloads возвращаются как JSON-строка внутри `content[0].text`. -- `search_employees` ищет через `ilike` по `full_name` и `canonical_url`. -- `get_employee` допускает частичный URL, поэтому строка `133709486` может найти `https://www.hse.ru/org/persons/133709486`. -- Временные значения сериализуются через `isoformat()`, display-поля для админских payload формируются в часовом поясе `Europe/Moscow`. diff --git a/README.md b/README.md index 1c0dc5a..d9ac005 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # MIEM Employees Server -Сервис собирает сотрудников МИЭМ с сайта ВШЭ, хранит карточки и историю обновлений в Postgres, показывает минимальную админку и отдает read-only MCP endpoint для ИИ-агентов. +Сервис собирает сотрудников МИЭМ с сайта ВШЭ, хранит карточки и историю обновлений в Postgres и показывает минимальную админку. ## Архитектура -- `api`: FastAPI, REST API, HTML-админка, MCP endpoint и healthcheck. +- `api`: FastAPI, REST API, HTML-админка и healthcheck. - `worker`: weekly scheduler, который запускает парсинг по `CRAWL_CRON`. - `postgres`: основная БД. @@ -107,36 +107,7 @@ curl -X POST http://localhost:8000/api/crawl-runs --cookie "miem_admin_session=. Во время выполнения парсинга `found_count`, `parsed_count`, `skipped_count` и `error_count` обновляются в базе. Админка опрашивает `/api/crawl-runs/latest` и показывает прогресс как `(parsed_count + skipped_count + error_count) / found_count`. -## MCP - -Endpoint: `POST /mcp`, без авторизации на уровне приложения. - -Поддерживаемые tools: - -- `get_service_info()` -- `sync_employees(client_hash?, include_data?)` -- `search_employees(query, status?, limit?)` -- `get_employee(profile_id_or_url)` -- `list_employee_publications(profile_id_or_url)` — публикации сотрудника; при наличии данных из `employee_publications` возвращает авторов, DOI, аннотацию, описание, citation text, год, тип, язык, статус и ссылку HSE Publications. -- `list_employee_courses(profile_id_or_url)` -- `get_crawl_status()` -- `get_crawl_run_details(run_id)` - -`get_service_info` возвращает метаданные сервиса, список tools и текущую версию набора сотрудников. `sync_employees` отдает полный snapshot или delta по `client_hash`; checksum набора строится по сотрудникам, их статусам и текущим checksums. Ответы tools возвращаются как JSON-строка внутри MCP `content[0].text`. - -Новости сотрудника отдельной MCP tool не имеют: они доступны в `get_employee(...).data.sections` и `sync_employees(include_data=true)` как секция `type = "news"` с массивом `news_links`. - -Пример локального запроса списка tools: - -```bash -curl http://localhost:8000/mcp \ - -H "Content-Type: application/json" \ - -d '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}' -``` - -Если MCP нужно ограничить, делайте это на сетевом уровне: localhost binding, VPN, firewall, reverse proxy или другой внешний контур доступа. - -## Обслуживание +## Обслуживание ```bash docker compose logs -f api diff --git a/app/main.py b/app/main.py index 7f34d48..046891c 100644 --- a/app/main.py +++ b/app/main.py @@ -4,14 +4,12 @@ from fastapi.staticfiles import StaticFiles from app.admin import router as admin_router from app.api import router as api_router from app.db import init_db -from app.mcp import router as mcp_router from app.version import BACKEND_VERSION app = FastAPI(title="MIEM Employees", version=BACKEND_VERSION) app.mount("/static", StaticFiles(directory="app/static"), name="static") app.include_router(api_router) app.include_router(admin_router) -app.include_router(mcp_router) @app.on_event("startup") diff --git a/app/mcp.py b/app/mcp.py deleted file mode 100644 index e63fbf3..0000000 --- a/app/mcp.py +++ /dev/null @@ -1,262 +0,0 @@ -import json - -from fastapi import APIRouter, Depends, Request -from sqlalchemy import desc, or_, select -from sqlalchemy.orm import Session - -from app.db import get_db -from app.models import CrawlRun, Employee, EmployeePublication -from app.services.admin_data import run_detail_payload -from app.services.dataset_versions import service_info_payload, sync_employees_payload -from app.version import BACKEND_VERSION - -router = APIRouter(prefix="/mcp") -PROTOCOL_VERSION = "2024-11-05" -SERVICE_NAME = "miem-employees" - - -TOOLS = [ - { - "name": "get_service_info", - "description": "Return service metadata, supported tools, and current dataset version.", - "inputSchema": {"type": "object", "properties": {}}, - }, - { - "name": "sync_employees", - "description": "Synchronize employees by dataset hash. Returns a full snapshot or a delta from client_hash.", - "inputSchema": { - "type": "object", - "properties": { - "client_hash": {"type": "string"}, - "include_data": {"type": "boolean", "default": True}, - }, - }, - }, - { - "name": "search_employees", - "description": "Search MIEM employees by name or profile URL.", - "inputSchema": { - "type": "object", - "properties": { - "query": {"type": "string"}, - "status": {"type": "string", "enum": ["active", "dismissed"]}, - "limit": {"type": "integer", "default": 20}, - }, - "required": ["query"], - }, - }, - { - "name": "get_employee", - "description": "Get one employee by profile id, profile key, or canonical URL.", - "inputSchema": {"type": "object", "properties": {"profile_id_or_url": {"type": "string"}}, "required": ["profile_id_or_url"]}, - }, - { - "name": "list_employee_publications", - "description": ( - "List employee publications with detailed fields when available: authors, DOI URL, annotation, " - "description, citation text, year, publication type, language, status, and HSE Publications URL." - ), - "inputSchema": {"type": "object", "properties": {"profile_id_or_url": {"type": "string"}}, "required": ["profile_id_or_url"]}, - }, - { - "name": "list_employee_courses", - "description": "List teaching courses parsed from an employee profile.", - "inputSchema": {"type": "object", "properties": {"profile_id_or_url": {"type": "string"}}, "required": ["profile_id_or_url"]}, - }, - { - "name": "get_crawl_status", - "description": "Return the latest crawl run status.", - "inputSchema": {"type": "object", "properties": {}}, - }, - { - "name": "get_crawl_run_details", - "description": "Return detailed employee changes and errors for one crawl run.", - "inputSchema": { - "type": "object", - "properties": {"run_id": {"type": "integer"}}, - "required": ["run_id"], - }, - }, -] - - -@router.post("") -async def mcp_http( - request: Request, - db: Session = Depends(get_db), -) -> dict: - payload = await request.json() - method = payload.get("method") - request_id = payload.get("id") - params = payload.get("params") or {} - - try: - if method == "initialize": - result = { - "protocolVersion": PROTOCOL_VERSION, - "serverInfo": {"name": SERVICE_NAME, "version": BACKEND_VERSION}, - "capabilities": {"tools": {}}, - } - elif method == "tools/list": - result = {"tools": TOOLS} - elif method == "tools/call": - result = _call_tool(db, params.get("name"), params.get("arguments") or {}) - else: - return {"jsonrpc": "2.0", "id": request_id, "error": {"code": -32601, "message": "Method not found"}} - return {"jsonrpc": "2.0", "id": request_id, "result": result} - except Exception as exc: - return {"jsonrpc": "2.0", "id": request_id, "error": {"code": -32000, "message": str(exc)}} - - -def _call_tool(db: Session, name: str, arguments: dict) -> dict: - if name == "get_service_info": - return _tool_response( - service_info_payload( - db, - tools=TOOLS, - service_name=SERVICE_NAME, - backend_version=BACKEND_VERSION, - protocol_version=PROTOCOL_VERSION, - ) - ) - if name == "sync_employees": - return _tool_response( - sync_employees_payload( - db, - client_hash=arguments.get("client_hash"), - include_data=bool(arguments.get("include_data", True)), - ) - ) - if name == "search_employees": - return _tool_response(_search_employees(db, arguments)) - if name == "get_employee": - employee = _find_employee(db, arguments["profile_id_or_url"]) - return _tool_response(_employee_payload(employee) if employee else {"error": "not_found"}) - if name == "list_employee_publications": - employee = _find_employee(db, arguments["profile_id_or_url"]) - return _tool_response(_collect_section_items(employee, "publications")) - if name == "list_employee_courses": - employee = _find_employee(db, arguments["profile_id_or_url"]) - return _tool_response(_collect_section_items(employee, "courses_by_year")) - if name == "get_crawl_status": - run = db.scalar(select(CrawlRun).order_by(desc(CrawlRun.started_at)).limit(1)) - return _tool_response(_run_payload(run) if run else {"status": "never_run"}) - if name == "get_crawl_run_details": - run = db.get(CrawlRun, int(arguments["run_id"])) - return _tool_response(run_detail_payload(db, run) if run else {"error": "not_found"}) - raise ValueError(f"Unknown tool: {name}") - - -def _search_employees(db: Session, arguments: dict) -> list[dict]: - query = arguments.get("query", "") - limit = min(int(arguments.get("limit") or 20), 100) - stmt = select(Employee) - if arguments.get("status"): - stmt = stmt.where(Employee.status == arguments["status"]) - if query: - pattern = f"%{query}%" - stmt = stmt.where(or_(Employee.full_name.ilike(pattern), Employee.canonical_url.ilike(pattern))) - employees = db.scalars(stmt.order_by(Employee.full_name).limit(limit)).all() - return [_employee_payload(employee, include_data=False) for employee in employees] - - -def _find_employee(db: Session, value: str) -> Employee | None: - pattern = value.strip() - stmt = select(Employee).where( - or_( - Employee.profile_key == pattern, - Employee.profile_id == pattern, - Employee.canonical_url == pattern, - Employee.canonical_url.ilike(f"%{pattern}%"), - ) - ) - return db.scalar(stmt.limit(1)) - - -def _collect_section_items(employee: Employee | None, section_type: str) -> dict: - if not employee: - return {"items": []} - if section_type == "publications": - publications = _stored_publications(employee) - if publications: - return {"employee": _employee_payload(employee, include_data=False), "items": publications} - if not employee.current_data: - return {"employee": _employee_payload(employee, include_data=False), "items": []} - items = [] - for section in employee.current_data.get("sections") or []: - if section.get("type") != section_type: - continue - if section_type == "publications": - items.extend(section.get("publications") or []) - elif section_type == "courses_by_year": - items.extend(section.get("courses") or []) - return {"employee": _employee_payload(employee, include_data=False), "items": items} - - -def _stored_publications(employee: Employee) -> list[dict]: - return [_publication_payload(publication) for publication in sorted(employee.publications, key=_publication_sort_key)] - - -def _publication_sort_key(publication: EmployeePublication) -> tuple: - return (publication.year or 0, publication.title or "", publication.id) - - -def _publication_payload(publication: EmployeePublication) -> dict: - text = publication.citation_text or publication.title - payload = { - "id": publication.publication_id, - "publication_id": publication.publication_id, - "title": publication.title, - "text": text, - "url": publication.url, - } - optional = { - "year": publication.year, - "type": publication.publication_type, - "publication_type": publication.publication_type, - "language": publication.language, - "status": publication.status, - "doi_url": publication.doi_url, - "other_url": publication.other_url, - "document_url": publication.document_url, - "citation_text": publication.citation_text, - "annotation": publication.annotation, - "description": publication.description, - "authors": publication.authors, - } - payload.update({key: value for key, value in optional.items() if value not in (None, [], {})}) - return payload - - -def _employee_payload(employee: Employee, include_data: bool = True) -> dict: - payload = { - "profile_key": employee.profile_key, - "profile_id": employee.profile_id, - "full_name": employee.full_name, - "status": employee.status, - "canonical_url": employee.canonical_url, - "last_seen_at": employee.last_seen_at.isoformat() if employee.last_seen_at else None, - "dismissed_at": employee.dismissed_at.isoformat() if employee.dismissed_at else None, - } - if include_data: - payload["data"] = employee.current_data - return payload - - -def _run_payload(run: CrawlRun) -> dict: - return { - "id": run.id, - "status": run.status, - "source_url": run.source_url, - "started_at": run.started_at.isoformat() if run.started_at else None, - "finished_at": run.finished_at.isoformat() if run.finished_at else None, - "found_count": run.found_count, - "parsed_count": run.parsed_count, - "skipped_count": run.skipped_count, - "error_count": run.error_count, - "dismissed_count": run.dismissed_count, - } - - -def _tool_response(data: object) -> dict: - return {"content": [{"type": "text", "text": json.dumps(data, ensure_ascii=False, default=str)}]} diff --git a/app/version.py b/app/version.py index 7e653b7..4732bd6 100644 --- a/app/version.py +++ b/app/version.py @@ -1,3 +1,3 @@ -APP_VERSION = "0.7.6" -FRONTEND_VERSION = "0.7.6" -BACKEND_VERSION = "0.7.6" +APP_VERSION = "0.7.7" +FRONTEND_VERSION = "0.7.7" +BACKEND_VERSION = "0.7.7" diff --git a/pyproject.toml b/pyproject.toml index ad1f45b..749af30 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "miem-workers" -version = "0.7.6" -description = "MIEM employees parser, admin API, and MCP server" +version = "0.7.7" +description = "MIEM employees parser, admin API, and web admin" requires-python = ">=3.11" dependencies = [ "apscheduler>=3.10.4", diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..84b79a0 --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,126 @@ +from datetime import datetime, timezone +from types import SimpleNamespace + +from fastapi.testclient import TestClient +from sqlalchemy import create_engine, select +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import StaticPool + +from app.config import Settings, get_settings +from app.db import Base, get_db +from app.main import app +from app.models import CrawlRun, CrawlRunEmployeeChange, Employee +from app.security import SESSION_COOKIE, sign_session + + +def test_health_returns_versions(): + response = TestClient(app).get("/api/health") + + assert response.status_code == 200 + assert response.json()["backend_version"] == "0.7.7" + + +def test_api_employees_and_stats_require_admin_session(): + engine = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False}, poolclass=StaticPool) + Base.metadata.create_all(engine) + session_factory = sessionmaker(bind=engine) + db = session_factory() + employee = Employee( + profile_key="staff:alpha", + profile_type="staff", + profile_id="alpha", + canonical_url="https://www.hse.ru/staff/alpha", + full_name="Alpha Person", + status="active", + first_seen_at=datetime.now(timezone.utc), + last_seen_at=datetime.now(timezone.utc), + current_data={"contacts": {"emails": ["alpha@hse.ru"]}, "sections": []}, + ) + run = CrawlRun(source_url="https://miem.hse.ru/persons", status="completed", new_count=1) + db.add_all([employee, run]) + db.commit() + db.add(CrawlRunEmployeeChange( + crawl_run_id=run.id, + employee_id=employee.id, + profile_key=employee.profile_key, + profile_url=employee.canonical_url, + full_name=employee.full_name, + change_type="new", + profile_available=True, + message="added", + )) + db.commit() + run_id = run.id + db.close() + + settings = Settings(admin_username="admin", admin_password="password", session_secret="session-secret") + + def override_db(): + session = session_factory() + try: + yield session + finally: + session.close() + + app.dependency_overrides[get_db] = override_db + app.dependency_overrides[get_settings] = lambda: settings + client = TestClient(app) + client.cookies.set(SESSION_COOKIE, sign_session("admin", settings)) + + employees = client.get("/api/employees", params={"q": "Alpha", "has_email": True}) + stats = client.get("/api/stats") + run_details = client.get(f"/api/crawl-runs/{run_id}") + + assert employees.status_code == 200 + assert employees.json()["total"] == 1 + assert stats.status_code == 200 + assert stats.json()["new_in_last_run"] == 1 + assert run_details.status_code == 200 + assert run_details.json()["changes"]["new"][0]["full_name"] == "Alpha Person" + app.dependency_overrides.clear() + + +def test_admin_refresh_employee_route_updates_only_requested_employee(monkeypatch): + engine = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False}, poolclass=StaticPool) + Base.metadata.create_all(engine) + session_factory = sessionmaker(bind=engine) + db = session_factory() + db.add(Employee( + profile_key="org_person:133709486", + profile_type="org_person", + profile_id="133709486", + canonical_url="https://www.hse.ru/org/persons/133709486", + full_name="Будков Юрий Алексеевич", + status="active", + )) + db.commit() + employee_id = db.scalar(select(Employee.id)) + db.close() + + settings = Settings(admin_username="admin", admin_password="password", session_secret="session-secret") + + def override_db(): + session = session_factory() + try: + yield session + finally: + session.close() + + calls = [] + + def fake_refresh_employee(db, refreshed_employee, route_settings): + calls.append((refreshed_employee.id, route_settings)) + return SimpleNamespace(status="completed") + + app.dependency_overrides[get_db] = override_db + app.dependency_overrides[get_settings] = lambda: settings + monkeypatch.setattr("app.admin.refresh_employee", fake_refresh_employee) + client = TestClient(app) + client.cookies.set(SESSION_COOKIE, sign_session("admin", settings)) + + response = client.post(f"/admin/employees/{employee_id}/refresh", follow_redirects=False) + + assert response.status_code == 303 + assert response.headers["location"] == f"/admin/employees/{employee_id}?refresh_status=success" + assert calls == [(employee_id, settings)] + app.dependency_overrides.clear() diff --git a/tests/test_api_mcp.py b/tests/test_api_mcp.py deleted file mode 100644 index 1bff029..0000000 --- a/tests/test_api_mcp.py +++ /dev/null @@ -1,532 +0,0 @@ -import json -from datetime import datetime, timezone -from types import SimpleNamespace - -from fastapi.testclient import TestClient -from sqlalchemy import create_engine, select -from sqlalchemy.orm import sessionmaker -from sqlalchemy.pool import StaticPool - -from app.config import Settings, get_settings -from app.db import Base, get_db -from app.main import app -from app.models import CrawlRun, CrawlRunEmployeeChange, Employee, EmployeePublication -from app.security import SESSION_COOKIE, sign_session - - -def test_health_returns_versions(): - client = TestClient(app) - - response = client.get("/api/health") - - assert response.status_code == 200 - assert response.json()["backend_version"] == "0.7.3" - - -def test_mcp_lists_tools_without_auth_and_ignores_auth_header(): - engine = create_engine( - "sqlite:///:memory:", - connect_args={"check_same_thread": False}, - poolclass=StaticPool, - ) - Base.metadata.create_all(engine) - Session = sessionmaker(bind=engine) - - def override_db(): - session = Session() - try: - yield session - finally: - session.close() - - app.dependency_overrides[get_db] = override_db - client = TestClient(app) - - without_auth = client.post("/mcp", json={"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}}) - with_auth = client.post( - "/mcp", - headers={"Authorization": "Bearer anything"}, - json={"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}}, - ) - - assert without_auth.status_code == 200 - assert with_auth.status_code == 200 - tool_names = {tool["name"] for tool in without_auth.json()["result"]["tools"]} - assert "search_employees" in tool_names - assert "get_service_info" in tool_names - assert "sync_employees" in tool_names - assert any(tool["name"] == "get_crawl_run_details" for tool in without_auth.json()["result"]["tools"]) - assert with_auth.json()["result"]["tools"] == without_auth.json()["result"]["tools"] - - app.dependency_overrides.clear() - - -def test_mcp_search_employees_returns_matching_employee(): - engine = create_engine( - "sqlite:///:memory:", - connect_args={"check_same_thread": False}, - poolclass=StaticPool, - ) - Base.metadata.create_all(engine) - Session = sessionmaker(bind=engine) - session = Session() - session.add( - Employee( - profile_key="staff:avsergeev", - profile_type="staff", - profile_id="avsergeev", - canonical_url="https://www.hse.ru/staff/avsergeev", - full_name="Сергеев Алексей Викторович", - status="active", - first_seen_at=datetime.now(timezone.utc), - last_seen_at=datetime.now(timezone.utc), - current_data={"sections": []}, - ) - ) - session.commit() - session.close() - - def override_db(): - db = Session() - try: - yield db - finally: - db.close() - - app.dependency_overrides[get_db] = override_db - client = TestClient(app) - - response = client.post( - "/mcp", - json={ - "jsonrpc": "2.0", - "id": 1, - "method": "tools/call", - "params": {"name": "search_employees", "arguments": {"query": "Сергеев"}}, - }, - ) - - assert response.status_code == 200 - assert "Сергеев Алексей Викторович" in response.json()["result"]["content"][0]["text"] - - app.dependency_overrides.clear() - - -def test_mcp_service_info_returns_tools_and_dataset_hash(): - engine = create_engine( - "sqlite:///:memory:", - connect_args={"check_same_thread": False}, - poolclass=StaticPool, - ) - Base.metadata.create_all(engine) - Session = sessionmaker(bind=engine) - session = Session() - session.add( - Employee( - profile_key="staff:alpha", - profile_type="staff", - profile_id="alpha", - canonical_url="https://www.hse.ru/staff/alpha", - full_name="Alpha Person", - status="active", - current_checksum="a" * 64, - current_data={"sections": []}, - ) - ) - session.commit() - session.close() - - def override_db(): - db = Session() - try: - yield db - finally: - db.close() - - app.dependency_overrides[get_db] = override_db - client = TestClient(app) - - response = client.post( - "/mcp", - json={"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": "get_service_info", "arguments": {}}}, - ) - - assert response.status_code == 200 - payload = json.loads(response.json()["result"]["content"][0]["text"]) - assert payload["service_name"] == "miem-employees" - assert payload["backend_version"] == "0.7.3" - assert payload["dataset"]["hash"] - assert any(tool["name"] == "sync_employees" for tool in payload["tools"]) - - app.dependency_overrides.clear() - - -def test_mcp_list_employee_publications_prefers_stored_publications_with_fallback(): - engine = create_engine( - "sqlite:///:memory:", - connect_args={"check_same_thread": False}, - poolclass=StaticPool, - ) - Base.metadata.create_all(engine) - Session = sessionmaker(bind=engine) - session = Session() - stored_employee = Employee( - profile_key="staff:stored", - profile_type="staff", - profile_id="stored", - canonical_url="https://www.hse.ru/staff/stored", - full_name="Stored Person", - status="active", - current_data={ - "sections": [ - { - "type": "publications", - "publications": [{"title": "Old JSON Publication", "url": "https://example.test/old"}], - } - ] - }, - ) - fallback_employee = Employee( - profile_key="staff:fallback", - profile_type="staff", - profile_id="fallback", - canonical_url="https://www.hse.ru/staff/fallback", - full_name="Fallback Person", - status="active", - current_data={ - "sections": [ - { - "type": "publications", - "publications": [{"title": "Fallback Publication", "url": "https://example.test/fallback"}], - } - ] - }, - ) - session.add_all([stored_employee, fallback_employee]) - session.commit() - session.add( - EmployeePublication( - employee_id=stored_employee.id, - publication_id="pub-1", - title="Stored Publication", - year=2024, - publication_type="ARTICLE", - url="https://publications.hse.ru/view/pub-1", - doi_url="https://doi.org/10.1/test", - citation_text="Stored Citation", - annotation={"ru": "Аннотация", "en": "Abstract"}, - description={"main": "Stored Citation"}, - authors=[{"id": "1", "title_ru": "Автор", "is_current_employee": True}], - source_hash="a" * 64, - ) - ) - session.commit() - session.close() - - def override_db(): - db = Session() - try: - yield db - finally: - db.close() - - app.dependency_overrides[get_db] = override_db - client = TestClient(app) - - stored_response = client.post( - "/mcp", - json={ - "jsonrpc": "2.0", - "id": 1, - "method": "tools/call", - "params": {"name": "list_employee_publications", "arguments": {"profile_id_or_url": "stored"}}, - }, - ) - fallback_response = client.post( - "/mcp", - json={ - "jsonrpc": "2.0", - "id": 2, - "method": "tools/call", - "params": {"name": "list_employee_publications", "arguments": {"profile_id_or_url": "fallback"}}, - }, - ) - - stored_payload = json.loads(stored_response.json()["result"]["content"][0]["text"]) - fallback_payload = json.loads(fallback_response.json()["result"]["content"][0]["text"]) - assert stored_payload["items"][0]["title"] == "Stored Publication" - assert stored_payload["items"][0]["doi_url"] == "https://doi.org/10.1/test" - assert stored_payload["items"][0]["annotation"] == {"ru": "Аннотация", "en": "Abstract"} - assert stored_payload["items"][0]["authors"] == [{"id": "1", "title_ru": "Автор", "is_current_employee": True}] - assert fallback_payload["items"][0]["title"] == "Fallback Publication" - - app.dependency_overrides.clear() - - -def test_mcp_sync_employees_full_empty_and_unknown_hash_modes(): - engine = create_engine( - "sqlite:///:memory:", - connect_args={"check_same_thread": False}, - poolclass=StaticPool, - ) - Base.metadata.create_all(engine) - Session = sessionmaker(bind=engine) - session = Session() - session.add( - Employee( - profile_key="staff:alpha", - profile_type="staff", - profile_id="alpha", - canonical_url="https://www.hse.ru/staff/alpha", - full_name="Alpha Person", - status="active", - current_checksum="a" * 64, - current_data={"sections": [{"type": "paragraphs"}]}, - ) - ) - session.commit() - session.close() - - def override_db(): - db = Session() - try: - yield db - finally: - db.close() - - app.dependency_overrides[get_db] = override_db - client = TestClient(app) - - full_response = client.post( - "/mcp", - json={"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": "sync_employees", "arguments": {}}}, - ) - full_payload = json.loads(full_response.json()["result"]["content"][0]["text"]) - current_hash = full_payload["to_hash"] - - empty_response = client.post( - "/mcp", - json={ - "jsonrpc": "2.0", - "id": 2, - "method": "tools/call", - "params": {"name": "sync_employees", "arguments": {"client_hash": current_hash}}, - }, - ) - empty_payload = json.loads(empty_response.json()["result"]["content"][0]["text"]) - - unknown_response = client.post( - "/mcp", - json={ - "jsonrpc": "2.0", - "id": 3, - "method": "tools/call", - "params": {"name": "sync_employees", "arguments": {"client_hash": "missing"}}, - }, - ) - unknown_payload = json.loads(unknown_response.json()["result"]["content"][0]["text"]) - - assert full_payload["mode"] == "full" - assert full_payload["items"][0]["data"] == {"sections": [{"type": "paragraphs"}]} - assert empty_payload["mode"] == "delta" - assert empty_payload["changes"] == {"added": [], "updated": [], "dismissed": [], "removed": []} - assert unknown_payload["mode"] == "full" - assert unknown_payload["reason"] == "unknown_client_hash" - - app.dependency_overrides.clear() - - -def test_mcp_get_crawl_run_details_returns_changes(): - engine = create_engine( - "sqlite:///:memory:", - connect_args={"check_same_thread": False}, - poolclass=StaticPool, - ) - Base.metadata.create_all(engine) - Session = sessionmaker(bind=engine) - session = Session() - run = CrawlRun(source_url="https://miem.hse.ru/persons", status="completed", new_count=1) - employee = Employee( - profile_key="staff:new", - profile_type="staff", - profile_id="new", - canonical_url="https://www.hse.ru/staff/new", - full_name="New Person", - status="active", - first_seen_at=datetime.now(timezone.utc), - last_seen_at=datetime.now(timezone.utc), - ) - session.add_all([run, employee]) - session.commit() - session.add( - CrawlRunEmployeeChange( - crawl_run_id=run.id, - employee_id=employee.id, - profile_key=employee.profile_key, - profile_url=employee.canonical_url, - full_name=employee.full_name, - change_type="new", - profile_available=True, - message="added", - ) - ) - session.commit() - run_id = run.id - session.close() - - def override_db(): - db = Session() - try: - yield db - finally: - db.close() - - app.dependency_overrides[get_db] = override_db - client = TestClient(app) - - response = client.post( - "/mcp", - json={ - "jsonrpc": "2.0", - "id": 1, - "method": "tools/call", - "params": {"name": "get_crawl_run_details", "arguments": {"run_id": run_id}}, - }, - ) - - assert response.status_code == 200 - text = response.json()["result"]["content"][0]["text"] - assert "New Person" in text - assert "changes_detail_available" in text - - app.dependency_overrides.clear() - - -def test_mcp_protected_resource_metadata_route_is_removed(): - client = TestClient(app) - - response = client.get("/.well-known/oauth-protected-resource") - - assert response.status_code == 404 - - -def test_api_employees_and_stats_require_admin_session(): - engine = create_engine( - "sqlite:///:memory:", - connect_args={"check_same_thread": False}, - poolclass=StaticPool, - ) - Base.metadata.create_all(engine) - Session = sessionmaker(bind=engine) - db = Session() - db.add( - Employee( - profile_key="staff:alpha", - profile_type="staff", - profile_id="alpha", - canonical_url="https://www.hse.ru/staff/alpha", - full_name="Alpha Person", - status="active", - first_seen_at=datetime.now(timezone.utc), - last_seen_at=datetime.now(timezone.utc), - current_data={"contacts": {"emails": ["alpha@hse.ru"]}, "sections": []}, - ) - ) - run = CrawlRun(source_url="https://miem.hse.ru/persons", status="completed", new_count=1) - db.add(run) - db.commit() - db.add( - CrawlRunEmployeeChange( - crawl_run_id=run.id, - employee_id=1, - profile_key="staff:alpha", - profile_url="https://www.hse.ru/staff/alpha", - full_name="Alpha Person", - change_type="new", - profile_available=True, - message="added", - ) - ) - db.commit() - run_id = run.id - db.close() - - settings = Settings(admin_username="admin", admin_password="password", session_secret="session-secret") - - def override_db(): - session = Session() - try: - yield session - finally: - session.close() - - app.dependency_overrides[get_db] = override_db - app.dependency_overrides[get_settings] = lambda: settings - client = TestClient(app) - client.cookies.set(SESSION_COOKIE, sign_session("admin", settings)) - - employees = client.get("/api/employees", params={"q": "Alpha", "has_email": True}) - stats = client.get("/api/stats") - run_details = client.get(f"/api/crawl-runs/{run_id}") - - assert employees.status_code == 200 - assert employees.json()["total"] == 1 - assert stats.status_code == 200 - assert stats.json()["new_in_last_run"] == 1 - assert run_details.status_code == 200 - assert run_details.json()["changes"]["new"][0]["full_name"] == "Alpha Person" - - app.dependency_overrides.clear() - - -def test_admin_refresh_employee_route_updates_only_requested_employee(monkeypatch): - engine = create_engine( - "sqlite:///:memory:", - connect_args={"check_same_thread": False}, - poolclass=StaticPool, - ) - Base.metadata.create_all(engine) - Session = sessionmaker(bind=engine) - db = Session() - db.add( - Employee( - profile_key="org_person:133709486", - profile_type="org_person", - profile_id="133709486", - canonical_url="https://www.hse.ru/org/persons/133709486", - full_name="Будков Юрий Алексеевич", - status="active", - ) - ) - db.commit() - employee_id = db.scalar(select(Employee.id)) - db.close() - - settings = Settings(admin_username="admin", admin_password="password", session_secret="session-secret") - - def override_db(): - session = Session() - try: - yield session - finally: - session.close() - - calls = [] - - def fake_refresh_employee(db, refreshed_employee, route_settings): - calls.append((refreshed_employee.id, route_settings)) - return SimpleNamespace(status="completed") - - app.dependency_overrides[get_db] = override_db - app.dependency_overrides[get_settings] = lambda: settings - monkeypatch.setattr("app.admin.refresh_employee", fake_refresh_employee) - client = TestClient(app) - client.cookies.set(SESSION_COOKIE, sign_session("admin", settings)) - - response = client.post(f"/admin/employees/{employee_id}/refresh", follow_redirects=False) - - assert response.status_code == 303 - assert response.headers["location"] == f"/admin/employees/{employee_id}?refresh_status=success" - assert calls == [(employee_id, settings)] - - app.dependency_overrides.clear() From e142138c1b5c227d4fa3328ae708476d09b7e17e Mon Sep 17 00:00:00 2001 From: admin Date: Thu, 27 Aug 2026 17:08:33 +0300 Subject: [PATCH 2/2] refactor: remove dataset sync leftovers --- README.md | 2 +- app/models.py | 40 ------ app/services/crawler.py | 4 - app/services/dataset_versions.py | 227 ------------------------------- tests/test_api.py | 4 + tests/test_dataset_versions.py | 88 ------------ 6 files changed, 5 insertions(+), 360 deletions(-) delete mode 100644 app/services/dataset_versions.py delete mode 100644 tests/test_dataset_versions.py diff --git a/README.md b/README.md index d9ac005..f55c69f 100644 --- a/README.md +++ b/README.md @@ -116,4 +116,4 @@ docker compose exec postgres pg_dump -U miem miem_workers > backup.sql docker compose down ``` -Версия сервиса: `0.7.6`. Админка всегда показывает версии backend и frontend в footer. +Версия сервиса: `0.7.7`. Админка всегда показывает версии backend и frontend в footer. diff --git a/app/models.py b/app/models.py index 2772f91..df4a67a 100644 --- a/app/models.py +++ b/app/models.py @@ -162,7 +162,6 @@ class CrawlRun(Base): message: Mapped[str | None] = mapped_column(Text) employee_changes: Mapped[list["CrawlRunEmployeeChange"]] = relationship(back_populates="crawl_run") - dataset_versions: Mapped[list["DatasetVersion"]] = relationship(back_populates="crawl_run") class CrawlRunEmployeeChange(Base): @@ -242,42 +241,3 @@ class ParseResourceCache(Base): body_snapshot: Mapped[bytes] = mapped_column(LargeBinary, nullable=False) parser_version: Mapped[str | None] = mapped_column(String(32)) fetched_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False) - - -class DatasetVersion(Base): - __tablename__ = "dataset_versions" - __table_args__ = ( - UniqueConstraint("hash", name="uq_dataset_versions_hash"), - Index("ix_dataset_versions_created_at", "created_at"), - ) - - id: Mapped[int] = mapped_column(Integer, primary_key=True) - hash: Mapped[str] = mapped_column(String(64), nullable=False) - previous_hash: Mapped[str | None] = mapped_column(String(64)) - crawl_run_id: Mapped[int | None] = mapped_column(ForeignKey("crawl_runs.id")) - employee_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False) - active_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False) - dismissed_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False) - created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False) - - crawl_run: Mapped[CrawlRun | None] = relationship(back_populates="dataset_versions") - items: Mapped[list["DatasetVersionItem"]] = relationship(back_populates="dataset_version", cascade="all, delete-orphan") - - -class DatasetVersionItem(Base): - __tablename__ = "dataset_version_items" - __table_args__ = ( - UniqueConstraint("dataset_version_id", "profile_key", name="uq_dataset_version_items_version_profile"), - Index("ix_dataset_version_items_hash", "dataset_version_id"), - Index("ix_dataset_version_items_profile_key", "profile_key"), - ) - - id: Mapped[int] = mapped_column(Integer, primary_key=True) - dataset_version_id: Mapped[int] = mapped_column(ForeignKey("dataset_versions.id"), nullable=False) - profile_key: Mapped[str] = mapped_column(String(255), nullable=False) - employee_id: Mapped[int | None] = mapped_column(ForeignKey("employees.id")) - status: Mapped[str] = mapped_column(String(32), nullable=False) - checksum: Mapped[str] = mapped_column(String(64), nullable=False) - - dataset_version: Mapped[DatasetVersion] = relationship(back_populates="items") - employee: Mapped[Employee | None] = relationship() diff --git a/app/services/crawler.py b/app/services/crawler.py index e458cb7..40074c1 100644 --- a/app/services/crawler.py +++ b/app/services/crawler.py @@ -26,7 +26,6 @@ from app.parser.collector import collect_profile_links from app.parser.profile import parse_person_profile from app.parser.profile_url import profile_key from app.services.academic_degrees import academic_degrees -from app.services.dataset_versions import get_or_create_current_version from app.services.resource_cache import ResourceCache HEADERS = { @@ -102,7 +101,6 @@ def run_crawl(db: Session, settings: Settings) -> CrawlRun: max_auto_dismissals=settings.max_auto_dismissals_per_run, ) run.status = "completed" - get_or_create_current_version(db, crawl_run_id=run.id) except Exception as exc: run.status = "failed" run.message = str(exc) @@ -146,7 +144,6 @@ def refresh_dismissed_status(db: Session, settings: Settings) -> CrawlRun: ) run.parsed_count += 1 run.status = "completed" - get_or_create_current_version(db, crawl_run_id=run.id) except Exception as exc: run.status = "failed" run.error_count = 1 @@ -194,7 +191,6 @@ def refresh_employee(db: Session, employee: Employee, settings: Settings) -> Cra else: run.skipped_count = 1 run.status = "completed" - get_or_create_current_version(db, crawl_run_id=run.id) except Exception as exc: run.status = "failed" run.error_count = 1 diff --git a/app/services/dataset_versions.py b/app/services/dataset_versions.py deleted file mode 100644 index 74bc59f..0000000 --- a/app/services/dataset_versions.py +++ /dev/null @@ -1,227 +0,0 @@ -import hashlib -import json -from dataclasses import dataclass - -from sqlalchemy import desc, select -from sqlalchemy.orm import Session - -from app.models import DatasetVersion, DatasetVersionItem, Employee - - -@dataclass(frozen=True) -class EmployeeMarker: - profile_key: str - employee_id: int | None - status: str - checksum: str - - -def get_or_create_current_version(db: Session, *, crawl_run_id: int | None = None) -> DatasetVersion: - employees = db.scalars(select(Employee).order_by(Employee.profile_key)).all() - markers = [_employee_marker(employee) for employee in employees] - dataset_hash = _dataset_hash(markers) - latest = get_latest_version(db) - if latest and latest.hash == dataset_hash: - return latest - - active_count = sum(1 for marker in markers if marker.status == "active") - dismissed_count = sum(1 for marker in markers if marker.status == "dismissed") - version = DatasetVersion( - hash=dataset_hash, - previous_hash=latest.hash if latest else None, - crawl_run_id=crawl_run_id, - employee_count=len(markers), - active_count=active_count, - dismissed_count=dismissed_count, - ) - db.add(version) - db.flush() - for marker in markers: - db.add( - DatasetVersionItem( - dataset_version_id=version.id, - profile_key=marker.profile_key, - employee_id=marker.employee_id, - status=marker.status, - checksum=marker.checksum, - ) - ) - db.flush() - return version - - -def get_latest_version(db: Session) -> DatasetVersion | None: - return db.scalar(select(DatasetVersion).order_by(desc(DatasetVersion.created_at), desc(DatasetVersion.id)).limit(1)) - - -def get_version_by_hash(db: Session, dataset_hash: str | None) -> DatasetVersion | None: - if not dataset_hash: - return None - return db.scalar(select(DatasetVersion).where(DatasetVersion.hash == dataset_hash).limit(1)) - - -def service_info_payload(db: Session, *, tools: list[dict], service_name: str, backend_version: str, protocol_version: str) -> dict: - version = get_or_create_current_version(db) - db.commit() - return { - "service_name": service_name, - "backend_version": backend_version, - "protocolVersion": protocol_version, - "tools": tools, - "dataset": _version_payload(version), - } - - -def sync_employees_payload(db: Session, *, client_hash: str | None = None, include_data: bool = True) -> dict: - current = get_or_create_current_version(db) - db.commit() - if not client_hash: - return _full_sync_payload(db, current, include_data=include_data, reason=None) - if client_hash == current.hash: - return { - "mode": "delta", - "from_hash": client_hash, - "to_hash": current.hash, - "dataset": _version_payload(current), - "changes": {"added": [], "updated": [], "dismissed": [], "removed": []}, - } - - previous = get_version_by_hash(db, client_hash) - if not previous: - return _full_sync_payload(db, current, include_data=include_data, reason="unknown_client_hash", from_hash=client_hash) - - return _delta_sync_payload(db, previous, current, include_data=include_data) - - -def _full_sync_payload( - db: Session, - current: DatasetVersion, - *, - include_data: bool, - reason: str | None, - from_hash: str | None = None, -) -> dict: - employees = db.scalars(select(Employee).order_by(Employee.profile_key)).all() - payload = { - "mode": "full", - "from_hash": from_hash, - "to_hash": current.hash, - "dataset": _version_payload(current), - "items": [_employee_payload(employee, include_data=include_data) for employee in employees], - } - if reason: - payload["reason"] = reason - return payload - - -def _delta_sync_payload(db: Session, previous: DatasetVersion, current: DatasetVersion, *, include_data: bool) -> dict: - previous_items = _items_by_profile_key(previous) - current_items = _items_by_profile_key(current) - employees = {employee.profile_key: employee for employee in db.scalars(select(Employee)).all()} - added = [] - updated = [] - dismissed = [] - removed = [] - - for profile_key, current_item in sorted(current_items.items()): - previous_item = previous_items.get(profile_key) - employee = employees.get(profile_key) - if not previous_item: - if employee: - added.append(_employee_payload(employee, include_data=include_data)) - continue - if previous_item.checksum == current_item.checksum and previous_item.status == current_item.status: - continue - if current_item.status == "dismissed": - dismissed.append(_tombstone(profile_key, current_item.status, employee)) - elif employee: - updated.append(_employee_payload(employee, include_data=include_data)) - - for profile_key, previous_item in sorted(previous_items.items()): - if profile_key not in current_items: - removed.append(_tombstone(profile_key, "removed", employees.get(profile_key), checksum=previous_item.checksum)) - - return { - "mode": "delta", - "from_hash": previous.hash, - "to_hash": current.hash, - "dataset": _version_payload(current), - "changes": { - "added": added, - "updated": updated, - "dismissed": dismissed, - "removed": removed, - }, - } - - -def _items_by_profile_key(version: DatasetVersion) -> dict[str, DatasetVersionItem]: - return {item.profile_key: item for item in version.items} - - -def _version_payload(version: DatasetVersion) -> dict: - return { - "hash": version.hash, - "previous_hash": version.previous_hash, - "created_at": version.created_at.isoformat() if version.created_at else None, - "crawl_run_id": version.crawl_run_id, - "employee_count": version.employee_count, - "active_count": version.active_count, - "dismissed_count": version.dismissed_count, - } - - -def _employee_marker(employee: Employee) -> EmployeeMarker: - return EmployeeMarker( - profile_key=employee.profile_key, - employee_id=employee.id, - status=employee.status, - checksum=employee.current_checksum or _payload_hash(employee.current_data or {}), - ) - - -def _dataset_hash(markers: list[EmployeeMarker]) -> str: - payload = [ - {"profile_key": marker.profile_key, "status": marker.status, "checksum": marker.checksum} - for marker in sorted(markers, key=lambda item: item.profile_key) - ] - return _payload_hash(payload) - - -def _payload_hash(value: object) -> str: - payload = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str) - return hashlib.sha256(payload.encode("utf-8")).hexdigest() - - -def _employee_payload(employee: Employee, *, include_data: bool) -> dict: - payload = { - "profile_key": employee.profile_key, - "profile_id": employee.profile_id, - "full_name": employee.full_name, - "status": employee.status, - "canonical_url": employee.canonical_url, - "last_seen_at": employee.last_seen_at.isoformat() if employee.last_seen_at else None, - "dismissed_at": employee.dismissed_at.isoformat() if employee.dismissed_at else None, - "checksum": employee.current_checksum or _payload_hash(employee.current_data or {}), - } - if include_data: - payload["data"] = employee.current_data - return payload - - -def _tombstone(profile_key: str, status: str, employee: Employee | None, *, checksum: str | None = None) -> dict: - payload = { - "profile_key": profile_key, - "status": status, - "checksum": checksum or (employee.current_checksum if employee else None), - } - if employee: - payload.update( - { - "profile_id": employee.profile_id, - "full_name": employee.full_name, - "canonical_url": employee.canonical_url, - "dismissed_at": employee.dismissed_at.isoformat() if employee.dismissed_at else None, - } - ) - return payload diff --git a/tests/test_api.py b/tests/test_api.py index 84b79a0..8465eff 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -20,6 +20,10 @@ def test_health_returns_versions(): assert response.json()["backend_version"] == "0.7.7" +def test_mcp_endpoint_is_removed(): + assert TestClient(app).get("/mcp").status_code == 404 + + def test_api_employees_and_stats_require_admin_session(): engine = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False}, poolclass=StaticPool) Base.metadata.create_all(engine) diff --git a/tests/test_dataset_versions.py b/tests/test_dataset_versions.py deleted file mode 100644 index f927795..0000000 --- a/tests/test_dataset_versions.py +++ /dev/null @@ -1,88 +0,0 @@ -from datetime import datetime, timezone - -from app.models import Employee -from app.services.dataset_versions import get_or_create_current_version, sync_employees_payload - - -def _employee(profile_key: str, checksum: str, *, status: str = "active") -> Employee: - return Employee( - profile_key=profile_key, - profile_type=profile_key.split(":", 1)[0], - profile_id=profile_key.split(":", 1)[1], - canonical_url=f"https://www.hse.ru/{profile_key}", - full_name=profile_key, - status=status, - first_seen_at=datetime.now(timezone.utc), - last_seen_at=datetime.now(timezone.utc), - current_data={"profile_key": profile_key}, - current_checksum=checksum, - ) - - -def test_dataset_version_hash_is_stable_for_same_employee_state(db_session): - db_session.add(_employee("staff:alpha", "a" * 64)) - db_session.commit() - - first = get_or_create_current_version(db_session) - db_session.commit() - second = get_or_create_current_version(db_session) - - assert second.id == first.id - assert second.hash == first.hash - assert second.employee_count == 1 - - -def test_dataset_version_hash_changes_when_employee_checksum_changes(db_session): - employee = _employee("staff:alpha", "a" * 64) - db_session.add(employee) - db_session.commit() - first = get_or_create_current_version(db_session) - db_session.commit() - - employee.current_checksum = "b" * 64 - db_session.commit() - second = get_or_create_current_version(db_session) - - assert second.hash != first.hash - assert second.previous_hash == first.hash - - -def test_sync_employees_diff_spans_multiple_intermediate_versions(db_session): - alpha = _employee("staff:alpha", "a" * 64) - db_session.add(alpha) - db_session.commit() - first = get_or_create_current_version(db_session) - db_session.commit() - - beta = _employee("staff:beta", "b" * 64) - db_session.add(beta) - db_session.commit() - get_or_create_current_version(db_session) - db_session.commit() - - alpha.current_checksum = "c" * 64 - alpha.current_data = {"profile_key": "staff:alpha", "changed": True} - db_session.commit() - - payload = sync_employees_payload(db_session, client_hash=first.hash, include_data=False) - - assert payload["mode"] == "delta" - assert [item["profile_key"] for item in payload["changes"]["added"]] == ["staff:beta"] - assert [item["profile_key"] for item in payload["changes"]["updated"]] == ["staff:alpha"] - assert payload["changes"]["dismissed"] == [] - assert payload["changes"]["removed"] == [] - - -def test_sync_employees_reports_dismissed_as_tombstone(db_session): - alpha = _employee("staff:alpha", "a" * 64) - db_session.add(alpha) - db_session.commit() - first = get_or_create_current_version(db_session) - db_session.commit() - - alpha.status = "dismissed" - db_session.commit() - payload = sync_employees_payload(db_session, client_hash=first.hash, include_data=False) - - assert payload["changes"]["dismissed"][0]["profile_key"] == "staff:alpha" - assert payload["changes"]["dismissed"][0]["status"] == "dismissed"