Compare commits

...

7 Commits

17 changed files with 263 additions and 167 deletions

View File

@@ -110,4 +110,4 @@ docker compose exec postgres pg_dump -U miem miem_workers > backup.sql
docker compose down
```
Версия сервиса: `0.2.2`. Админка всегда показывает версии backend и frontend в footer.
Версия сервиса: `0.2.6`. Админка всегда показывает версии backend и frontend в footer.

View File

@@ -1,14 +1,14 @@
from fastapi import APIRouter, BackgroundTasks, Depends, Form, Request
from fastapi.responses import HTMLResponse, RedirectResponse
from fastapi.templating import Jinja2Templates
from sqlalchemy import desc, func, or_, select
from sqlalchemy import desc, func, select
from sqlalchemy.orm import Session
from app.config import Settings, get_settings
from app.db import SessionLocal, get_db
from app.models import CrawlError, CrawlRun, Employee
from app.security import SESSION_COOKIE, require_admin, sign_session, verify_admin
from app.services.admin_data import employee_detail_payload, list_employees_page, run_payload, stats_payload
from app.services.admin_data import employee_detail_payload, format_admin_datetime, list_employees_page, run_payload, stats_payload
from app.services.crawl_control import get_running_run, run_crawl_if_idle
from app.version import BACKEND_VERSION, FRONTEND_VERSION
@@ -22,8 +22,9 @@ def dashboard(request: Request, db: Session = Depends(get_db), settings: Setting
counts = stats_payload(db)
counts["runs"] = db.scalar(select(func.count()).select_from(CrawlRun)) or 0
counts["errors"] = db.scalar(select(func.count()).select_from(CrawlError)) or 0
runs = db.scalars(select(CrawlRun).order_by(desc(CrawlRun.started_at)).limit(10)).all()
return _render(request, "dashboard.html", {"counts": counts, "runs": runs, "latest_run": run_payload(runs[0]) if runs else None})
run_models = db.scalars(select(CrawlRun).order_by(desc(CrawlRun.started_at)).limit(10)).all()
runs = [run_payload(run) for run in run_models]
return _render(request, "dashboard.html", {"counts": counts, "runs": runs, "latest_run": runs[0] if runs else None})
@router.get("/login", response_class=HTMLResponse)
@@ -57,18 +58,10 @@ def employees(
request: Request,
status: str | None = None,
q: str | None = None,
db: Session = Depends(get_db),
settings: Settings = Depends(get_settings),
):
require_admin(request, settings)
stmt = select(Employee)
if status:
stmt = stmt.where(Employee.status == status)
if q:
pattern = f"%{q}%"
stmt = stmt.where(or_(Employee.full_name.ilike(pattern), Employee.canonical_url.ilike(pattern)))
items = db.scalars(stmt.order_by(Employee.full_name).limit(200)).all()
return _render(request, "employees.html", {"employees": items, "status": status or "", "q": q or ""})
return RedirectResponse("/admin/directory", status_code=303)
@router.get("/directory", response_class=HTMLResponse)
@@ -133,7 +126,14 @@ def employee_detail(
employee = db.get(Employee, employee_id)
if not employee:
return RedirectResponse("/admin/employees", status_code=303)
snapshots = sorted(employee.snapshots, key=lambda item: item.captured_at, reverse=True)[:20]
snapshots = [
{
"captured_display": format_admin_datetime(snapshot.captured_at),
"checksum": snapshot.checksum,
"parser_version": snapshot.parser_version,
}
for snapshot in sorted(employee.snapshots, key=lambda item: item.captured_at, reverse=True)[:20]
]
return _render(
request,
"employee_detail.html",
@@ -144,7 +144,8 @@ def employee_detail(
@router.get("/runs", response_class=HTMLResponse)
def runs(request: Request, db: Session = Depends(get_db), settings: Settings = Depends(get_settings)):
require_admin(request, settings)
items = db.scalars(select(CrawlRun).order_by(desc(CrawlRun.started_at)).limit(50)).all()
run_models = db.scalars(select(CrawlRun).order_by(desc(CrawlRun.started_at)).limit(50)).all()
items = [run_payload(run) for run in run_models]
errors = db.scalars(select(CrawlError).order_by(desc(CrawlError.created_at)).limit(50)).all()
return _render(request, "runs.html", {"runs": items, "errors": errors})

View File

@@ -3,6 +3,7 @@ from __future__ import annotations
from datetime import date, datetime, time
from math import ceil
from typing import Any
from zoneinfo import ZoneInfo
from sqlalchemy import Select, Text, and_, desc, func, or_, select
from sqlalchemy.orm import Session
@@ -20,18 +21,20 @@ EMPLOYEE_SORTS = {
def employee_display_payload(employee: Employee) -> dict[str, Any]:
data = employee.current_data or {}
contacts = data.get("contacts") or {}
sections = data.get("sections") or []
emails = contacts.get("emails") or []
phones = contacts.get("phones") or []
data = _as_dict(employee.current_data)
contacts = _as_dict(data.get("contacts"))
sections = _as_list(data.get("sections"))
positions = _clean_list(data.get("positions"))
emails = _clean_list(contacts.get("emails"))
phones = _clean_list(contacts.get("phones"))
return {
"id": employee.id,
"full_name": employee.full_name,
"status": employee.status,
"status_display": _employee_status_display(employee.status),
"canonical_url": employee.canonical_url,
"positions": data.get("positions") or [],
"positions_text": "; ".join(data.get("positions") or []),
"positions": positions,
"positions_text": "; ".join(positions),
"hse_start_year": data.get("hse_start_year"),
"emails": emails,
"email_text": ", ".join(emails),
@@ -43,12 +46,15 @@ def employee_display_payload(employee: Employee) -> dict[str, Any]:
"first_seen_at": employee.first_seen_at.isoformat() if employee.first_seen_at else None,
"last_seen_at": employee.last_seen_at.isoformat() if employee.last_seen_at else None,
"dismissed_at": employee.dismissed_at.isoformat() if employee.dismissed_at else None,
"first_seen_display": format_admin_datetime(employee.first_seen_at),
"last_seen_display": format_admin_datetime(employee.last_seen_at),
"dismissed_display": format_admin_datetime(employee.dismissed_at),
}
def employee_detail_payload(employee: Employee) -> dict[str, Any]:
data = employee.current_data or {}
contacts = data.get("contacts") or {}
data = _as_dict(employee.current_data)
contacts = _as_dict(data.get("contacts"))
return {
**employee_display_payload(employee),
"profile_type": employee.profile_type or data.get("profile_type"),
@@ -58,10 +64,10 @@ def employee_detail_payload(employee: Employee) -> dict[str, Any]:
"emails": _clean_list(contacts.get("emails")),
"phones": _clean_list(contacts.get("phones")),
"address": contacts.get("address"),
"items": _normalize_contact_items(contacts.get("items")),
"contact_items": _normalize_contact_items(contacts.get("items")),
},
"external_ids": _normalize_external_ids(data.get("external_ids")),
"sections": [_normalize_section(section) for section in data.get("sections") or []],
"sections": [_normalize_section(section) for section in _as_list(data.get("sections"))],
}
@@ -120,7 +126,7 @@ def list_employees_page(
order = desc(sort_column) if direction == "desc" else sort_column
employees = db.scalars(base_stmt.order_by(order).limit(limit).offset(offset)).all()
return {
"items": [employee_display_payload(employee) for employee in employees],
"employees": [employee_display_payload(employee) for employee in employees],
"total": total,
"limit": limit,
"offset": offset,
@@ -153,8 +159,11 @@ def run_payload(run: CrawlRun | None) -> dict[str, Any] | None:
"id": run.id,
"source_url": run.source_url,
"status": run.status,
"status_display": _run_status_display(run.status),
"started_at": run.started_at.isoformat() if run.started_at else None,
"finished_at": run.finished_at.isoformat() if run.finished_at else None,
"started_display": format_admin_datetime(run.started_at),
"finished_display": format_admin_datetime(run.finished_at),
"found_count": run.found_count,
"parsed_count": run.parsed_count,
"new_count": run.new_count,
@@ -166,6 +175,31 @@ def run_payload(run: CrawlRun | None) -> dict[str, Any] | None:
}
def format_admin_datetime(value: Any) -> str:
if not value:
return "Не указано"
if isinstance(value, str):
try:
value = datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError:
return value
if not isinstance(value, datetime):
return str(value)
if value.tzinfo:
value = value.astimezone(ZoneInfo("Europe/Moscow"))
return value.strftime("%d.%m.%Y %H:%M")
def _employee_status_display(status: str | None) -> str:
labels = {"active": "Работает", "dismissed": "Уволен"}
return labels.get(status or "", status or "Не указано")
def _run_status_display(status: str | None) -> str:
labels = {"running": "Выполняется", "completed": "Завершен", "failed": "Ошибка"}
return labels.get(status or "", status or "Не указано")
def _count_section_items(sections: list[dict[str, Any]], section_type: str) -> int:
total = 0
for section in sections:
@@ -179,11 +213,23 @@ def _count_section_items(sections: list[dict[str, Any]], section_type: str) -> i
def _clean_list(values: Any) -> list[str]:
if not isinstance(values, list):
if values is None:
return []
if not isinstance(values, list):
values = [values]
return [str(value).strip() for value in values if str(value or "").strip()]
def _as_dict(value: Any) -> dict[str, Any]:
return value if isinstance(value, dict) else {}
def _as_list(value: Any) -> list[Any]:
if value is None:
return []
return value if isinstance(value, list) else [value]
def _normalize_contact_items(items: Any) -> list[str]:
normalized = []
if not isinstance(items, list):
@@ -230,7 +276,7 @@ def _normalize_section(section: Any) -> dict[str, Any]:
"type": section_type,
"raw_text": raw_text,
"paragraphs": paragraphs,
"items": items,
"list_items": items,
"links": _normalize_links(section.get("links")),
"year_entries": _normalize_year_entries(section.get("year_entries")),
"publications": _normalize_publications(section.get("publications")),

View File

@@ -79,7 +79,7 @@
const errors = document.querySelector("[data-progress-errors]");
const fill = document.querySelector("[data-progress-fill]");
const percent = document.querySelector("[data-progress-percent]");
if (status) status.textContent = run.status;
if (status) status.textContent = run.status_display || run.status;
if (processed) processed.textContent = run.processed_count;
if (found) found.textContent = run.found_count;
if (errors) errors.textContent = run.error_count;

View File

@@ -10,12 +10,11 @@
<header class="admin__header">
<h1 class="admin__brand">MIEM Employees</h1>
<nav class="admin__nav">
<a class="admin__link" href="/admin">Dashboard</a>
<a class="admin__link" href="/admin/directory">Directory</a>
<a class="admin__link" href="/admin/employees">Employees</a>
<a class="admin__link" href="/admin/runs">Runs</a>
<a class="admin__link" href="/admin">Обзор</a>
<a class="admin__link" href="/admin/directory">Сотрудники</a>
<a class="admin__link" href="/admin/runs">Запуски</a>
<form method="post" action="/admin/logout">
<button class="button button--ghost" type="submit">Logout</button>
<button class="button button--ghost" type="submit">Выйти</button>
</form>
</nav>
</header>

View File

@@ -1,43 +1,43 @@
{% extends "base.html" %}
{% block title %}Dashboard · MIEM Employees{% endblock %}
{% block title %}Обзор · MIEM Employees{% endblock %}
{% block content %}
<section class="admin__grid">
<div class="metric"><div class="metric__label">Total</div><div class="metric__value">{{ counts.total }}</div></div>
<div class="metric"><div class="metric__label">Active</div><div class="metric__value">{{ counts.active }}</div></div>
<div class="metric"><div class="metric__label">New in last run</div><div class="metric__value">{{ counts.new_in_last_run }}</div></div>
<div class="metric"><div class="metric__label">Dismissed</div><div class="metric__value">{{ counts.dismissed }}</div></div>
<div class="metric"><div class="metric__label">Всего в базе</div><div class="metric__value">{{ counts.total }}</div></div>
<div class="metric"><div class="metric__label">Работают</div><div class="metric__value">{{ counts.active }}</div></div>
<div class="metric"><div class="metric__label">Новые за запуск</div><div class="metric__value">{{ counts.new_in_last_run }}</div></div>
<div class="metric"><div class="metric__label">Уволены</div><div class="metric__value">{{ counts.dismissed }}</div></div>
</section>
<section class="stats-strip">
<div class="stats-strip__item">
<span class="stats-strip__label">Latest added</span>
<span class="stats-strip__label">Последний добавленный</span>
{% if counts.latest_added %}
<a class="stats-strip__value" href="/admin/employees/{{ counts.latest_added.id }}">{{ counts.latest_added.full_name or counts.latest_added.canonical_url }}</a>
{% else %}
<span class="stats-strip__value">No employees yet</span>
<span class="stats-strip__value">Сотрудников пока нет</span>
{% endif %}
</div>
<div class="stats-strip__item">
<span class="stats-strip__label">Runs</span>
<span class="stats-strip__label">Запуски</span>
<span class="stats-strip__value">{{ counts.runs }}</span>
</div>
<div class="stats-strip__item">
<span class="stats-strip__label">Errors</span>
<span class="stats-strip__label">Ошибки</span>
<span class="stats-strip__value">{{ counts.errors }}</span>
</div>
</section>
<section class="panel progress-panel" data-progress-panel>
<div class="progress-panel__header">
<h2 class="panel__title">Parsing progress</h2>
<h2 class="panel__title">Прогресс парсинга</h2>
<form method="post" action="/admin/crawl-now">
<button class="button" type="submit">Start crawl now</button>
<button class="button" type="submit">Запустить парсинг</button>
</form>
</div>
{% set run = counts.current_running_run or latest_run %}
<div class="progress-panel__body" data-progress-body>
<div class="progress-panel__meta">
<span data-progress-status>{{ run.status if run else "idle" }}</span>
<span><span data-progress-processed>{{ run.processed_count if run else 0 }}</span> / <span data-progress-found>{{ run.found_count if run else 0 }}</span> processed</span>
<span><span data-progress-errors>{{ run.error_count if run else 0 }}</span> errors</span>
<span data-progress-status>{{ run.status_display if run else "Ожидание" }}</span>
<span>обработано: <span data-progress-processed>{{ run.processed_count if run else 0 }}</span> / <span data-progress-found>{{ run.found_count if run else 0 }}</span></span>
<span>ошибок: <span data-progress-errors>{{ run.error_count if run else 0 }}</span></span>
</div>
<div class="progress-bar" aria-label="Parsing progress">
<div class="progress-bar__fill" data-progress-fill style="width: {{ run.progress_percent if run else 0 }}%"></div>
@@ -46,12 +46,12 @@
</div>
</section>
<section class="panel">
<h2 class="panel__title">Latest runs</h2>
<h2 class="panel__title">Последние запуски</h2>
<table class="table">
<thead><tr><th class="table__head">ID</th><th class="table__head">Status</th><th class="table__head">Parsed</th><th class="table__head">Errors</th><th class="table__head">Started</th></tr></thead>
<thead><tr><th class="table__head">ID</th><th class="table__head">Статус</th><th class="table__head">Обработано</th><th class="table__head">Ошибки</th><th class="table__head">Старт</th></tr></thead>
<tbody>
{% for run in runs %}
<tr><td class="table__cell">{{ run.id }}</td><td class="table__cell">{{ run.status }}</td><td class="table__cell">{{ run.parsed_count }}</td><td class="table__cell">{{ run.error_count }}</td><td class="table__cell">{{ run.started_at }}</td></tr>
<tr><td class="table__cell">{{ run.id }}</td><td class="table__cell">{{ run.status_display }}</td><td class="table__cell">{{ run.parsed_count }}</td><td class="table__cell">{{ run.error_count }}</td><td class="table__cell">{{ run.started_display }}</td></tr>
{% endfor %}
</tbody>
</table>

View File

@@ -1,65 +1,65 @@
{% extends "base.html" %}
{% block title %}Directory · MIEM Employees{% endblock %}
{% block title %}Сотрудники · MIEM Employees{% endblock %}
{% block content %}
<section class="directory">
<div class="directory__header">
<div>
<h2 class="directory__title">Directory</h2>
<p class="directory__summary">{{ page.total }} employees found</p>
<h2 class="directory__title">Сотрудники</h2>
<p class="directory__summary">Найдено: {{ page.total }}</p>
</div>
<button class="button" type="button" data-columns-open>Columns</button>
<button class="button" type="button" data-columns-open>Колонки</button>
</div>
<form class="directory__filters" method="get" action="/admin/directory">
<input class="directory__input" name="q" value="{{ filters.q }}" placeholder="Name or URL">
<input class="directory__input" name="q" value="{{ filters.q }}" placeholder="ФИО или ссылка">
<select class="directory__input" name="status">
<option value="" {% if not filters.status %}selected{% endif %}>All statuses</option>
<option value="active" {% if filters.status == "active" %}selected{% endif %}>Active</option>
<option value="dismissed" {% if filters.status == "dismissed" %}selected{% endif %}>Dismissed</option>
<option value="" {% if not filters.status %}selected{% endif %}>Все статусы</option>
<option value="active" {% if filters.status == "active" %}selected{% endif %}>Работает</option>
<option value="dismissed" {% if filters.status == "dismissed" %}selected{% endif %}>Уволен</option>
</select>
<select class="directory__input" name="has_email">
<option value="" {% if not filters.has_email %}selected{% endif %}>Any email</option>
<option value="true" {% if filters.has_email == "true" %}selected{% endif %}>Has email</option>
<option value="false" {% if filters.has_email == "false" %}selected{% endif %}>No email</option>
<option value="" {% if not filters.has_email %}selected{% endif %}>Любой email</option>
<option value="true" {% if filters.has_email == "true" %}selected{% endif %}>Есть email</option>
<option value="false" {% if filters.has_email == "false" %}selected{% endif %}>Нет email</option>
</select>
<input class="directory__input" type="date" name="started_from" value="{{ filters.started_from }}" aria-label="First seen from">
<input class="directory__input" type="date" name="started_to" value="{{ filters.started_to }}" aria-label="First seen to">
<input class="directory__input" type="date" name="started_from" value="{{ filters.started_from }}" aria-label="Впервые найден с">
<input class="directory__input" type="date" name="started_to" value="{{ filters.started_to }}" aria-label="Впервые найден по">
<select class="directory__input" name="sort">
{% for value, label in [("full_name", "Name"), ("status", "Status"), ("hse_start_year", "HSE start"), ("first_seen_at", "First seen"), ("last_seen_at", "Last seen"), ("dismissed_at", "Dismissed")] %}
<option value="{{ value }}" {% if filters.sort == value %}selected{% endif %}>Sort: {{ label }}</option>
{% for value, label in [("full_name", "ФИО"), ("status", "Статус"), ("hse_start_year", "Год начала"), ("first_seen_at", "Впервые найден"), ("last_seen_at", "Последний раз найден"), ("dismissed_at", "Дата увольнения")] %}
<option value="{{ value }}" {% if filters.sort == value %}selected{% endif %}>Сортировка: {{ label }}</option>
{% endfor %}
</select>
<select class="directory__input" name="direction">
<option value="asc" {% if filters.direction == "asc" %}selected{% endif %}>Ascending</option>
<option value="desc" {% if filters.direction == "desc" %}selected{% endif %}>Descending</option>
<option value="asc" {% if filters.direction == "asc" %}selected{% endif %}>По возрастанию</option>
<option value="desc" {% if filters.direction == "desc" %}selected{% endif %}>По убыванию</option>
</select>
<button class="button" type="submit">Apply</button>
<button class="button" type="submit">Применить</button>
</form>
<div class="directory__table-wrap">
<table class="directory-table" data-directory-table>
<thead>
<tr>
<th class="directory-table__head" data-column="full_name">Name</th>
<th class="directory-table__head" data-column="status">Status</th>
<th class="directory-table__head" data-column="positions">Positions</th>
<th class="directory-table__head" data-column="hse_start_year">HSE start</th>
<th class="directory-table__head" data-column="full_name">ФИО</th>
<th class="directory-table__head" data-column="status">Статус</th>
<th class="directory-table__head" data-column="positions">Должности</th>
<th class="directory-table__head" data-column="hse_start_year">Год начала</th>
<th class="directory-table__head" data-column="email">Email</th>
<th class="directory-table__head" data-column="phone">Phone</th>
<th class="directory-table__head" data-column="address">Address</th>
<th class="directory-table__head" data-column="publications_count">Publications</th>
<th class="directory-table__head" data-column="courses_count">Courses</th>
<th class="directory-table__head" data-column="first_seen_at">First seen</th>
<th class="directory-table__head" data-column="last_seen_at">Last seen</th>
<th class="directory-table__head" data-column="dismissed_at">Dismissed</th>
<th class="directory-table__head" data-column="profile">Profile</th>
<th class="directory-table__head" data-column="phone">Телефон</th>
<th class="directory-table__head" data-column="address">Адрес</th>
<th class="directory-table__head" data-column="publications_count">Публикации</th>
<th class="directory-table__head" data-column="courses_count">Курсы</th>
<th class="directory-table__head" data-column="first_seen_at">Впервые найден</th>
<th class="directory-table__head" data-column="last_seen_at">Последний раз найден</th>
<th class="directory-table__head" data-column="dismissed_at">Дата увольнения</th>
<th class="directory-table__head" data-column="profile">Профиль</th>
</tr>
</thead>
<tbody>
{% for employee in page.items %}
{% for employee in page.employees %}
<tr class="directory-table__row" data-row-href="/admin/employees/{{ employee.id }}">
<td class="directory-table__cell" data-column="full_name">{{ employee.full_name or "No name" }}</td>
<td class="directory-table__cell" data-column="status"><span class="badge {% if employee.status == "dismissed" %}badge--dismissed{% endif %}">{{ employee.status }}</span></td>
<td class="directory-table__cell" data-column="full_name">{{ employee.full_name or "Без имени" }}</td>
<td class="directory-table__cell" data-column="status"><span class="badge {% if employee.status == "dismissed" %}badge--dismissed{% endif %}">{{ employee.status_display }}</span></td>
<td class="directory-table__cell" data-column="positions">{{ employee.positions_text }}</td>
<td class="directory-table__cell" data-column="hse_start_year">{{ employee.hse_start_year or "" }}</td>
<td class="directory-table__cell" data-column="email">{{ employee.email_text }}</td>
@@ -67,13 +67,13 @@
<td class="directory-table__cell" data-column="address">{{ employee.address or "" }}</td>
<td class="directory-table__cell" data-column="publications_count">{{ employee.publications_count }}</td>
<td class="directory-table__cell" data-column="courses_count">{{ employee.courses_count }}</td>
<td class="directory-table__cell" data-column="first_seen_at">{{ employee.first_seen_at or "" }}</td>
<td class="directory-table__cell" data-column="last_seen_at">{{ employee.last_seen_at or "" }}</td>
<td class="directory-table__cell" data-column="dismissed_at">{{ employee.dismissed_at or "" }}</td>
<td class="directory-table__cell" data-column="profile"><a class="admin__link" href="{{ employee.canonical_url }}">Open</a></td>
<td class="directory-table__cell" data-column="first_seen_at">{{ employee.first_seen_display }}</td>
<td class="directory-table__cell" data-column="last_seen_at">{{ employee.last_seen_display }}</td>
<td class="directory-table__cell" data-column="dismissed_at">{{ employee.dismissed_display }}</td>
<td class="directory-table__cell" data-column="profile"><a class="admin__link" href="{{ employee.canonical_url }}">Открыть</a></td>
</tr>
{% else %}
<tr><td class="directory-table__empty" colspan="13">No employees match these filters.</td></tr>
<tr><td class="directory-table__empty" colspan="13">По этим фильтрам сотрудники не найдены.</td></tr>
{% endfor %}
</tbody>
</table>
@@ -83,24 +83,24 @@
{% set prev_offset = filters.offset - filters.limit %}
{% set next_offset = filters.offset + filters.limit %}
{% if filters.offset > 0 %}
<a class="admin__link" href="{{ request.url.include_query_params(offset=prev_offset) }}">Previous</a>
<a class="admin__link" href="{{ request.url.include_query_params(offset=prev_offset) }}">Назад</a>
{% endif %}
<span class="directory__page">Page {{ page.page }}{% if page.pages %} of {{ page.pages }}{% endif %}</span>
<span class="directory__page">Страница {{ page.page }}{% if page.pages %} из {{ page.pages }}{% endif %}</span>
{% if next_offset < page.total %}
<a class="admin__link" href="{{ request.url.include_query_params(offset=next_offset) }}">Next</a>
<a class="admin__link" href="{{ request.url.include_query_params(offset=next_offset) }}">Вперед</a>
{% endif %}
</div>
</section>
<div class="columns-modal" data-columns-modal hidden>
<div class="columns-modal__backdrop" data-columns-close></div>
<section class="columns-modal__panel" aria-label="Column settings">
<section class="columns-modal__panel" aria-label="Настройка колонок">
<div class="columns-modal__header">
<h3 class="columns-modal__title">Visible columns</h3>
<button class="button button--ghost" type="button" data-columns-close>Close</button>
<h3 class="columns-modal__title">Отображаемые колонки</h3>
<button class="button button--ghost" type="button" data-columns-close>Закрыть</button>
</div>
<div class="columns-modal__grid">
{% for key, label in [("full_name", "Name"), ("status", "Status"), ("positions", "Positions"), ("hse_start_year", "HSE start"), ("email", "Email"), ("phone", "Phone"), ("address", "Address"), ("publications_count", "Publications"), ("courses_count", "Courses"), ("first_seen_at", "First seen"), ("last_seen_at", "Last seen"), ("dismissed_at", "Dismissed"), ("profile", "Profile")] %}
{% for key, label in [("full_name", "ФИО"), ("status", "Статус"), ("positions", "Должности"), ("hse_start_year", "Год начала"), ("email", "Email"), ("phone", "Телефон"), ("address", "Адрес"), ("publications_count", "Публикации"), ("courses_count", "Курсы"), ("first_seen_at", "Впервые найден"), ("last_seen_at", "Последний раз найден"), ("dismissed_at", "Дата увольнения"), ("profile", "Профиль")] %}
<label class="columns-modal__option"><input class="columns-modal__checkbox" type="checkbox" value="{{ key }}" data-column-toggle> {{ label }}</label>
{% endfor %}
</div>

View File

@@ -5,7 +5,7 @@
<div class="employee-card__header">
<div class="employee-card__identity">
<h2 class="employee-card__title">{{ employee_view.full_name or employee.profile_key }}</h2>
<span class="badge {% if employee_view.status == "dismissed" %}badge--dismissed{% endif %}">{{ employee_view.status }}</span>
<span class="badge {% if employee_view.status == "dismissed" %}badge--dismissed{% endif %}">{{ employee_view.status_display }}</span>
</div>
<a class="admin__link" href="{{ employee_view.canonical_url }}">{{ employee_view.canonical_url }}</a>
</div>
@@ -28,12 +28,11 @@
</dd>
</div>
<div class="employee-card__meta-item"><dt class="employee-card__meta-label">Год начала работы в ВШЭ</dt><dd class="employee-card__meta-value">{{ employee_view.hse_start_year or "Не указано" }}</dd></div>
<div class="employee-card__meta-item"><dt class="employee-card__meta-label">Profile type</dt><dd class="employee-card__meta-value">{{ employee_view.profile_type or "Не указано" }}</dd></div>
<div class="employee-card__meta-item"><dt class="employee-card__meta-label">Profile ID</dt><dd class="employee-card__meta-value">{{ employee_view.profile_id or "Не указано" }}</dd></div>
<div class="employee-card__meta-item"><dt class="employee-card__meta-label">First seen</dt><dd class="employee-card__meta-value">{{ employee_view.first_seen_at or "Не указано" }}</dd></div>
<div class="employee-card__meta-item"><dt class="employee-card__meta-label">Last seen</dt><dd class="employee-card__meta-value">{{ employee_view.last_seen_at or "Не указано" }}</dd></div>
<div class="employee-card__meta-item"><dt class="employee-card__meta-label">Dismissed at</dt><dd class="employee-card__meta-value">{{ employee_view.dismissed_at or "Не указано" }}</dd></div>
<div class="employee-card__meta-item"><dt class="employee-card__meta-label">Parser version</dt><dd class="employee-card__meta-value">{{ employee_view.parser_version or "Не указано" }}</dd></div>
<div class="employee-card__meta-item"><dt class="employee-card__meta-label">Тип профиля</dt><dd class="employee-card__meta-value">{{ employee_view.profile_type or "Не указано" }}</dd></div>
<div class="employee-card__meta-item"><dt class="employee-card__meta-label">ID профиля</dt><dd class="employee-card__meta-value">{{ employee_view.profile_id or "Не указано" }}</dd></div>
<div class="employee-card__meta-item"><dt class="employee-card__meta-label">Впервые найден</dt><dd class="employee-card__meta-value">{{ employee_view.first_seen_display }}</dd></div>
<div class="employee-card__meta-item"><dt class="employee-card__meta-label">Последний раз найден</dt><dd class="employee-card__meta-value">{{ employee_view.last_seen_display }}</dd></div>
<div class="employee-card__meta-item"><dt class="employee-card__meta-label">Дата увольнения</dt><dd class="employee-card__meta-value">{{ employee_view.dismissed_display }}</dd></div>
</dl>
</section>
@@ -62,12 +61,12 @@
<dt class="employee-card__meta-label">Адрес</dt>
<dd class="employee-card__meta-value">{{ employee_view.contacts.address or "Не указано" }}</dd>
</div>
{% if employee_view.contacts.items %}
{% if employee_view.contacts.contact_items %}
<div class="employee-card__meta-item employee-card__meta-item--wide">
<dt class="employee-card__meta-label">Прочее</dt>
<dd class="employee-card__meta-value">
<ul class="employee-card__list">
{% for item in employee_view.contacts.items %}
{% for item in employee_view.contacts.contact_items %}
<li class="employee-card__list-item">{{ item }}</li>
{% endfor %}
</ul>
@@ -162,9 +161,9 @@
<p class="employee-section__text">{{ paragraph }}</p>
{% endfor %}
{% endif %}
{% if section.items %}
{% if section.list_items %}
<ul class="employee-card__list">
{% for item in section.items %}
{% for item in section.list_items %}
<li class="employee-card__list-item">{{ item }}</li>
{% endfor %}
</ul>
@@ -188,12 +187,12 @@
</section>
<section class="panel">
<h2 class="panel__title">Snapshots</h2>
<h2 class="panel__title">Снапшоты</h2>
<table class="table">
<thead><tr><th class="table__head">Captured</th><th class="table__head">Checksum</th><th class="table__head">Parser</th></tr></thead>
<thead><tr><th class="table__head">Дата</th><th class="table__head">Checksum</th><th class="table__head">Парсер</th></tr></thead>
<tbody>
{% for snapshot in snapshots %}
<tr><td class="table__cell">{{ snapshot.captured_at }}</td><td class="table__cell">{{ snapshot.checksum }}</td><td class="table__cell">{{ snapshot.parser_version }}</td></tr>
<tr><td class="table__cell">{{ snapshot.captured_display }}</td><td class="table__cell">{{ snapshot.checksum }}</td><td class="table__cell">{{ snapshot.parser_version }}</td></tr>
{% endfor %}
</tbody>
</table>

View File

@@ -1,29 +0,0 @@
{% extends "base.html" %}
{% block title %}Employees · MIEM Employees{% endblock %}
{% block content %}
<section class="panel">
<h2 class="panel__title">Employees</h2>
<form class="form" method="get" action="/admin/employees">
<input class="form__input" name="q" value="{{ q }}" placeholder="Name or URL">
<select class="form__select" name="status">
<option value="" {% if not status %}selected{% endif %}>All</option>
<option value="active" {% if status == "active" %}selected{% endif %}>Active</option>
<option value="dismissed" {% if status == "dismissed" %}selected{% endif %}>Dismissed</option>
</select>
<button class="button" type="submit">Search</button>
</form>
<table class="table">
<thead><tr><th class="table__head">Name</th><th class="table__head">Status</th><th class="table__head">Last seen</th><th class="table__head">Profile</th></tr></thead>
<tbody>
{% for employee in employees %}
<tr>
<td class="table__cell"><a class="admin__link" href="/admin/employees/{{ employee.id }}">{{ employee.full_name or employee.profile_key }}</a></td>
<td class="table__cell"><span class="badge {% if employee.status == "dismissed" %}badge--dismissed{% endif %}">{{ employee.status }}</span></td>
<td class="table__cell">{{ employee.last_seen_at }}</td>
<td class="table__cell"><a class="admin__link" href="{{ employee.canonical_url }}">{{ employee.canonical_url }}</a></td>
</tr>
{% endfor %}
</tbody>
</table>
</section>
{% endblock %}

View File

@@ -3,18 +3,18 @@
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Login · MIEM Employees</title>
<title>Вход · MIEM Employees</title>
<link rel="stylesheet" href="/static/admin.css">
</head>
<body class="admin">
<main class="admin__main">
<section class="panel">
<h1 class="panel__title">Admin login</h1>
<h1 class="panel__title">Вход в админку</h1>
{% if error %}<p>{{ error }}</p>{% endif %}
<form class="form" method="post" action="/admin/login">
<label class="form__label">Login <input class="form__input" name="username" autocomplete="username"></label>
<label class="form__label">Password <input class="form__input" name="password" type="password" autocomplete="current-password"></label>
<button class="button" type="submit">Sign in</button>
<label class="form__label">Логин <input class="form__input" name="username" autocomplete="username"></label>
<label class="form__label">Пароль <input class="form__input" name="password" type="password" autocomplete="current-password"></label>
<button class="button" type="submit">Войти</button>
</form>
</section>
</main>

View File

@@ -1,10 +1,10 @@
{% extends "base.html" %}
{% block title %}Runs · MIEM Employees{% endblock %}
{% block title %}Запуски · MIEM Employees{% endblock %}
{% block content %}
<section class="panel">
<div class="progress-panel__header">
<h2 class="panel__title">Crawl runs</h2>
<form method="post" action="/admin/runs"><button class="button" type="submit">Start crawl now</button></form>
<h2 class="panel__title">Запуски парсинга</h2>
<form method="post" action="/admin/runs"><button class="button" type="submit">Запустить парсинг</button></form>
</div>
{% set run = runs[0] if runs else none %}
{% if run %}
@@ -12,9 +12,9 @@
{% set percent = ((processed / run.found_count) * 100) | round(1) if run.found_count else 0 %}
<div class="progress-panel" data-progress-panel>
<div class="progress-panel__meta">
<span data-progress-status>{{ run.status }}</span>
<span><span data-progress-processed>{{ processed }}</span> / <span data-progress-found>{{ run.found_count }}</span> processed</span>
<span><span data-progress-errors>{{ run.error_count }}</span> errors</span>
<span data-progress-status>{{ run.status_display }}</span>
<span>обработано: <span data-progress-processed>{{ processed }}</span> / <span data-progress-found>{{ run.found_count }}</span></span>
<span>ошибок: <span data-progress-errors>{{ run.error_count }}</span></span>
</div>
<div class="progress-bar" aria-label="Parsing progress">
<div class="progress-bar__fill" data-progress-fill style="width: {{ percent }}%"></div>
@@ -24,9 +24,9 @@
{% else %}
<div class="progress-panel" data-progress-panel>
<div class="progress-panel__meta">
<span data-progress-status>idle</span>
<span><span data-progress-processed>0</span> / <span data-progress-found>0</span> processed</span>
<span><span data-progress-errors>0</span> errors</span>
<span data-progress-status>Ожидание</span>
<span>обработано: <span data-progress-processed>0</span> / <span data-progress-found>0</span></span>
<span>ошибок: <span data-progress-errors>0</span></span>
</div>
<div class="progress-bar" aria-label="Parsing progress">
<div class="progress-bar__fill" data-progress-fill style="width: 0%"></div>
@@ -35,18 +35,18 @@
</div>
{% endif %}
<table class="table">
<thead><tr><th class="table__head">ID</th><th class="table__head">Status</th><th class="table__head">Found</th><th class="table__head">Parsed</th><th class="table__head">New</th><th class="table__head">Errors</th><th class="table__head">Dismissed</th></tr></thead>
<thead><tr><th class="table__head">ID</th><th class="table__head">Статус</th><th class="table__head">Найдено</th><th class="table__head">Обработано</th><th class="table__head">Новые</th><th class="table__head">Ошибки</th><th class="table__head">Уволены</th><th class="table__head">Старт</th></tr></thead>
<tbody>
{% for run in runs %}
<tr><td class="table__cell">{{ run.id }}</td><td class="table__cell">{{ run.status }}</td><td class="table__cell">{{ run.found_count }}</td><td class="table__cell">{{ run.parsed_count }}</td><td class="table__cell">{{ run.new_count }}</td><td class="table__cell">{{ run.error_count }}</td><td class="table__cell">{{ run.dismissed_count }}</td></tr>
<tr><td class="table__cell">{{ run.id }}</td><td class="table__cell">{{ run.status_display }}</td><td class="table__cell">{{ run.found_count }}</td><td class="table__cell">{{ run.parsed_count }}</td><td class="table__cell">{{ run.new_count }}</td><td class="table__cell">{{ run.error_count }}</td><td class="table__cell">{{ run.dismissed_count }}</td><td class="table__cell">{{ run.started_display }}</td></tr>
{% endfor %}
</tbody>
</table>
</section>
<section class="panel">
<h2 class="panel__title">Recent errors</h2>
<h2 class="panel__title">Последние ошибки</h2>
<table class="table">
<thead><tr><th class="table__head">Run</th><th class="table__head">Profile</th><th class="table__head">Error</th></tr></thead>
<thead><tr><th class="table__head">Запуск</th><th class="table__head">Профиль</th><th class="table__head">Ошибка</th></tr></thead>
<tbody>
{% for error in errors %}
<tr><td class="table__cell">{{ error.crawl_run_id }}</td><td class="table__cell">{{ error.profile_url }}</td><td class="table__cell">{{ error.error_type }}: {{ error.message }}</td></tr>

View File

@@ -1,3 +1,3 @@
APP_VERSION = "0.2.2"
FRONTEND_VERSION = "0.2.2"
BACKEND_VERSION = "0.2.2"
APP_VERSION = "0.2.6"
FRONTEND_VERSION = "0.2.6"
BACKEND_VERSION = "0.2.6"

View File

@@ -1,6 +1,6 @@
[project]
name = "miem-workers"
version = "0.1.0"
version = "0.2.6"
description = "MIEM employees parser, admin API, and MCP server"
requires-python = ">=3.11"
dependencies = [

View File

@@ -4,12 +4,21 @@ from app.models import CrawlRun, Employee
from app.services.admin_data import (
employee_detail_payload,
employee_display_payload,
format_admin_datetime,
list_employees_page,
run_payload,
stats_payload,
)
def test_format_admin_datetime_handles_datetime_string_and_none():
value = datetime(2026, 4, 28, 17, 13, 34, tzinfo=timezone.utc)
assert format_admin_datetime(value) == "28.04.2026 20:13"
assert format_admin_datetime("2026-04-28T17:13:34.448605+00:00") == "28.04.2026 20:13"
assert format_admin_datetime(None) == "Не указано"
def test_employee_display_payload_extracts_common_fields(db_session):
employee = Employee(
profile_key="staff:person",
@@ -32,9 +41,11 @@ def test_employee_display_payload_extracts_common_fields(db_session):
payload = employee_display_payload(employee)
assert payload["positions_text"] == "Professor"
assert payload["status_display"] == "Работает"
assert payload["email_text"] == "person@hse.ru"
assert payload["publications_count"] == 1
assert payload["courses_count"] == 1
assert payload["first_seen_display"] != "Не указано"
def test_employee_detail_payload_normalizes_human_readable_sections(db_session):
@@ -86,7 +97,7 @@ def test_employee_detail_payload_normalizes_human_readable_sections(db_session):
payload = employee_detail_payload(employee)
assert payload["contacts"]["emails"] == ["person@hse.ru"]
assert payload["contacts"]["items"] == ["consultation hours"]
assert payload["contacts"]["contact_items"] == ["consultation hours"]
assert payload["external_ids"][0]["system"] == "ORCID"
assert payload["sections"][0]["year_entries"][0]["text"] == "Master degree"
assert payload["sections"][1]["publications"][0]["title"] == "Paper"
@@ -94,6 +105,27 @@ def test_employee_detail_payload_normalizes_human_readable_sections(db_session):
assert payload["sections"][3]["paragraphs"] == ["Fallback text"]
def test_employee_payloads_tolerate_malformed_current_data(db_session):
employee = Employee(
profile_key="staff:broken",
canonical_url="https://www.hse.ru/staff/broken",
full_name="Broken Data",
status="active",
first_seen_at=datetime.now(timezone.utc),
last_seen_at=datetime.now(timezone.utc),
current_data="not-a-dict",
)
display = employee_display_payload(employee)
detail = employee_detail_payload(employee)
assert display["positions"] == []
assert display["email_text"] == ""
assert detail["contacts"]["emails"] == []
assert detail["contacts"]["contact_items"] == []
assert detail["sections"] == []
def test_list_employees_page_filters_sorts_and_paginates(db_session):
db_session.add(
Employee(
@@ -122,7 +154,7 @@ def test_list_employees_page_filters_sorts_and_paginates(db_session):
page = list_employees_page(db_session, status="active", sort="full_name", direction="asc", limit=10)
assert page["total"] == 1
assert page["items"][0]["full_name"] == "Alpha"
assert page["employees"][0]["full_name"] == "Alpha"
def test_stats_payload_uses_latest_run_new_count(db_session):
@@ -159,3 +191,4 @@ def test_run_payload_calculates_progress():
assert payload["processed_count"] == 5
assert payload["progress_percent"] == 50.0
assert payload["status_display"] == "Выполняется"

View File

@@ -0,0 +1,32 @@
from pathlib import Path
def test_base_navigation_is_russian_and_has_no_legacy_employees_link():
template = Path("app/templates/base.html").read_text(encoding="utf-8")
assert "Обзор" in template
assert "Сотрудники" in template
assert "Запуски" in template
assert "Выйти" in template
assert ">Employees<" not in template
assert "/admin/employees" not in template
def test_directory_template_is_russian_and_uses_display_dates():
template = Path("app/templates/directory.html").read_text(encoding="utf-8")
assert "Сотрудники" in template
assert "Колонки" in template
assert "Применить" in template
assert "Найдено:" in template
assert "employee.first_seen_display" in template
assert "employee.last_seen_display" in template
assert "employee.dismissed_display" in template
assert "Directory" not in template
assert "employees found" not in template
def test_admin_employees_route_redirects_to_directory():
source = Path("app/admin.py").read_text(encoding="utf-8")
assert 'RedirectResponse("/admin/directory", status_code=303)' in source

View File

@@ -18,7 +18,7 @@ def test_health_returns_versions():
response = client.get("/api/health")
assert response.status_code == 200
assert response.json()["backend_version"] == "0.2.2"
assert response.json()["backend_version"] == "0.2.6"
def test_mcp_requires_token_and_lists_tools():

View File

@@ -7,7 +7,22 @@ def test_employee_detail_template_is_human_readable():
assert "Current data" not in template
assert "<pre class=\"code\"" not in template
assert ">Tabs<" not in template
assert "contacts.items" not in template
assert "contacts.contact_items" in template
assert "section.items" not in template
assert "section.list_items" in template
assert "Основная информация" in template
assert "Контакты" in template
assert "Разделы профиля" in template
assert "Snapshots" in template
assert "Parser version" not in template
assert "First seen" not in template
assert "Last seen" not in template
assert "Dismissed at" not in template
assert "Profile type" not in template
assert "Profile ID" not in template
assert "Впервые найден" in template
assert "Последний раз найден" in template
assert "Дата увольнения" in template
assert "Тип профиля" in template
assert "ID профиля" in template
assert "Снапшоты" in template