feature: improve admin directory and crawl progress

app/admin.py

@@ -1,4 +1,4 @@
-from fastapi import APIRouter, BackgroundTasks, Depends, Form, Request, Response
+from fastapi import APIRouter, BackgroundTasks, Depends, Form, Request
 from fastapi.responses import HTMLResponse, RedirectResponse
 from fastapi.templating import Jinja2Templates
 from sqlalchemy import desc, func, or_, select
@@ -8,7 +8,8 @@ from app.config import Settings, get_settings
 from app.db import SessionLocal, get_db
 from app.models import CrawlError, CrawlRun, Employee
 from app.security import SESSION_COOKIE, require_admin, sign_session, verify_admin
-from app.services.crawler import run_crawl
+from app.services.admin_data import list_employees_page, run_payload, stats_payload
+from app.services.crawl_control import get_running_run, run_crawl_if_idle
 from app.version import BACKEND_VERSION, FRONTEND_VERSION
 
 router = APIRouter(prefix="/admin")
@@ -18,14 +19,11 @@ templates = Jinja2Templates(directory="app/templates")
 @router.get("", response_class=HTMLResponse)
 def dashboard(request: Request, db: Session = Depends(get_db), settings: Settings = Depends(get_settings)):
     require_admin(request, settings)
-    counts = {
-        "active": db.scalar(select(func.count()).select_from(Employee).where(Employee.status == "active")) or 0,
-        "dismissed": db.scalar(select(func.count()).select_from(Employee).where(Employee.status == "dismissed")) or 0,
-        "runs": db.scalar(select(func.count()).select_from(CrawlRun)) or 0,
-        "errors": db.scalar(select(func.count()).select_from(CrawlError)) or 0,
-    }
+    counts = stats_payload(db)
+    counts["runs"] = db.scalar(select(func.count()).select_from(CrawlRun)) or 0
+    counts["errors"] = db.scalar(select(func.count()).select_from(CrawlError)) or 0
     runs = db.scalars(select(CrawlRun).order_by(desc(CrawlRun.started_at)).limit(10)).all()
-    return _render(request, "dashboard.html", {"counts": counts, "runs": runs})
+    return _render(request, "dashboard.html", {"counts": counts, "runs": runs, "latest_run": run_payload(runs[0]) if runs else None})
 
 
 @router.get("/login", response_class=HTMLResponse)
@@ -35,7 +33,6 @@ def login_form(request: Request):
 
 @router.post("/login")
 def login(
-    response: Response,
     request: Request,
     username: str = Form(...),
     password: str = Form(...),
@@ -74,6 +71,57 @@ def employees(
     return _render(request, "employees.html", {"employees": items, "status": status or "", "q": q or ""})
 
 
+@router.get("/directory", response_class=HTMLResponse)
+def directory(
+    request: Request,
+    status: str | None = None,
+    q: str | None = None,
+    started_from: str | None = None,
+    started_to: str | None = None,
+    has_email: str | None = None,
+    sort: str = "full_name",
+    direction: str = "asc",
+    limit: int = 50,
+    offset: int = 0,
+    db: Session = Depends(get_db),
+    settings: Settings = Depends(get_settings),
+):
+    require_admin(request, settings)
+    parsed_started_from = _parse_date(started_from)
+    parsed_started_to = _parse_date(started_to)
+    parsed_has_email = None if has_email in (None, "") else has_email == "true"
+    page = list_employees_page(
+        db,
+        status=status,
+        q=q,
+        started_from=parsed_started_from,
+        started_to=parsed_started_to,
+        has_email=parsed_has_email,
+        sort=sort,
+        direction=direction,
+        limit=limit,
+        offset=offset,
+    )
+    return _render(
+        request,
+        "directory.html",
+        {
+            "page": page,
+            "filters": {
+                "status": status or "",
+                "q": q or "",
+                "started_from": started_from or "",
+                "started_to": started_to or "",
+                "has_email": has_email or "",
+                "sort": sort,
+                "direction": direction,
+                "limit": limit,
+                "offset": offset,
+            },
+        },
+    )
+
+
 @router.get("/employees/{employee_id}", response_class=HTMLResponse)
 def employee_detail(
     employee_id: int,
@@ -101,18 +149,40 @@ def runs(request: Request, db: Session = Depends(get_db), settings: Settings = D
 def trigger_run(
     request: Request,
     background_tasks: BackgroundTasks,
+    db: Session = Depends(get_db),
     settings: Settings = Depends(get_settings),
 ):
     require_admin(request, settings)
+    if get_running_run(db):
+        return RedirectResponse("/admin/runs", status_code=303)
 
     def _crawl() -> None:
         with SessionLocal() as db:
-            run_crawl(db, settings)
+            run_crawl_if_idle(db, settings)
 
     background_tasks.add_task(_crawl)
     return RedirectResponse("/admin/runs", status_code=303)
 
 
+@router.post("/crawl-now")
+def crawl_now(
+    request: Request,
+    background_tasks: BackgroundTasks,
+    db: Session = Depends(get_db),
+    settings: Settings = Depends(get_settings),
+):
+    require_admin(request, settings)
+    if get_running_run(db):
+        return RedirectResponse("/admin", status_code=303)
+
+    def _crawl() -> None:
+        with SessionLocal() as db:
+            run_crawl_if_idle(db, settings)
+
+    background_tasks.add_task(_crawl)
+    return RedirectResponse("/admin", status_code=303)
+
+
 def _render(request: Request, template: str, context: dict, status_code: int = 200) -> HTMLResponse:
     payload = {
         "request": request,
@@ -121,3 +191,14 @@ def _render(request: Request, template: str, context: dict, status_code: int = 2
         **context,
     }
     return templates.TemplateResponse(template, payload, status_code=status_code)
+
+
+def _parse_date(value: str | None):
+    if not value:
+        return None
+    try:
+        from datetime import date
+
+        return date.fromisoformat(value)
+    except ValueError:
+        return None

app/api.py

@@ -1,12 +1,15 @@
+from datetime import date
+
 from fastapi import APIRouter, BackgroundTasks, Depends, Request
-from sqlalchemy import desc, or_, select
+from sqlalchemy import desc, select
 from sqlalchemy.orm import Session
 
 from app.config import Settings, get_settings
 from app.db import SessionLocal, get_db
 from app.models import CrawlRun, Employee
 from app.security import require_admin
-from app.services.crawler import run_crawl
+from app.services.admin_data import employee_display_payload, list_employees_page, run_payload, stats_payload
+from app.services.crawl_control import get_running_run, run_crawl_if_idle
 from app.version import BACKEND_VERSION, FRONTEND_VERSION
 
 router = APIRouter(prefix="/api")
@@ -22,20 +25,29 @@ def list_employees(
     request: Request,
     status: str | None = None,
     q: str | None = None,
+    started_from: date | None = None,
+    started_to: date | None = None,
+    has_email: bool | None = None,
+    sort: str = "full_name",
+    direction: str = "asc",
     limit: int = 50,
     offset: int = 0,
     db: Session = Depends(get_db),
     settings: Settings = Depends(get_settings),
 ) -> dict:
     require_admin(request, settings)
-    stmt = select(Employee)
-    if status:
-        stmt = stmt.where(Employee.status == status)
-    if q:
-        pattern = f"%{q}%"
-        stmt = stmt.where(or_(Employee.full_name.ilike(pattern), Employee.canonical_url.ilike(pattern)))
-    employees = db.scalars(stmt.order_by(Employee.full_name).limit(limit).offset(offset)).all()
-    return {"items": [_employee_summary(item) for item in employees], "limit": limit, "offset": offset}
+    return list_employees_page(
+        db,
+        status=status,
+        q=q,
+        started_from=started_from,
+        started_to=started_to,
+        has_email=has_email,
+        sort=sort,
+        direction=direction,
+        limit=limit,
+        offset=offset,
+    )
 
 
 @router.get("/employees/{employee_id}")
@@ -61,34 +73,53 @@ def list_crawl_runs(
 ) -> dict:
     require_admin(request, settings)
     runs = db.scalars(select(CrawlRun).order_by(desc(CrawlRun.started_at)).limit(limit)).all()
-    return {"items": [_run_summary(run) for run in runs]}
+    return {"items": [run_payload(run) for run in runs]}
+
+
+@router.get("/crawl-runs/latest")
+def latest_crawl_run(
+    request: Request,
+    db: Session = Depends(get_db),
+    settings: Settings = Depends(get_settings),
+) -> dict:
+    require_admin(request, settings)
+    running = get_running_run(db)
+    latest = db.scalar(select(CrawlRun).order_by(desc(CrawlRun.started_at)).limit(1))
+    return {"running": run_payload(running), "latest": run_payload(latest)}
 
 
 @router.post("/crawl-runs")
 def trigger_crawl(
     request: Request,
     background_tasks: BackgroundTasks,
+    db: Session = Depends(get_db),
     settings: Settings = Depends(get_settings),
 ) -> dict:
     require_admin(request, settings)
+    running = get_running_run(db)
+    if running:
+        return {"status": "already_running", "run": run_payload(running)}
 
     def _crawl() -> None:
         with SessionLocal() as db:
-            run_crawl(db, settings)
+            run_crawl_if_idle(db, settings)
 
     background_tasks.add_task(_crawl)
     return {"status": "scheduled"}
 
 
+@router.get("/stats")
+def stats(
+    request: Request,
+    db: Session = Depends(get_db),
+    settings: Settings = Depends(get_settings),
+) -> dict:
+    require_admin(request, settings)
+    return stats_payload(db)
+
+
 def _employee_summary(employee: Employee) -> dict:
-    return {
-        "id": employee.id,
-        "full_name": employee.full_name,
-        "status": employee.status,
-        "canonical_url": employee.canonical_url,
-        "last_seen_at": employee.last_seen_at.isoformat() if employee.last_seen_at else None,
-        "dismissed_at": employee.dismissed_at.isoformat() if employee.dismissed_at else None,
-    }
+    return employee_display_payload(employee)
 
 
 def _employee_detail(employee: Employee) -> dict:
@@ -99,15 +130,4 @@ def _employee_detail(employee: Employee) -> dict:
 
 
 def _run_summary(run: CrawlRun) -> dict:
-    return {
-        "id": run.id,
-        "source_url": run.source_url,
-        "status": run.status,
-        "started_at": run.started_at.isoformat() if run.started_at else None,
-        "finished_at": run.finished_at.isoformat() if run.finished_at else None,
-        "found_count": run.found_count,
-        "parsed_count": run.parsed_count,
-        "error_count": run.error_count,
-        "dismissed_count": run.dismissed_count,
-        "message": run.message,
-    }
+    return run_payload(run) or {}

app/models.py

@@ -69,6 +69,7 @@ class CrawlRun(Base):
     finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     found_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
     parsed_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
+    new_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
     error_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
     dismissed_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
     message: Mapped[str | None] = mapped_column(Text)

app/services/admin_data.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+from datetime import date, datetime, time
+from math import ceil
+from typing import Any
+
+from sqlalchemy import Select, Text, and_, desc, func, or_, select
+from sqlalchemy.orm import Session
+
+from app.models import CrawlRun, Employee
+
+EMPLOYEE_SORTS = {
+    "full_name": Employee.full_name,
+    "status": Employee.status,
+    "first_seen_at": Employee.first_seen_at,
+    "last_seen_at": Employee.last_seen_at,
+    "dismissed_at": Employee.dismissed_at,
+    "hse_start_year": Employee.current_data["hse_start_year"].as_integer(),
+}
+
+
+def employee_display_payload(employee: Employee) -> dict[str, Any]:
+    data = employee.current_data or {}
+    contacts = data.get("contacts") or {}
+    sections = data.get("sections") or []
+    emails = contacts.get("emails") or []
+    phones = contacts.get("phones") or []
+    return {
+        "id": employee.id,
+        "full_name": employee.full_name,
+        "status": employee.status,
+        "canonical_url": employee.canonical_url,
+        "positions": data.get("positions") or [],
+        "positions_text": "; ".join(data.get("positions") or []),
+        "hse_start_year": data.get("hse_start_year"),
+        "emails": emails,
+        "email_text": ", ".join(emails),
+        "phones": phones,
+        "phone_text": ", ".join(phones),
+        "address": contacts.get("address"),
+        "publications_count": _count_section_items(sections, "publications"),
+        "courses_count": _count_section_items(sections, "courses_by_year"),
+        "first_seen_at": employee.first_seen_at.isoformat() if employee.first_seen_at else None,
+        "last_seen_at": employee.last_seen_at.isoformat() if employee.last_seen_at else None,
+        "dismissed_at": employee.dismissed_at.isoformat() if employee.dismissed_at else None,
+    }
+
+
+def build_employee_query(
+    *,
+    status: str | None = None,
+    q: str | None = None,
+    started_from: date | None = None,
+    started_to: date | None = None,
+    has_email: bool | None = None,
+) -> Select[tuple[Employee]]:
+    stmt = select(Employee)
+    filters = []
+    if status:
+        filters.append(Employee.status == status)
+    if q:
+        pattern = f"%{q}%"
+        filters.append(or_(Employee.full_name.ilike(pattern), Employee.canonical_url.ilike(pattern)))
+    if started_from:
+        filters.append(Employee.first_seen_at >= datetime.combine(started_from, time.min))
+    if started_to:
+        filters.append(Employee.first_seen_at <= datetime.combine(started_to, time.max))
+    if has_email is True:
+        filters.append(Employee.current_data.cast(Text).ilike("%@%"))
+    elif has_email is False:
+        filters.append(or_(Employee.current_data.is_(None), ~Employee.current_data.cast(Text).ilike("%@%")))
+    if filters:
+        stmt = stmt.where(and_(*filters))
+    return stmt
+
+
+def list_employees_page(
+    db: Session,
+    *,
+    status: str | None = None,
+    q: str | None = None,
+    started_from: date | None = None,
+    started_to: date | None = None,
+    has_email: bool | None = None,
+    sort: str = "full_name",
+    direction: str = "asc",
+    limit: int = 50,
+    offset: int = 0,
+) -> dict[str, Any]:
+    limit = max(1, min(limit, 200))
+    offset = max(0, offset)
+    base_stmt = build_employee_query(
+        status=status,
+        q=q,
+        started_from=started_from,
+        started_to=started_to,
+        has_email=has_email,
+    )
+    total = db.scalar(select(func.count()).select_from(base_stmt.subquery())) or 0
+    sort_column = EMPLOYEE_SORTS.get(sort, Employee.full_name)
+    order = desc(sort_column) if direction == "desc" else sort_column
+    employees = db.scalars(base_stmt.order_by(order).limit(limit).offset(offset)).all()
+    return {
+        "items": [employee_display_payload(employee) for employee in employees],
+        "total": total,
+        "limit": limit,
+        "offset": offset,
+        "pages": ceil(total / limit) if total else 0,
+        "page": (offset // limit) + 1,
+    }
+
+
+def stats_payload(db: Session) -> dict[str, Any]:
+    latest_run = db.scalar(select(CrawlRun).order_by(desc(CrawlRun.started_at)).limit(1))
+    running_run = db.scalar(select(CrawlRun).where(CrawlRun.status == "running").order_by(desc(CrawlRun.started_at)).limit(1))
+    latest_added = db.scalar(select(Employee).order_by(desc(Employee.first_seen_at)).limit(1))
+    return {
+        "total": db.scalar(select(func.count()).select_from(Employee)) or 0,
+        "active": db.scalar(select(func.count()).select_from(Employee).where(Employee.status == "active")) or 0,
+        "dismissed": db.scalar(select(func.count()).select_from(Employee).where(Employee.status == "dismissed")) or 0,
+        "new_in_last_run": latest_run.new_count if latest_run else 0,
+        "latest_added": employee_display_payload(latest_added) if latest_added else None,
+        "latest_run": run_payload(latest_run) if latest_run else None,
+        "current_running_run": run_payload(running_run) if running_run else None,
+    }
+
+
+def run_payload(run: CrawlRun | None) -> dict[str, Any] | None:
+    if not run:
+        return None
+    processed = run.parsed_count + run.error_count
+    percent = round((processed / run.found_count) * 100, 1) if run.found_count else 0
+    return {
+        "id": run.id,
+        "source_url": run.source_url,
+        "status": run.status,
+        "started_at": run.started_at.isoformat() if run.started_at else None,
+        "finished_at": run.finished_at.isoformat() if run.finished_at else None,
+        "found_count": run.found_count,
+        "parsed_count": run.parsed_count,
+        "new_count": run.new_count,
+        "error_count": run.error_count,
+        "dismissed_count": run.dismissed_count,
+        "processed_count": processed,
+        "progress_percent": percent,
+        "message": run.message,
+    }
+
+
+def _count_section_items(sections: list[dict[str, Any]], section_type: str) -> int:
+    total = 0
+    for section in sections:
+        if section.get("type") != section_type:
+            continue
+        if section_type == "publications":
+            total += len(section.get("publications") or section.get("items") or [])
+        elif section_type == "courses_by_year":
+            total += len(section.get("courses") or [])
+    return total

app/services/crawl_control.py

@@ -0,0 +1,17 @@
+from sqlalchemy import desc, select
+from sqlalchemy.orm import Session
+
+from app.config import Settings
+from app.models import CrawlRun
+from app.services.crawler import run_crawl
+
+
+def get_running_run(db: Session) -> CrawlRun | None:
+    return db.scalar(select(CrawlRun).where(CrawlRun.status == "running").order_by(desc(CrawlRun.started_at)).limit(1))
+
+
+def run_crawl_if_idle(db: Session, settings: Settings) -> tuple[CrawlRun, bool]:
+    running = get_running_run(db)
+    if running:
+        return running, False
+    return run_crawl(db, settings), True

app/services/crawler.py

@@ -106,6 +106,7 @@ def _upsert_employee(db: Session, run: CrawlRun, parsed: dict) -> Employee:
             first_seen_at=now,
         )
         db.add(employee)
+        run.new_count += 1
 
     employee.full_name = parsed.get("full_name")
     employee.status = "active"

app/static/admin.css

@@ -151,3 +151,262 @@
   border-radius: 8px;
   white-space: pre-wrap;
 }
+
+.stats-strip {
+  display: grid;
+  grid-template-columns: repeat(auto-fit, minmax(220px, 1fr));
+  gap: 14px;
+  margin-top: 16px;
+}
+
+.stats-strip__item {
+  padding: 14px 16px;
+  background: #ffffff;
+  border: 1px solid #d9dee7;
+  border-radius: 8px;
+}
+
+.stats-strip__label {
+  display: block;
+  color: #6b7280;
+  font-size: 12px;
+  text-transform: uppercase;
+}
+
+.stats-strip__value {
+  display: block;
+  margin-top: 6px;
+  color: #1f2937;
+  font-weight: 700;
+}
+
+.progress-panel {
+  display: grid;
+  gap: 12px;
+}
+
+.progress-panel__header {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  gap: 16px;
+}
+
+.progress-panel__body {
+  display: grid;
+  gap: 10px;
+}
+
+.progress-panel__meta {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 12px;
+  color: #4b5563;
+  font-size: 14px;
+}
+
+.progress-panel__percent {
+  color: #0f766e;
+  font-weight: 700;
+}
+
+.progress-panel__empty {
+  margin: 0;
+  color: #6b7280;
+}
+
+.progress-bar {
+  height: 12px;
+  overflow: hidden;
+  background: #e5e7eb;
+  border-radius: 999px;
+}
+
+.progress-bar__fill {
+  height: 100%;
+  width: 0;
+  background: #0f766e;
+  transition: width 0.25s ease;
+}
+
+.directory {
+  display: grid;
+  gap: 18px;
+}
+
+.directory__header {
+  display: flex;
+  align-items: end;
+  justify-content: space-between;
+  gap: 16px;
+}
+
+.directory__title {
+  margin: 0;
+  font-size: 24px;
+}
+
+.directory__summary {
+  margin: 6px 0 0;
+  color: #6b7280;
+}
+
+.directory__filters {
+  display: grid;
+  grid-template-columns: minmax(220px, 1.7fr) repeat(6, minmax(120px, 1fr));
+  gap: 10px;
+  padding: 16px;
+  background: #ffffff;
+  border: 1px solid #d9dee7;
+  border-radius: 8px;
+}
+
+.directory__input {
+  min-width: 0;
+  padding: 10px 12px;
+  border: 1px solid #cbd5e1;
+  border-radius: 6px;
+}
+
+.directory__table-wrap {
+  overflow-x: auto;
+  background: #ffffff;
+  border: 1px solid #d9dee7;
+  border-radius: 8px;
+}
+
+.directory__pagination {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  gap: 16px;
+}
+
+.directory__page {
+  color: #4b5563;
+  font-weight: 700;
+}
+
+.directory-table {
+  width: 100%;
+  min-width: 1120px;
+  border-collapse: collapse;
+}
+
+.directory-table__head {
+  padding: 12px 10px;
+  color: #374151;
+  background: #f9fafb;
+  border-bottom: 1px solid #e5e7eb;
+  font-size: 13px;
+  text-align: left;
+  white-space: nowrap;
+}
+
+.directory-table__cell {
+  max-width: 280px;
+  padding: 12px 10px;
+  border-bottom: 1px solid #e5e7eb;
+  vertical-align: top;
+}
+
+.directory-table__row {
+  cursor: pointer;
+}
+
+.directory-table__row:hover {
+  background: #f0fdfa;
+}
+
+.directory-table__empty {
+  padding: 28px;
+  color: #6b7280;
+  text-align: center;
+}
+
+.directory-table__cell--hidden,
+.directory-table__head--hidden {
+  display: none;
+}
+
+.columns-modal {
+  position: fixed;
+  inset: 0;
+  z-index: 50;
+  display: grid;
+  place-items: center;
+  padding: 20px;
+}
+
+.columns-modal[hidden] {
+  display: none;
+}
+
+.columns-modal__backdrop {
+  position: absolute;
+  inset: 0;
+  background: rgba(17, 24, 39, 0.54);
+}
+
+.columns-modal__panel {
+  position: relative;
+  width: min(620px, 100%);
+  max-height: min(720px, calc(100vh - 40px));
+  overflow: auto;
+  padding: 20px;
+  background: #ffffff;
+  border-radius: 8px;
+  box-shadow: 0 24px 80px rgba(15, 23, 42, 0.22);
+}
+
+.columns-modal__header {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  gap: 12px;
+}
+
+.columns-modal__title {
+  margin: 0;
+  font-size: 18px;
+}
+
+.columns-modal__grid {
+  display: grid;
+  grid-template-columns: repeat(auto-fit, minmax(180px, 1fr));
+  gap: 10px;
+  margin-top: 18px;
+}
+
+.columns-modal__option {
+  display: flex;
+  align-items: center;
+  gap: 8px;
+  padding: 10px;
+  background: #f9fafb;
+  border: 1px solid #e5e7eb;
+  border-radius: 6px;
+}
+
+.columns-modal__checkbox {
+  width: 16px;
+  height: 16px;
+}
+
+@media (max-width: 920px) {
+  .directory__filters {
+    grid-template-columns: 1fr 1fr;
+  }
+
+  .progress-panel__header,
+  .directory__header {
+    align-items: stretch;
+    flex-direction: column;
+  }
+}
+
+@media (max-width: 620px) {
+  .directory__filters {
+    grid-template-columns: 1fr;
+  }
+}

app/static/admin.js

@@ -0,0 +1,111 @@
+(function () {
+  const columnDefaults = [
+    "full_name",
+    "status",
+    "positions",
+    "hse_start_year",
+    "email",
+    "last_seen_at",
+    "dismissed_at",
+    "profile",
+  ];
+  const storageKey = "miem.directory.columns";
+
+  function readColumns() {
+    try {
+      const stored = JSON.parse(localStorage.getItem(storageKey) || "[]");
+      return Array.isArray(stored) && stored.length ? stored : columnDefaults;
+    } catch (_error) {
+      return columnDefaults;
+    }
+  }
+
+  function writeColumns(columns) {
+    localStorage.setItem(storageKey, JSON.stringify(columns));
+  }
+
+  function applyColumns(columns) {
+    document.querySelectorAll("[data-column]").forEach((node) => {
+      const visible = columns.includes(node.dataset.column);
+      node.classList.toggle("directory-table__cell--hidden", !visible && node.classList.contains("directory-table__cell"));
+      node.classList.toggle("directory-table__head--hidden", !visible && node.classList.contains("directory-table__head"));
+    });
+    document.querySelectorAll("[data-column-toggle]").forEach((checkbox) => {
+      checkbox.checked = columns.includes(checkbox.value);
+    });
+  }
+
+  function setupColumns() {
+    if (!document.querySelector("[data-directory-table]")) return;
+    let columns = readColumns();
+    const modal = document.querySelector("[data-columns-modal]");
+    applyColumns(columns);
+
+    document.querySelectorAll("[data-columns-open]").forEach((button) => {
+      button.addEventListener("click", () => {
+        modal.hidden = false;
+      });
+    });
+    document.querySelectorAll("[data-columns-close]").forEach((button) => {
+      button.addEventListener("click", () => {
+        modal.hidden = true;
+      });
+    });
+    document.querySelectorAll("[data-column-toggle]").forEach((checkbox) => {
+      checkbox.addEventListener("change", () => {
+        columns = Array.from(document.querySelectorAll("[data-column-toggle]:checked")).map((item) => item.value);
+        if (!columns.length) columns = ["full_name"];
+        writeColumns(columns);
+        applyColumns(columns);
+      });
+    });
+    document.querySelectorAll("[data-row-href]").forEach((row) => {
+      row.addEventListener("click", (event) => {
+        if (event.target.closest("a, button, input, select, label")) return;
+        window.location.href = row.dataset.rowHref;
+      });
+    });
+  }
+
+  function setupProgress() {
+    const panel = document.querySelector("[data-progress-panel]");
+    if (!panel) return;
+
+    const update = (run) => {
+      if (!run) return;
+      const status = document.querySelector("[data-progress-status]");
+      const processed = document.querySelector("[data-progress-processed]");
+      const found = document.querySelector("[data-progress-found]");
+      const errors = document.querySelector("[data-progress-errors]");
+      const fill = document.querySelector("[data-progress-fill]");
+      const percent = document.querySelector("[data-progress-percent]");
+      if (status) status.textContent = run.status;
+      if (processed) processed.textContent = run.processed_count;
+      if (found) found.textContent = run.found_count;
+      if (errors) errors.textContent = run.error_count;
+      if (fill) fill.style.width = `${run.progress_percent}%`;
+      if (percent) percent.textContent = run.progress_percent;
+    };
+
+    const poll = async () => {
+      try {
+        const response = await fetch("/api/crawl-runs/latest", { credentials: "same-origin" });
+        if (!response.ok) return false;
+        const data = await response.json();
+        const run = data.running || data.latest;
+        update(run);
+        return Boolean(data.running);
+      } catch (_error) {
+        return false;
+      }
+    };
+
+    const interval = window.setInterval(async () => {
+      const keepGoing = await poll();
+      if (!keepGoing) window.clearInterval(interval);
+    }, 4000);
+  }
+
+  setupColumns();
+  setupProgress();
+})();

app/templates/base.html

@@ -11,6 +11,7 @@
       <h1 class="admin__brand">MIEM Employees</h1>
       <nav class="admin__nav">
         <a class="admin__link" href="/admin">Dashboard</a>
+        <a class="admin__link" href="/admin/directory">Directory</a>
         <a class="admin__link" href="/admin/employees">Employees</a>
         <a class="admin__link" href="/admin/runs">Runs</a>
         <form method="post" action="/admin/logout">
@@ -24,5 +25,6 @@
     <footer class="admin__footer">
       Backend {{ backend_version }} · Frontend {{ frontend_version }}
     </footer>
+    {% block scripts %}{% endblock %}
   </body>
 </html>

app/templates/dashboard.html

@@ -2,10 +2,48 @@
 {% block title %}Dashboard · MIEM Employees{% endblock %}
 {% block content %}
 <section class="admin__grid">
+  <div class="metric"><div class="metric__label">Total</div><div class="metric__value">{{ counts.total }}</div></div>
   <div class="metric"><div class="metric__label">Active</div><div class="metric__value">{{ counts.active }}</div></div>
+  <div class="metric"><div class="metric__label">New in last run</div><div class="metric__value">{{ counts.new_in_last_run }}</div></div>
   <div class="metric"><div class="metric__label">Dismissed</div><div class="metric__value">{{ counts.dismissed }}</div></div>
-  <div class="metric"><div class="metric__label">Runs</div><div class="metric__value">{{ counts.runs }}</div></div>
-  <div class="metric"><div class="metric__label">Errors</div><div class="metric__value">{{ counts.errors }}</div></div>
+</section>
+<section class="stats-strip">
+  <div class="stats-strip__item">
+    <span class="stats-strip__label">Latest added</span>
+    {% if counts.latest_added %}
+    <a class="stats-strip__value" href="/admin/employees/{{ counts.latest_added.id }}">{{ counts.latest_added.full_name or counts.latest_added.canonical_url }}</a>
+    {% else %}
+    <span class="stats-strip__value">No employees yet</span>
+    {% endif %}
+  </div>
+  <div class="stats-strip__item">
+    <span class="stats-strip__label">Runs</span>
+    <span class="stats-strip__value">{{ counts.runs }}</span>
+  </div>
+  <div class="stats-strip__item">
+    <span class="stats-strip__label">Errors</span>
+    <span class="stats-strip__value">{{ counts.errors }}</span>
+  </div>
+</section>
+<section class="panel progress-panel" data-progress-panel>
+  <div class="progress-panel__header">
+    <h2 class="panel__title">Parsing progress</h2>
+    <form method="post" action="/admin/crawl-now">
+      <button class="button" type="submit">Start crawl now</button>
+    </form>
+  </div>
+  {% set run = counts.current_running_run or latest_run %}
+  <div class="progress-panel__body" data-progress-body>
+    <div class="progress-panel__meta">
+      <span data-progress-status>{{ run.status if run else "idle" }}</span>
+      <span><span data-progress-processed>{{ run.processed_count if run else 0 }}</span> / <span data-progress-found>{{ run.found_count if run else 0 }}</span> processed</span>
+      <span><span data-progress-errors>{{ run.error_count if run else 0 }}</span> errors</span>
+    </div>
+    <div class="progress-bar" aria-label="Parsing progress">
+      <div class="progress-bar__fill" data-progress-fill style="width: {{ run.progress_percent if run else 0 }}%"></div>
+    </div>
+    <div class="progress-panel__percent"><span data-progress-percent>{{ run.progress_percent if run else 0 }}</span>%</div>
+  </div>
 </section>
 <section class="panel">
   <h2 class="panel__title">Latest runs</h2>
@@ -19,3 +57,6 @@
   </table>
 </section>
 {% endblock %}
+{% block scripts %}
+<script src="/static/admin.js"></script>
+{% endblock %}

app/templates/directory.html

@@ -0,0 +1,112 @@
+{% extends "base.html" %}
+{% block title %}Directory · MIEM Employees{% endblock %}
+{% block content %}
+<section class="directory">
+  <div class="directory__header">
+    <div>
+      <h2 class="directory__title">Directory</h2>
+      <p class="directory__summary">{{ page.total }} employees found</p>
+    </div>
+    <button class="button" type="button" data-columns-open>Columns</button>
+  </div>
+
+  <form class="directory__filters" method="get" action="/admin/directory">
+    <input class="directory__input" name="q" value="{{ filters.q }}" placeholder="Name or URL">
+    <select class="directory__input" name="status">
+      <option value="" {% if not filters.status %}selected{% endif %}>All statuses</option>
+      <option value="active" {% if filters.status == "active" %}selected{% endif %}>Active</option>
+      <option value="dismissed" {% if filters.status == "dismissed" %}selected{% endif %}>Dismissed</option>
+    </select>
+    <select class="directory__input" name="has_email">
+      <option value="" {% if not filters.has_email %}selected{% endif %}>Any email</option>
+      <option value="true" {% if filters.has_email == "true" %}selected{% endif %}>Has email</option>
+      <option value="false" {% if filters.has_email == "false" %}selected{% endif %}>No email</option>
+    </select>
+    <input class="directory__input" type="date" name="started_from" value="{{ filters.started_from }}" aria-label="First seen from">
+    <input class="directory__input" type="date" name="started_to" value="{{ filters.started_to }}" aria-label="First seen to">
+    <select class="directory__input" name="sort">
+      {% for value, label in [("full_name", "Name"), ("status", "Status"), ("hse_start_year", "HSE start"), ("first_seen_at", "First seen"), ("last_seen_at", "Last seen"), ("dismissed_at", "Dismissed")] %}
+      <option value="{{ value }}" {% if filters.sort == value %}selected{% endif %}>Sort: {{ label }}</option>
+      {% endfor %}
+    </select>
+    <select class="directory__input" name="direction">
+      <option value="asc" {% if filters.direction == "asc" %}selected{% endif %}>Ascending</option>
+      <option value="desc" {% if filters.direction == "desc" %}selected{% endif %}>Descending</option>
+    </select>
+    <button class="button" type="submit">Apply</button>
+  </form>
+
+  <div class="directory__table-wrap">
+    <table class="directory-table" data-directory-table>
+      <thead>
+        <tr>
+          <th class="directory-table__head" data-column="full_name">Name</th>
+          <th class="directory-table__head" data-column="status">Status</th>
+          <th class="directory-table__head" data-column="positions">Positions</th>
+          <th class="directory-table__head" data-column="hse_start_year">HSE start</th>
+          <th class="directory-table__head" data-column="email">Email</th>
+          <th class="directory-table__head" data-column="phone">Phone</th>
+          <th class="directory-table__head" data-column="address">Address</th>
+          <th class="directory-table__head" data-column="publications_count">Publications</th>
+          <th class="directory-table__head" data-column="courses_count">Courses</th>
+          <th class="directory-table__head" data-column="first_seen_at">First seen</th>
+          <th class="directory-table__head" data-column="last_seen_at">Last seen</th>
+          <th class="directory-table__head" data-column="dismissed_at">Dismissed</th>
+          <th class="directory-table__head" data-column="profile">Profile</th>
+        </tr>
+      </thead>
+      <tbody>
+        {% for employee in page.items %}
+        <tr class="directory-table__row" data-row-href="/admin/employees/{{ employee.id }}">
+          <td class="directory-table__cell" data-column="full_name">{{ employee.full_name or "No name" }}</td>
+          <td class="directory-table__cell" data-column="status"><span class="badge {% if employee.status == "dismissed" %}badge--dismissed{% endif %}">{{ employee.status }}</span></td>
+          <td class="directory-table__cell" data-column="positions">{{ employee.positions_text }}</td>
+          <td class="directory-table__cell" data-column="hse_start_year">{{ employee.hse_start_year or "" }}</td>
+          <td class="directory-table__cell" data-column="email">{{ employee.email_text }}</td>
+          <td class="directory-table__cell" data-column="phone">{{ employee.phone_text }}</td>
+          <td class="directory-table__cell" data-column="address">{{ employee.address or "" }}</td>
+          <td class="directory-table__cell" data-column="publications_count">{{ employee.publications_count }}</td>
+          <td class="directory-table__cell" data-column="courses_count">{{ employee.courses_count }}</td>
+          <td class="directory-table__cell" data-column="first_seen_at">{{ employee.first_seen_at or "" }}</td>
+          <td class="directory-table__cell" data-column="last_seen_at">{{ employee.last_seen_at or "" }}</td>
+          <td class="directory-table__cell" data-column="dismissed_at">{{ employee.dismissed_at or "" }}</td>
+          <td class="directory-table__cell" data-column="profile"><a class="admin__link" href="{{ employee.canonical_url }}">Open</a></td>
+        </tr>
+        {% else %}
+        <tr><td class="directory-table__empty" colspan="13">No employees match these filters.</td></tr>
+        {% endfor %}
+      </tbody>
+    </table>
+  </div>
+
+  <div class="directory__pagination">
+    {% set prev_offset = filters.offset - filters.limit %}
+    {% set next_offset = filters.offset + filters.limit %}
+    {% if filters.offset > 0 %}
+    <a class="admin__link" href="{{ request.url.include_query_params(offset=prev_offset) }}">Previous</a>
+    {% endif %}
+    <span class="directory__page">Page {{ page.page }}{% if page.pages %} of {{ page.pages }}{% endif %}</span>
+    {% if next_offset < page.total %}
+    <a class="admin__link" href="{{ request.url.include_query_params(offset=next_offset) }}">Next</a>
+    {% endif %}
+  </div>
+</section>
+
+<div class="columns-modal" data-columns-modal hidden>
+  <div class="columns-modal__backdrop" data-columns-close></div>
+  <section class="columns-modal__panel" aria-label="Column settings">
+    <div class="columns-modal__header">
+      <h3 class="columns-modal__title">Visible columns</h3>
+      <button class="button button--ghost" type="button" data-columns-close>Close</button>
+    </div>
+    <div class="columns-modal__grid">
+      {% for key, label in [("full_name", "Name"), ("status", "Status"), ("positions", "Positions"), ("hse_start_year", "HSE start"), ("email", "Email"), ("phone", "Phone"), ("address", "Address"), ("publications_count", "Publications"), ("courses_count", "Courses"), ("first_seen_at", "First seen"), ("last_seen_at", "Last seen"), ("dismissed_at", "Dismissed"), ("profile", "Profile")] %}
+      <label class="columns-modal__option"><input class="columns-modal__checkbox" type="checkbox" value="{{ key }}" data-column-toggle> {{ label }}</label>
+      {% endfor %}
+    </div>
+  </section>
+</div>
+{% endblock %}
+{% block scripts %}
+<script src="/static/admin.js"></script>
+{% endblock %}

app/templates/runs.html

@@ -2,13 +2,43 @@
 {% block title %}Runs · MIEM Employees{% endblock %}
 {% block content %}
 <section class="panel">
-  <h2 class="panel__title">Crawl runs</h2>
-  <form method="post" action="/admin/runs"><button class="button" type="submit">Start crawl</button></form>
+  <div class="progress-panel__header">
+    <h2 class="panel__title">Crawl runs</h2>
+    <form method="post" action="/admin/runs"><button class="button" type="submit">Start crawl now</button></form>
+  </div>
+  {% set run = runs[0] if runs else none %}
+  {% if run %}
+  {% set processed = run.parsed_count + run.error_count %}
+  {% set percent = ((processed / run.found_count) * 100) | round(1) if run.found_count else 0 %}
+  <div class="progress-panel" data-progress-panel>
+    <div class="progress-panel__meta">
+      <span data-progress-status>{{ run.status }}</span>
+      <span><span data-progress-processed>{{ processed }}</span> / <span data-progress-found>{{ run.found_count }}</span> processed</span>
+      <span><span data-progress-errors>{{ run.error_count }}</span> errors</span>
+    </div>
+    <div class="progress-bar" aria-label="Parsing progress">
+      <div class="progress-bar__fill" data-progress-fill style="width: {{ percent }}%"></div>
+    </div>
+  <div class="progress-panel__percent"><span data-progress-percent>{{ percent }}</span>%</div>
+  </div>
+  {% else %}
+  <div class="progress-panel" data-progress-panel>
+    <div class="progress-panel__meta">
+      <span data-progress-status>idle</span>
+      <span><span data-progress-processed>0</span> / <span data-progress-found>0</span> processed</span>
+      <span><span data-progress-errors>0</span> errors</span>
+    </div>
+    <div class="progress-bar" aria-label="Parsing progress">
+      <div class="progress-bar__fill" data-progress-fill style="width: 0%"></div>
+    </div>
+    <div class="progress-panel__percent"><span data-progress-percent>0</span>%</div>
+  </div>
+  {% endif %}
   <table class="table">
-    <thead><tr><th class="table__head">ID</th><th class="table__head">Status</th><th class="table__head">Found</th><th class="table__head">Parsed</th><th class="table__head">Errors</th><th class="table__head">Dismissed</th></tr></thead>
+    <thead><tr><th class="table__head">ID</th><th class="table__head">Status</th><th class="table__head">Found</th><th class="table__head">Parsed</th><th class="table__head">New</th><th class="table__head">Errors</th><th class="table__head">Dismissed</th></tr></thead>
     <tbody>
       {% for run in runs %}
-      <tr><td class="table__cell">{{ run.id }}</td><td class="table__cell">{{ run.status }}</td><td class="table__cell">{{ run.found_count }}</td><td class="table__cell">{{ run.parsed_count }}</td><td class="table__cell">{{ run.error_count }}</td><td class="table__cell">{{ run.dismissed_count }}</td></tr>
+      <tr><td class="table__cell">{{ run.id }}</td><td class="table__cell">{{ run.status }}</td><td class="table__cell">{{ run.found_count }}</td><td class="table__cell">{{ run.parsed_count }}</td><td class="table__cell">{{ run.new_count }}</td><td class="table__cell">{{ run.error_count }}</td><td class="table__cell">{{ run.dismissed_count }}</td></tr>
       {% endfor %}
     </tbody>
   </table>
@@ -25,3 +55,6 @@
   </table>
 </section>
 {% endblock %}
+{% block scripts %}
+<script src="/static/admin.js"></script>
+{% endblock %}

app/version.py

@@ -1,3 +1,3 @@
-APP_VERSION = "0.1.0"
-FRONTEND_VERSION = "0.1.0"
-BACKEND_VERSION = "0.1.0"
+APP_VERSION = "0.2.0"
+FRONTEND_VERSION = "0.2.0"
+BACKEND_VERSION = "0.2.0"