feat: add employee news links parsing and storage

app/db.py

@@ -37,6 +37,10 @@ def _ensure_runtime_schema() -> None:
         models.EmployeePublication.__table__.create(bind=engine, checkfirst=True)
         inspector = inspect(engine)
         table_names = set(inspector.get_table_names())
+    if "employees" in table_names and "employee_news_links" not in table_names:
+        models.EmployeeNewsLink.__table__.create(bind=engine, checkfirst=True)
+        inspector = inspect(engine)
+        table_names = set(inspector.get_table_names())
     if "crawl_runs" not in table_names:
         return
     crawl_run_columns = {column["name"] for column in inspector.get_columns("crawl_runs")}

app/models.py

@@ -42,6 +42,7 @@ class Employee(Base):
     snapshots: Mapped[list["EmployeeSnapshot"]] = relationship(back_populates="employee")
     tabs: Mapped[list["ProfileTab"]] = relationship(back_populates="employee", cascade="all, delete-orphan")
     publications: Mapped[list["EmployeePublication"]] = relationship(back_populates="employee", cascade="all, delete-orphan")
+    news_links: Mapped[list["EmployeeNewsLink"]] = relationship(back_populates="employee", cascade="all, delete-orphan")
     crawl_run_changes: Mapped[list["CrawlRunEmployeeChange"]] = relationship(back_populates="employee")
 
 
@@ -97,6 +98,32 @@ class EmployeePublication(Base):
     employee: Mapped[Employee] = relationship(back_populates="publications")
 
 
+class EmployeeNewsLink(Base):
+    __tablename__ = "employee_news_links"
+    __table_args__ = (
+        UniqueConstraint("employee_id", "url", name="uq_employee_news_links_employee_url"),
+        UniqueConstraint("employee_id", "source_hash", name="uq_employee_news_links_employee_source_hash"),
+        Index("ix_employee_news_links_employee_id", "employee_id"),
+        Index("ix_employee_news_links_url", "url"),
+        Index("ix_employee_news_links_published_at", "published_at"),
+        Index("ix_employee_news_links_published_year", "published_year"),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    employee_id: Mapped[int] = mapped_column(ForeignKey("employees.id", ondelete="CASCADE"), nullable=False)
+    title: Mapped[str] = mapped_column(Text, nullable=False)
+    url: Mapped[str | None] = mapped_column(Text)
+    summary: Mapped[str | None] = mapped_column(Text)
+    published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    published_year: Mapped[int | None] = mapped_column(Integer)
+    source_hash: Mapped[str] = mapped_column(String(64), nullable=False)
+    raw_data: Mapped[dict | None] = mapped_column(json_type)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False)
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow, nullable=False)
+
+    employee: Mapped[Employee] = relationship(back_populates="news_links")
+
+
 class CrawlRun(Base):
     __tablename__ = "crawl_runs"
 

app/parser/profile.py

@@ -1,6 +1,7 @@
 import hashlib
 import json
 import re
+from datetime import datetime, timezone
 from urllib.parse import urljoin
 
 from bs4 import BeautifulSoup, NavigableString, Tag
@@ -101,6 +102,8 @@ def extract_person_header(soup: BeautifulSoup, source_url: str) -> dict:
 def extract_sections(soup: BeautifulSoup, source_url: str) -> list[dict]:
     sections = []
     for h2 in soup.select("h2"):
+        if h2.find_parent(class_="post") or h2.find_parent(attrs={"data-tab": "press_links_news"}):
+            continue
         title = normalize_ws(h2.get_text(" ", strip=True))
         if not title or "расписание занятий" in title.lower():
             continue
@@ -142,6 +145,21 @@ def extract_sections(soup: BeautifulSoup, source_url: str) -> list[dict]:
             if section_type in {"generic", "paragraphs"}:
                 section["type"] = "year_blocks"
         sections.append(section)
+    news_links = _parse_news_links(soup, source_url)
+    if news_links:
+        sections.append(
+            {
+                "title": "В новостях",
+                "slug": "v_novostyah",
+                "type": "news",
+                "raw_text": "",
+                "paragraphs": [],
+                "items": [item["title"] for item in news_links if item.get("title")],
+                "links": [{"text": item["title"], "url": item["url"]} for item in news_links if item.get("title") and item.get("url")],
+                "news_count": len(news_links),
+                "news_links": news_links,
+            }
+        )
     return sections
 
 
@@ -575,6 +593,95 @@ def _parse_vkr_items(nodes: list) -> list[str]:
     return [item for item in dict.fromkeys(items) if item]
 
 
+def _parse_news_links(soup: BeautifulSoup, source_url: str) -> list[dict]:
+    news = []
+    for post in soup.select('[data-tab="press_links_news"] .post'):
+        if not isinstance(post, Tag):
+            continue
+        anchor = post.select_one(".post__content h2 a[href], h2 a[href], a[href]")
+        title = normalize_ws(anchor.get_text(" ", strip=True)) if anchor else ""
+        href = normalize_ws(anchor.get("href")) if anchor else ""
+        summary_node = post.select_one(".post__text")
+        summary = normalize_ws(summary_node.get_text(" ", strip=True)) if summary_node else ""
+        published_at = _parse_post_date(post)
+        if not title and not href:
+            continue
+        item = {
+            "title": title or href,
+            "url": urljoin(source_url, href) if href else None,
+            "summary": summary or None,
+            "published_at": published_at.isoformat() if published_at else None,
+            "published_year": published_at.year if published_at else _int_or_none(normalize_ws(_select_text(post, ".post-meta__year"))),
+            "raw_data": {
+                "title": title or href,
+                "url": href or None,
+                "summary": summary or None,
+                "date_text": normalize_ws(_select_text(post, ".post-meta__date")),
+            },
+        }
+        news.append(item)
+    return _dedupe_news_links(news)
+
+
+def _select_text(node: Tag, selector: str) -> str:
+    selected = node.select_one(selector)
+    return selected.get_text(" ", strip=True) if selected else ""
+
+
+def _parse_post_date(post: Tag) -> datetime | None:
+    day = _int_or_none(normalize_ws(_select_text(post, ".post-meta__day")))
+    month = _month_number(normalize_ws(_select_text(post, ".post-meta__month")))
+    year = _int_or_none(normalize_ws(_select_text(post, ".post-meta__year")))
+    if not day or not month or not year:
+        return None
+    try:
+        return datetime(year, month, day, tzinfo=timezone.utc)
+    except ValueError:
+        return None
+
+
+def _month_number(value: str) -> int | None:
+    lowered = value.lower().strip(".")
+    months = {
+        "янв": 1,
+        "январь": 1,
+        "января": 1,
+        "фев": 2,
+        "февр": 2,
+        "февраль": 2,
+        "февраля": 2,
+        "март": 3,
+        "мар": 3,
+        "марта": 3,
+        "апр": 4,
+        "апрель": 4,
+        "апреля": 4,
+        "май": 5,
+        "мая": 5,
+        "июнь": 6,
+        "июня": 6,
+        "июль": 7,
+        "июля": 7,
+        "авг": 8,
+        "август": 8,
+        "августа": 8,
+        "сент": 9,
+        "сен": 9,
+        "сентябрь": 9,
+        "сентября": 9,
+        "окт": 10,
+        "октябрь": 10,
+        "октября": 10,
+        "нояб": 11,
+        "ноябрь": 11,
+        "ноября": 11,
+        "дек": 12,
+        "декабрь": 12,
+        "декабря": 12,
+    }
+    return months.get(lowered)
+
+
 def _normalize_publication_item(item: dict, current_author_id: str | None = None) -> dict:
     publication_id = str(item.get("id") or "").strip()
     title = _html_to_text(item.get("title"))
@@ -698,6 +805,17 @@ def _dedupe_publications(items: list[dict]) -> list[dict]:
     return unique
 
 
+def _dedupe_news_links(items: list[dict]) -> list[dict]:
+    seen = set()
+    unique = []
+    for item in items:
+        key = item.get("url") or item.get("title")
+        if key and key not in seen:
+            seen.add(key)
+            unique.append(item)
+    return unique
+
+
 def _html_to_text(value: object) -> str:
     return normalize_ws(BeautifulSoup(str(value or ""), "html.parser").get_text(" ", strip=True))
 

app/services/admin_data.py

@@ -8,7 +8,7 @@ from zoneinfo import ZoneInfo
 from sqlalchemy import Select, Text, and_, desc, func, or_, select
 from sqlalchemy.orm import Session
 
-from app.models import CrawlError, CrawlRun, CrawlRunEmployeeChange, Employee
+from app.models import CrawlError, CrawlRun, CrawlRunEmployeeChange, Employee, EmployeeNewsLink
 
 EMPLOYEE_SORTS = {
     "full_name": Employee.full_name,
@@ -24,6 +24,7 @@ def employee_display_payload(employee: Employee) -> dict[str, Any]:
     data = _as_dict(employee.current_data)
     contacts = _as_dict(data.get("contacts"))
     sections = _as_list(data.get("sections"))
+    stored_news_links = _stored_news_links(employee)
     positions = _clean_list(data.get("positions"))
     emails = _clean_list(contacts.get("emails"))
     phones = _clean_list(contacts.get("phones"))
@@ -43,6 +44,7 @@ def employee_display_payload(employee: Employee) -> dict[str, Any]:
         "address": contacts.get("address"),
         "publications_count": _count_section_items(sections, "publications"),
         "courses_count": _count_section_items(sections, "courses_by_year"),
+        "news_count": len(stored_news_links) or _count_section_items(sections, "news"),
         "first_seen_at": employee.first_seen_at.isoformat() if employee.first_seen_at else None,
         "last_seen_at": employee.last_seen_at.isoformat() if employee.last_seen_at else None,
         "dismissed_at": employee.dismissed_at.isoformat() if employee.dismissed_at else None,
@@ -67,6 +69,7 @@ def employee_detail_payload(employee: Employee) -> dict[str, Any]:
             "contact_items": _normalize_contact_items(contacts.get("items")),
         },
         "external_ids": _normalize_external_ids(data.get("external_ids")),
+        "news_links": _detail_news_links(employee, data),
         "sections": [_normalize_section(section) for section in _as_list(data.get("sections"))],
     }
 
@@ -276,6 +279,8 @@ def _count_section_items(sections: list[dict[str, Any]], section_type: str) -> i
             total += len(section.get("publications") or section.get("items") or [])
         elif section_type == "courses_by_year":
             total += len(section.get("courses") or [])
+        elif section_type == "news":
+            total += len(section.get("news_links") or section.get("items") or [])
     return total
 
 
@@ -348,6 +353,8 @@ def _normalize_section(section: Any) -> dict[str, Any]:
         "year_entries": _normalize_year_entries(section.get("year_entries")),
         "publications": _normalize_publications(section.get("publications")),
         "publications_count": section.get("publications_count"),
+        "news_links": _normalize_news_links(section.get("news_links")),
+        "news_count": section.get("news_count"),
         "theses": _normalize_theses(section.get("theses")),
         "theses_count": section.get("theses_count"),
         "academic_year": section.get("academic_year"),
@@ -370,6 +377,77 @@ def _normalize_links(items: Any) -> list[dict[str, str | None]]:
     return normalized
 
 
+def _stored_news_links(employee: Employee) -> list[dict[str, Any]]:
+    return [_stored_news_link_payload(item) for item in sorted(employee.news_links, key=_news_link_sort_key)]
+
+
+def _news_link_sort_key(item: EmployeeNewsLink) -> tuple:
+    timestamp = item.published_at.timestamp() if item.published_at else 0
+    return (-timestamp, item.title or "", item.id)
+
+
+def _stored_news_link_payload(item: EmployeeNewsLink) -> dict[str, Any]:
+    return {
+        "title": item.title,
+        "url": item.url,
+        "summary": item.summary,
+        "published_at": item.published_at.isoformat() if item.published_at else None,
+        "published_year": item.published_year,
+        "published_display": format_admin_date(item.published_at) if item.published_at else str(item.published_year or ""),
+    }
+
+
+def _detail_news_links(employee: Employee, data: dict[str, Any]) -> list[dict[str, Any]]:
+    stored = _stored_news_links(employee)
+    if stored:
+        return stored
+    for section in _as_list(data.get("sections")):
+        if isinstance(section, dict) and section.get("type") == "news":
+            return _normalize_news_links(section.get("news_links"))
+    return []
+
+
+def format_admin_date(value: Any) -> str:
+    if not value:
+        return ""
+    if isinstance(value, str):
+        try:
+            value = datetime.fromisoformat(value.replace("Z", "+00:00"))
+        except ValueError:
+            return value
+    if not isinstance(value, datetime):
+        return str(value)
+    if value.tzinfo:
+        value = value.astimezone(ZoneInfo("Europe/Moscow"))
+    return value.strftime("%d.%m.%Y")
+
+
+def _normalize_news_links(items: Any) -> list[dict[str, Any]]:
+    normalized = []
+    if not isinstance(items, list):
+        return normalized
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        title = str(item.get("title") or item.get("url") or "").strip()
+        url = str(item.get("url") or "").strip()
+        summary = str(item.get("summary") or "").strip()
+        published_at = str(item.get("published_at") or "").strip()
+        published_year = item.get("published_year")
+        if title or url:
+            normalized.append(
+                {
+                    "title": title or url,
+                    "url": url or None,
+                    "summary": summary or None,
+                    "published_at": published_at or None,
+                    "published_year": published_year,
+                    "published_display": format_admin_date(published_at) if published_at else str(published_year or ""),
+                }
+            )
+    return normalized
+
+
 def _normalize_year_entries(items: Any) -> list[dict[str, Any]]:
     normalized = []
     if not isinstance(items, list):

app/services/crawler.py

@@ -15,6 +15,7 @@ from app.models import (
     CrawlRun,
     CrawlRunEmployeeChange,
     Employee,
+    EmployeeNewsLink,
     EmployeePublication,
     EmployeeSnapshot,
     ParserSource,
@@ -230,6 +231,7 @@ def _upsert_employee(db: Session, run: CrawlRun, parsed: dict) -> tuple[Employee
         )
     db.flush()
     _try_sync_employee_publications(db, run, employee, parsed)
+    _try_sync_employee_news_links(db, run, employee, parsed)
     return employee, changed
 
 
@@ -349,6 +351,108 @@ def _int_or_none(value: object) -> int | None:
         return None
 
 
+def _try_sync_employee_news_links(db: Session, run: CrawlRun, employee: Employee, parsed: dict) -> None:
+    try:
+        if not _news_link_payloads(parsed):
+            return
+        if not _employee_news_links_table_exists(db):
+            return
+        with db.begin_nested():
+            _sync_employee_news_links(db, employee, parsed)
+    except Exception as exc:
+        db.add(
+            CrawlError(
+                crawl_run_id=run.id,
+                profile_url=employee.canonical_url,
+                error_type=type(exc).__name__,
+                message=f"Не удалось сохранить новости сотрудника: {exc}",
+            )
+        )
+
+
+def _employee_news_links_table_exists(db: Session) -> bool:
+    return inspect(db.connection()).has_table(EmployeeNewsLink.__tablename__)
+
+
+def _sync_employee_news_links(db: Session, employee: Employee, parsed: dict) -> None:
+    news_links = _news_link_payloads(parsed)
+    seen_hashes = set()
+    for news_link in news_links:
+        source_hash = _news_link_hash(news_link)
+        seen_hashes.add(source_hash)
+        url = _clean_optional(news_link.get("url"))
+        existing = None
+        if url:
+            existing = db.scalar(
+                select(EmployeeNewsLink).where(
+                    EmployeeNewsLink.employee_id == employee.id,
+                    EmployeeNewsLink.url == url,
+                )
+            )
+        if not existing:
+            existing = db.scalar(
+                select(EmployeeNewsLink).where(
+                    EmployeeNewsLink.employee_id == employee.id,
+                    EmployeeNewsLink.source_hash == source_hash,
+                )
+            )
+        if not existing:
+            existing = EmployeeNewsLink(employee_id=employee.id, source_hash=source_hash, title=_news_link_title(news_link))
+            db.add(existing)
+        _apply_news_link(existing, news_link, source_hash)
+
+    if seen_hashes:
+        stale = db.scalars(
+            select(EmployeeNewsLink).where(
+                EmployeeNewsLink.employee_id == employee.id,
+                EmployeeNewsLink.source_hash.not_in(seen_hashes),
+            )
+        ).all()
+        for item in stale:
+            db.delete(item)
+
+
+def _news_link_payloads(parsed: dict) -> list[dict]:
+    news_links = []
+    for section in parsed.get("sections") or []:
+        if not isinstance(section, dict) or section.get("type") != "news":
+            continue
+        for item in section.get("news_links") or []:
+            if isinstance(item, dict):
+                news_links.append(item)
+    return news_links
+
+
+def _apply_news_link(target: EmployeeNewsLink, news_link: dict, source_hash: str) -> None:
+    target.title = _news_link_title(news_link)
+    target.url = _clean_optional(news_link.get("url"))
+    target.summary = _clean_optional(news_link.get("summary"))
+    target.published_at = _datetime_or_none(news_link.get("published_at"))
+    target.published_year = _int_or_none(news_link.get("published_year"))
+    target.raw_data = news_link.get("raw_data") if isinstance(news_link.get("raw_data"), dict) else news_link
+    target.source_hash = source_hash
+
+
+def _news_link_hash(news_link: dict) -> str:
+    return _payload_hash(news_link.get("raw_data") if isinstance(news_link.get("raw_data"), dict) else news_link)
+
+
+def _news_link_title(news_link: dict) -> str:
+    return _clean_optional(news_link.get("title") or news_link.get("url")) or "Untitled news"
+
+
+def _datetime_or_none(value: object) -> datetime | None:
+    if isinstance(value, datetime):
+        return value
+    if not value:
+        return None
+    try:
+        parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+    except ValueError:
+        return None
+    return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)
+
+
 def _mark_dismissed(db: Session, run: CrawlRun, found_keys: set[str], session: requests.Session, timeout: int) -> int:
     dismissed = 0
     active = db.scalars(select(Employee).where(Employee.status == "active")).all()

app/templates/directory.html

@@ -55,6 +55,7 @@
           <th class="directory-table__head" data-column="address">Адрес</th>
           <th class="directory-table__head" data-column="publications_count">Публикации</th>
           <th class="directory-table__head" data-column="courses_count">Курсы</th>
+          <th class="directory-table__head" data-column="news_count">Новости</th>
           <th class="directory-table__head" data-column="first_seen_at">Впервые найден</th>
           <th class="directory-table__head" data-column="last_seen_at">Последний раз найден</th>
           <th class="directory-table__head" data-column="dismissed_at">Дата увольнения</th>
@@ -73,13 +74,14 @@
           <td class="directory-table__cell" data-column="address">{{ employee.address or "" }}</td>
           <td class="directory-table__cell" data-column="publications_count">{{ employee.publications_count }}</td>
           <td class="directory-table__cell" data-column="courses_count">{{ employee.courses_count }}</td>
+          <td class="directory-table__cell" data-column="news_count">{{ employee.news_count }}</td>
           <td class="directory-table__cell" data-column="first_seen_at">{{ employee.first_seen_display }}</td>
           <td class="directory-table__cell" data-column="last_seen_at">{{ employee.last_seen_display }}</td>
           <td class="directory-table__cell" data-column="dismissed_at">{{ employee.dismissed_display }}</td>
           <td class="directory-table__cell" data-column="profile"><a class="admin__link" href="{{ employee.canonical_url }}">Открыть</a></td>
         </tr>
         {% else %}
-        <tr><td class="directory-table__empty" colspan="13">По этим фильтрам сотрудники не найдены.</td></tr>
+        <tr><td class="directory-table__empty" colspan="14">По этим фильтрам сотрудники не найдены.</td></tr>
         {% endfor %}
       </tbody>
     </table>
@@ -106,7 +108,7 @@
       <button class="button button--ghost" type="button" data-columns-close>Закрыть</button>
     </div>
     <div class="columns-modal__grid">
-      {% for key, label in [("full_name", "ФИО"), ("status", "Статус"), ("positions", "Должности"), ("hse_start_year", "Год начала"), ("email", "Email"), ("phone", "Телефон"), ("address", "Адрес"), ("publications_count", "Публикации"), ("courses_count", "Курсы"), ("first_seen_at", "Впервые найден"), ("last_seen_at", "Последний раз найден"), ("dismissed_at", "Дата увольнения"), ("profile", "Профиль")] %}
+      {% for key, label in [("full_name", "ФИО"), ("status", "Статус"), ("positions", "Должности"), ("hse_start_year", "Год начала"), ("email", "Email"), ("phone", "Телефон"), ("address", "Адрес"), ("publications_count", "Публикации"), ("courses_count", "Курсы"), ("news_count", "Новости"), ("first_seen_at", "Впервые найден"), ("last_seen_at", "Последний раз найден"), ("dismissed_at", "Дата увольнения"), ("profile", "Профиль")] %}
       <label class="columns-modal__option"><input class="columns-modal__checkbox" type="checkbox" value="{{ key }}" data-column-toggle> {{ label }}</label>
       {% endfor %}
     </div>

app/templates/employee_detail.html

@@ -104,6 +104,25 @@
   </section>
   {% endif %}
 
+  {% if employee_view.news_links %}
+  <section class="employee-card__section">
+    <h3 class="employee-section__title">В новостях</h3>
+    <ul class="employee-card__list">
+      {% for news in employee_view.news_links %}
+      <li class="employee-card__list-item">
+        {% if news.published_display %}<div class="employee-section__meta"><span class="employee-section__meta-item">{{ news.published_display }}</span></div>{% endif %}
+        {% if news.url %}
+        <a class="admin__link" href="{{ news.url }}">{{ news.title }}</a>
+        {% else %}
+        {{ news.title }}
+        {% endif %}
+        {% if news.summary %}<div class="employee-section__text">{{ news.summary }}</div>{% endif %}
+      </li>
+      {% endfor %}
+    </ul>
+  </section>
+  {% endif %}
+
   <section class="employee-card__section">
     <h3 class="employee-section__title">Разделы профиля</h3>
     {% if employee_view.sections %}

app/version.py

@@ -1,3 +1,3 @@
-APP_VERSION = "0.6.2"
-FRONTEND_VERSION = "0.6.2"
-BACKEND_VERSION = "0.6.2"
+APP_VERSION = "0.7.0"
+FRONTEND_VERSION = "0.7.0"
+BACKEND_VERSION = "0.7.0"