diff --git a/CHANGELOG.md b/CHANGELOG.md index 905f46b..fa710c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.7.3 + +- Восстановлена проверка профилей сотрудников и хранение истории URL профиля. + ## 0.7.2 - В каталоге сотрудников добавлены фильтр и колонка учёной степени. diff --git a/README.md b/README.md index 2fb00f0..2fc4514 100644 --- a/README.md +++ b/README.md @@ -147,4 +147,4 @@ docker compose exec postgres pg_dump -U miem miem_workers > backup.sql docker compose down ``` -Версия сервиса: `0.7.1`. Админка всегда показывает версии backend и frontend в footer. +Версия сервиса: `0.7.3`. Админка всегда показывает версии backend и frontend в footer. diff --git a/app/db.py b/app/db.py index 095a666..8bfeb4f 100644 --- a/app/db.py +++ b/app/db.py @@ -41,6 +41,20 @@ def _ensure_runtime_schema() -> None: models.EmployeeNewsLink.__table__.create(bind=engine, checkfirst=True) inspector = inspect(engine) table_names = set(inspector.get_table_names()) + if "employees" in table_names and "employee_profile_urls" not in table_names: + models.EmployeeProfileUrl.__table__.create(bind=engine, checkfirst=True) + inspector = inspect(engine) + if "employees" in table_names: + employee_columns = {column["name"] for column in inspector.get_columns("employees")} + missing_columns = [] + if "profile_unavailable_streak" not in employee_columns: + missing_columns.append("profile_unavailable_streak INTEGER NOT NULL DEFAULT 0") + if "last_profile_check_at" not in employee_columns: + missing_columns.append("last_profile_check_at TIMESTAMPTZ") + if missing_columns: + with engine.begin() as connection: + for column in missing_columns: + connection.execute(text(f"ALTER TABLE employees ADD COLUMN {column}")) if "crawl_runs" not in table_names: return crawl_run_columns = {column["name"] for column in inspector.get_columns("crawl_runs")} diff --git a/app/models.py b/app/models.py index 1251612..f00cff6 100644 --- a/app/models.py +++ b/app/models.py @@ -33,6 +33,8 @@ class Employee(Base): first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False) last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False) dismissed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + profile_unavailable_streak: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + last_profile_check_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) parser_version: Mapped[str | None] = mapped_column(String(32)) current_data: Mapped[dict | None] = mapped_column(json_type) current_checksum: Mapped[str | None] = mapped_column(String(64)) @@ -40,12 +42,29 @@ class Employee(Base): updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow, nullable=False) snapshots: Mapped[list["EmployeeSnapshot"]] = relationship(back_populates="employee") + profile_urls: Mapped[list["EmployeeProfileUrl"]] = relationship(back_populates="employee", cascade="all, delete-orphan") tabs: Mapped[list["ProfileTab"]] = relationship(back_populates="employee", cascade="all, delete-orphan") publications: Mapped[list["EmployeePublication"]] = relationship(back_populates="employee", cascade="all, delete-orphan") news_links: Mapped[list["EmployeeNewsLink"]] = relationship(back_populates="employee", cascade="all, delete-orphan") crawl_run_changes: Mapped[list["CrawlRunEmployeeChange"]] = relationship(back_populates="employee") +class EmployeeProfileUrl(Base): + __tablename__ = "employee_profile_urls" + __table_args__ = ( + UniqueConstraint("employee_id", "url", name="uq_employee_profile_urls_employee_url"), + Index("ix_employee_profile_urls_employee_id", "employee_id"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + employee_id: Mapped[int] = mapped_column(ForeignKey("employees.id", ondelete="CASCADE"), nullable=False) + url: Mapped[str] = mapped_column(Text, nullable=False) + first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False) + last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, nullable=False) + + employee: Mapped[Employee] = relationship(back_populates="profile_urls") + + class EmployeeSnapshot(Base): __tablename__ = "employee_snapshots" __table_args__ = (Index("ix_employee_snapshots_employee_id", "employee_id"),) diff --git a/app/services/admin_data.py b/app/services/admin_data.py index 15d68f3..3394b61 100644 --- a/app/services/admin_data.py +++ b/app/services/admin_data.py @@ -162,6 +162,10 @@ def stats_payload(db: Session) -> dict[str, Any]: return { "total": db.scalar(select(func.count()).select_from(Employee)) or 0, "active": db.scalar(select(func.count()).select_from(Employee).where(Employee.status == "active")) or 0, + "verification_required": db.scalar( + select(func.count()).select_from(Employee).where(Employee.status == "verification_required") + ) + or 0, "dismissed": db.scalar(select(func.count()).select_from(Employee).where(Employee.status == "dismissed")) or 0, "new_in_last_run": latest_run.new_count if latest_run else 0, "latest_added": employee_display_payload(latest_added) if latest_added else None, @@ -205,7 +209,7 @@ def run_detail_payload(db: Session, run: CrawlRun | None) -> dict[str, Any] | No .order_by(CrawlRunEmployeeChange.created_at, CrawlRunEmployeeChange.id) ).all() errors = db.scalars(select(CrawlError).where(CrawlError.crawl_run_id == run.id).order_by(CrawlError.created_at)).all() - grouped_changes = {"new": [], "missing_from_source": [], "dismissed": []} + grouped_changes = {"new": [], "missing_from_source": [], "verification_required": [], "dismissed": []} for change in changes: grouped_changes.setdefault(change.change_type, []).append(_change_payload(change)) return { @@ -262,7 +266,7 @@ def _json_text_contains(data_text: Any, value: str) -> Any: def _employee_status_display(status: str | None) -> str: - labels = {"active": "Работает", "dismissed": "Уволен"} + labels = {"active": "Работает", "verification_required": "Требует проверки", "dismissed": "Уволен"} return labels.get(status or "", status or "Не указано") diff --git a/app/templates/directory.html b/app/templates/directory.html index 8436bf9..d198c2b 100644 --- a/app/templates/directory.html +++ b/app/templates/directory.html @@ -15,6 +15,7 @@