feature: add MIEM employees parser service with admin UI and MCP

This commit is contained in:
Anton
2026-04-28 16:20:51 +03:00
parent 6480f31e8f
commit d512580960
29 changed files with 1883 additions and 0 deletions

45
app/worker.py Normal file
View File

@@ -0,0 +1,45 @@
import logging
import signal
import time
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.triggers.cron import CronTrigger
from app.config import get_settings
from app.db import SessionLocal, init_db
from app.services.crawler import run_crawl
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
logger = logging.getLogger(__name__)
def crawl_once() -> None:
settings = get_settings()
with SessionLocal() as db:
run = run_crawl(db, settings)
logger.info("crawl finished: id=%s status=%s parsed=%s errors=%s", run.id, run.status, run.parsed_count, run.error_count)
def main() -> None:
init_db()
settings = get_settings()
scheduler = BackgroundScheduler(timezone="Europe/Moscow")
scheduler.add_job(crawl_once, CronTrigger.from_crontab(settings.crawl_cron), id="weekly_miem_crawl", replace_existing=True)
scheduler.start()
logger.info("worker started with cron=%s", settings.crawl_cron)
stop = False
def _stop(*_: object) -> None:
nonlocal stop
stop = True
signal.signal(signal.SIGTERM, _stop)
signal.signal(signal.SIGINT, _stop)
while not stop:
time.sleep(1)
scheduler.shutdown()
if __name__ == "__main__":
main()