feature: add MIEM employees parser service with admin UI and MCP
This commit is contained in:
45
app/worker.py
Normal file
45
app/worker.py
Normal file
@@ -0,0 +1,45 @@
|
||||
import logging
|
||||
import signal
|
||||
import time
|
||||
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
|
||||
from app.config import get_settings
|
||||
from app.db import SessionLocal, init_db
|
||||
from app.services.crawler import run_crawl
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def crawl_once() -> None:
|
||||
settings = get_settings()
|
||||
with SessionLocal() as db:
|
||||
run = run_crawl(db, settings)
|
||||
logger.info("crawl finished: id=%s status=%s parsed=%s errors=%s", run.id, run.status, run.parsed_count, run.error_count)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
init_db()
|
||||
settings = get_settings()
|
||||
scheduler = BackgroundScheduler(timezone="Europe/Moscow")
|
||||
scheduler.add_job(crawl_once, CronTrigger.from_crontab(settings.crawl_cron), id="weekly_miem_crawl", replace_existing=True)
|
||||
scheduler.start()
|
||||
logger.info("worker started with cron=%s", settings.crawl_cron)
|
||||
|
||||
stop = False
|
||||
|
||||
def _stop(*_: object) -> None:
|
||||
nonlocal stop
|
||||
stop = True
|
||||
|
||||
signal.signal(signal.SIGTERM, _stop)
|
||||
signal.signal(signal.SIGINT, _stop)
|
||||
while not stop:
|
||||
time.sleep(1)
|
||||
scheduler.shutdown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user