53 lines
1.4 KiB
Python
53 lines
1.4 KiB
Python
import logging
|
|
import signal
|
|
import time
|
|
|
|
from apscheduler.schedulers.background import BackgroundScheduler
|
|
from apscheduler.triggers.cron import CronTrigger
|
|
|
|
from app.config import get_settings
|
|
from app.db import SessionLocal, init_db
|
|
from app.services.crawler import run_crawl
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def crawl_once() -> None:
|
|
settings = get_settings()
|
|
with SessionLocal() as db:
|
|
run = run_crawl(db, settings)
|
|
logger.info(
|
|
"crawl finished: id=%s status=%s parsed=%s skipped=%s errors=%s",
|
|
run.id,
|
|
run.status,
|
|
run.parsed_count,
|
|
run.skipped_count,
|
|
run.error_count,
|
|
)
|
|
|
|
|
|
def main() -> None:
|
|
init_db()
|
|
settings = get_settings()
|
|
scheduler = BackgroundScheduler(timezone="Europe/Moscow")
|
|
scheduler.add_job(crawl_once, CronTrigger.from_crontab(settings.crawl_cron), id="weekly_miem_crawl", replace_existing=True)
|
|
scheduler.start()
|
|
logger.info("worker started with cron=%s", settings.crawl_cron)
|
|
|
|
stop = False
|
|
|
|
def _stop(*_: object) -> None:
|
|
nonlocal stop
|
|
stop = True
|
|
|
|
signal.signal(signal.SIGTERM, _stop)
|
|
signal.signal(signal.SIGINT, _stop)
|
|
while not stop:
|
|
time.sleep(1)
|
|
scheduler.shutdown()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|