fix: recover expired Ministry document lists

backend/ingestion/minjust_cbd.py

@@ -21,7 +21,7 @@ from datetime import datetime, timezone
 from pathlib import Path
 from typing import Callable, Iterable
 
-APP_VERSION = "0.1.1"
+APP_VERSION = "0.1.2"
 API_BASE_URL = "https://cbd.minjust.gov.kg/api/v1/OpenData/"
 LANGUAGES = {"Rus": "ru", "Kyr": "ky"}
 IMAGE_LANGUAGES = {"Russian": "ru", "Kyrgyz": "ky"}
@@ -87,10 +87,17 @@ class CbdClient:
         self.request_json("CheckAvailable")
 
     def document_codes(self, limit: int | None = None, page_size: int = 1000):
-        first = self.request_json(
-            "GetDocumentListByQuery",
-            (("Property", "Code"), ("PageSize", page_size), ("PageNumber", 1)),
-        )
+        def query_page(page_number: int):
+            return self.request_json(
+                "GetDocumentListByQuery",
+                (
+                    ("Property", "Code"),
+                    ("PageSize", page_size),
+                    ("PageNumber", page_number),
+                ),
+            )
+
+        first = query_page(1)
         yielded = 0
         page = first
         page_number = 1
@@ -104,15 +111,23 @@ class CbdClient:
             if yielded >= page["TotalCount"]:
                 return
             page_number += 1
-            page = self.request_json(
-                "GetDocumentListById",
-                (
-                    ("DocumentListId", first["Id"]),
-                    ("Property", "Code"),
-                    ("PageSize", page_size),
-                    ("PageNumber", page_number),
-                ),
-            )
+            try:
+                page = self.request_json(
+                    "GetDocumentListById",
+                    (
+                        ("DocumentListId", first["Id"]),
+                        ("Property", "Code"),
+                        ("PageSize", page_size),
+                        ("PageNumber", page_number),
+                    ),
+                )
+            except urllib.error.HTTPError as error:
+                if error.code != 404:
+                    raise
+                first = page = query_page(page_number)
+                self.total_documents = min(
+                    first["TotalCount"], limit or first["TotalCount"]
+                )
 
     def document(self, code: int) -> dict:
         return self.request_json(
@@ -306,7 +321,7 @@ def sync_archive(
                 SyncResult(discovered, downloaded, skipped, failed),
                 client.total_documents,
             )
-        if discovered % 100 == 0:
+        if not progress and discovered % 100 == 0:
             LOGGER.info(
                 "discovered=%s downloaded=%s skipped=%s failed=%s",
                 discovered,