import base64 import tempfile import unittest import urllib.error from email.message import Message from pathlib import Path from unittest.mock import Mock, patch from ingestion.minjust_cbd import CbdClient, SyncResult, progress_line, sync_archive class MinjustCbdTest(unittest.TestCase): def test_archives_document_and_resumes(self): image = base64.b64encode(b"image").decode() document = { "Code": 1, "Name": {"Rus": "Закон", "Kyr": "Мыйзам"}, "Editions": [ { "Code": 10, "Name": {"Rus": "01.01.2026", "Kyr": "01.01.2026"}, "Data": {"Rus": "

Закон

", "Kyr": "

Мыйзам

"}, "Images": [ { "Lang": "Russian", "Name": "image001.jpg", "Data": image, } ], } ], } client = Mock() client.total_documents = 1 client.document_codes.return_value = [1] client.document.return_value = document with tempfile.TemporaryDirectory() as directory: output = Path(directory) first = sync_archive(output, client) second = sync_archive(output, client) refreshed = sync_archive(output, client, refresh=True) self.assertEqual(first.downloaded, 1) self.assertEqual(second.skipped, 1) self.assertEqual(refreshed.downloaded, 1) self.assertEqual(client.document.call_count, 2) self.assertEqual( (output / "documents/1/editions/10/ru.html").read_text(), "

Закон

", ) self.assertEqual( (output / "documents/1/editions/10/images/ru/image001.jpg").read_bytes(), b"image", ) def test_uses_stable_list_id_for_next_page(self): client = CbdClient(requests_per_second=1000) client.request_json = Mock( side_effect=[ { "Id": "list-id", "TotalCount": 3, "Documents": [{"Code": 1}, {"Code": 2}], }, { "Id": "list-id", "TotalCount": 3, "Documents": [{"Code": 3}], }, ] ) self.assertEqual(list(client.document_codes(page_size=2)), [1, 2, 3]) self.assertEqual( client.request_json.call_args_list[1].args[0], "GetDocumentListById" ) def test_recreates_expired_list_at_current_page(self): client = CbdClient(requests_per_second=1000) client.request_json = Mock( side_effect=[ { "Id": "expired-list", "TotalCount": 3, "Documents": [{"Code": 1}, {"Code": 2}], }, urllib.error.HTTPError("https://example.test", 404, "", {}, None), { "Id": "new-list", "TotalCount": 3, "Documents": [{"Code": 3}], }, ] ) self.assertEqual(list(client.document_codes(page_size=2)), [1, 2, 3]) recovery_call = client.request_json.call_args_list[2] self.assertEqual(recovery_call.args[0], "GetDocumentListByQuery") self.assertIn(("PageNumber", 2), recovery_call.args[1]) def test_recreates_list_after_page_retries_are_exhausted(self): client = CbdClient(requests_per_second=1000) client.request_json = Mock( side_effect=[ { "Id": "failed-list", "TotalCount": 3, "Documents": [{"Code": 1}, {"Code": 2}], }, RuntimeError("Ministry of Justice API request failed"), { "Id": "new-list", "TotalCount": 3, "Documents": [{"Code": 3}], }, ] ) self.assertEqual(list(client.document_codes(page_size=2)), [1, 2, 3]) recovery_call = client.request_json.call_args_list[2] self.assertEqual(recovery_call.args[0], "GetDocumentListByQuery") self.assertIn(("PageNumber", 2), recovery_call.args[1]) def test_formats_progress_with_rate_and_eta(self): line = progress_line( SyncResult(discovered=50, downloaded=48, skipped=1, failed=1), total=100, elapsed=25, ) self.assertIn("50.00%", line) self.assertIn("rate=2.00/s", line) self.assertIn("ETA=00:00:25", line) @patch("ingestion.minjust_cbd.time.sleep") @patch("ingestion.minjust_cbd.urllib.request.urlopen") def test_respects_retry_after_on_rate_limit(self, urlopen, sleep): headers = Message() headers["Retry-After"] = "7" response = Mock() response.read.return_value = b"{}" response.__enter__ = Mock(return_value=response) response.__exit__ = Mock(return_value=False) urlopen.side_effect = [ urllib.error.HTTPError("https://example.test", 429, "", headers, None), response, ] CbdClient(requests_per_second=1000, retries=2).request_json("Example") self.assertIn(((7.0,), {}), [(call.args, call.kwargs) for call in sleep.call_args_list]) if __name__ == "__main__": unittest.main()