fix: add back-off on requests retry to handle HTTP 429 status

terracatalogueclient/__init__.py

@@ -1,5 +1,5 @@
 __title__ = "terracatalogueclient"
-__version__ = "0.1.16"
+__version__ = "0.1.17"
 __author__ = "Stijn Caerts"
 
 from terracatalogueclient.client import Catalogue, Collection, Product, ProductFile, ProductFileType

terracatalogueclient/client.py

@@ -173,19 +173,27 @@ def __init__(self, config: CatalogueConfig = None):
         self._auth = None
         self.s3 = None
 
-        adapter = requests.adapters.HTTPAdapter(max_retries=requests.adapters.Retry(total=5))
+        adapter = requests.adapters.HTTPAdapter(
+            max_retries=requests.adapters.Retry(
+                total=5,
+                backoff_factor=2,
+                status_forcelist=[429, 500, 502, 503, 504]
+            )
+        )
 
         self._session_search = requests.Session()
         self._session_search.headers.update(_DEFAULT_REQUEST_HEADERS)
         self._session_search.headers.update({
             "Accept": "application/json, application/geo+json"
         })
-        self._session_search.mount("http", adapter)
+        self._session_search.mount("http://", adapter)
+        self._session_search.mount("https://", adapter)
 
         self._session_download = requests.Session()
         self._session_download.headers.update(_DEFAULT_REQUEST_HEADERS)
         self._session_download.headers.update({"Accept": "application/json"})
-        self._session_download.mount("http", adapter)
+        self._session_download.mount("http://", adapter)
+        self._session_download.mount("https://", adapter)
 
     def authenticate(self) -> 'Catalogue':
         """

tests/test_cgls.py

@@ -48,3 +48,13 @@ def test_download(self):
         with tempfile.TemporaryDirectory() as tmpdir:
             catalogue.download_product(next(products), tmpdir)
 
+    def test_download_multiple(self):
+        catalogue = Catalogue(self.config_cgls)
+        params = {
+            "collection": "clms_global_lst_5km_v2_hourly_netcdf",
+            "start": date(2023, 1, 1),
+            "end": date(2023, 1, 2)
+        }
+        products = catalogue.get_products(**params)
+        with tempfile.TemporaryDirectory() as tmpdir:
+            catalogue.download_products(products, tmpdir, force=True)