HDXDSYS-2507 Add optional retriever parameter to resource download method of HDX Python API (#94)

* Allow retriever to be passed to download method

Author: mcarans

requirements.txt

@@ -12,7 +12,7 @@ attrs==25.4.0
     #   referencing
 babel==2.17.0
     # via mkdocs-material
-backrefs==5.9
+backrefs==6.0.1
     # via mkdocs-material
 cachetools==6.2.1
     # via google-auth
@@ -22,17 +22,17 @@ cfgv==3.4.0
     # via pre-commit
 chardet==5.2.0
     # via frictionless
-charset-normalizer==3.4.3
+charset-normalizer==3.4.4
     # via requests
-ckanapi==4.8
+ckanapi==4.9
     # via hdx-python-api (pyproject.toml)
 click==8.3.0
     # via
     #   mkdocs
     #   typer
 colorama==0.4.6
     # via mkdocs-material
-coverage==7.10.7
+coverage==7.11.3
     # via pytest-cov
 defopt==7.0.0
     # via hdx-python-api (pyproject.toml)
@@ -44,7 +44,7 @@ docopt==0.6.2
     # via
     #   ckanapi
     #   num2words
-docutils==0.22.2
+docutils==0.22.3
     # via defopt
 email-validator==2.3.0
     # via hdx-python-api (pyproject.toml)
@@ -56,7 +56,7 @@ frictionless==5.18.1
     # via hdx-python-utilities
 ghp-import==2.1.0
     # via mkdocs
-google-auth==2.41.1
+google-auth==2.43.0
     # via
     #   google-auth-oauthlib
     #   gspread
@@ -70,7 +70,7 @@ hdx-python-utilities==3.9.5
     # via
     #   hdx-python-api (pyproject.toml)
     #   hdx-python-country
-humanize==4.13.0
+humanize==4.14.0
     # via frictionless
 identify==2.6.15
     # via pre-commit
@@ -82,7 +82,7 @@ ijson==3.4.0.post0
     # via hdx-python-utilities
 inflect==7.5.0
     # via quantulum3
-iniconfig==2.1.0
+iniconfig==2.3.0
     # via pytest
 isodate==0.7.2
     # via frictionless
@@ -110,14 +110,14 @@ loguru==0.7.3
     # via hdx-python-utilities
 makefun==1.16.0
     # via hdx-python-api (pyproject.toml)
-markdown==3.9
+markdown==3.10
     # via
     #   mkdocs
     #   mkdocs-material
     #   pymdown-extensions
 markdown-it-py==4.0.0
     # via rich
-marko==2.2.0
+marko==2.2.1
     # via frictionless
 markupsafe==3.0.3
     # via
@@ -137,7 +137,7 @@ mkdocs==1.6.1
     #   mkdocs-material
 mkdocs-get-deps==0.2.0
     # via mkdocs
-mkdocs-material==9.6.21
+mkdocs-material==9.7.0
     # via mkapi
 mkdocs-material-extensions==1.3.1
     # via mkdocs-material
@@ -175,28 +175,28 @@ ply==3.11
     #   libhxl
 pockets==0.9.1
     # via sphinxcontrib-napoleon
-pre-commit==4.3.0
+pre-commit==4.4.0
     # via hdx-python-api (pyproject.toml)
 pyasn1==0.6.1
     # via
     #   pyasn1-modules
     #   rsa
 pyasn1-modules==0.4.2
     # via google-auth
-pydantic==2.12.0
+pydantic==2.12.4
     # via frictionless
-pydantic-core==2.41.1
+pydantic-core==2.41.5
     # via pydantic
 pygments==2.19.2
     # via
     #   mkdocs-material
     #   pytest
     #   rich
-pymdown-extensions==10.16.1
+pymdown-extensions==10.17.1
     # via mkdocs-material
 pyphonetics==0.5.3
     # via hdx-python-utilities
-pytest==8.4.2
+pytest==9.0.0
     # via
     #   hdx-python-api (pyproject.toml)
     #   pytest-check
@@ -232,7 +232,7 @@ quantulum3==0.9.2
     # via hdx-python-api (pyproject.toml)
 ratelimit==2.2.1
     # via hdx-python-utilities
-referencing==0.36.2
+referencing==0.37.0
     # via
     #   jsonschema
     #   jsonschema-specifications
@@ -245,21 +245,21 @@ requests==2.32.5
     #   mkdocs-material
     #   requests-file
     #   requests-oauthlib
-requests-file==2.1.0
+requests-file==3.0.1
     # via hdx-python-utilities
 requests-oauthlib==2.0.0
     # via google-auth-oauthlib
 rfc3986==2.0.0
     # via frictionless
 rich==14.2.0
     # via typer
-rpds-py==0.27.1
+rpds-py==0.28.0
     # via
     #   jsonschema
     #   referencing
 rsa==4.9.1
     # via google-auth
-ruamel-yaml==0.18.15
+ruamel-yaml==0.18.16
     # via hdx-python-utilities
 ruamel-yaml-clib==0.2.14
     # via ruamel-yaml
@@ -279,7 +279,7 @@ six==1.17.0
     #   sphinxcontrib-napoleon
 sphinxcontrib-napoleon==0.7
     # via defopt
-structlog==25.4.0
+structlog==25.5.0
     # via libhxl
 tableschema-to-template==0.0.13
     # via hdx-python-utilities
@@ -291,7 +291,7 @@ text-unidecode==1.3
     # via python-slugify
 typeguard==4.4.4
     # via inflect
-typer==0.19.2
+typer==0.20.0
     # via frictionless
 typing-extensions==4.15.0
     # via
@@ -313,7 +313,7 @@ urllib3==2.5.0
     #   requests
 validators==0.35.0
     # via frictionless
-virtualenv==20.35.3
+virtualenv==20.35.4
     # via pre-commit
 watchdog==6.0.0
     # via mkdocs

src/hdx/data/resource.py

@@ -15,6 +15,7 @@
 from hdx.data.resource_view import ResourceView
 from hdx.utilities.dateparse import now_utc, now_utc_notz, parse_date
 from hdx.utilities.downloader import Download
+from hdx.utilities.retriever import Retrieve
 from hdx.utilities.typehint import ListTuple
 from hdx.utilities.uuid import is_valid_uuid
 
@@ -591,12 +592,15 @@ def search_in_hdx(
             logger.debug(result)
         return resources
 
-    def download(self, folder: Optional[str] = None) -> Tuple[str, str]:
+    def download(
+        self, folder: Optional[str] = None, retriever: Optional[Retrieve] = None
+    ) -> Tuple[str, str]:
         """Download resource store to provided folder or temporary folder if no folder
         supplied
 
         Args:
             folder (Optional[str]): Folder to download resource to. Defaults to None.
+            retriever (Optional[Retrieve]): Retrieve object to use. Defaults to None.
 
         Returns:
             Tuple[str, str]: (URL downloaded, Path to downloaded file)
@@ -613,14 +617,16 @@ def download(self, folder: Optional[str] = None) -> Tuple[str, str]:
             filename = f"{filename}{file_format}"
         apikey = self.configuration.get_api_key()
         if apikey:
-            headers = {"Authorization": self.configuration.get_api_key()}
+            headers = {"Authorization": apikey}
         else:
             headers = None
         with Download(
             full_agent=self.configuration.get_user_agent(),
             use_env=False,
             headers=headers,
         ) as downloader:
+            if retriever:
+                downloader = retriever.clone(downloader)
             path = downloader.download_file(url, folder=folder, filename=filename)
             return url, path
 

tests/hdx/data/test_resource.py

@@ -16,7 +16,9 @@
 from hdx.data.resource import Resource
 from hdx.utilities.dateparse import parse_date
 from hdx.utilities.dictandlist import merge_two_dictionaries
-from hdx.utilities.downloader import DownloadError
+from hdx.utilities.downloader import Download, DownloadError
+from hdx.utilities.path import get_temp_dir
+from hdx.utilities.retriever import Retrieve
 
 resultdict = {
     "cache_last_updated": None,
@@ -940,17 +942,26 @@ def test_search_in_hdx(self, configuration, search):
 
     def test_download(self, configuration, read):
         resource = Resource.read_from_hdx("74b74ae1-df0c-4716-829f-4f939a046811")
-        resource2 = Resource.read_from_hdx("74b74ae1-df0c-4716-829f-4f939a046814")
         url, path = resource.download()
         remove(path)
         assert (
             url
             == "https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/main/tests/fixtures/test_data.csv"
         )
         assert basename(path) == "MyResource1.csv"
+        tempdir = get_temp_dir()
+        retriever = Retrieve(Download(user_agent="test"), tempdir, tempdir, tempdir)
+        url, path = resource.download(retriever=retriever)
+        remove(path)
+        assert (
+            url
+            == "https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/main/tests/fixtures/test_data.csv"
+        )
+
         resource["url"] = ""
         with pytest.raises(HDXError):
             resource.download()
+        resource2 = Resource.read_from_hdx("74b74ae1-df0c-4716-829f-4f939a046814")
         with pytest.raises(DownloadError):
             resource2.download()