New HDX state class

Author: mcarans

requirements.txt

@@ -32,7 +32,7 @@ click==8.1.8
     #   typer
 colorama==0.4.6
     # via mkdocs-material
-coverage==7.8.0
+coverage==7.8.1
     # via pytest-cov
 defopt==6.4.0
     # via hdx-python-api (pyproject.toml)
@@ -56,7 +56,7 @@ frictionless==5.18.1
     # via hdx-python-utilities
 ghp-import==2.1.0
     # via mkdocs
-google-auth==2.40.1
+google-auth==2.40.2
     # via
     #   google-auth-oauthlib
     #   gspread
@@ -250,7 +250,7 @@ rfc3986==2.0.0
     # via frictionless
 rich==14.0.0
     # via typer
-rpds-py==0.25.0
+rpds-py==0.25.1
     # via
     #   jsonschema
     #   referencing
@@ -296,7 +296,7 @@ typing-extensions==4.13.2
     #   typeguard
     #   typer
     #   typing-inspection
-typing-inspection==0.4.0
+typing-inspection==0.4.1
     # via pydantic
 unidecode==1.4.0
     # via

src/hdx/api/utilities/hdx_state.py

@@ -0,0 +1,63 @@
+"""Utility to save state to a dataset and read it back."""
+
+import logging
+from os.path import join
+from typing import Any, Callable
+
+from hdx.data.dataset import Dataset
+from hdx.utilities.loader import load_text
+from hdx.utilities.saver import save_text
+from hdx.utilities.state import State
+
+logger = logging.getLogger(__name__)
+
+
+class HDXState(State):
+    """State class that allows the reading and writing of state to a given HDX
+    dataset. Input and output state transformations can be supplied in read_fn and
+    write_fn respectively. The input state transformation takes in a string
+    while the output transformation outputs a string.
+
+    Args:
+        dataset_name_or_id (str): Dataset name or ID
+        path (str): Path to temporary folder for state
+        read_fn (Callable[[str], Any]): Input state transformation. Defaults to lambda x: x.
+        write_fn: Callable[[Any], str]: Output state transformation. Defaults to lambda x: x.
+    """
+
+    def __init__(
+        self,
+        dataset_name_or_id: str,
+        path: str,
+        read_fn: Callable[[str], Any] = lambda x: x,
+        write_fn: Callable[[Any], str] = lambda x: x,
+    ) -> None:
+        self.dataset_name_or_id = dataset_name_or_id
+        self.resource = None
+        super().__init__(path, read_fn, write_fn)
+
+    def read(self) -> Any:
+        """Read state from HDX dataset
+
+        Returns:
+            Any: State
+        """
+        dataset = Dataset.read_from_hdx(self.dataset_name_or_id)
+        self.resource = dataset.get_resource()
+        _, path = self.resource.download()
+        value = self.read_fn(load_text(path))
+        logger.info(f"State read from {self.dataset_name_or_id} = {value}")
+        return value
+
+    def write(self) -> None:
+        """Write state to HDX dataset
+
+        Returns:
+            None
+        """
+        logger.info(f"State written to {self.dataset_name_or_id} = {self.state}")
+        filename = self.resource["name"]
+        file_to_upload = join(self.path, filename)
+        save_text(self.write_fn(self.state), file_to_upload)
+        self.resource.set_file_to_upload(file_to_upload)
+        self.resource.update_in_hdx()

tests/fixtures/state/analysis_dates.txt

@@ -0,0 +1 @@
+DEFAULT=2020-09-23

tests/fixtures/state/last_build_date.txt

@@ -0,0 +1 @@
+2020-09-23

tests/hdx/utilities/test_hdx_state.py

@@ -0,0 +1,163 @@
+"""HDXState Utility Tests"""
+
+import json
+from copy import deepcopy
+from datetime import datetime, timezone
+from os import makedirs
+from os.path import exists, join
+from shutil import copyfile, rmtree
+from tempfile import gettempdir
+
+import pytest
+
+from tests.hdx.data import MockResponse, dataset_resultdict
+from tests.hdx.data.test_resource import resultdict
+
+from hdx.api.configuration import Configuration
+from hdx.api.utilities.hdx_state import HDXState
+from hdx.utilities.dateparse import iso_string_from_datetime, parse_date
+
+
+class TestState:
+    @pytest.fixture(scope="class")
+    def tempfolder(self):
+        return join(gettempdir(), "test_state")
+
+    @pytest.fixture(scope="class")
+    def statefolder(self, fixturesfolder):
+        return join(fixturesfolder, "state")
+
+    @pytest.fixture(scope="class")
+    def statefile(self):
+        return "last_build_date.txt"
+
+    @pytest.fixture(scope="class")
+    def multidatestatefile(self):
+        return "analysis_dates.txt"
+
+    @pytest.fixture(scope="class")
+    def date1(self):
+        return datetime(2020, 9, 23, 0, 0, tzinfo=timezone.utc)
+
+    @pytest.fixture(scope="class")
+    def date2(self):
+        return datetime(2022, 5, 12, 10, 15, tzinfo=timezone.utc)
+
+    @pytest.fixture(scope="function")
+    def do_state(self, tempfolder, statefile):
+        class MockSession:
+            @staticmethod
+            def post(url, data, headers, files, allow_redirects, auth=None):
+                if "resource" in url:
+                    result = json.dumps(resultdict)
+                    return MockResponse(
+                        200,
+                        '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_show"}'
+                        % result,
+                    )
+                else:
+                    myresultdict = deepcopy(dataset_resultdict)
+                    resource = myresultdict["resources"][0]
+                    resource["name"] = statefile
+                    resource["url"] = join(tempfolder, statefile)
+                    result = json.dumps(myresultdict)
+                    return MockResponse(
+                        200,
+                        '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=package_show"}'
+                        % result,
+                    )
+
+        Configuration.read().remoteckan().session = MockSession()
+
+    @pytest.fixture(scope="function")
+    def do_state_multi(self, tempfolder, multidatestatefile):
+        class MockSession:
+            @staticmethod
+            def post(url, data, headers, files, allow_redirects, auth=None):
+                if "resource" in url:
+                    result = json.dumps(resultdict)
+                    return MockResponse(
+                        200,
+                        '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_show"}'
+                        % result,
+                    )
+                else:
+                    myresultdict = deepcopy(dataset_resultdict)
+                    resource = myresultdict["resources"][0]
+                    resource["name"] = multidatestatefile
+                    resource["url"] = join(tempfolder, multidatestatefile)
+                    result = json.dumps(myresultdict)
+                    return MockResponse(
+                        200,
+                        '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=package_show"}'
+                        % result,
+                    )
+
+        Configuration.read().remoteckan().session = MockSession()
+
+    def test_state(
+        self, tempfolder, statefolder, statefile, date1, date2, configuration, do_state
+    ):
+        if not exists(tempfolder):
+            makedirs(tempfolder)
+        statepath = join(tempfolder, statefile)
+        copyfile(join(statefolder, statefile), statepath)
+        with HDXState(
+            "test_dataset", tempfolder, parse_date, iso_string_from_datetime
+        ) as state:
+            assert state.get() == date1
+        with HDXState(
+            "test_dataset", tempfolder, parse_date, iso_string_from_datetime
+        ) as state:
+            assert state.get() == date1
+            state.set(date2)
+        with HDXState(
+            "test_dataset", tempfolder, parse_date, iso_string_from_datetime
+        ) as state:
+            assert state.get() == date2.replace(hour=0, minute=0)
+        rmtree(tempfolder)
+
+    def test_multi_date_state(
+        self,
+        tempfolder,
+        statefolder,
+        multidatestatefile,
+        date1,
+        date2,
+        configuration,
+        do_state_multi,
+    ):
+        if not exists(tempfolder):
+            makedirs(tempfolder)
+        statepath = join(tempfolder, multidatestatefile)
+        copyfile(join(statefolder, multidatestatefile), statepath)
+        with HDXState(
+            "test_dataset",
+            tempfolder,
+            HDXState.dates_str_to_country_date_dict,
+            HDXState.country_date_dict_to_dates_str,
+        ) as state:
+            state_dict = state.get()
+            assert state_dict == {"DEFAULT": date1}
+        with HDXState(
+            "test_dataset",
+            tempfolder,
+            HDXState.dates_str_to_country_date_dict,
+            HDXState.country_date_dict_to_dates_str,
+        ) as state:
+            state_dict = state.get()
+            assert state_dict == {"DEFAULT": date1}
+            state_dict["AFG"] = date2
+            state.set(state_dict)
+        with HDXState(
+            "test_dataset",
+            tempfolder,
+            HDXState.dates_str_to_country_date_dict,
+            HDXState.country_date_dict_to_dates_str,
+        ) as state:
+            state_dict = state.get()
+            assert state_dict == {
+                "DEFAULT": date1,
+                "AFG": date2.replace(hour=0, minute=0),
+            }
+        rmtree(tempfolder)