diff --git a/requirements.txt b/requirements.txt index 0c8033ff..499a608f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,7 +32,7 @@ click==8.1.8 # typer colorama==0.4.6 # via mkdocs-material -coverage==7.8.0 +coverage==7.8.1 # via pytest-cov defopt==6.4.0 # via hdx-python-api (pyproject.toml) @@ -56,7 +56,7 @@ frictionless==5.18.1 # via hdx-python-utilities ghp-import==2.1.0 # via mkdocs -google-auth==2.40.1 +google-auth==2.40.2 # via # google-auth-oauthlib # gspread @@ -181,7 +181,7 @@ pyasn1==0.6.1 # rsa pyasn1-modules==0.4.2 # via google-auth -pydantic==2.11.4 +pydantic==2.11.5 # via frictionless pydantic-core==2.33.2 # via pydantic @@ -250,7 +250,7 @@ rfc3986==2.0.0 # via frictionless rich==14.0.0 # via typer -rpds-py==0.25.0 +rpds-py==0.25.1 # via # jsonschema # referencing @@ -296,7 +296,7 @@ typing-extensions==4.13.2 # typeguard # typer # typing-inspection -typing-inspection==0.4.0 +typing-inspection==0.4.1 # via pydantic unidecode==1.4.0 # via diff --git a/src/hdx/api/configuration.py b/src/hdx/api/configuration.py index 7c97dbce..f0d704c1 100755 --- a/src/hdx/api/configuration.py +++ b/src/hdx/api/configuration.py @@ -10,7 +10,7 @@ import ckanapi import requests -from . import __version__ +from hdx.api import __version__ from hdx.utilities.dictandlist import merge_two_dictionaries from hdx.utilities.email import Email from hdx.utilities.loader import load_json, load_yaml diff --git a/src/hdx/api/utilities/hdx_state.py b/src/hdx/api/utilities/hdx_state.py new file mode 100644 index 00000000..889beb19 --- /dev/null +++ b/src/hdx/api/utilities/hdx_state.py @@ -0,0 +1,69 @@ +"""Utility to save state to a dataset and read it back.""" + +import logging +from os.path import join +from typing import Any, Callable, Optional + +from hdx.api.configuration import Configuration +from hdx.data.dataset import Dataset +from hdx.utilities.loader import load_text +from hdx.utilities.saver import save_text +from hdx.utilities.state import State + +logger = logging.getLogger(__name__) + + +class HDXState(State): + """State class that allows the reading and writing of state to a given HDX + dataset. Input and output state transformations can be supplied in read_fn and + write_fn respectively. The input state transformation takes in a string + while the output transformation outputs a string. + + Args: + dataset_name_or_id (str): Dataset name or ID + path (str): Path to temporary folder for state + read_fn (Callable[[str], Any]): Input state transformation. Defaults to lambda x: x. + write_fn: Callable[[Any], str]: Output state transformation. Defaults to lambda x: x. + configuration (Optional[Configuration]): HDX configuration. Defaults to global configuration. + """ + + def __init__( + self, + dataset_name_or_id: str, + path: str, + read_fn: Callable[[str], Any] = lambda x: x, + write_fn: Callable[[Any], str] = lambda x: x, + configuration: Optional[Configuration] = None, + ) -> None: + self._dataset_name_or_id = dataset_name_or_id + self._resource = None + self._configuration = configuration + super().__init__(path, read_fn, write_fn) + + def read(self) -> Any: + """Read state from HDX dataset + + Returns: + Any: State + """ + dataset = Dataset.read_from_hdx( + self._dataset_name_or_id, configuration=self._configuration + ) + self._resource = dataset.get_resource() + _, path = self._resource.download() + value = self.read_fn(load_text(path)) + logger.info(f"State read from {self._dataset_name_or_id} = {value}") + return value + + def write(self) -> None: + """Write state to HDX dataset + + Returns: + None + """ + logger.info(f"State written to {self._dataset_name_or_id} = {self.state}") + filename = self._resource["name"] + file_to_upload = join(self.path, filename) + save_text(self.write_fn(self.state), file_to_upload) + self._resource.set_file_to_upload(file_to_upload) + self._resource.update_in_hdx() diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/fixtures/state/analysis_dates.txt b/tests/fixtures/state/analysis_dates.txt new file mode 100644 index 00000000..b52fb270 --- /dev/null +++ b/tests/fixtures/state/analysis_dates.txt @@ -0,0 +1 @@ +DEFAULT=2020-09-23 diff --git a/tests/fixtures/state/last_build_date.txt b/tests/fixtures/state/last_build_date.txt new file mode 100644 index 00000000..34c8ca46 --- /dev/null +++ b/tests/fixtures/state/last_build_date.txt @@ -0,0 +1 @@ +2020-09-23 diff --git a/tests/hdx/__init__.py b/tests/hdx/__init__.py new file mode 100755 index 00000000..78bf8982 --- /dev/null +++ b/tests/hdx/__init__.py @@ -0,0 +1,315 @@ +import copy +import json + + +class MockResponse: + def __init__(self, status_code, text): + self.status_code = status_code + self.text = text + + def json(self): + return json.loads(self.text) + + +resultgroups = [ + { + "description": "", + "name": "dza", + "image_display_url": "", + "display_name": "Algeria", + "id": "dza", + "title": "Algeria", + }, + { + "description": "", + "name": "zwe", + "image_display_url": "", + "display_name": "Zimbabwe", + "id": "zwe", + "title": "Zimbabwe", + }, +] + +resulttags = [ + { + "state": "active", + "display_name": "conflict", + "vocabulary_id": "4381925f-0ae9-44a3-b30d-cae35598757b", + "id": "1dae41e5-eacd-4fa5-91df-8d80cf579e53", + "name": "conflict", + }, + { + "state": "active", + "display_name": "political violence", + "vocabulary_id": "4381925f-0ae9-44a3-b30d-cae35598757b", + "id": "aaafc63b-2234-48e3-8ccc-198d7cf0f3f3", + "name": "political violence", + }, + { + "state": "active", + "display_name": "crisis-somewhere", + "vocabulary_id": "4381925f-0ae9-44a3-b30d-cae35598757b", + "id": "9dae41e5-eacd-4fa5-91df-8d80cf579e52", + "name": "crisis-somewhere", + }, +] + +dataset_data = { + "name": "MyDataset1", + "title": "MyDataset1", + "dataset_date": "06/04/2016", # has to be MM/DD/YYYY + "groups": resultgroups, + "owner_org": "My Org", + "author": "AN Other", + "author_email": "another@another.com", + "maintainer": "AN Other", + "maintainer_email": "another@another.com", + "license_id": "cc-by-sa", + "subnational": "1", + "notes": "some notes", + "caveats": "some caveats", + "data_update_frequency": "7", + "methodology": "other", + "methodology_other": "methodology description", + "dataset_source": "Mr Org", + "package_creator": "AN Other", + "private": False, + "url": None, + "state": "active", + "tags": resulttags, +} + +resources_data = [ + { + "id": "de6549d8-268b-4dfe-adaf-a4ae5c8510d5", + "description": "Resource1", + "name": "Resource1", + "url": "http://resource1.xlsx", + "format": "xlsx", + }, + { + "id": "3d777226-96aa-4239-860a-703389d16d1f", + "description": "Resource2", + "name": "Resource2", + "url": "http://resource2.csv", + "format": "csv", + }, + { + "id": "3d777226-96aa-4239-860a-703389d16d1g", + "description": "Resource2", + "name": "Resource2", + "url": "http://resource2.xls", + "format": "xls", + }, +] + +resource_data = { + "name": "MyResource1", + "package_id": "6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d", + "format": "xlsx", + "url": "http://test/spreadsheet.xlsx", + "description": "My Resource", + "api_type": "api", + "resource_type": "api", +} + +organization_data = { + "name": "MyOrganization1", + "title": "Humanitarian Organization", + "description": "We do humanitarian work", +} + +user_data = { + "name": "MyUser1", + "email": "xxx@yyy.com", + "password": "xxx", + "fullname": "xxx xxx", + "about": "Data Scientist", + "capacity": "admin", +} + +dataset_resultdict = { + "resources": [ + { + "revision_id": "43765383-1fce-471f-8166-d6c8660cc8a9", + "cache_url": None, + "datastore_active": False, + "format": "xlsx", + "webstore_url": None, + "last_modified": None, + "tracking_summary": {"recent": 0, "total": 0}, + "id": "de6549d8-268b-4dfe-adaf-a4ae5c8510d5", + "webstore_last_updated": None, + "mimetype": None, + "state": "active", + "created": "2016-06-07T08:57:27.367939", + "description": "Resource1", + "position": 0, + "hash": "", + "package_id": "6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d", + "name": "Resource1", + "url": "http://resource1.xlsx", + "resource_type": "api", + "url_type": "api", + "size": None, + "mimetype_inner": None, + "cache_last_updated": None, + }, + { + "revision_id": "387e5d1a-50ca-4039-b5a7-f7b6b88d0f2b", + "cache_url": None, + "datastore_active": False, + "format": "csv", + "webstore_url": None, + "last_modified": None, + "tracking_summary": {"recent": 0, "total": 0}, + "id": "3d777226-96aa-4239-860a-703389d16d1f", + "webstore_last_updated": None, + "mimetype": None, + "state": "active", + "created": "2016-06-07T08:57:27.367959", + "description": "Resource2", + "position": 1, + "hash": "", + "package_id": "6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d", + "name": "Resource2", + "url": "http://resource2_csv.zip", + "resource_type": "api", + "url_type": "api", + "size": None, + "mimetype_inner": None, + "cache_last_updated": None, + }, + { + "revision_id": "387e5d1a-50ca-4039-b5a7-f7b6b88d0f2c", + "cache_url": None, + "datastore_active": False, + "format": "geojson", + "webstore_url": None, + "last_modified": None, + "tracking_summary": {"recent": 0, "total": 0}, + "id": "3d777226-96aa-4239-860a-703389d16d1g", + "webstore_last_updated": None, + "mimetype": None, + "state": "active", + "created": "2016-06-07T08:57:27.367959", + "description": "Resource2", + "position": 2, + "hash": "", + "package_id": "6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d", + "name": "Resource2", + "url": "http://resource2_csv.zip", + "resource_type": "api", + "url_type": "api", + "size": None, + "mimetype_inner": None, + "cache_last_updated": None, + }, + ], + "isopen": True, + "caveats": "Various", + "revision_id": "032833ca-c403-40cc-8b86-69d5a6eecb1b", + "url": None, + "author": "acled", + "metadata_created": "2016-03-23T14:28:48.664205", + "license_url": "http://www.opendefinition.org/licenses/cc-by-sa", + "relationships_as_object": [], + "creator_user_id": "154de241-38d6-47d3-a77f-0a9848a61df3", + "methodology_other": "This page contains information.", + "subnational": "1", + "maintainer_email": "me@me.com", + "license_title": "Creative Commons Attribution Share-Alike", + "title": "MyDataset", + "private": False, + "maintainer": "8b84230c-e04a-43ec-99e5-41307a203a2f", + "methodology": "Other", + "num_tags": 4, + "license_id": "cc-by-sa", + "tracking_summary": {"recent": 32, "total": 178}, + "relationships_as_subject": [], + "owner_org": "b67e6c74-c185-4f43-b561-0e114a736f19", + "id": "6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d", + "dataset_source": "ACLED", + "type": "dataset", + "notes": "Notes", + "organization": { + "revision_id": "684f3eee-b708-4f91-bd22-7860d4eca423", + "description": "MyOrganisation", + "name": "acled", + "type": "organization", + "image_url": "", + "approval_status": "approved", + "state": "active", + "title": "MyOrganisation", + "created": "2015-01-09T14:44:54.006612", + "id": "b67e6c74-c185-4f43-b561-0e114a736f19", + "is_organization": True, + }, + "state": "active", + "author_email": "me@me.com", + "package_creator": "someone", + "num_resources": 2, + "total_res_downloads": 4, + "name": "MyDataset1", + "metadata_modified": "2016-06-09T12:49:33.854367", + "groups": resultgroups, + "data_update_frequency": "7", + "tags": resulttags, + "version": None, + "solr_additions": '{"countries": ["Algeria", "Zimbabwe"]}', + "dataset_date": "06/04/2016", +} + + +def dataset_mockshow(url, datadict): + if "show" not in url: + return MockResponse( + 404, + '{"success": false, "error": {"message": "TEST ERROR: Not show", "__type": "TEST ERROR: Not Show Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=package_show"}', + ) + if "resource_show" in url: + result = json.dumps(resources_data[0]) + return MockResponse( + 200, + '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_show"}' + % result, + ) + else: + if datadict["id"] == "TEST1": + result = json.dumps(dataset_resultdict) + return MockResponse( + 200, + '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=package_show"}' + % result, + ) + if datadict["id"] == "DatasetExist": + result = json.dumps(dataset_resultdict) + return MockResponse( + 200, + '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=package_show"}' + % result, + ) + if datadict["id"] == "TEST2": + return MockResponse( + 404, + '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=package_show"}', + ) + if datadict["id"] == "TEST3": + return MockResponse( + 200, + '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=package_show"}', + ) + if datadict["id"] == "TEST4": + resultdictcopy = copy.deepcopy(dataset_resultdict) + resultdictcopy["id"] = "TEST4" + result = json.dumps(resultdictcopy) + return MockResponse( + 200, + '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=package_show"}' + % result, + ) + + return MockResponse( + 404, + '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=package_show"}', + ) diff --git a/tests/hdx/api/__init__.py b/tests/hdx/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/hdx/api/utilities/__init__.py b/tests/hdx/api/utilities/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/hdx/utilities/test_dataset_title_helper.py b/tests/hdx/api/utilities/test_dataset_title_helper.py similarity index 100% rename from tests/hdx/utilities/test_dataset_title_helper.py rename to tests/hdx/api/utilities/test_dataset_title_helper.py diff --git a/tests/hdx/utilities/test_filestore_helper.py b/tests/hdx/api/utilities/test_filestore_helper.py similarity index 100% rename from tests/hdx/utilities/test_filestore_helper.py rename to tests/hdx/api/utilities/test_filestore_helper.py diff --git a/tests/hdx/utilities/test_hdx_error_handler.py b/tests/hdx/api/utilities/test_hdx_error_handler.py similarity index 100% rename from tests/hdx/utilities/test_hdx_error_handler.py rename to tests/hdx/api/utilities/test_hdx_error_handler.py diff --git a/tests/hdx/api/utilities/test_hdx_state.py b/tests/hdx/api/utilities/test_hdx_state.py new file mode 100644 index 00000000..f362618d --- /dev/null +++ b/tests/hdx/api/utilities/test_hdx_state.py @@ -0,0 +1,162 @@ +"""HDXState Utility Tests""" + +import json +from copy import deepcopy +from datetime import datetime, timezone +from os import makedirs +from os.path import exists, join +from shutil import copyfile, rmtree +from tempfile import gettempdir + +import pytest + +from ... import MockResponse, dataset_resultdict +from ...data.test_resource import resultdict +from hdx.api.configuration import Configuration +from hdx.api.utilities.hdx_state import HDXState +from hdx.utilities.dateparse import iso_string_from_datetime, parse_date + + +class TestState: + @pytest.fixture(scope="class") + def tempfolder(self): + return join(gettempdir(), "test_state") + + @pytest.fixture(scope="class") + def statefolder(self, fixturesfolder): + return join(fixturesfolder, "state") + + @pytest.fixture(scope="class") + def statefile(self): + return "last_build_date.txt" + + @pytest.fixture(scope="class") + def multidatestatefile(self): + return "analysis_dates.txt" + + @pytest.fixture(scope="class") + def date1(self): + return datetime(2020, 9, 23, 0, 0, tzinfo=timezone.utc) + + @pytest.fixture(scope="class") + def date2(self): + return datetime(2022, 5, 12, 10, 15, tzinfo=timezone.utc) + + @pytest.fixture(scope="function") + def do_state(self, tempfolder, statefile): + class MockSession: + @staticmethod + def post(url, data, headers, files, allow_redirects, auth=None): + if "resource" in url: + result = json.dumps(resultdict) + return MockResponse( + 200, + '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_show"}' + % result, + ) + else: + myresultdict = deepcopy(dataset_resultdict) + resource = myresultdict["resources"][0] + resource["name"] = statefile + resource["url"] = join(tempfolder, statefile) + result = json.dumps(myresultdict) + return MockResponse( + 200, + '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=package_show"}' + % result, + ) + + Configuration.read().remoteckan().session = MockSession() + + @pytest.fixture(scope="function") + def do_state_multi(self, tempfolder, multidatestatefile): + class MockSession: + @staticmethod + def post(url, data, headers, files, allow_redirects, auth=None): + if "resource" in url: + result = json.dumps(resultdict) + return MockResponse( + 200, + '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_show"}' + % result, + ) + else: + myresultdict = deepcopy(dataset_resultdict) + resource = myresultdict["resources"][0] + resource["name"] = multidatestatefile + resource["url"] = join(tempfolder, multidatestatefile) + result = json.dumps(myresultdict) + return MockResponse( + 200, + '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=package_show"}' + % result, + ) + + Configuration.read().remoteckan().session = MockSession() + + def test_state( + self, tempfolder, statefolder, statefile, date1, date2, configuration, do_state + ): + if not exists(tempfolder): + makedirs(tempfolder) + statepath = join(tempfolder, statefile) + copyfile(join(statefolder, statefile), statepath) + with HDXState( + "test_dataset", tempfolder, parse_date, iso_string_from_datetime + ) as state: + assert state.get() == date1 + with HDXState( + "test_dataset", tempfolder, parse_date, iso_string_from_datetime + ) as state: + assert state.get() == date1 + state.set(date2) + with HDXState( + "test_dataset", tempfolder, parse_date, iso_string_from_datetime + ) as state: + assert state.get() == date2.replace(hour=0, minute=0) + rmtree(tempfolder) + + def test_multi_date_state( + self, + tempfolder, + statefolder, + multidatestatefile, + date1, + date2, + configuration, + do_state_multi, + ): + if not exists(tempfolder): + makedirs(tempfolder) + statepath = join(tempfolder, multidatestatefile) + copyfile(join(statefolder, multidatestatefile), statepath) + with HDXState( + "test_dataset", + tempfolder, + HDXState.dates_str_to_country_date_dict, + HDXState.country_date_dict_to_dates_str, + ) as state: + state_dict = state.get() + assert state_dict == {"DEFAULT": date1} + with HDXState( + "test_dataset", + tempfolder, + HDXState.dates_str_to_country_date_dict, + HDXState.country_date_dict_to_dates_str, + ) as state: + state_dict = state.get() + assert state_dict == {"DEFAULT": date1} + state_dict["AFG"] = date2 + state.set(state_dict) + with HDXState( + "test_dataset", + tempfolder, + HDXState.dates_str_to_country_date_dict, + HDXState.country_date_dict_to_dates_str, + ) as state: + state_dict = state.get() + assert state_dict == { + "DEFAULT": date1, + "AFG": date2.replace(hour=0, minute=0), + } + rmtree(tempfolder) diff --git a/tests/hdx/data/__init__.py b/tests/hdx/data/__init__.py old mode 100755 new mode 100644 index 78bf8982..e69de29b --- a/tests/hdx/data/__init__.py +++ b/tests/hdx/data/__init__.py @@ -1,315 +0,0 @@ -import copy -import json - - -class MockResponse: - def __init__(self, status_code, text): - self.status_code = status_code - self.text = text - - def json(self): - return json.loads(self.text) - - -resultgroups = [ - { - "description": "", - "name": "dza", - "image_display_url": "", - "display_name": "Algeria", - "id": "dza", - "title": "Algeria", - }, - { - "description": "", - "name": "zwe", - "image_display_url": "", - "display_name": "Zimbabwe", - "id": "zwe", - "title": "Zimbabwe", - }, -] - -resulttags = [ - { - "state": "active", - "display_name": "conflict", - "vocabulary_id": "4381925f-0ae9-44a3-b30d-cae35598757b", - "id": "1dae41e5-eacd-4fa5-91df-8d80cf579e53", - "name": "conflict", - }, - { - "state": "active", - "display_name": "political violence", - "vocabulary_id": "4381925f-0ae9-44a3-b30d-cae35598757b", - "id": "aaafc63b-2234-48e3-8ccc-198d7cf0f3f3", - "name": "political violence", - }, - { - "state": "active", - "display_name": "crisis-somewhere", - "vocabulary_id": "4381925f-0ae9-44a3-b30d-cae35598757b", - "id": "9dae41e5-eacd-4fa5-91df-8d80cf579e52", - "name": "crisis-somewhere", - }, -] - -dataset_data = { - "name": "MyDataset1", - "title": "MyDataset1", - "dataset_date": "06/04/2016", # has to be MM/DD/YYYY - "groups": resultgroups, - "owner_org": "My Org", - "author": "AN Other", - "author_email": "another@another.com", - "maintainer": "AN Other", - "maintainer_email": "another@another.com", - "license_id": "cc-by-sa", - "subnational": "1", - "notes": "some notes", - "caveats": "some caveats", - "data_update_frequency": "7", - "methodology": "other", - "methodology_other": "methodology description", - "dataset_source": "Mr Org", - "package_creator": "AN Other", - "private": False, - "url": None, - "state": "active", - "tags": resulttags, -} - -resources_data = [ - { - "id": "de6549d8-268b-4dfe-adaf-a4ae5c8510d5", - "description": "Resource1", - "name": "Resource1", - "url": "http://resource1.xlsx", - "format": "xlsx", - }, - { - "id": "3d777226-96aa-4239-860a-703389d16d1f", - "description": "Resource2", - "name": "Resource2", - "url": "http://resource2.csv", - "format": "csv", - }, - { - "id": "3d777226-96aa-4239-860a-703389d16d1g", - "description": "Resource2", - "name": "Resource2", - "url": "http://resource2.xls", - "format": "xls", - }, -] - -resource_data = { - "name": "MyResource1", - "package_id": "6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d", - "format": "xlsx", - "url": "http://test/spreadsheet.xlsx", - "description": "My Resource", - "api_type": "api", - "resource_type": "api", -} - -organization_data = { - "name": "MyOrganization1", - "title": "Humanitarian Organization", - "description": "We do humanitarian work", -} - -user_data = { - "name": "MyUser1", - "email": "xxx@yyy.com", - "password": "xxx", - "fullname": "xxx xxx", - "about": "Data Scientist", - "capacity": "admin", -} - -dataset_resultdict = { - "resources": [ - { - "revision_id": "43765383-1fce-471f-8166-d6c8660cc8a9", - "cache_url": None, - "datastore_active": False, - "format": "xlsx", - "webstore_url": None, - "last_modified": None, - "tracking_summary": {"recent": 0, "total": 0}, - "id": "de6549d8-268b-4dfe-adaf-a4ae5c8510d5", - "webstore_last_updated": None, - "mimetype": None, - "state": "active", - "created": "2016-06-07T08:57:27.367939", - "description": "Resource1", - "position": 0, - "hash": "", - "package_id": "6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d", - "name": "Resource1", - "url": "http://resource1.xlsx", - "resource_type": "api", - "url_type": "api", - "size": None, - "mimetype_inner": None, - "cache_last_updated": None, - }, - { - "revision_id": "387e5d1a-50ca-4039-b5a7-f7b6b88d0f2b", - "cache_url": None, - "datastore_active": False, - "format": "csv", - "webstore_url": None, - "last_modified": None, - "tracking_summary": {"recent": 0, "total": 0}, - "id": "3d777226-96aa-4239-860a-703389d16d1f", - "webstore_last_updated": None, - "mimetype": None, - "state": "active", - "created": "2016-06-07T08:57:27.367959", - "description": "Resource2", - "position": 1, - "hash": "", - "package_id": "6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d", - "name": "Resource2", - "url": "http://resource2_csv.zip", - "resource_type": "api", - "url_type": "api", - "size": None, - "mimetype_inner": None, - "cache_last_updated": None, - }, - { - "revision_id": "387e5d1a-50ca-4039-b5a7-f7b6b88d0f2c", - "cache_url": None, - "datastore_active": False, - "format": "geojson", - "webstore_url": None, - "last_modified": None, - "tracking_summary": {"recent": 0, "total": 0}, - "id": "3d777226-96aa-4239-860a-703389d16d1g", - "webstore_last_updated": None, - "mimetype": None, - "state": "active", - "created": "2016-06-07T08:57:27.367959", - "description": "Resource2", - "position": 2, - "hash": "", - "package_id": "6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d", - "name": "Resource2", - "url": "http://resource2_csv.zip", - "resource_type": "api", - "url_type": "api", - "size": None, - "mimetype_inner": None, - "cache_last_updated": None, - }, - ], - "isopen": True, - "caveats": "Various", - "revision_id": "032833ca-c403-40cc-8b86-69d5a6eecb1b", - "url": None, - "author": "acled", - "metadata_created": "2016-03-23T14:28:48.664205", - "license_url": "http://www.opendefinition.org/licenses/cc-by-sa", - "relationships_as_object": [], - "creator_user_id": "154de241-38d6-47d3-a77f-0a9848a61df3", - "methodology_other": "This page contains information.", - "subnational": "1", - "maintainer_email": "me@me.com", - "license_title": "Creative Commons Attribution Share-Alike", - "title": "MyDataset", - "private": False, - "maintainer": "8b84230c-e04a-43ec-99e5-41307a203a2f", - "methodology": "Other", - "num_tags": 4, - "license_id": "cc-by-sa", - "tracking_summary": {"recent": 32, "total": 178}, - "relationships_as_subject": [], - "owner_org": "b67e6c74-c185-4f43-b561-0e114a736f19", - "id": "6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d", - "dataset_source": "ACLED", - "type": "dataset", - "notes": "Notes", - "organization": { - "revision_id": "684f3eee-b708-4f91-bd22-7860d4eca423", - "description": "MyOrganisation", - "name": "acled", - "type": "organization", - "image_url": "", - "approval_status": "approved", - "state": "active", - "title": "MyOrganisation", - "created": "2015-01-09T14:44:54.006612", - "id": "b67e6c74-c185-4f43-b561-0e114a736f19", - "is_organization": True, - }, - "state": "active", - "author_email": "me@me.com", - "package_creator": "someone", - "num_resources": 2, - "total_res_downloads": 4, - "name": "MyDataset1", - "metadata_modified": "2016-06-09T12:49:33.854367", - "groups": resultgroups, - "data_update_frequency": "7", - "tags": resulttags, - "version": None, - "solr_additions": '{"countries": ["Algeria", "Zimbabwe"]}', - "dataset_date": "06/04/2016", -} - - -def dataset_mockshow(url, datadict): - if "show" not in url: - return MockResponse( - 404, - '{"success": false, "error": {"message": "TEST ERROR: Not show", "__type": "TEST ERROR: Not Show Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=package_show"}', - ) - if "resource_show" in url: - result = json.dumps(resources_data[0]) - return MockResponse( - 200, - '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_show"}' - % result, - ) - else: - if datadict["id"] == "TEST1": - result = json.dumps(dataset_resultdict) - return MockResponse( - 200, - '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=package_show"}' - % result, - ) - if datadict["id"] == "DatasetExist": - result = json.dumps(dataset_resultdict) - return MockResponse( - 200, - '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=package_show"}' - % result, - ) - if datadict["id"] == "TEST2": - return MockResponse( - 404, - '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=package_show"}', - ) - if datadict["id"] == "TEST3": - return MockResponse( - 200, - '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=package_show"}', - ) - if datadict["id"] == "TEST4": - resultdictcopy = copy.deepcopy(dataset_resultdict) - resultdictcopy["id"] = "TEST4" - result = json.dumps(resultdictcopy) - return MockResponse( - 200, - '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=package_show"}' - % result, - ) - - return MockResponse( - 404, - '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=package_show"}', - ) diff --git a/tests/hdx/data/test_dataset_add_hapi_error.py b/tests/hdx/data/test_dataset_add_hapi_error.py index 267324c1..ac6cf0ef 100644 --- a/tests/hdx/data/test_dataset_add_hapi_error.py +++ b/tests/hdx/data/test_dataset_add_hapi_error.py @@ -3,7 +3,7 @@ import pytest -from . import MockResponse, dataset_data, dataset_resultdict, resources_data +from .. import MockResponse, dataset_data, dataset_resultdict, resources_data from hdx.api.configuration import Configuration from hdx.data.dataset import Dataset diff --git a/tests/hdx/data/test_dataset_core.py b/tests/hdx/data/test_dataset_core.py index 53de7c79..8e58e4f2 100755 --- a/tests/hdx/data/test_dataset_core.py +++ b/tests/hdx/data/test_dataset_core.py @@ -10,7 +10,7 @@ import pytest from pytest_check import check -from . import ( +from .. import ( MockResponse, dataset_data, dataset_mockshow, diff --git a/tests/hdx/data/test_dataset_noncore.py b/tests/hdx/data/test_dataset_noncore.py index 8e6ea946..4f083060 100755 --- a/tests/hdx/data/test_dataset_noncore.py +++ b/tests/hdx/data/test_dataset_noncore.py @@ -7,7 +7,7 @@ import pytest -from . import ( +from .. import ( MockResponse, dataset_data, dataset_mockshow, diff --git a/tests/hdx/data/test_organization.py b/tests/hdx/data/test_organization.py index 42800744..623d96a5 100755 --- a/tests/hdx/data/test_organization.py +++ b/tests/hdx/data/test_organization.py @@ -6,7 +6,7 @@ import pytest -from . import MockResponse, organization_data, user_data +from .. import MockResponse, organization_data, user_data from .test_user import user_mockshow from hdx.api.configuration import Configuration from hdx.data.hdxobject import HDXError diff --git a/tests/hdx/data/test_resource.py b/tests/hdx/data/test_resource.py index 5dd4528c..7096f812 100755 --- a/tests/hdx/data/test_resource.py +++ b/tests/hdx/data/test_resource.py @@ -9,7 +9,7 @@ import pytest -from . import MockResponse, dataset_resultdict, resource_data +from .. import MockResponse, dataset_resultdict, resource_data from .test_resource_view import resource_view_list, resource_view_mocklist from hdx.api.configuration import Configuration from hdx.data.hdxobject import HDXError diff --git a/tests/hdx/data/test_resource_view.py b/tests/hdx/data/test_resource_view.py index ab5914ee..ef376a25 100755 --- a/tests/hdx/data/test_resource_view.py +++ b/tests/hdx/data/test_resource_view.py @@ -6,7 +6,7 @@ import pytest -from . import MockResponse +from .. import MockResponse from hdx.api.configuration import Configuration from hdx.data.hdxobject import HDXError from hdx.data.resource_view import ResourceView diff --git a/tests/hdx/data/test_showcase.py b/tests/hdx/data/test_showcase.py index e2e06ed8..21d16d9b 100755 --- a/tests/hdx/data/test_showcase.py +++ b/tests/hdx/data/test_showcase.py @@ -6,7 +6,7 @@ import pytest -from . import MockResponse +from .. import MockResponse from .test_vocabulary import vocabulary_mockshow from hdx.api.configuration import Configuration from hdx.data.hdxobject import HDXError diff --git a/tests/hdx/data/test_user.py b/tests/hdx/data/test_user.py index 0956fdee..2f9447ed 100755 --- a/tests/hdx/data/test_user.py +++ b/tests/hdx/data/test_user.py @@ -6,7 +6,7 @@ import pytest -from . import MockResponse, user_data +from .. import MockResponse, user_data from hdx.api.configuration import Configuration from hdx.data.hdxobject import HDXError from hdx.data.user import User diff --git a/tests/hdx/data/test_vocabulary.py b/tests/hdx/data/test_vocabulary.py index c526a793..12591da6 100755 --- a/tests/hdx/data/test_vocabulary.py +++ b/tests/hdx/data/test_vocabulary.py @@ -7,7 +7,7 @@ import pytest from requests.exceptions import RetryError -from . import MockResponse +from .. import MockResponse from hdx.api.configuration import Configuration from hdx.data.hdxobject import HDXError from hdx.data.vocabulary import ChainRuleError, Vocabulary