From a1ee2e5ae67f20d3d05afb26d994ed3f61452276 Mon Sep 17 00:00:00 2001 From: mcarans Date: Fri, 14 Feb 2025 17:14:40 +1300 Subject: [PATCH 1/2] Add move resource method to Dataset with test --- .config/pre-commit-config.yaml | 4 +- pyproject.toml | 8 +- requirements.txt | 36 ++++--- src/hdx/data/dataset.py | 33 ++++++ tests/hdx/data/test_dataset_core.py | 161 ++++++++++++++++++++++++++++ tests/hdx/data/test_showcase.py | 2 +- tests/hdx/data/test_update_logic.py | 6 +- 7 files changed, 224 insertions(+), 26 deletions(-) diff --git a/.config/pre-commit-config.yaml b/.config/pre-commit-config.yaml index 177b3e19..1f135149 100644 --- a/.config/pre-commit-config.yaml +++ b/.config/pre-commit-config.yaml @@ -8,7 +8,7 @@ repos: - id: end-of-file-fixer - id: check-ast - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.6 + rev: v0.9.6 hooks: # Run the linter. - id: ruff @@ -17,7 +17,7 @@ repos: - id: ruff-format args: [--config, .config/ruff.toml] - repo: https://github.com/astral-sh/uv-pre-commit - rev: 0.5.15 + rev: 0.5.29 hooks: # Run the pip compile - id: pip-compile diff --git a/pyproject.toml b/pyproject.toml index 17500fff..ea10c601 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,8 +37,8 @@ dependencies = [ "ckanapi>=4.8", "defopt>=6.4.0", "email_validator", - "hdx-python-country>=3.8.6", - "hdx-python-utilities>=3.8.2", + "hdx-python-country>=3.8.8", + "hdx-python-utilities>=3.8.3", "libhxl>=5.2.2", "makefun", "quantulum3", @@ -54,7 +54,7 @@ content-type = "text/markdown" Homepage = "https://github.com/OCHA-DAP/hdx-python-api" [project.optional-dependencies] -test = ["pytest", "pytest-cov", "gspread"] +test = ["pytest", "pytest-check", "pytest-cov", "gspread"] dev = ["pre-commit"] @@ -98,7 +98,7 @@ run = """ """ [tool.hatch.envs.hatch-static-analysis] -dependencies = ["ruff==0.8.6"] +dependencies = ["ruff==0.9.6"] [tool.hatch.envs.hatch-static-analysis.scripts] format-check = ["ruff format --config .config/ruff.toml --check --diff {args:.}",] diff --git a/requirements.txt b/requirements.txt index e935b77b..152e4bed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,15 +2,15 @@ # uv pip compile pyproject.toml --resolver=backtracking --all-extras -o requirements.txt annotated-types==0.7.0 # via pydantic -attrs==24.3.0 +attrs==25.1.0 # via # frictionless # jsonlines # jsonschema # referencing -cachetools==5.5.0 +cachetools==5.5.1 # via google-auth -certifi==2024.12.14 +certifi==2025.1.31 # via requests cfgv==3.4.0 # via pre-commit @@ -22,7 +22,7 @@ ckanapi==4.8 # via hdx-python-api (pyproject.toml) click==8.1.8 # via typer -coverage==7.6.10 +coverage==7.6.12 # via pytest-cov defopt==6.4.0 # via hdx-python-api (pyproject.toml) @@ -40,11 +40,11 @@ email-validator==2.2.0 # via hdx-python-api (pyproject.toml) et-xmlfile==2.0.0 # via openpyxl -filelock==3.16.1 +filelock==3.17.0 # via virtualenv frictionless==5.18.0 # via hdx-python-utilities -google-auth==2.37.0 +google-auth==2.38.0 # via # google-auth-oauthlib # gspread @@ -52,15 +52,15 @@ google-auth-oauthlib==1.2.1 # via gspread gspread==6.1.4 # via hdx-python-api (pyproject.toml) -hdx-python-country==3.8.6 +hdx-python-country==3.8.8 # via hdx-python-api (pyproject.toml) -hdx-python-utilities==3.8.2 +hdx-python-utilities==3.8.3 # via # hdx-python-api (pyproject.toml) # hdx-python-country humanize==4.11.0 # via frictionless -identify==2.6.5 +identify==2.6.7 # via pre-commit idna==3.10 # via @@ -102,7 +102,7 @@ markupsafe==3.0.2 # via jinja2 mdurl==0.1.2 # via markdown-it-py -more-itertools==10.5.0 +more-itertools==10.6.0 # via inflect nodeenv==1.9.1 # via pre-commit @@ -126,7 +126,7 @@ ply==3.11 # libhxl pockets==0.9.1 # via sphinxcontrib-napoleon -pre-commit==4.0.1 +pre-commit==4.1.0 # via hdx-python-api (pyproject.toml) pyasn1==0.6.1 # via @@ -134,7 +134,7 @@ pyasn1==0.6.1 # rsa pyasn1-modules==0.4.1 # via google-auth -pydantic==2.10.5 +pydantic==2.10.6 # via frictionless pydantic-core==2.27.2 # via pydantic @@ -145,7 +145,10 @@ pyphonetics==0.5.3 pytest==8.3.4 # via # hdx-python-api (pyproject.toml) + # pytest-check # pytest-cov +pytest-check==2.5.0 + # via hdx-python-api (pyproject.toml) pytest-cov==6.0.0 # via hdx-python-api (pyproject.toml) python-dateutil==2.9.0.post0 @@ -168,7 +171,7 @@ quantulum3==0.9.2 # via hdx-python-api (pyproject.toml) ratelimit==2.2.1 # via hdx-python-utilities -referencing==0.35.1 +referencing==0.36.2 # via # jsonschema # jsonschema-specifications @@ -216,7 +219,7 @@ sphinxcontrib-napoleon==0.7 # via defopt stringcase==1.2.0 # via frictionless -structlog==24.4.0 +structlog==25.1.0 # via libhxl tableschema-to-template==0.0.13 # via hdx-python-utilities @@ -235,6 +238,7 @@ typing-extensions==4.12.2 # frictionless # pydantic # pydantic-core + # referencing # typeguard # typer unidecode==1.3.8 @@ -247,7 +251,7 @@ urllib3==2.3.0 # requests validators==0.34.0 # via frictionless -virtualenv==20.28.1 +virtualenv==20.29.2 # via pre-commit wheel==0.45.1 # via libhxl @@ -257,7 +261,7 @@ xlrd3==1.1.0 # via libhxl xlsx2csv==0.8.4 # via hdx-python-utilities -xlsxwriter==3.2.0 +xlsxwriter==3.2.2 # via tableschema-to-template xlwt==1.3.0 # via hdx-python-utilities diff --git a/src/hdx/data/dataset.py b/src/hdx/data/dataset.py index 4619f438..1d0d7536 100755 --- a/src/hdx/data/dataset.py +++ b/src/hdx/data/dataset.py @@ -423,6 +423,39 @@ def reorder_resources( if hxl_update: self.hxl_update() + def move_resource( + self, + resource_name: str, + insert_before: str, + ) -> "Resource": + """Move resource in dataset to be before the resource whose name starts + with the value of insert_before. + + Args: + resource_name (str): Name of resource to move + insert_before (str): Resource to insert before + + Returns: + Resource: The resource that was moved + """ + from_index = None + to_index = None + for i, resource in enumerate(self.resources): + res_name = resource["name"] + if res_name == resource_name: + from_index = i + elif res_name.startswith(insert_before): + to_index = i + if to_index is None: + # insert at the start if a manual resource for year cannot be found + to_index = 0 + resource = self.resources.pop(from_index) + if from_index < to_index: + # to index was calculated while element was in front + to_index -= 1 + self.resources.insert(to_index, resource) + return resource + def update_from_yaml( self, path: str = join("config", "hdx_dataset_static.yaml") ) -> None: diff --git a/tests/hdx/data/test_dataset_core.py b/tests/hdx/data/test_dataset_core.py index c6f7ec93..e79a238b 100755 --- a/tests/hdx/data/test_dataset_core.py +++ b/tests/hdx/data/test_dataset_core.py @@ -8,6 +8,7 @@ from os.path import join import pytest +from pytest_check import check from . import ( MockResponse, @@ -1043,6 +1044,166 @@ def test_reorder_resources(self, configuration, post_reorder): ] ) + def test_move_resources(self, configuration): + datasetdata = copy.deepcopy(dataset_data) + resourcesdata = copy.deepcopy(resources_data) + dataset = Dataset(datasetdata) + dataset.add_update_resources(resourcesdata) + resource_ids = [x["id"] for x in dataset.get_resources()] + check.equal( + resource_ids, + [ + "de6549d8-268b-4dfe-adaf-a4ae5c8510d5", + "3d777226-96aa-4239-860a-703389d16d1f", + "3d777226-96aa-4239-860a-703389d16d1g", + ], + ) + resources = dataset.get_resources() + resource_name = "Resource1" + insert_before = "resource2xls" + resources[2]["name"] = insert_before + resource = dataset.move_resource(resource_name, insert_before) + check.equal(resource["name"], resource_name) + resources = dataset.get_resources() + resource_ids = [x["id"] for x in resources] + check.equal( + resource_ids, + [ + "3d777226-96aa-4239-860a-703389d16d1f", + "de6549d8-268b-4dfe-adaf-a4ae5c8510d5", + "3d777226-96aa-4239-860a-703389d16d1g", + ], + ) + + input_resource_names = [ + "afg_hpc_needs_2024.xlsx", + "afg_hpc_needs_2023.xlsx", + "afg_hpc_needs_2022.xlsx", + "afg_hpc_needs_2021.xlsx", + "afg_hpc_needs_api_2024.csv", + ] + resources = [Resource({"name": x}) for x in input_resource_names] + dataset.resources = resources + resource_name = "afg_hpc_needs_api_2024.csv" + insert_before = "afg_hpc_needs_2024" + resource = dataset.move_resource(resource_name, insert_before) + check.equal(resource["name"], "afg_hpc_needs_api_2024.csv") + resource_names = [x["name"] for x in resources] + check.equal( + resource_names, + [ + "afg_hpc_needs_api_2024.csv", + "afg_hpc_needs_2024.xlsx", + "afg_hpc_needs_2023.xlsx", + "afg_hpc_needs_2022.xlsx", + "afg_hpc_needs_2021.xlsx", + ], + ) + resource = dataset.move_resource(resource_name, insert_before) + check.equal(resource["name"], "afg_hpc_needs_api_2024.csv") + resource_names = [x["name"] for x in resources] + check.equal( + resource_names, + [ + "afg_hpc_needs_api_2024.csv", + "afg_hpc_needs_2024.xlsx", + "afg_hpc_needs_2023.xlsx", + "afg_hpc_needs_2022.xlsx", + "afg_hpc_needs_2021.xlsx", + ], + ) + + input_resource_names = [ + "afg_hpc_needs_2024.xlsx", + "afg_hpc_needs_2023.xlsx", + "afg_hpc_needs_2022.xlsx", + "afg_hpc_needs_2021.xlsx", + "afg_hpc_needs_api_2021.csv", + ] + resources = [Resource({"name": x}) for x in input_resource_names] + dataset.resources = resources + resource_name = "afg_hpc_needs_api_2021.csv" + insert_before = "afg_hpc_needs_2021" + _ = dataset.move_resource(resource_name, insert_before) + resource_names = [x["name"] for x in resources] + check.equal( + resource_names, + [ + "afg_hpc_needs_2024.xlsx", + "afg_hpc_needs_2023.xlsx", + "afg_hpc_needs_2022.xlsx", + "afg_hpc_needs_api_2021.csv", + "afg_hpc_needs_2021.xlsx", + ], + ) + _ = dataset.move_resource(resource_name, insert_before) + resource_names = [x["name"] for x in resources] + check.equal( + resource_names, + [ + "afg_hpc_needs_2024.xlsx", + "afg_hpc_needs_2023.xlsx", + "afg_hpc_needs_2022.xlsx", + "afg_hpc_needs_api_2021.csv", + "afg_hpc_needs_2021.xlsx", + ], + ) + + input_resource_names = [ + "afg_hpc_needs_2023.xlsx", + "afg_hpc_needs_2022.xlsx", + "afg_hpc_needs_2021.xlsx", + "afg_hpc_needs_api_2024.csv", + ] + resources = [Resource({"name": x}) for x in input_resource_names] + dataset.resources = resources + resource_name = "afg_hpc_needs_api_2024.csv" + insert_before = "afg_hpc_needs_2024" + _ = dataset.move_resource(resource_name, insert_before) + resource_names = [x["name"] for x in resources] + check.equal( + resource_names, + [ + "afg_hpc_needs_api_2024.csv", + "afg_hpc_needs_2023.xlsx", + "afg_hpc_needs_2022.xlsx", + "afg_hpc_needs_2021.xlsx", + ], + ) + _ = dataset.move_resource(resource_name, insert_before) + resource_names = [x["name"] for x in resources] + check.equal( + resource_names, + [ + "afg_hpc_needs_api_2024.csv", + "afg_hpc_needs_2023.xlsx", + "afg_hpc_needs_2022.xlsx", + "afg_hpc_needs_2021.xlsx", + ], + ) + input_resource_names = [ + "afg_hpc_needs_2024.xlsx", + "afg_hpc_needs_2023.xlsx", + "afg_hpc_needs_api_2024.csv", + "afg_hpc_needs_2022.xlsx", + "afg_hpc_needs_2021.xlsx", + ] + resources = [Resource({"name": x}) for x in input_resource_names] + dataset.resources = resources + _ = dataset.move_resource(resource_name, insert_before) + check.equal(resource["name"], "afg_hpc_needs_api_2024.csv") + resource_names = [x["name"] for x in resources] + check.equal( + resource_names, + [ + "afg_hpc_needs_api_2024.csv", + "afg_hpc_needs_2024.xlsx", + "afg_hpc_needs_2023.xlsx", + "afg_hpc_needs_2022.xlsx", + "afg_hpc_needs_2021.xlsx", + ], + ) + def test_search_in_hdx(self, configuration, search): datasets = Dataset.search_in_hdx("ACLED") assert len(datasets) == 10 diff --git a/tests/hdx/data/test_showcase.py b/tests/hdx/data/test_showcase.py index 89255e5a..0d5c24ff 100755 --- a/tests/hdx/data/test_showcase.py +++ b/tests/hdx/data/test_showcase.py @@ -147,7 +147,7 @@ def mockallsearch(url, datadict): copy.deepcopy(newsearchdict["results"]) ) for i, x in enumerate(newsearchdict["results"]): - x["id"] = f'{x["id"]}{i}' + x["id"] = f"{x['id']}{i}" return MockResponse( 200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=package_search"}' diff --git a/tests/hdx/data/test_update_logic.py b/tests/hdx/data/test_update_logic.py index 9feac265..98fbfcf7 100644 --- a/tests/hdx/data/test_update_logic.py +++ b/tests/hdx/data/test_update_logic.py @@ -142,9 +142,9 @@ def check_resources(update, results): ) name = f"/tmp/WHO/{name}" fu = files_to_upload[key] - assert ( - name == fu - ), f'Mismatch at index {index}: "{rn}", file to upload is "{fu}"' + assert name == fu, ( + f'Mismatch at index {index}: "{rn}", file to upload is "{fu}"' + ) def test_update_logic_1( self, From c3124c7ec507d267707ee03efbaf25e2b014b380 Mon Sep 17 00:00:00 2001 From: mcarans Date: Fri, 14 Feb 2025 17:15:23 +1300 Subject: [PATCH 2/2] Add move resource method to Dataset with test --- src/hdx/data/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hdx/data/dataset.py b/src/hdx/data/dataset.py index 1d0d7536..29f417c9 100755 --- a/src/hdx/data/dataset.py +++ b/src/hdx/data/dataset.py @@ -447,7 +447,7 @@ def move_resource( elif res_name.startswith(insert_before): to_index = i if to_index is None: - # insert at the start if a manual resource for year cannot be found + # insert at the start if resource cannot be found to_index = 0 resource = self.resources.pop(from_index) if from_index < to_index: