From bb22edfacf35f2b616146c05c37ae514f8730334 Mon Sep 17 00:00:00 2001 From: mcarans Date: Fri, 30 May 2025 13:52:03 +1200 Subject: [PATCH 1/3] Accept a variety of types in write_to_hdx and also check env var --- requirements.txt | 18 +++++++------- src/hdx/api/utilities/hdx_error_handler.py | 12 +++++++--- .../api/utilities/test_hdx_error_handler.py | 24 +++++++++++++++++++ 3 files changed, 43 insertions(+), 11 deletions(-) diff --git a/requirements.txt b/requirements.txt index 499a608f..d5e1fb41 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,13 +26,13 @@ charset-normalizer==3.4.2 # via requests ckanapi==4.8 # via hdx-python-api (pyproject.toml) -click==8.1.8 +click==8.2.1 # via # mkdocs # typer colorama==0.4.6 # via mkdocs-material -coverage==7.8.1 +coverage==7.8.2 # via pytest-cov defopt==6.4.0 # via hdx-python-api (pyproject.toml) @@ -72,7 +72,7 @@ hdx-python-utilities==3.8.7 # hdx-python-country humanize==4.12.3 # via frictionless -identify==2.6.10 +identify==2.6.12 # via pre-commit idna==3.10 # via @@ -96,7 +96,7 @@ jsonlines==4.0.0 # via hdx-python-utilities jsonpath-ng==1.7.0 # via libhxl -jsonschema==4.23.0 +jsonschema==4.24.0 # via # frictionless # tableschema-to-template @@ -129,7 +129,7 @@ mergedeep==1.3.4 # via # mkdocs # mkdocs-get-deps -mkapi==4.4.0 +mkapi==4.4.1 # via hdx-python-api (pyproject.toml) mkdocs==1.6.1 # via @@ -256,9 +256,11 @@ rpds-py==0.25.1 # referencing rsa==4.9.1 # via google-auth -ruamel-yaml==0.18.10 +ruamel-yaml==0.18.11 # via hdx-python-utilities -setuptools==80.8.0 +ruamel-yaml-clib==0.2.12 + # via ruamel-yaml +setuptools==80.9.0 # via ckanapi shellingham==1.5.4 # via typer @@ -286,7 +288,7 @@ text-unidecode==1.3 # via python-slugify typeguard==4.4.2 # via inflect -typer==0.15.4 +typer==0.16.0 # via frictionless typing-extensions==4.13.2 # via diff --git a/src/hdx/api/utilities/hdx_error_handler.py b/src/hdx/api/utilities/hdx_error_handler.py index 95250f63..9e46abf0 100644 --- a/src/hdx/api/utilities/hdx_error_handler.py +++ b/src/hdx/api/utilities/hdx_error_handler.py @@ -1,4 +1,5 @@ import logging +from os import getenv from typing import Any, Tuple from hdx.data.dataset import Dataset @@ -19,17 +20,22 @@ class HDXErrorHandler(ErrorHandler): Args: should_exit_on_error (bool): Whether to exit with a 1 code if there are errors. Default is False. - write_to_hdx (bool): Whether to write errors to HDX resources. Default is False. + write_to_hdx (Any): Whether to write errors to HDX resources. Default is None (write errors). """ def __init__( self, should_exit_on_error: bool = False, - write_to_hdx: bool = False, + write_to_hdx: Any = None, ): super().__init__(should_exit_on_error) - self._write_to_hdx = write_to_hdx + if write_to_hdx is None: + write_to_hdx = getenv("ERR_TO_HDX", True) + if write_to_hdx in (False, 0, "false", "False", "FALSE", "N", "n", ""): + self._write_to_hdx = False + else: + self._write_to_hdx = True self.shared_errors["hdx_error"] = {} @staticmethod diff --git a/tests/hdx/api/utilities/test_hdx_error_handler.py b/tests/hdx/api/utilities/test_hdx_error_handler.py index 027eb274..f089564c 100644 --- a/tests/hdx/api/utilities/test_hdx_error_handler.py +++ b/tests/hdx/api/utilities/test_hdx_error_handler.py @@ -12,6 +12,30 @@ class TestHDXErrorHandler: def test_hdx_error_handler(self, caplog): + error_handler = HDXErrorHandler() + assert error_handler._write_to_hdx is True + error_handler = HDXErrorHandler(write_to_hdx=None) + assert error_handler._write_to_hdx is True + error_handler = HDXErrorHandler(write_to_hdx="true") + assert error_handler._write_to_hdx is True + error_handler = HDXErrorHandler(write_to_hdx="Y") + assert error_handler._write_to_hdx is True + error_handler = HDXErrorHandler(write_to_hdx=True) + assert error_handler._write_to_hdx is True + error_handler = HDXErrorHandler(write_to_hdx=1) + assert error_handler._write_to_hdx is True + error_handler = HDXErrorHandler(write_to_hdx="") + assert error_handler._write_to_hdx is False + error_handler = HDXErrorHandler(write_to_hdx="false") + assert error_handler._write_to_hdx is False + error_handler = HDXErrorHandler(write_to_hdx="FALSE") + assert error_handler._write_to_hdx is False + error_handler = HDXErrorHandler(write_to_hdx="n") + assert error_handler._write_to_hdx is False + error_handler = HDXErrorHandler(write_to_hdx=False) + assert error_handler._write_to_hdx is False + error_handler = HDXErrorHandler(write_to_hdx=0) + assert error_handler._write_to_hdx is False with pytest.raises(SystemExit): with caplog.at_level(logging.ERROR): with HDXErrorHandler(should_exit_on_error=True) as errors: From 2602173098ba2f6e0d19ef0820e37195620a366d Mon Sep 17 00:00:00 2001 From: mcarans Date: Fri, 30 May 2025 13:55:18 +1200 Subject: [PATCH 2/3] Accept a variety of types in write_to_hdx and also check env var --- src/hdx/api/utilities/hdx_error_handler.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/hdx/api/utilities/hdx_error_handler.py b/src/hdx/api/utilities/hdx_error_handler.py index 9e46abf0..f6480a43 100644 --- a/src/hdx/api/utilities/hdx_error_handler.py +++ b/src/hdx/api/utilities/hdx_error_handler.py @@ -34,8 +34,10 @@ def __init__( write_to_hdx = getenv("ERR_TO_HDX", True) if write_to_hdx in (False, 0, "false", "False", "FALSE", "N", "n", ""): self._write_to_hdx = False + logger.info("Errors won't be written to HDX") else: self._write_to_hdx = True + logger.info("Errors will be written to HDX") self.shared_errors["hdx_error"] = {} @staticmethod From 1a779c71b9816cb9642556a5ae35b36e307361ac Mon Sep 17 00:00:00 2001 From: mcarans Date: Fri, 30 May 2025 14:01:25 +1200 Subject: [PATCH 3/3] Accept a variety of types in write_to_hdx and also check env var --- tests/hdx/api/utilities/test_hdx_error_handler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/hdx/api/utilities/test_hdx_error_handler.py b/tests/hdx/api/utilities/test_hdx_error_handler.py index f089564c..03fe9ce9 100644 --- a/tests/hdx/api/utilities/test_hdx_error_handler.py +++ b/tests/hdx/api/utilities/test_hdx_error_handler.py @@ -38,7 +38,9 @@ def test_hdx_error_handler(self, caplog): assert error_handler._write_to_hdx is False with pytest.raises(SystemExit): with caplog.at_level(logging.ERROR): - with HDXErrorHandler(should_exit_on_error=True) as errors: + with HDXErrorHandler( + should_exit_on_error=True, write_to_hdx=False + ) as errors: errors.add_message( "pipeline1", "dataset1",