diff --git a/.config/coveragerc b/.coveragerc similarity index 100% rename from .config/coveragerc rename to .coveragerc diff --git a/.config/pre-commit-config.yaml b/.pre-commit-config.yaml similarity index 84% rename from .config/pre-commit-config.yaml rename to .pre-commit-config.yaml index 1f135149..e66d3ee3 100644 --- a/.config/pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,16 +8,15 @@ repos: - id: end-of-file-fixer - id: check-ast - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.9.6 + rev: v0.9.10 hooks: # Run the linter. - id: ruff - args: [--config, .config/ruff.toml, --fix] + args: [ --fix ] # Run the formatter. - id: ruff-format - args: [--config, .config/ruff.toml] - repo: https://github.com/astral-sh/uv-pre-commit - rev: 0.5.29 + rev: 0.6.5 hooks: # Run the pip compile - id: pip-compile diff --git a/documentation/.readthedocs.yaml b/documentation/.readthedocs.yaml index 3c9949f1..ffc61b37 100644 --- a/documentation/.readthedocs.yaml +++ b/documentation/.readthedocs.yaml @@ -1,16 +1,14 @@ +# Required version: 2 -mkdocs: - configuration: docs/mkdocs.yml - fail_on_warning: false - +# Set the version of Python and other tools you might need build: - os: ubuntu-22.04 + os: ubuntu-24.04 tools: - python: "3.11" + python: "3.12" jobs: pre_build: - - pip install --upgrade mkdocs - - pip install mkdocs-material - - pip install pydoc-markdown - - pydoc-markdown documentation/pydoc-markdown.yaml + - pip install .[docs] + +mkdocs: + configuration: documentation/mkdocs.yaml diff --git a/documentation/main.md b/documentation/index.md similarity index 100% rename from documentation/main.md rename to documentation/index.md diff --git a/documentation/mkdocs.yaml b/documentation/mkdocs.yaml new file mode 100644 index 00000000..54058e8f --- /dev/null +++ b/documentation/mkdocs.yaml @@ -0,0 +1,25 @@ +site_name: HDX Python API +repo_url: https://github.com/OCHA-DAP/hdx-python-api/ +repo_name: OCHA-DAP/hdx-python-api +docs_dir: . +site_dir: ../site +theme: + name: material + highlightjs: true +plugins: + - search + - mkapi +nav: + - Home: index.md + - API Documentation: + - Configuration: $src/hdx.api.configuration.* + - Locations: $src/hdx.api.locations.* + - Dataset: $src/hdx.data.dataset.* + - Resource: $src/hdx.data.resource.* + - Resource View: $src/hdx.data.resource_view.* + - Showcase: $src/hdx.data.showcase.* + - User: $src/hdx.data.user.* + - Organization: $src/hdx.data.organization.* + - Vocabulary: $src/hdx.data.vocabulary.* + - Simple Facade: $src/hdx.facades.simple.* + - Keyword Arguments Facade: $src/hdx.facades.keyword_arguments.* diff --git a/documentation/pydoc-markdown.yaml b/documentation/pydoc-markdown.yaml deleted file mode 100644 index f7906e27..00000000 --- a/documentation/pydoc-markdown.yaml +++ /dev/null @@ -1,58 +0,0 @@ -loaders: - - type: python - search_path: - - ../src - packages: - - hdx.api - - hdx.data - - hdx.facades -renderer: - type: mkdocs - output_directory: docs - mkdocs_config: - site_name: HDX Python API - theme: material - repo_url: "https://github.com/OCHA-DAP/hdx-python-api" - markdown: - source_linker: - type: github - repo: OCHA-DAP/hdx-python-api - pages: - - title: Home - name: index - source: main.md - - title: API Documentation - children: - - title: Configuration - contents: - - hdx.api.configuration.* - - title: Locations - contents: - - hdx.api.locations.* - - title: Dataset - contents: - - hdx.data.dataset.* - - title: Resource - contents: - - hdx.data.resource.* - - title: Resource View - contents: - - hdx.data.resource_view.* - - title: Showcase - contents: - - hdx.data.showcase.* - - title: User - contents: - - hdx.data.user.* - - title: Organization - contents: - - hdx.data.organization.* - - title: Vocabulary - contents: - - hdx.data.vocabulary.* - - title: Simple Facade - contents: - - hdx.facades.simple.* - - title: Keyword Arguments Facade - contents: - - hdx.facades.keyword_arguments.* diff --git a/hatch.toml b/hatch.toml new file mode 100644 index 00000000..1ce9ac89 --- /dev/null +++ b/hatch.toml @@ -0,0 +1,37 @@ +# Build + +[build.targets.wheel] +packages = ["src/hdx"] + +[build.hooks.vcs] +version-file = "src/hdx/api/_version.py" + +[metadata] +allow-direct-references = true + +# Versioning + +[version] +source = "vcs" + +[version.raw-options] +local_scheme = "no-local-version" +version_scheme = "python-simplified-semver" + +# Tests + +[envs.hatch-test] +features = ["test"] + +[[envs.hatch-test.matrix]] +python = ["3.12"] + +[envs.hatch-test.scripts] +run = """ + pytest --rootdir=. --junitxml=test-results.xml --cov --no-cov-on-fail \ + --cov-report=lcov --cov-report=term-missing + """ + +[envs.hatch-static-analysis] +config-path = "none" +dependencies = ["ruff==0.9.10"] diff --git a/pyproject.toml b/pyproject.toml index ea10c601..01523859 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,8 +37,8 @@ dependencies = [ "ckanapi>=4.8", "defopt>=6.4.0", "email_validator", - "hdx-python-country>=3.8.8", - "hdx-python-utilities>=3.8.3", + "hdx-python-country>=3.9.2", + "hdx-python-utilities>=3.8.6", "libhxl>=5.2.2", "makefun", "quantulum3", @@ -56,52 +56,4 @@ Homepage = "https://github.com/OCHA-DAP/hdx-python-api" [project.optional-dependencies] test = ["pytest", "pytest-check", "pytest-cov", "gspread"] dev = ["pre-commit"] - - -######### -# Hatch # -######### - -# Build - -[tool.hatch.build.targets.wheel] -packages = ["src/hdx"] - -[tool.hatch.build.hooks.vcs] -version-file = "src/hdx/api/_version.py" - -[tool.hatch.metadata] -allow-direct-references = true - -# Versioning - -[tool.hatch.version] -source = "vcs" - -[tool.hatch.version.raw-options] -local_scheme = "no-local-version" -version_scheme = "python-simplified-semver" - -# Tests - -[tool.hatch.envs.hatch-test] -features = ["test"] - -[[tool.hatch.envs.hatch-test.matrix]] -python = ["3.12"] - -[tool.hatch.envs.hatch-test.scripts] -run = """ - pytest -c .config/pytest.ini --rootdir=. --junitxml=test-results.xml \ - --cov --cov-config=.config/coveragerc --no-cov-on-fail \ - --cov-report=lcov --cov-report=term-missing - """ - -[tool.hatch.envs.hatch-static-analysis] -dependencies = ["ruff==0.9.6"] - -[tool.hatch.envs.hatch-static-analysis.scripts] -format-check = ["ruff format --config .config/ruff.toml --check --diff {args:.}",] -format-fix = ["ruff format --config .config/ruff.toml {args:.}",] -lint-check = ["ruff check --config .config/ruff.toml {args:.}",] -lint-fix = ["ruff check --config .config/ruff.toml --fix {args:.}",] +docs = ["mkapi"] diff --git a/.config/pytest.ini b/pytest.ini similarity index 69% rename from .config/pytest.ini rename to pytest.ini index bb28fd3a..3009ece7 100644 --- a/.config/pytest.ini +++ b/pytest.ini @@ -1,4 +1,4 @@ [pytest] -pythonpath = ../src +pythonpath = src addopts = "--color=yes" log_cli = 1 diff --git a/requirements.txt b/requirements.txt index 152e4bed..168b3fa3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,28 +2,30 @@ # uv pip compile pyproject.toml --resolver=backtracking --all-extras -o requirements.txt annotated-types==0.7.0 # via pydantic -attrs==25.1.0 +attrs==25.3.0 # via # frictionless # jsonlines # jsonschema # referencing -cachetools==5.5.1 +cachetools==5.5.2 # via google-auth -certifi==2025.1.31 +certifi==2025.4.26 # via requests cfgv==3.4.0 # via pre-commit chardet==5.2.0 # via frictionless -charset-normalizer==3.4.1 +charset-normalizer==3.4.2 # via requests ckanapi==4.8 # via hdx-python-api (pyproject.toml) click==8.1.8 # via typer -coverage==7.6.12 +coverage==7.8.0 # via pytest-cov +cydifflib==1.2.0 + # via hdx-python-utilities defopt==6.4.0 # via hdx-python-api (pyproject.toml) distlib==0.3.9 @@ -40,27 +42,27 @@ email-validator==2.2.0 # via hdx-python-api (pyproject.toml) et-xmlfile==2.0.0 # via openpyxl -filelock==3.17.0 +filelock==3.18.0 # via virtualenv -frictionless==5.18.0 +frictionless==5.18.1 # via hdx-python-utilities -google-auth==2.38.0 +google-auth==2.40.0 # via # google-auth-oauthlib # gspread -google-auth-oauthlib==1.2.1 +google-auth-oauthlib==1.2.2 # via gspread -gspread==6.1.4 +gspread==6.2.0 # via hdx-python-api (pyproject.toml) -hdx-python-country==3.8.8 +hdx-python-country==3.9.2 # via hdx-python-api (pyproject.toml) -hdx-python-utilities==3.8.3 +hdx-python-utilities==3.8.6 # via # hdx-python-api (pyproject.toml) # hdx-python-country -humanize==4.11.0 +humanize==4.12.3 # via frictionless -identify==2.6.7 +identify==2.6.10 # via pre-commit idna==3.10 # via @@ -70,11 +72,11 @@ ijson==3.3.0 # via hdx-python-utilities inflect==7.5.0 # via quantulum3 -iniconfig==2.0.0 +iniconfig==2.1.0 # via pytest isodate==0.7.2 # via frictionless -jinja2==3.1.5 +jinja2==3.1.6 # via frictionless jsonlines==4.0.0 # via hdx-python-utilities @@ -84,7 +86,7 @@ jsonschema==4.23.0 # via # frictionless # tableschema-to-template -jsonschema-specifications==2024.10.1 +jsonschema-specifications==2025.4.1 # via jsonschema libhxl==5.2.2 # via @@ -96,13 +98,13 @@ makefun==1.15.6 # via hdx-python-api (pyproject.toml) markdown-it-py==3.0.0 # via rich -marko==2.1.2 +marko==2.1.3 # via frictionless markupsafe==3.0.2 # via jinja2 mdurl==0.1.2 # via markdown-it-py -more-itertools==10.6.0 +more-itertools==10.7.0 # via inflect nodeenv==1.9.1 # via pre-commit @@ -112,11 +114,11 @@ oauthlib==3.2.2 # via requests-oauthlib openpyxl==3.1.5 # via hdx-python-utilities -packaging==24.2 +packaging==25.0 # via pytest -petl==1.7.15 +petl==1.7.16 # via frictionless -platformdirs==4.3.6 +platformdirs==4.3.7 # via virtualenv pluggy==1.5.0 # via pytest @@ -126,30 +128,30 @@ ply==3.11 # libhxl pockets==0.9.1 # via sphinxcontrib-napoleon -pre-commit==4.1.0 +pre-commit==4.2.0 # via hdx-python-api (pyproject.toml) pyasn1==0.6.1 # via # pyasn1-modules # rsa -pyasn1-modules==0.4.1 +pyasn1-modules==0.4.2 # via google-auth -pydantic==2.10.6 +pydantic==2.11.4 # via frictionless -pydantic-core==2.27.2 +pydantic-core==2.33.2 # via pydantic pygments==2.19.1 # via rich pyphonetics==0.5.3 # via hdx-python-utilities -pytest==8.3.4 +pytest==8.3.5 # via # hdx-python-api (pyproject.toml) # pytest-check # pytest-cov -pytest-check==2.5.0 +pytest-check==2.5.3 # via hdx-python-api (pyproject.toml) -pytest-cov==6.0.0 +pytest-cov==6.1.1 # via hdx-python-api (pyproject.toml) python-dateutil==2.9.0.post0 # via @@ -189,25 +191,25 @@ requests-oauthlib==2.0.0 # via google-auth-oauthlib rfc3986==2.0.0 # via frictionless -rich==13.9.4 +rich==14.0.0 # via typer -rpds-py==0.22.3 +rpds-py==0.24.0 # via # jsonschema # referencing -rsa==4.9 +rsa==4.9.1 # via google-auth ruamel-yaml==0.18.10 # via hdx-python-utilities ruamel-yaml-clib==0.2.12 # via ruamel-yaml -setuptools==75.8.0 +setuptools==80.3.1 # via ckanapi shellingham==1.5.4 # via typer simpleeval==1.0.3 # via frictionless -simplejson==3.19.3 +simplejson==3.20.1 # via ckanapi six==1.17.0 # via @@ -217,23 +219,21 @@ six==1.17.0 # sphinxcontrib-napoleon sphinxcontrib-napoleon==0.7 # via defopt -stringcase==1.2.0 - # via frictionless -structlog==25.1.0 +structlog==25.3.0 # via libhxl tableschema-to-template==0.0.13 # via hdx-python-utilities tabulate==0.9.0 # via frictionless -tenacity==9.0.0 +tenacity==9.1.2 # via hdx-python-country text-unidecode==1.3 # via python-slugify -typeguard==4.4.1 +typeguard==4.4.2 # via inflect -typer==0.15.1 +typer==0.15.3 # via frictionless -typing-extensions==4.12.2 +typing-extensions==4.13.2 # via # frictionless # pydantic @@ -241,17 +241,20 @@ typing-extensions==4.12.2 # referencing # typeguard # typer -unidecode==1.3.8 + # typing-inspection +typing-inspection==0.4.0 + # via pydantic +unidecode==1.4.0 # via # libhxl # pyphonetics -urllib3==2.3.0 +urllib3==2.4.0 # via # libhxl # requests -validators==0.34.0 +validators==0.35.0 # via frictionless -virtualenv==20.29.2 +virtualenv==20.31.1 # via pre-commit wheel==0.45.1 # via libhxl @@ -261,7 +264,7 @@ xlrd3==1.1.0 # via libhxl xlsx2csv==0.8.4 # via hdx-python-utilities -xlsxwriter==3.2.2 +xlsxwriter==3.2.3 # via tableschema-to-template xlwt==1.3.0 # via hdx-python-utilities diff --git a/.config/ruff.toml b/ruff.toml similarity index 93% rename from .config/ruff.toml rename to ruff.toml index 4625a347..cf4db097 100644 --- a/.config/ruff.toml +++ b/ruff.toml @@ -1,4 +1,3 @@ -line-length = 79 exclude = ["_version.py"] [lint] diff --git a/src/hdx/api/configuration.py b/src/hdx/api/configuration.py index c463e264..7c97dbce 100755 --- a/src/hdx/api/configuration.py +++ b/src/hdx/api/configuration.py @@ -73,32 +73,22 @@ def __init__(self, **kwargs: Any) -> None: hdx_base_config_json = kwargs.get("hdx_base_config_json", "") if hdx_base_config_json: if hdx_base_config_found: - raise ConfigurationError( - "More than one HDX base configuration given!" - ) + raise ConfigurationError("More than one HDX base configuration given!") hdx_base_config_found = True - logger.info( - f"Loading HDX base configuration from: {hdx_base_config_json}" - ) + logger.info(f"Loading HDX base configuration from: {hdx_base_config_json}") hdx_base_config_dict = load_json(hdx_base_config_json) hdx_base_config_yaml = kwargs.get("hdx_base_config_yaml", "") if hdx_base_config_found: if hdx_base_config_yaml: - raise ConfigurationError( - "More than one HDX base configuration given!" - ) + raise ConfigurationError("More than one HDX base configuration given!") else: if not hdx_base_config_yaml: - hdx_base_config_yaml = ( - Configuration.default_hdx_base_config_yaml - ) + hdx_base_config_yaml = Configuration.default_hdx_base_config_yaml logger.info( f"No HDX base configuration parameter. Using default base configuration file: {hdx_base_config_yaml}." ) - logger.info( - f"Loading HDX base configuration from: {hdx_base_config_yaml}" - ) + logger.info(f"Loading HDX base configuration from: {hdx_base_config_yaml}") hdx_base_config_dict = load_yaml(hdx_base_config_yaml) hdx_config_found = False @@ -110,9 +100,7 @@ def __init__(self, **kwargs: Any) -> None: hdx_config_json = kwargs.get("hdx_config_json", "") if hdx_config_json: if hdx_config_found: - raise ConfigurationError( - "More than one HDX configuration given!" - ) + raise ConfigurationError("More than one HDX configuration given!") hdx_config_found = True logger.info(f"Loading HDX configuration from: {hdx_config_json}") hdx_config_dict = load_json(hdx_config_json) @@ -120,9 +108,7 @@ def __init__(self, **kwargs: Any) -> None: hdx_config_yaml = kwargs.get("hdx_config_yaml", "") if hdx_config_found: if hdx_config_yaml: - raise ConfigurationError( - "More than one HDX configuration given!" - ) + raise ConfigurationError("More than one HDX configuration given!") else: if not hdx_config_yaml: hdx_config_yaml = Configuration.default_hdx_config_yaml @@ -139,14 +125,10 @@ def __init__(self, **kwargs: Any) -> None: hdx_config_yaml = None hdx_config_dict = {} if hdx_config_yaml: - logger.info( - f"Loading HDX configuration from: {hdx_config_yaml}" - ) + logger.info(f"Loading HDX configuration from: {hdx_config_yaml}") hdx_config_dict = load_yaml(hdx_config_yaml) - self.data = merge_two_dictionaries( - hdx_base_config_dict, hdx_config_dict - ) + self.data = merge_two_dictionaries(hdx_base_config_dict, hdx_config_dict) project_config_found = False project_config_dict = kwargs.get("project_config_dict") @@ -157,21 +139,15 @@ def __init__(self, **kwargs: Any) -> None: project_config_json = kwargs.get("project_config_json", "") if project_config_json: if project_config_found: - raise ConfigurationError( - "More than one project configuration given!" - ) + raise ConfigurationError("More than one project configuration given!") project_config_found = True - logger.info( - f"Loading project configuration from: {project_config_json}" - ) + logger.info(f"Loading project configuration from: {project_config_json}") project_config_dict = load_json(project_config_json) project_config_yaml = kwargs.get("project_config_yaml", "") if project_config_found: if project_config_yaml: - raise ConfigurationError( - "More than one project configuration given!" - ) + raise ConfigurationError("More than one project configuration given!") else: if project_config_yaml: logger.info( @@ -181,18 +157,14 @@ def __init__(self, **kwargs: Any) -> None: else: project_config_dict = {} - self.data = merge_two_dictionaries( - hdx_base_config_dict, project_config_dict - ) + self.data = merge_two_dictionaries(hdx_base_config_dict, project_config_dict) ua = kwargs.get("full_agent") if ua: self.user_agent = ua else: try: - self.user_agent = UserAgent.get( - prefix=Configuration.prefix, **kwargs - ) + self.user_agent = UserAgent.get(prefix=Configuration.prefix, **kwargs) except UserAgentError: self.user_agent = UserAgent.get( prefix=Configuration.prefix, **self.data @@ -631,9 +603,7 @@ def create( """ if cls._configuration is not None: raise ConfigurationError("Configuration already created!") - return cls._create( - configuration=configuration, remoteckan=remoteckan, **kwargs - ) + return cls._create(configuration=configuration, remoteckan=remoteckan, **kwargs) @classmethod def delete(cls) -> None: diff --git a/src/hdx/api/locations.py b/src/hdx/api/locations.py index e28bfe27..9a573e5c 100755 --- a/src/hdx/api/locations.py +++ b/src/hdx/api/locations.py @@ -115,9 +115,7 @@ def get_HDX_code_from_location_partial( Returns: Tuple[Optional[str], bool]: HDX code and if the match is exact or (None, False) for no match """ - hdx_code = cls.get_HDX_code_from_location( - location, locations, configuration - ) + hdx_code = cls.get_HDX_code_from_location(location, locations, configuration) if hdx_code is not None: return hdx_code, True diff --git a/src/hdx/api/remotehdx.py b/src/hdx/api/remotehdx.py index 6c4e167b..611f143f 100644 --- a/src/hdx/api/remotehdx.py +++ b/src/hdx/api/remotehdx.py @@ -43,9 +43,7 @@ def _request_fn(self, url, data, headers, files, requests_kwargs): return r.status_code, r.text def _request_fn_get(self, url, data_dict, headers, requests_kwargs): - r = self.session.get( - url, params=data_dict, headers=headers, **requests_kwargs - ) + r = self.session.get(url, params=data_dict, headers=headers, **requests_kwargs) if r.status_code == 429: retryafter = r.headers.get("Retry-After") if retryafter: diff --git a/src/hdx/api/utilities/dataset_title_helper.py b/src/hdx/api/utilities/dataset_title_helper.py index f5a3eeaf..5ada66e6 100755 --- a/src/hdx/api/utilities/dataset_title_helper.py +++ b/src/hdx/api/utilities/dataset_title_helper.py @@ -101,9 +101,7 @@ def fuzzy_match_dates_in_title( ranges.append((startdaterl, enddaterl)) else: date_components = year - ranges.append( - parse_date_range(year, zero_time=True, max_endtime=True) - ) + ranges.append(parse_date_range(year, zero_time=True, max_endtime=True)) newtitle = title for date_component in date_components: newtitle = remove_string( @@ -211,9 +209,7 @@ def get_dates_from_title( ) ranges.append((startdate, enddate)) newtitle = remove_string(title, match.group(0)) - logger.info( - f"Removing date range from title: {title} -> {newtitle}" - ) + logger.info(f"Removing date range from title: {title} -> {newtitle}") title = newtitle for match in cls.YEAR_RANGE_PATTERN2.finditer(title): @@ -236,14 +232,10 @@ def get_dates_from_title( ) ranges.append((startdate, enddate)) newtitle = remove_string(title, match.group(0)) - logger.info( - f"Removing date range from title: {title} -> {newtitle}" - ) + logger.info(f"Removing date range from title: {title} -> {newtitle}") title = newtitle - title = cls.fuzzy_match_dates_in_title( - title, ranges, ignore_wrong_years - ) + title = cls.fuzzy_match_dates_in_title(title, ranges, ignore_wrong_years) for match in cls.WORD_RIGHT_BRACKET_PATTERN.finditer(title): word = match.group(2) diff --git a/src/hdx/api/utilities/filestore_helper.py b/src/hdx/api/utilities/filestore_helper.py index 66cee014..446f7acf 100755 --- a/src/hdx/api/utilities/filestore_helper.py +++ b/src/hdx/api/utilities/filestore_helper.py @@ -27,11 +27,7 @@ def resource_check_required_fields( """ if "ignore_check" in kwargs: # allow ignoring of field checks return - if ( - check_upload - and resource.get_file_to_upload() - and "url" in resource.data - ): + if check_upload and resource.get_file_to_upload() and "url" in resource.data: del resource.data["url"] ignore_fields = kwargs.get("ignore_fields", list()) resource_ignore_fields = [] @@ -92,7 +88,7 @@ def dataset_update_filestore_resource( data_updated = resource_data_to_update.is_marked_data_updated() if data_updated: # Should not output timezone info here - resource_data_to_update["last_modified"] = ( - now_utc_notz().isoformat(timespec="microseconds") + resource_data_to_update["last_modified"] = now_utc_notz().isoformat( + timespec="microseconds" ) resource_data_to_update.data_updated = False diff --git a/src/hdx/api/utilities/hdx_error_handler.py b/src/hdx/api/utilities/hdx_error_handler.py index 54480048..95250f63 100644 --- a/src/hdx/api/utilities/hdx_error_handler.py +++ b/src/hdx/api/utilities/hdx_error_handler.py @@ -96,9 +96,7 @@ def add_message( None """ self.add(text, self.get_category(pipeline, identifier), message_type) - self.errors_to_hdx( - pipeline, identifier, text, resource_name, err_to_hdx - ) + self.errors_to_hdx(pipeline, identifier, text, resource_name, err_to_hdx) def add_missing_value_message( self, @@ -227,9 +225,7 @@ def write_errors_to_resource( error_text = ", ".join(sorted(errors)) dataset = Dataset.read_from_hdx(dataset_name) try: - success = dataset.add_hapi_error( - error_text, resource_name=resource_name - ) + success = dataset.add_hapi_error(error_text, resource_name=resource_name) except (HDXError, AttributeError): logger.error(f"Could not write error to {dataset_name}") return False diff --git a/src/hdx/data/dataset.py b/src/hdx/data/dataset.py index 29f417c9..05016ae2 100755 --- a/src/hdx/data/dataset.py +++ b/src/hdx/data/dataset.py @@ -249,9 +249,7 @@ def _get_resource_from_obj( resource, configuration=self.configuration ) elif isinstance(resource, dict): - resource = res_module.Resource( - resource, configuration=self.configuration - ) + resource = res_module.Resource(resource, configuration=self.configuration) if not isinstance(resource, res_module.Resource): raise HDXError( f"Type {type(resource).__name__} cannot be added as a resource!" @@ -279,9 +277,7 @@ def add_update_resource( f"Resource {resource['name']} being added already has a dataset id!" ) resource.check_url_filetoupload() - resource_index = ResourceMatcher.match_resource_list( - self.resources, resource - ) + resource_index = ResourceMatcher.match_resource_list(self.resources, resource) if resource_index is None: self.resources.append(resource) else: @@ -289,9 +285,7 @@ def add_update_resource( self.resources[resource_index], resource ) if resource.get_file_to_upload(): - updated_resource.set_file_to_upload( - resource.get_file_to_upload() - ) + updated_resource.set_file_to_upload(resource.get_file_to_upload()) if resource.is_marked_data_updated(): updated_resource.mark_data_updated() @@ -323,9 +317,7 @@ def add_update_resources( updated_resource_matches, _, updated_resource_no_matches, - ) = ResourceMatcher.match_resource_lists( - self.resources, resource_objects - ) + ) = ResourceMatcher.match_resource_lists(self.resources, resource_objects) for i, resource_index in enumerate(resource_matches): resource = resource_objects[updated_resource_matches[i]] resource.check_url_filetoupload() @@ -333,9 +325,7 @@ def add_update_resources( self.resources[resource_index], resource ) if resource.get_file_to_upload(): - updated_resource.set_file_to_upload( - resource.get_file_to_upload() - ) + updated_resource.set_file_to_upload(resource.get_file_to_upload()) if resource.is_marked_data_updated(): updated_resource.mark_data_updated() for resource_index in updated_resource_no_matches: @@ -415,9 +405,7 @@ def reorder_resources( ordered_ids = results["order"] reordered_resources = [] for resource_id in ordered_ids: - resource = next( - x for x in self.resources if x["id"] == resource_id - ) + resource = next(x for x in self.resources if x["id"] == resource_id) reordered_resources.append(resource) self.resources = reordered_resources if hxl_update: @@ -553,9 +541,7 @@ def check_required_fields( if not ignore_field.startswith("resource:"): dataset_ignore_fields.append(ignore_field) if self.is_requestable(): - self._check_required_fields( - "dataset-requestable", dataset_ignore_fields - ) + self._check_required_fields("dataset-requestable", dataset_ignore_fields) else: self._check_required_fields("dataset", dataset_ignore_fields) if len(self.resources) == 0 and not allow_no_resources: @@ -679,11 +665,7 @@ def _revise_filter( el_name = orig_value.get("name") present = False for value_dict in value: - if ( - el_id - and "id" in value_dict - and value_dict["id"] == el_id - ): + if el_id and "id" in value_dict and value_dict["id"] == el_id: present = True break if ( @@ -740,9 +722,9 @@ def _revise_files_to_upload_resource_deletions( resource_index, file_to_upload, ) in filestore_resources.items(): - files_to_upload[ - f"update__resources__{resource_index}__upload" - ] = file_to_upload + files_to_upload[f"update__resources__{resource_index}__upload"] = ( + file_to_upload + ) return files_to_upload def _revise_dataset( @@ -784,13 +766,9 @@ def _revise_dataset( if ( "skip_validation" in kwargs ): # Whether or not CKAN should perform validation steps (checking fields present) - dataset_data_to_update["skip_validation"] = kwargs[ - "skip_validation" - ] + dataset_data_to_update["skip_validation"] = kwargs["skip_validation"] dataset_data_to_update["state"] = "active" - revise_filter = self._revise_filter( - dataset_data_to_update, keys_to_delete - ) + revise_filter = self._revise_filter(dataset_data_to_update, keys_to_delete) files_to_upload = self._revise_files_to_upload_resource_deletions( revise_filter, resources_to_update, @@ -819,9 +797,7 @@ def _revise_dataset( "ignore_check" ): # allow ignoring of field checks ignore_fields = kwargs.get("ignore_fields", list()) - ignore_field = self.configuration["dataset"].get( - "ignore_on_update" - ) + ignore_field = self.configuration["dataset"].get("ignore_on_update") if ignore_field and ignore_field not in ignore_fields: ignore_fields.append(ignore_field) ignore_field = kwargs.get("ignore_field") @@ -829,9 +805,7 @@ def _revise_dataset( ignore_fields.append(ignore_field) self.check_required_fields(ignore_fields=ignore_fields) if new_resource_order: - existing_order = [ - (x["name"], x["format"].lower()) for x in self.resources - ] + existing_order = [(x["name"], x["format"].lower()) for x in self.resources] if existing_order != new_resource_order: sorted_resources = sorted( self.resources, @@ -899,9 +873,7 @@ def _dataset_update_resources( resource_data_to_update = resources_metadata_to_update[ updated_resource_matches[match_index] ] - logger.warning( - f"Resource exists. Updating {resource['name']}" - ) + logger.warning(f"Resource exists. Updating {resource['name']}") FilestoreHelper.dataset_update_filestore_resource( resource_data_to_update, filestore_resources, @@ -936,9 +908,7 @@ def _dataset_update_resources( if len(self.resources) > i: updated_resource_name = resource_data_to_update["name"] resource_name = self.resources[i]["name"] - logger.warning( - f"Resource exists. Updating {resource_name}" - ) + logger.warning(f"Resource exists. Updating {resource_name}") if resource_name != updated_resource_name: logger.warning( f"Changing resource name to: {updated_resource_name}" @@ -968,8 +938,7 @@ def _dataset_update_resources( resources_to_delete = list(reversed(resources_to_delete)) if match_resource_order: new_resource_order = [ - (x["name"], x["format"].lower()) - for x in resources_metadata_to_update + (x["name"], x["format"].lower()) for x in resources_metadata_to_update ] else: new_resource_order = None @@ -1081,9 +1050,7 @@ def update_in_hdx( if self._dataset_load_from_hdx(self.data["id"]): loaded = True else: - logger.warning( - f"Failed to load dataset with id {self.data['id']}" - ) + logger.warning(f"Failed to load dataset with id {self.data['id']}") if not loaded: self._check_existing_object("dataset", "name") if not self._dataset_load_from_hdx(self.data["name"]): @@ -1133,9 +1100,7 @@ def create_in_hdx( None """ if "ignore_check" not in kwargs: # allow ignoring of field checks - self.check_required_fields( - allow_no_resources=allow_no_resources, **kwargs - ) + self.check_required_fields(allow_no_resources=allow_no_resources, **kwargs) # No need to check again after revising dataset kwargs["ignore_check"] = True loadedid = None @@ -1143,9 +1108,7 @@ def create_in_hdx( if self._dataset_load_from_hdx(self.data["id"]): loadedid = self.data["id"] else: - logger.warning( - f"Failed to load dataset with id {self.data['id']}" - ) + logger.warning(f"Failed to load dataset with id {self.data['id']}") if not loadedid: if self._dataset_load_from_hdx(self.data["name"]): loadedid = self.data["name"] @@ -1204,9 +1167,7 @@ def hxl_update(self) -> None: Returns: None """ - self._read_from_hdx( - "dataset", self.data["id"], action=self.actions()["hxl"] - ) + self._read_from_hdx("dataset", self.data["id"], action=self.actions()["hxl"]) @classmethod def search_in_hdx( @@ -1316,9 +1277,7 @@ def search_in_hdx( all_datasets = None attempts += 1 if attempts == cls.max_attempts and all_datasets is None: - raise HDXError( - "Maximum attempts reached for searching for datasets!" - ) + raise HDXError("Maximum attempts reached for searching for datasets!") return all_datasets @staticmethod @@ -1507,9 +1466,7 @@ def set_time_period_year_range( ( self.data["dataset_date"], retval, - ) = DateHelper.get_hdx_time_period_from_years( - dataset_year, dataset_end_year - ) + ) = DateHelper.get_hdx_time_period_from_years(dataset_year, dataset_end_year) return retval def set_reference_period_year_range( @@ -1533,9 +1490,7 @@ def list_valid_update_frequencies(cls) -> List[str]: return list(cls.update_frequencies.keys()) @classmethod - def transform_update_frequency( - cls, frequency: Union[str, int] - ) -> Optional[str]: + def transform_update_frequency(cls, frequency: Union[str, int]) -> Optional[str]: """Get numeric update frequency (as string since that is required field format) from textual representation or vice versa (eg. 'Every month' = '30', '30' or 30 = 'Every month') @@ -1561,9 +1516,7 @@ def get_expected_update_frequency(self) -> Optional[str]: else: return None - def set_expected_update_frequency( - self, update_frequency: Union[str, int] - ) -> None: + def set_expected_update_frequency(self, update_frequency: Union[str, int]) -> None: """Set expected update frequency. You can pass frequencies like "Every week" or '7' or 7. Valid values for update frequency can be found from Dataset.list_valid_update_frequencies(). @@ -1578,9 +1531,7 @@ def set_expected_update_frequency( try: int(update_frequency) except ValueError: - update_frequency = Dataset.transform_update_frequency( - update_frequency - ) + update_frequency = Dataset.transform_update_frequency(update_frequency) if update_frequency not in Dataset.update_frequencies.keys(): raise HDXError("Invalid update frequency supplied!") self.data["data_update_frequency"] = update_frequency @@ -1605,9 +1556,7 @@ def add_tag( Returns: Tuple[List[str], List[str]]: Tuple containing list of added tags and list of deleted tags and tags not added """ - return vocabulary.Vocabulary.add_mapped_tag( - self, tag, log_deleted=log_deleted - ) + return vocabulary.Vocabulary.add_mapped_tag(self, tag, log_deleted=log_deleted) def add_tags( self, tags: ListTuple[str], log_deleted: bool = True @@ -1625,9 +1574,7 @@ def add_tags( self, tags, log_deleted=log_deleted ) - def clean_tags( - self, log_deleted: bool = True - ) -> Tuple[List[str], List[str]]: + def clean_tags(self, log_deleted: bool = True) -> Tuple[List[str], List[str]]: """Clean tags in an HDX object according to tags cleanup spreadsheet, deleting invalid tags that cannot be mapped Args: @@ -1742,9 +1689,7 @@ def add_country_location( Returns: bool: True if country added or False if country already present """ - iso3, match = Country.get_iso3_country_code_fuzzy( - country, use_live=use_live - ) + iso3, match = Country.get_iso3_country_code_fuzzy(country, use_live=use_live) if iso3 is None: raise HDXError(f"Country: {country} - cannot find iso3 code!") return self.add_other_location( @@ -1851,9 +1796,7 @@ def remove_location(self, location: str) -> bool: Returns: bool: True if location removed or False if not """ - res = self._remove_hdxobject( - self.data.get("groups"), location, matchon="name" - ) + res = self._remove_hdxobject(self.data.get("groups"), location, matchon="name") if not res: res = self._remove_hdxobject( self.data.get("groups"), location.upper(), matchon="name" @@ -1893,9 +1836,7 @@ def set_maintainer(self, maintainer: Union["User", Dict, str]) -> None: f"Type {type(maintainer).__name__} cannot be added as a maintainer!" ) if is_valid_uuid(maintainer) is False: - raise HDXError( - f"{maintainer} is not a valid user id for a maintainer!" - ) + raise HDXError(f"{maintainer} is not a valid user id for a maintainer!") self.data["maintainer"] = maintainer def get_organization(self) -> "Organization": @@ -1967,9 +1908,7 @@ def _get_dataset_showcase_dict( Returns: dict: Dataset showcase dict """ - if isinstance(showcase, sc_module.Showcase) or isinstance( - showcase, dict - ): + if isinstance(showcase, sc_module.Showcase) or isinstance(showcase, dict): if "id" not in showcase: showcase = sc_module.Showcase.read_from_hdx(showcase["name"]) showcase = showcase["id"] @@ -2026,9 +1965,7 @@ def add_showcases( showcases_to_check = self.get_showcases() allshowcasesadded = True for showcase in showcases: - if not self.add_showcase( - showcase, showcases_to_check=showcases_to_check - ): + if not self.add_showcase(showcase, showcases_to_check=showcases_to_check): allshowcasesadded = False return allshowcasesadded @@ -2212,9 +2149,7 @@ def set_quickchart_resource( """ if isinstance(resource, int) and not isinstance(resource, bool): resource = self.resources[resource] - if isinstance(resource, res_module.Resource) or isinstance( - resource, dict - ): + if isinstance(resource, res_module.Resource) or isinstance(resource, dict): res = resource.get("id") if res is None: resource = resource["name"] @@ -2230,10 +2165,7 @@ def set_quickchart_resource( search = "name" preview_resource = None for dataset_resource in self.resources: - if ( - preview_resource is None - and dataset_resource[search] == resource - ): + if preview_resource is None and dataset_resource[search] == resource: dataset_resource.enable_dataset_preview() self.preview_resource() preview_resource = dataset_resource @@ -2253,9 +2185,7 @@ def quickcharts_resource_last(self) -> bool: return True return False - def create_default_views( - self, create_datastore_views: bool = False - ) -> None: + def create_default_views(self, create_datastore_views: bool = False) -> None: """Create default resource views for all resources in dataset Args: @@ -2278,10 +2208,7 @@ def _create_preview_resourceview(self) -> None: """ if self.preview_resourceview: for resource in self.resources: - if ( - resource["name"] - == self.preview_resourceview["resource_name"] - ): + if resource["name"] == self.preview_resourceview["resource_name"]: del self.preview_resourceview["resource_name"] self.preview_resourceview["resource_id"] = resource["id"] self.preview_resourceview.create_in_hdx() @@ -2390,9 +2317,7 @@ def replace_indicator(qc_config, index): qc_config, f"TITLE_VALUE_{ind_str}", indicator["title"] ) replace = indicator.get("unit", "") - qc_config = replace_string( - qc_config, f"UNIT_VALUE_{ind_str}", replace - ) + qc_config = replace_string(qc_config, f"UNIT_VALUE_{ind_str}", replace) qc_config = replace_col( qc_config, f"CODE_COL_{ind_str}", indicator, "code_col" ) @@ -2736,9 +2661,7 @@ def generate_resource_from_iterable( numeric_hashtag = quickcharts.get("numeric_hashtag") if numeric_hashtag: qc["numeric"] = next( - key - for key, value in hxltags.items() - if value == numeric_hashtag + key for key, value in hxltags.items() if value == numeric_hashtag ) # reverse lookup else: qc["numeric"] = None @@ -2757,9 +2680,7 @@ def generate_resource_from_iterable( if numeric_hashtag and qc["numeric"] not in cutdownhashtags: cutdownhashtags.append(qc["numeric"]) qc["headers"] = [x for x in headers if x in cutdownhashtags] - qc["rows"] = [ - Download.hxl_row(qc["headers"], hxltags, dict_form=True) - ] + qc["rows"] = [Download.hxl_row(qc["headers"], hxltags, dict_form=True)] if yearcol is not None: diff --git a/src/hdx/data/hdxobject.py b/src/hdx/data/hdxobject.py index 6575b79b..9cd85f6c 100755 --- a/src/hdx/data/hdxobject.py +++ b/src/hdx/data/hdxobject.py @@ -178,15 +178,11 @@ def _read_from_hdx_class( return hdxobject return None - def _check_existing_object( - self, object_type: str, id_field_name: str - ) -> None: + def _check_existing_object(self, object_type: str, id_field_name: str) -> None: if not self.data: raise HDXError(f"No data in {object_type}!") if id_field_name not in self.data: - raise HDXError( - f"No {id_field_name} field (mandatory) in {object_type}!" - ) + raise HDXError(f"No {id_field_name} field (mandatory) in {object_type}!") def _check_load_existing_object( self, object_type: str, id_field_name: str, operation: str = "update" @@ -206,9 +202,7 @@ def _check_load_existing_object( raise HDXError(f"No existing {object_type} to {operation}!") @abstractmethod - def check_required_fields( - self, ignore_fields: ListTuple[str] = [] - ) -> None: + def check_required_fields(self, ignore_fields: ListTuple[str] = []) -> None: """Abstract method to check that metadata for HDX object is complete. The parameter ignore_fields should be set if required to any fields that should be ignored for the particular operation. @@ -233,12 +227,8 @@ def _check_required_fields( for field in self.configuration[object_type]["required_fields"]: if field not in ignore_fields: if field not in self.data: - raise HDXError( - f"Field {field} is missing in {object_type}!" - ) - if not self.data[field] and not isinstance( - self.data[field], bool - ): + raise HDXError(f"Field {field} is missing in {object_type}!") + if not self.data[field] and not isinstance(self.data[field], bool): raise HDXError(f"Field {field} is empty in {object_type}!") def _check_kwargs_fields(self, object_type: str, **kwargs: Any) -> None: @@ -265,9 +255,7 @@ def _check_kwargs_fields(self, object_type: str, **kwargs: Any) -> None: "ignore_check" ): # allow ignoring of field checks ignore_fields = kwargs.get("ignore_fields", list()) - ignore_field = self.configuration[object_type].get( - "ignore_on_update" - ) + ignore_field = self.configuration[object_type].get("ignore_on_update") if ignore_field and ignore_field not in ignore_fields: ignore_fields.append(ignore_field) ignore_field = kwargs.get("ignore_field") @@ -299,9 +287,7 @@ def _hdx_update( """ self._check_kwargs_fields(object_type, **kwargs) operation = kwargs.pop("operation", "update") - self._save_to_hdx( - operation, id_field_name, files_to_upload, force_active - ) + self._save_to_hdx(operation, id_field_name, files_to_upload, force_active) def _merge_hdx_update( self, @@ -406,9 +392,7 @@ def _write_to_hdx( idstr = f" {data[id_field_name]}" else: idstr = "" - raise HDXError( - f"Failed when trying to {action}{idstr}! (POST)" - ) from e + raise HDXError(f"Failed when trying to {action}{idstr}! (POST)") from e finally: for file in files_to_upload.values(): if isinstance(file, str): @@ -436,9 +420,7 @@ def _save_to_hdx( """ if force_active: self.data["state"] = "active" - result = self._write_to_hdx( - action, self.data, id_field_name, files_to_upload - ) + result = self._write_to_hdx(action, self.data, id_field_name, files_to_upload) self.old_data = self.data self.data = result @@ -477,9 +459,7 @@ def _create_in_hdx( if id_field_name in self.data and self._load_from_hdx( object_type, self.data[id_field_name] ): - logger.warning( - f"{object_type} exists. Updating {self.data[id_field_name]}" - ) + logger.warning(f"{object_type} exists. Updating {self.data[id_field_name]}") self._merge_hdx_update( object_type, id_field_name, @@ -488,9 +468,7 @@ def _create_in_hdx( **kwargs, ) else: - self._save_to_hdx( - "create", name_field_name, files_to_upload, force_active - ) + self._save_to_hdx("create", name_field_name, files_to_upload, force_active) @abstractmethod def delete_from_hdx(self) -> None: @@ -511,9 +489,7 @@ def _delete_from_hdx(self, object_type: str, id_field_name: str) -> None: None """ if id_field_name not in self.data: - raise HDXError( - f"No {id_field_name} field (mandatory) in {object_type}!" - ) + raise HDXError(f"No {id_field_name} field (mandatory) in {object_type}!") self._save_to_hdx("delete", id_field_name) @classmethod @@ -602,9 +578,7 @@ def _remove_hdxobject( return True return False - def _convert_hdxobjects( - self, hdxobjects: ListTuple["HDXObject"] - ) -> List[Dict]: + def _convert_hdxobjects(self, hdxobjects: ListTuple["HDXObject"]) -> List[Dict]: """Helper function to convert supplied list of HDX objects to a list of dict Args: @@ -680,9 +654,7 @@ def _separate_hdxobjects( for new_hdxobject in new_hdxobjects: if new_hdxobject[id_field] not in hdxobject_names: hdxobjects.append( - hdxobjectclass( - new_hdxobject, configuration=self.configuration - ) + hdxobjectclass(new_hdxobject, configuration=self.configuration) ) del self.data[hdxobjects_name] @@ -772,9 +744,7 @@ def _add_string_to_commastring(self, field: str, string: str) -> bool: self.data[field] = strings return True - def _add_strings_to_commastring( - self, field: str, strings: ListTuple[str] - ) -> bool: + def _add_strings_to_commastring(self, field: str, strings: ListTuple[str]) -> bool: """Add a list of strings to a comma separated list of strings Args: diff --git a/src/hdx/data/organization.py b/src/hdx/data/organization.py index e3fbbac8..65dc8bf0 100755 --- a/src/hdx/data/organization.py +++ b/src/hdx/data/organization.py @@ -88,13 +88,9 @@ def read_from_hdx( Returns: Optional[Organization]: Organization object if successful read, None if not """ - return cls._read_from_hdx_class( - "organization", identifier, configuration - ) + return cls._read_from_hdx_class("organization", identifier, configuration) - def check_required_fields( - self, ignore_fields: ListTuple[str] = tuple() - ) -> None: + def check_required_fields(self, ignore_fields: ListTuple[str] = tuple()) -> None: """Check that metadata for organization is complete. The parameter ignore_fields should be set if required to any fields that should be ignored for the particular operation. @@ -186,9 +182,7 @@ def add_update_user( user["capacity"] = capacity self._addupdate_hdxobject(users, "name", user) return - raise HDXError( - f"Type {type(user).__name__} cannot be added as a user!" - ) + raise HDXError(f"Type {type(user).__name__} cannot be added as a user!") def add_update_users( self, @@ -220,9 +214,7 @@ def remove_user(self, user: Union["User", Dict, str]) -> bool: """ return self._remove_hdxobject(self.data.get("users"), user) - def get_datasets( - self, query: str = "*:*", **kwargs: Any - ) -> List["Dataset"]: # noqa: F821 + def get_datasets(self, query: str = "*:*", **kwargs: Any) -> List["Dataset"]: # noqa: F821 """Get list of datasets in organization Args: diff --git a/src/hdx/data/resource.py b/src/hdx/data/resource.py index c8a92f46..c1cbea73 100755 --- a/src/hdx/data/resource.py +++ b/src/hdx/data/resource.py @@ -256,9 +256,7 @@ def set_format(self, format: str) -> str: Returns: str: Format that was set """ - file_format = self.get_mapped_format( - format, configuration=self.configuration - ) + file_format = self.get_mapped_format(format, configuration=self.configuration) if not file_format: raise HDXError( f"Supplied file type {file_format} is invalid and could not be mapped to a known type!" @@ -331,9 +329,7 @@ def check_url_filetoupload(self) -> None: if "url_type" not in self.data: self.data["url_type"] = "api" else: - raise HDXError( - "Either a url or a file to upload must be supplied!" - ) + raise HDXError("Either a url or a file to upload must be supplied!") else: if "url" in self.data: if self.data["url"] != FilestoreHelper.temporary_url: @@ -348,9 +344,7 @@ def check_url_filetoupload(self) -> None: del self.data["tracking_summary"] self.clean_format() - def check_required_fields( - self, ignore_fields: ListTuple[str] = tuple() - ) -> None: + def check_required_fields(self, ignore_fields: ListTuple[str] = tuple()) -> None: """Check that metadata for resource is complete. The parameter ignore_fields should be set if required to any fields that should be ignored for the particular operation. @@ -397,9 +391,7 @@ def _resource_merge_hdx_update( ) self.data_updated = False # old_data will be merged into data in the next step - self._merge_hdx_update( - "resource", "id", self._get_files(), True, **kwargs - ) + self._merge_hdx_update("resource", "id", self._get_files(), True, **kwargs) def update_in_hdx(self, **kwargs: Any) -> None: """Check if resource exists in HDX and if so, update it. To indicate @@ -498,9 +490,7 @@ def search_in_hdx( count = result.get("count", None) if count: for resourcedict in result["results"]: - resource = Resource( - resourcedict, configuration=configuration - ) + resource = Resource(resourcedict, configuration=configuration) resources.append(resource) else: logger.debug(result) @@ -536,9 +526,7 @@ def download(self, folder: Optional[str] = None) -> Tuple[str, str]: use_env=False, headers=headers, ) as downloader: - path = downloader.download_file( - url, folder=folder, filename=filename - ) + path = downloader.download_file(url, folder=folder, filename=filename) return url, path @staticmethod @@ -684,9 +672,7 @@ def reorder_resource_views( else: resource_view_id = resource_view["id"] if is_valid_uuid(resource_view_id) is False: - raise HDXError( - f"{resource_view} is not a valid resource view id!" - ) + raise HDXError(f"{resource_view} is not a valid resource view id!") ids.append(resource_view_id) _, result = self._read_from_hdx( "resource view", @@ -709,9 +695,7 @@ def delete_resource_view( """ if isinstance(resource_view, str): if is_valid_uuid(resource_view) is False: - raise HDXError( - f"{resource_view} is not a valid resource view id!" - ) + raise HDXError(f"{resource_view} is not a valid resource view id!") resource_view = ResourceView( {"id": resource_view}, configuration=self.configuration ) @@ -795,9 +779,7 @@ def get_date_data_updated(self) -> datetime: Returns: datetime: Date resource data was updated """ - return parse_date( - self.data["last_modified"], include_microseconds=True - ) + return parse_date(self.data["last_modified"], include_microseconds=True) def set_date_data_updated( self, date: Union[datetime, str], ignore_timeinfo: bool = False diff --git a/src/hdx/data/resource_matcher.py b/src/hdx/data/resource_matcher.py index ab27b75d..a53125aa 100755 --- a/src/hdx/data/resource_matcher.py +++ b/src/hdx/data/resource_matcher.py @@ -42,9 +42,7 @@ def match_resource_list( format2 = resource2["format"].lower() dupnames = { - item - for item, count in collections.Counter(names1).items() - if count > 1 + item for item, count in collections.Counter(names1).items() if count > 1 } for i, name1 in enumerate(names1): if name1 != name2: @@ -91,14 +89,10 @@ def match_resource_lists( index1_matches.append(i) index2_matches.append(j) dupnames1 = { - item - for item, count in collections.Counter(names1).items() - if count > 1 + item for item, count in collections.Counter(names1).items() if count > 1 } dupnames2 = { - item - for item, count in collections.Counter(names2).items() - if count > 1 + item for item, count in collections.Counter(names2).items() if count > 1 } dupnames = dupnames1.union(dupnames2) for i, name1 in enumerate(names1): @@ -115,12 +109,8 @@ def match_resource_lists( if groups1[i] == groups2[j]: index1_matches.append(i) index2_matches.append(j) - index1_nomatches = [ - i for i, _ in enumerate(ids1) if i not in index1_matches - ] - index2_nomatches = [ - i for i, _ in enumerate(ids2) if i not in index2_matches - ] + index1_nomatches = [i for i, _ in enumerate(ids1) if i not in index1_matches] + index2_nomatches = [i for i, _ in enumerate(ids2) if i not in index2_matches] return ( index1_matches, index2_matches, diff --git a/src/hdx/data/resource_view.py b/src/hdx/data/resource_view.py index a9b8fd9d..61d78c4c 100755 --- a/src/hdx/data/resource_view.py +++ b/src/hdx/data/resource_view.py @@ -84,9 +84,7 @@ def read_from_hdx( Returns: Optional[ResourceView]: ResourceView object if successful read, None if not """ - return cls._read_from_hdx_class( - "resource view", identifier, configuration - ) + return cls._read_from_hdx_class("resource view", identifier, configuration) @staticmethod def get_all_for_resource( @@ -115,9 +113,7 @@ def get_all_for_resource( resourceviews.append(resourceview) return resourceviews - def check_required_fields( - self, ignore_fields: ListTuple[str] = tuple() - ) -> None: + def check_required_fields(self, ignore_fields: ListTuple[str] = tuple()) -> None: """Check that metadata for resource view is complete. The parameter ignore_fields should be set if required to any fields that should be ignored for the particular operation. @@ -136,15 +132,11 @@ def _update_resource_view(self, log: bool = False, **kwargs: Any) -> bool: bool: True if updated and False if not """ update = False - if "id" in self.data and self._load_from_hdx( - "resource view", self.data["id"] - ): + if "id" in self.data and self._load_from_hdx("resource view", self.data["id"]): update = True else: if "resource_id" in self.data: - resource_views = self.get_all_for_resource( - self.data["resource_id"] - ) + resource_views = self.get_all_for_resource(self.data["resource_id"]) for resource_view in resource_views: if self.data["title"] == resource_view["title"]: self.old_data = self.data @@ -153,9 +145,7 @@ def _update_resource_view(self, log: bool = False, **kwargs: Any) -> bool: break if update: if log: - logger.warning( - f"resource view exists. Updating {self.data['id']}" - ) + logger.warning(f"resource view exists. Updating {self.data['id']}") self._merge_hdx_update("resource view", "id", **kwargs) return update @@ -198,9 +188,7 @@ def copy(self, resource_view: Union["ResourceView", Dict, str]) -> None: """ if isinstance(resource_view, str): if is_valid_uuid(resource_view) is False: - raise HDXError( - f"{resource_view} is not a valid resource view id!" - ) + raise HDXError(f"{resource_view} is not a valid resource view id!") resource_view = ResourceView.read_from_hdx(resource_view) if not isinstance(resource_view, dict) and not isinstance( resource_view, ResourceView diff --git a/src/hdx/data/showcase.py b/src/hdx/data/showcase.py index 7f3b2b21..597ae97e 100755 --- a/src/hdx/data/showcase.py +++ b/src/hdx/data/showcase.py @@ -96,9 +96,7 @@ def read_from_hdx( """ return cls._read_from_hdx_class("showcase", identifier, configuration) - def check_required_fields( - self, ignore_fields: ListTuple[str] = tuple() - ) -> None: + def check_required_fields(self, ignore_fields: ListTuple[str] = tuple()) -> None: """Check that metadata for showcase is complete. The parameter ignore_fields should be set if required to any fields that should be ignored for the particular operation. @@ -133,12 +131,8 @@ def create_in_hdx(self, **kwargs: Any) -> None: """ if "ignore_check" not in kwargs: # allow ignoring of field checks self.check_required_fields() - if "name" in self.data and self._load_from_hdx( - "showcase", self.data["name"] - ): - logger.warning( - f"{'showcase'} exists. Updating {self.data['name']}" - ) + if "name" in self.data and self._load_from_hdx("showcase", self.data["name"]): + logger.warning(f"{'showcase'} exists. Updating {self.data['name']}") merge_two_dictionaries(self.data, self.old_data) self.clean_tags() self._hdx_update("showcase", "name", force_active=True, **kwargs) @@ -196,9 +190,7 @@ def add_tags( self, tags, log_deleted=log_deleted ) - def clean_tags( - self, log_deleted: bool = True - ) -> Tuple[List[str], List[str]]: + def clean_tags(self, log_deleted: bool = True) -> Tuple[List[str], List[str]]: """Clean tags in an HDX object according to tags cleanup spreadsheet Args: @@ -207,9 +199,7 @@ def clean_tags( Returns: Tuple[List[str], List[str]]: Tuple containing list of mapped tags and list of deleted tags and tags not added """ - return hdx.data.vocabulary.Vocabulary.clean_tags( - self, log_deleted=log_deleted - ) + return hdx.data.vocabulary.Vocabulary.clean_tags(self, log_deleted=log_deleted) def remove_tag(self, tag: str) -> bool: """Remove a tag @@ -257,22 +247,16 @@ def _get_showcase_dataset_dict( Returns: Dict: showcase dataset dict """ - if isinstance(dataset, hdx.data.dataset.Dataset) or isinstance( - dataset, dict - ): + if isinstance(dataset, hdx.data.dataset.Dataset) or isinstance(dataset, dict): if "id" not in dataset: - dataset = hdx.data.dataset.Dataset.read_from_hdx( - dataset["name"] - ) + dataset = hdx.data.dataset.Dataset.read_from_hdx(dataset["name"]) dataset = dataset["id"] elif not isinstance(dataset, str): raise hdx.data.hdxobject.HDXError( f"Type {type(dataset).__name__} cannot be added as a dataset!" ) if is_valid_uuid(dataset) is False: - raise hdx.data.hdxobject.HDXError( - f"{dataset} is not a valid dataset id!" - ) + raise hdx.data.hdxobject.HDXError(f"{dataset} is not a valid dataset id!") return {"showcase_id": self.data["id"], "package_id": dataset} def add_dataset( @@ -316,9 +300,7 @@ def add_datasets( datasets_to_check = self.get_datasets() alldatasetsadded = True for dataset in datasets: - if not self.add_dataset( - dataset, datasets_to_check=datasets_to_check - ): + if not self.add_dataset(dataset, datasets_to_check=datasets_to_check): alldatasetsadded = False return alldatasetsadded diff --git a/src/hdx/data/user.py b/src/hdx/data/user.py index 821ea933..e6b8c86a 100755 --- a/src/hdx/data/user.py +++ b/src/hdx/data/user.py @@ -88,9 +88,7 @@ def read_from_hdx( """ return cls._read_from_hdx_class("user", identifier, configuration) - def check_required_fields( - self, ignore_fields: ListTuple[str] = tuple() - ) -> None: + def check_required_fields(self, ignore_fields: ListTuple[str] = tuple()) -> None: """Check that metadata for user is complete. The parameter ignore_fields should be set if required to any fields that should be ignored for the particular operation. @@ -269,9 +267,7 @@ def get_organization_dicts(self, permission: str = "read") -> List[Dict]: # noq return result return [] - def get_organizations( - self, permission: str = "read" - ) -> List["Organization"]: # noqa: F821 + def get_organizations(self, permission: str = "read") -> List["Organization"]: # noqa: F821 """Get organizations in HDX that this user is a member of. Args: @@ -301,9 +297,7 @@ def check_organization_access( Returns: bool: True if the logged in user is a member of the organization. """ - for organization_dict in self.get_organization_dicts( - permission=permission - ): + for organization_dict in self.get_organization_dicts(permission=permission): if organization_dict["id"] == organization: return True if organization_dict["name"] == organization: @@ -348,9 +342,7 @@ def get_current_user_organizations( Returns: List[Organization]: List of organizations in HDX that logged in user is a member of """ - result = cls.get_current_user_organization_dicts( - permission, configuration - ) + result = cls.get_current_user_organization_dicts(permission, configuration) organizations = [] for organizationdict in result: org = hdx.data.organization.Organization.read_from_hdx( diff --git a/src/hdx/data/vocabulary.py b/src/hdx/data/vocabulary.py index 14949a68..c7af4a59 100755 --- a/src/hdx/data/vocabulary.py +++ b/src/hdx/data/vocabulary.py @@ -100,9 +100,7 @@ def read_from_hdx( Returns: Optional[Vocabulary]: Vocabulary object if successful read, None if not """ - return cls._read_from_hdx_class( - "vocabulary", identifier, configuration - ) + return cls._read_from_hdx_class("vocabulary", identifier, configuration) @staticmethod def get_all_vocabularies( @@ -119,19 +117,13 @@ def get_all_vocabularies( """ vocabulary = Vocabulary(configuration=configuration) - vocabulary["id"] = ( - "all vocabulary names" # only for error message if produced - ) + vocabulary["id"] = "all vocabulary names" # only for error message if produced vocabularies = [] for vocabularydict in vocabulary._write_to_hdx("list", {}): - vocabularies.append( - Vocabulary(vocabularydict, configuration=configuration) - ) + vocabularies.append(Vocabulary(vocabularydict, configuration=configuration)) return vocabularies - def check_required_fields( - self, ignore_fields: ListTuple[str] = tuple() - ) -> None: + def check_required_fields(self, ignore_fields: ListTuple[str] = tuple()) -> None: """Check that metadata for vocabulary is complete. The parameter ignore_fields should be set if required to any fields that should be ignored for the particular operation. @@ -157,9 +149,7 @@ def create_in_hdx(self, **kwargs: Any) -> None: Returns: None """ - self._create_in_hdx( - "vocabulary", "id", "name", force_active=False, **kwargs - ) + self._create_in_hdx("vocabulary", "id", "name", force_active=False, **kwargs) def delete_from_hdx(self, empty: bool = True) -> None: """Deletes a vocabulary from HDX. First tags are removed then vocabulary is deleted. @@ -343,9 +333,7 @@ def delete_approved_vocabulary( vocabulary.delete_from_hdx() @classmethod - def approved_tags( - cls, configuration: Optional[Configuration] = None - ) -> List[str]: + def approved_tags(cls, configuration: Optional[Configuration] = None) -> List[str]: """ Return list of approved tags @@ -357,9 +345,7 @@ def approved_tags( """ return [ x["name"] - for x in cls.get_approved_vocabulary(configuration=configuration)[ - "tags" - ] + for x in cls.get_approved_vocabulary(configuration=configuration)["tags"] ] @classmethod @@ -589,9 +575,7 @@ def add_mapped_tags( ) added_tags = hdxobject._add_tags( new_tags, - cls.get_approved_vocabulary(configuration=hdxobject.configuration)[ - "id" - ], + cls.get_approved_vocabulary(configuration=hdxobject.configuration)["id"], ) unadded_tags = [x for x in new_tags if x not in added_tags] unadded_tags.extend(deleted_tags) diff --git a/src/hdx/facades/infer_arguments.py b/src/hdx/facades/infer_arguments.py index 34af66f6..3cfe4b7e 100755 --- a/src/hdx/facades/infer_arguments.py +++ b/src/hdx/facades/infer_arguments.py @@ -40,9 +40,7 @@ def facade(projectmainfn: Callable[[Any], None], **kwargs: Any): main_doc = [f"{parsed_main_doc.first_line}\n\nArgs:"] no_main_params = len(parsed_main_doc.params) for param_name, param_info in parsed_main_doc.params.items(): - main_doc.append( - f"\n {param_name} ({param_info.type}): {param_info.text}" - ) + main_doc.append(f"\n {param_name} ({param_info.type}): {param_info.text}") create_config_doc = getdoc(Configuration.create) kwargs_index = create_config_doc.index("**kwargs") kwargs_index = create_config_doc.index("\n", kwargs_index) @@ -75,9 +73,7 @@ def facade(projectmainfn: Callable[[Any], None], **kwargs: Any): "Configuration.create has new parameter with unknown type!" ) param_names.append(f"{param_name}: {param_type} = {default}") - main_doc.append( - f"\n {param_name} ({param_type}): {param_info.text}" - ) + main_doc.append(f"\n {param_name} ({param_type}): {param_info.text}") main_doc = "".join(main_doc) projectmainname = projectmainfn.__name__ @@ -101,9 +97,7 @@ def gen_func(*args, **kwargs): gen_func.__doc__ = main_doc configuration_create = defopt.bind(gen_func, argv=argv, cli_options="all") - main_func, _ = defopt.bind_known( - projectmainfn, argv=argv, cli_options="all" - ) + main_func, _ = defopt.bind_known(projectmainfn, argv=argv, cli_options="all") configuration_create() UserAgent.user_agent = Configuration.read().user_agent diff --git a/tests/hdx/api/test_ckan.py b/tests/hdx/api/test_ckan.py index 7c7923f6..76d5125b 100644 --- a/tests/hdx/api/test_ckan.py +++ b/tests/hdx/api/test_ckan.py @@ -59,9 +59,7 @@ def params(self): def gclient(self): gsheet_auth = getenv("GSHEET_AUTH") if not gsheet_auth: - auth_file_path = os.path.join( - os.path.expanduser("~"), ".gsheet_auth.json" - ) + auth_file_path = os.path.join(os.path.expanduser("~"), ".gsheet_auth.json") if os.path.exists(auth_file_path): with open(auth_file_path, encoding="utf-8") as auth_file: gsheet_auth = auth_file.read() @@ -91,9 +89,7 @@ def setup_teardown_folder(self, configuration, gclient, params): finally: payload = {"trashed": True} url = f"{DRIVE_FILES_API_V3_URL}/{folderid}" - gclient.http_client.request( - "patch", url, json=payload, params=params - ) + gclient.http_client.request("patch", url, json=payload, params=params) Vocabulary._approved_vocabulary = None Vocabulary._tags_dict = None Configuration.delete() @@ -182,9 +178,7 @@ def create_resource(): assert dataset["title"] == title assert dataset.get_tags() == tags assert dataset.get_maintainer()["id"] == maintainer_id - assert ( - dataset.get_organization()["display_name"] == "INNAGO (inactive)" - ) + assert dataset.get_organization()["display_name"] == "INNAGO (inactive)" resources = dataset.get_resources() for i, resource in enumerate(resources): assert resource["name"] == f"test_resource_{i}" @@ -225,9 +219,7 @@ def create_resource(): dataset.add_update_resources(resources) sleep(2) - dataset.update_in_hdx( - hxl_update=False, remove_additional_resources=True - ) + dataset.update_in_hdx(hxl_update=False, remove_additional_resources=True) sleep(2) # check updated dataset @@ -238,9 +230,7 @@ def create_resource(): assert dataset["caveats"] == caveats assert dataset.get_tags() == tags assert dataset.get_maintainer()["id"] == maintainer_id - assert ( - dataset.get_organization()["display_name"] == "INNAGO (inactive)" - ) + assert dataset.get_organization()["display_name"] == "INNAGO (inactive)" updated_resources = dataset.get_resources() for i, updated_resource in enumerate(updated_resources): resource = resources[i] @@ -285,9 +275,7 @@ def create_resource(): assert "caveats" not in dataset assert dataset.get_tags() == tags assert dataset.get_maintainer()["id"] == maintainer_id - assert ( - dataset.get_organization()["display_name"] == "INNAGO (inactive)" - ) + assert dataset.get_organization()["display_name"] == "INNAGO (inactive)" updated_resources = dataset.get_resources() for i, updated_resource in enumerate(updated_resources): resource = resources[i] diff --git a/tests/hdx/api/test_configuration.py b/tests/hdx/api/test_configuration.py index ff63f566..d355b148 100755 --- a/tests/hdx/api/test_configuration.py +++ b/tests/hdx/api/test_configuration.py @@ -70,10 +70,7 @@ def test_init( configuration.get_hdx_site_url() == "https://stage.data-humdata-org.ahconu.org" ) - assert ( - configuration.get_user_agent() - == f"HDXPythonLibrary/{__version__}-test" - ) + assert configuration.get_user_agent() == f"HDXPythonLibrary/{__version__}-test" configuration = Configuration(full_agent="test", hdx_key="test") assert configuration.get_user_agent() == "test" Configuration.default_hdx_config_yaml = default_config_file @@ -236,10 +233,7 @@ def test_hdx_configuration_dict(self, project_config_yaml, mocksmtp): configuration = Configuration.read() assert configuration == expected_configuration - assert ( - configuration.get_user_agent() - == f"HDXPythonLibrary/{__version__}-test" - ) + assert configuration.get_user_agent() == f"HDXPythonLibrary/{__version__}-test" Configuration._create( hdx_config_dict={ @@ -259,10 +253,7 @@ def test_hdx_configuration_dict(self, project_config_yaml, mocksmtp): configuration = Configuration.read() expected_configuration["user_agent"] = "test" assert configuration == expected_configuration - assert ( - configuration.get_user_agent() - == f"HDXPythonLibrary/{__version__}-test" - ) + assert configuration.get_user_agent() == f"HDXPythonLibrary/{__version__}-test" smtp_initargs = { "host": "localhost", @@ -388,9 +379,7 @@ def test_hdx_configuration_yaml( "dataset_date", ] }, - "resource": { - "required_fields": ["package_id", "name", "description"] - }, + "resource": {"required_fields": ["package_id", "name", "description"]}, "showcase": {"required_fields": ["name", "title"]}, "approved_tags_vocabulary": "Topics", "tags_list_url": "tests/fixtures/Accepted_Tags.csv", @@ -403,9 +392,7 @@ def test_hdx_configuration_yaml( assert configuration.hdx_key == "12345" def test_project_configuration_dict(self, hdx_config_yaml): - Configuration._create( - user_agent="test", hdx_config_yaml=hdx_config_yaml - ) + Configuration._create(user_agent="test", hdx_config_yaml=hdx_config_yaml) expected_configuration = { "hdx_site": "prod", "hdx_read_only": False, @@ -523,9 +510,7 @@ def test_project_configuration_dict(self, hdx_config_yaml): expected_configuration["abc"] = "123" assert Configuration.read() == expected_configuration - def test_project_configuration_json( - self, hdx_config_yaml, project_config_json - ): + def test_project_configuration_json(self, hdx_config_yaml, project_config_json): Configuration._create( user_agent="test", hdx_config_yaml=hdx_config_yaml, @@ -642,9 +627,7 @@ def test_project_configuration_json( } assert Configuration.read() == expected_configuration - def test_project_configuration_yaml( - self, hdx_config_yaml, project_config_yaml - ): + def test_project_configuration_yaml(self, hdx_config_yaml, project_config_yaml): Configuration._create( user_agent="test", hdx_config_yaml=hdx_config_yaml, @@ -769,10 +752,7 @@ def test_get_hdx_key_site(self, hdx_config_yaml, project_config_yaml): ) actual_configuration = Configuration.read() assert actual_configuration.get_api_key() == "12345" - assert ( - actual_configuration.get_hdx_site_url() - == "https://data.humdata.org" - ) + assert actual_configuration.get_hdx_site_url() == "https://data.humdata.org" assert actual_configuration._get_credentials() is None assert ( actual_configuration.get_dataset_url("mydataset") diff --git a/tests/hdx/api/test_locations.py b/tests/hdx/api/test_locations.py index 47f3b8ad..75bb6901 100755 --- a/tests/hdx/api/test_locations.py +++ b/tests/hdx/api/test_locations.py @@ -15,18 +15,13 @@ def test_validlocations(self, project_config_yaml): Country.countriesdata(use_live=False) validlocations = [{"name": "shn", "title": "St. Helena"}] assert ( - Locations.get_HDX_code_from_location( - "sh", locations=validlocations - ) - is None + Locations.get_HDX_code_from_location("sh", locations=validlocations) is None ) assert Locations.get_HDX_code_from_location_partial( "sh", locations=validlocations ) == (None, False) assert ( - Locations.get_location_from_HDX_code( - "shn", locations=validlocations - ) + Locations.get_location_from_HDX_code("shn", locations=validlocations) == "St. Helena" ) validlocations = [ @@ -59,25 +54,18 @@ def test_validlocations(self, project_config_yaml): False, ) assert ( - Locations.get_location_from_HDX_code( - "zmb", locations=validlocations - ) + Locations.get_location_from_HDX_code("zmb", locations=validlocations) == "Zambia" ) validlocations = [{"name": "shn", "title": "St. Helena"}] assert ( - Locations.get_HDX_code_from_location( - "sh", locations=validlocations - ) - is None + Locations.get_HDX_code_from_location("sh", locations=validlocations) is None ) assert Locations.get_HDX_code_from_location_partial( "sh", locations=validlocations ) == (None, False) assert ( - Locations.get_location_from_HDX_code( - "shn", locations=validlocations - ) + Locations.get_location_from_HDX_code("shn", locations=validlocations) == "St. Helena" ) Configuration.setup(MyConfiguration()) diff --git a/tests/hdx/data/test_dataset_core.py b/tests/hdx/data/test_dataset_core.py index e79a238b..53de7c79 100755 --- a/tests/hdx/data/test_dataset_core.py +++ b/tests/hdx/data/test_dataset_core.py @@ -33,9 +33,7 @@ from hdx.utilities.dictandlist import merge_two_dictionaries from hdx.utilities.loader import load_yaml -searchdict = load_yaml( - join("tests", "fixtures", "dataset_search_results.yaml") -) +searchdict = load_yaml(join("tests", "fixtures", "dataset_search_results.yaml")) dataset_list = [ "acled-conflict-data-for-libya", "acled-conflict-data-for-liberia", @@ -143,9 +141,7 @@ def mockall(url, datadict): if datadict["start"] == 2: newsearchdict["results"] = newsearchdict["results"][2:9] else: - newsearchdict["results"] = newsearchdict["results"][ - 4:5 - ] # repeated dataset + newsearchdict["results"] = newsearchdict["results"][4:5] # repeated dataset elif datadict["rows"] == 5: newsearchdict["count"] = 6 if datadict["sort"] == "metadata_modified desc": @@ -218,8 +214,7 @@ class MockSession: def post(url, data, headers, files, allow_redirects, auth=None): if isinstance(data, dict): datadict = { - k.decode("utf8"): v.decode("utf8") - for k, v in data.items() + k.decode("utf8"): v.decode("utf8") for k, v in data.items() } else: datadict = json.loads(data.decode("utf-8")) @@ -247,8 +242,7 @@ class MockSession: def post(url, data, headers, files, allow_redirects, auth=None): if isinstance(data, dict): datadict = { - k.decode("utf8"): v.decode("utf8") - for k, v in data.items() + k.decode("utf8"): v.decode("utf8") for k, v in data.items() } else: datadict = json.loads(data.decode("utf-8")) @@ -321,8 +315,7 @@ class MockSession: def post(url, data, headers, files, allow_redirects, auth=None): if isinstance(data, dict): datadict = { - k.decode("utf8"): v.decode("utf8") - for k, v in data.items() + k.decode("utf8"): v.decode("utf8") for k, v in data.items() } else: datadict = json.loads(data.decode("utf-8")) @@ -371,14 +364,12 @@ def post(url, data, headers, files, allow_redirects, auth=None): resultdictcopy = copy.deepcopy(dataset_resultdict) for i, resource in enumerate(datadict["resources"]): if not resource: - datadict["resources"][i] = resultdictcopy[ - "resources" - ][i] + datadict["resources"][i] = resultdictcopy["resources"][ + i + ] merge_two_dictionaries(resultdictcopy, datadict) - for i, resource in enumerate( - resultdictcopy["resources"] - ): + for i, resource in enumerate(resultdictcopy["resources"]): resource["package_id"] = resultdictcopy["id"] resultdictcopy = {"package": resultdictcopy} result = json.dumps(resultdictcopy) diff --git a/tests/hdx/data/test_dataset_noncore.py b/tests/hdx/data/test_dataset_noncore.py index fec66095..c8aab55c 100755 --- a/tests/hdx/data/test_dataset_noncore.py +++ b/tests/hdx/data/test_dataset_noncore.py @@ -44,9 +44,7 @@ class TestDatasetNoncore: @pytest.fixture(scope="class") def static_resource_view_yaml(self): - return join( - "tests", "fixtures", "config", "hdx_resource_view_static.yaml" - ) + return join("tests", "fixtures", "config", "hdx_resource_view_static.yaml") @pytest.fixture(scope="function") def vocabulary_read(self): @@ -119,8 +117,7 @@ class MockSession: def post(url, data, headers, files, allow_redirects, auth=None): if isinstance(data, dict): datadict = { - k.decode("utf8"): v.decode("utf8") - for k, v in data.items() + k.decode("utf8"): v.decode("utf8") for k, v in data.items() } else: datadict = json.loads(data.decode("utf-8")) @@ -146,9 +143,7 @@ def post(url, data, headers, files, allow_redirects, auth=None): Configuration.read().remoteckan().session = MockSession() - def test_get_name_or_id( - self, configuration, hdx_config_yaml, project_config_yaml - ): + def test_get_name_or_id(self, configuration, hdx_config_yaml, project_config_yaml): dataset = Dataset() assert dataset.get_name_or_id() is None datasetdata = copy.deepcopy(dataset_resultdict) @@ -159,9 +154,7 @@ def test_get_name_or_id( == "6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d" ) del dataset["name"] - assert ( - dataset.get_name_or_id() == "6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d" - ) + assert dataset.get_name_or_id() == "6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d" assert ( dataset.get_name_or_id(prefer_name=False) == "6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d" @@ -172,17 +165,12 @@ def test_get_name_or_id( assert dataset.get_name_or_id() == "MyDataset1" assert dataset.get_name_or_id(prefer_name=False) == "MyDataset1" - def test_get_hdx_url( - self, configuration, hdx_config_yaml, project_config_yaml - ): + def test_get_hdx_url(self, configuration, hdx_config_yaml, project_config_yaml): dataset = Dataset() assert dataset.get_hdx_url() is None datasetdata = copy.deepcopy(dataset_data) dataset = Dataset(datasetdata) - assert ( - dataset.get_hdx_url() - == "https://data.humdata.org/dataset/MyDataset1" - ) + assert dataset.get_hdx_url() == "https://data.humdata.org/dataset/MyDataset1" Configuration.delete() Configuration._create( hdx_site="feature", @@ -196,9 +184,7 @@ def test_get_hdx_url( == "https://feature.data-humdata-org.ahconu.org/dataset/MyDataset1" ) - def test_get_api_url( - self, configuration, hdx_config_yaml, project_config_yaml - ): + def test_get_api_url(self, configuration, hdx_config_yaml, project_config_yaml): dataset = Dataset() assert dataset.get_api_url() is None datasetdata = copy.deepcopy(dataset_data) @@ -249,9 +235,7 @@ def test_get_set_date_of_dataset(self): "ongoing": False, } dataset.set_time_period("2020-02-09", ongoing=True) - result = dataset.get_time_period( - "%d/%m/%Y", today=datetime(2020, 3, 9, 0, 0) - ) + result = dataset.get_time_period("%d/%m/%Y", today=datetime(2020, 3, 9, 0, 0)) assert result == { "startdate": datetime(2020, 2, 9, 0, 0, tzinfo=timezone.utc), "enddate": datetime(2020, 3, 9, 23, 59, 59, tzinfo=timezone.utc), @@ -696,10 +680,7 @@ def test_add_update_delete_showcase(self, configuration, showcase_read): dataset.remove_showcase(showcases[0]) assert TestDatasetNoncore.association == "delete" TestDatasetNoncore.association = None - assert ( - dataset.add_showcase("15e392bf-04e0-4ca6-848c-4e87bba10745") - is True - ) + assert dataset.add_showcase("15e392bf-04e0-4ca6-848c-4e87bba10745") is True assert TestDatasetNoncore.association == "create" TestDatasetNoncore.association = None dataset.add_showcases([{"id": "15e392bf-04e0-4ca6-848c-4e87bba10745"}]) @@ -731,9 +712,9 @@ def test_set_quickchart_resource(self, configuration): dataset = Dataset(datasetdata) assert "dataset_preview" not in dataset assert ( - dataset.set_quickchart_resource( - "3d777226-96aa-4239-860a-703389d16d1f" - )["id"] + dataset.set_quickchart_resource("3d777226-96aa-4239-860a-703389d16d1f")[ + "id" + ] == "3d777226-96aa-4239-860a-703389d16d1f" ) assert dataset["dataset_preview"] == "resource_id" @@ -800,9 +781,7 @@ def test_generate_resource_view( datasetdata["resources"] = resourcesdata dataset = Dataset(datasetdata) assert "dataset_preview" not in dataset - resourceview = dataset.generate_quickcharts( - path=static_resource_view_yaml - ) + resourceview = dataset.generate_quickcharts(path=static_resource_view_yaml) hxl_preview_config = json.loads(resourceview["hxl_preview_config"]) assert resourceview["id"] == "c06b5a0d-1d41-4a74-a196-41c251c76023" assert hxl_preview_config["bites"][0]["title"] == "Sum of fatalities" @@ -853,9 +832,9 @@ def test_generate_resource_view( hxl_preview_config = json.loads(resourceview["hxl_preview_config"]) assert resourceview["id"] == "c06b5a0d-1d41-4a74-a196-41c251c76023" assert ( - hxl_preview_config["bites"][0]["ingredient"]["filters"][ - "filterWith" - ][0]["#indicator+code"] + hxl_preview_config["bites"][0]["ingredient"]["filters"]["filterWith"][0][ + "#indicator+code" + ] == "1" ) assert ( @@ -864,49 +843,34 @@ def test_generate_resource_view( ) assert hxl_preview_config["bites"][0]["uiProperties"]["title"] == "My1" assert ( - hxl_preview_config["bites"][0]["computedProperties"]["dataTitle"] - == "ones" + hxl_preview_config["bites"][0]["computedProperties"]["dataTitle"] == "ones" ) assert ( - hxl_preview_config["bites"][1]["ingredient"]["filters"][ - "filterWith" - ][0]["#indicator+code"] + hxl_preview_config["bites"][1]["ingredient"]["filters"]["filterWith"][0][ + "#indicator+code" + ] == "2" ) - assert ( - hxl_preview_config["bites"][1]["ingredient"]["description"] == "" - ) + assert hxl_preview_config["bites"][1]["ingredient"]["description"] == "" assert hxl_preview_config["bites"][1]["uiProperties"]["title"] == "My2" assert ( - hxl_preview_config["bites"][1]["computedProperties"]["dataTitle"] - == "twos" - ) - assert ( - hxl_preview_config["bites"][1]["ingredient"]["aggregateColumn"] - == "Agg2" + hxl_preview_config["bites"][1]["computedProperties"]["dataTitle"] == "twos" ) + assert hxl_preview_config["bites"][1]["ingredient"]["aggregateColumn"] == "Agg2" assert ( - hxl_preview_config["bites"][2]["ingredient"]["filters"][ - "filterWith" - ][0]["#indicator+code"] + hxl_preview_config["bites"][2]["ingredient"]["filters"]["filterWith"][0][ + "#indicator+code" + ] == "3" ) assert ( hxl_preview_config["bites"][2]["ingredient"]["description"] == "This is my three!" ) - assert ( - hxl_preview_config["bites"][2]["ingredient"]["dateColumn"] == "dt3" - ) + assert hxl_preview_config["bites"][2]["ingredient"]["dateColumn"] == "dt3" assert hxl_preview_config["bites"][2]["uiProperties"]["title"] == "My3" - assert ( - hxl_preview_config["bites"][2]["computedProperties"]["dataTitle"] - == "" - ) - assert ( - hxl_preview_config["bites"][2]["uiProperties"]["dateFormat"] - == "%b %Y" - ) + assert hxl_preview_config["bites"][2]["computedProperties"]["dataTitle"] == "" + assert hxl_preview_config["bites"][2]["uiProperties"]["dateFormat"] == "%b %Y" resourceview = dataset.generate_quickcharts( indicators=indicators, findreplace={ @@ -917,29 +881,20 @@ def test_generate_resource_view( hxl_preview_config = json.loads(resourceview["hxl_preview_config"]) assert resourceview["id"] == "c06b5a0d-1d41-4a74-a196-41c251c76023" assert ( - hxl_preview_config["bites"][0]["ingredient"]["filters"][ - "filterWith" - ][0]["#item+code"] + hxl_preview_config["bites"][0]["ingredient"]["filters"]["filterWith"][0][ + "#item+code" + ] == "1" ) - assert ( - hxl_preview_config["bites"][0]["ingredient"]["valueColumn"] - == "#value" - ) + assert hxl_preview_config["bites"][0]["ingredient"]["valueColumn"] == "#value" assert dataset.generate_quickcharts(indicators=[]) is None + assert dataset.generate_quickcharts(indicators=[None, None, None]) is None assert ( - dataset.generate_quickcharts(indicators=[None, None, None]) is None - ) - assert ( - dataset.generate_quickcharts( - resource="123", path=static_resource_view_yaml - ) + dataset.generate_quickcharts(resource="123", path=static_resource_view_yaml) is None ) del dataset.get_resources()[0]["id"] - resourceview = dataset.generate_quickcharts( - path=static_resource_view_yaml - ) + resourceview = dataset.generate_quickcharts(path=static_resource_view_yaml) assert "id" not in resourceview assert "resource_id" not in resourceview assert resourceview["resource_name"] == "Resource1" @@ -977,14 +932,9 @@ def test_remove_dates_from_title(self): assert dataset["title"] == newtitle assert "dataset_date" not in dataset dataset["title"] = title - assert ( - dataset.remove_dates_from_title(set_time_period=True) == expected - ) + assert dataset.remove_dates_from_title(set_time_period=True) == expected assert dataset["title"] == newtitle - assert ( - dataset["dataset_date"] - == "[1981-01-01T00:00:00 TO 2015-12-31T23:59:59]" - ) + assert dataset["dataset_date"] == "[1981-01-01T00:00:00 TO 2015-12-31T23:59:59]" assert dataset.remove_dates_from_title() == [] dataset["title"] = "Mon_State_Village_Tract_Boundaries 9999 2001" expected = [ @@ -993,32 +943,17 @@ def test_remove_dates_from_title(self): datetime(2001, 12, 31, 23, 59, 59, tzinfo=timezone.utc), ) ] - assert ( - dataset.remove_dates_from_title(set_time_period=True) == expected - ) + assert dataset.remove_dates_from_title(set_time_period=True) == expected assert dataset["title"] == "Mon_State_Village_Tract_Boundaries 9999" - assert ( - dataset["dataset_date"] - == "[2001-01-01T00:00:00 TO 2001-12-31T23:59:59]" - ) + assert dataset["dataset_date"] == "[2001-01-01T00:00:00 TO 2001-12-31T23:59:59]" dataset["title"] = "Mon_State_Village_Tract_Boundaries 2001 99" - assert ( - dataset.remove_dates_from_title(set_time_period=True) == expected - ) + assert dataset.remove_dates_from_title(set_time_period=True) == expected assert dataset["title"] == "Mon_State_Village_Tract_Boundaries 99" - assert ( - dataset["dataset_date"] - == "[2001-01-01T00:00:00 TO 2001-12-31T23:59:59]" - ) + assert dataset["dataset_date"] == "[2001-01-01T00:00:00 TO 2001-12-31T23:59:59]" dataset["title"] = "Mon_State_Village_Tract_Boundaries 9999 2001 99" - assert ( - dataset.remove_dates_from_title(set_time_period=True) == expected - ) + assert dataset.remove_dates_from_title(set_time_period=True) == expected assert dataset["title"] == "Mon_State_Village_Tract_Boundaries 9999 99" - assert ( - dataset["dataset_date"] - == "[2001-01-01T00:00:00 TO 2001-12-31T23:59:59]" - ) + assert dataset["dataset_date"] == "[2001-01-01T00:00:00 TO 2001-12-31T23:59:59]" def test_load_save_to_json(self, vocabulary_read): with temp_dir( @@ -1027,9 +962,7 @@ def test_load_save_to_json(self, vocabulary_read): delete_on_failure=False, ) as temp_folder: name = "mydatasetname" - dataset = Dataset( - {"name": name, "title": "title", "notes": "description"} - ) + dataset = Dataset({"name": name, "title": "title", "notes": "description"}) maintainer = "196196be-6037-4488-8b71-d786adf4c081" dataset.set_maintainer(maintainer) dataset.set_organization("fb7c2910-6080-4b66-8b4f-0be9b6dc4d8e") @@ -1057,10 +990,7 @@ def test_load_save_to_json(self, vocabulary_read): dateinfo = dataset.get_time_period() assert dateinfo["startdate_str"][:10] == start_date assert dateinfo["enddate_str"][:10] == end_date - assert ( - dataset.get_expected_update_frequency() - == expected_update_frequency - ) + assert dataset.get_expected_update_frequency() == expected_update_frequency assert dataset.get_tags() == tags resource = dataset.get_resource() assert resource["name"] == resource_name diff --git a/tests/hdx/data/test_dataset_resource_generation.py b/tests/hdx/data/test_dataset_resource_generation.py index c7a1c624..87e2f7e2 100644 --- a/tests/hdx/data/test_dataset_resource_generation.py +++ b/tests/hdx/data/test_dataset_resource_generation.py @@ -139,12 +139,8 @@ def process_row(headers, row): ) assert success is True assert results == { - "startdate": datetime( - 2001, 1, 1, 0, 0, tzinfo=timezone.utc - ), - "enddate": datetime( - 2002, 12, 31, 23, 59, 59, tzinfo=timezone.utc - ), + "startdate": datetime(2001, 1, 1, 0, 0, tzinfo=timezone.utc), + "enddate": datetime(2002, 12, 31, 23, 59, 59, tzinfo=timezone.utc), "bites_disabled": [False, True, False], "resource": { "description": "Conflict data with HXL tags", @@ -476,12 +472,8 @@ def process_row(headers, row): ) assert success is True assert results == { - "startdate": datetime( - 2001, 1, 1, 0, 0, tzinfo=timezone.utc - ), - "enddate": datetime( - 2002, 12, 31, 23, 59, 59, tzinfo=timezone.utc - ), + "startdate": datetime(2001, 1, 1, 0, 0, tzinfo=timezone.utc), + "enddate": datetime(2002, 12, 31, 23, 59, 59, tzinfo=timezone.utc), "bites_disabled": [False, True, False], "resource": { "description": "Conflict data with HXL tags", diff --git a/tests/hdx/data/test_organization.py b/tests/hdx/data/test_organization.py index 221c6d3b..42800744 100755 --- a/tests/hdx/data/test_organization.py +++ b/tests/hdx/data/test_organization.py @@ -15,9 +15,7 @@ from hdx.utilities.dictandlist import merge_two_dictionaries from hdx.utilities.loader import load_yaml -resultdict = load_yaml( - join("tests", "fixtures", "organization_show_results.yaml") -) +resultdict = load_yaml(join("tests", "fixtures", "organization_show_results.yaml")) organization_list = [ "acaps", @@ -31,9 +29,7 @@ "afdb", "afghanistan-protection-cluster", ] -searchdict = load_yaml( - join("tests", "fixtures", "dataset_search_results.yaml") -) +searchdict = load_yaml(join("tests", "fixtures", "dataset_search_results.yaml")) organization_autocomplete = [ { @@ -106,15 +102,11 @@ def mockgetdatasets(url, datadict): class TestOrganization: @pytest.fixture(scope="class") def static_yaml(self): - return join( - "tests", "fixtures", "config", "hdx_organization_static.yaml" - ) + return join("tests", "fixtures", "config", "hdx_organization_static.yaml") @pytest.fixture(scope="class") def static_json(self): - return join( - "tests", "fixtures", "config", "hdx_organization_static.json" - ) + return join("tests", "fixtures", "config", "hdx_organization_static.json") @pytest.fixture(scope="function") def read(self): diff --git a/tests/hdx/data/test_resource.py b/tests/hdx/data/test_resource.py index 14cec238..da5aa4af 100755 --- a/tests/hdx/data/test_resource.py +++ b/tests/hdx/data/test_resource.py @@ -330,15 +330,11 @@ def static_json(self): @pytest.fixture(scope="class") def topline_yaml(self): - return join( - "tests", "fixtures", "config", "hdx_datasource_topline.yaml" - ) + return join("tests", "fixtures", "config", "hdx_datasource_topline.yaml") @pytest.fixture(scope="class") def topline_json(self): - return join( - "tests", "fixtures", "config", "hdx_datasource_topline.json" - ) + return join("tests", "fixtures", "config", "hdx_datasource_topline.json") @pytest.fixture(scope="function") def read(self): @@ -357,8 +353,7 @@ class MockSession: def post(url, data, headers, files, allow_redirects, auth=None): if isinstance(data, dict): datadict = { - k.decode("utf8"): v.decode("utf8") - for k, v in data.items() + k.decode("utf8"): v.decode("utf8") for k, v in data.items() } else: datadict = json.loads(data.decode("utf-8")) @@ -419,8 +414,7 @@ class MockSession: def post(url, data, headers, files, allow_redirects, auth=None): if isinstance(data, dict): datadict = { - k.decode("utf8"): v.decode("utf8") - for k, v in data.items() + k.decode("utf8"): v.decode("utf8") for k, v in data.items() } else: datadict = json.loads(data.decode("utf-8")) @@ -556,10 +550,7 @@ def post(url, data, headers, files, allow_redirects, auth=None): 404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=datastore_create"}', ) - if ( - "search" in url - and datadict["resource_id"] == "_table_metadata" - ): + if "search" in url and datadict["resource_id"] == "_table_metadata": return MockResponse( 200, '{"success": true, "result": {"include_total": true, "resource_id": "_table_metadata", "fields": [{"type": "int", "id": "_id"}, {"type": "name", "id": "name"}, {"type": "oid", "id": "oid"}, {"type": "name", "id": "alias_of"}], "records_format": "objects", "records": [{"_id":"f9cd60f3d7f2f6d0","name":"f9228459-d808-4b51-948f-68a5850abfde","oid":"919290","alias_of":null},{"_id":"7ae63490de9b7d7b","name":"af618a0b-09b8-42c8-836f-2be597e1ea34","oid":"135294","alias_of":null},{"_id":"1dc37f4e89988644","name":"748b40dd-7bd3-40a3-941b-e76f0bfbe0eb","oid":"117144","alias_of":null},{"_id":"2a554a61bd366206","name":"91c78d24-eab3-40b5-ba91-6b29bcda7178","oid":"116963","alias_of":null},{"_id":"fd787575143afe90","name":"9320cfce-4620-489a-bcbe-25c73867d4fc","oid":"107430","alias_of":null},{"_id":"a70093abd230f647","name":"b9d2eb36-e65c-417a-bc28-f4dadb149302","oid":"107409","alias_of":null},{"_id":"95fbdd2d06c07aea","name":"ca6a0891-8395-4d58-9168-6c44e17e0193","oid":"107385","alias_of":null}], "limit": 10000, "_links": {"start": "/api/action/datastore_search?limit=10000&resource_id=_table_metadata", "next": "/api/action/datastore_search?offset=10000&limit=10000&resource_id=_table_metadata"}, "total": 7}}', @@ -569,9 +560,7 @@ def post(url, data, headers, files, allow_redirects, auth=None): or "insert" in url or "upsert" in url or "search" in url - ) and datadict[ - "resource_id" - ] == "de6549d8-268b-4dfe-adaf-a4ae5c8510d5": + ) and datadict["resource_id"] == "de6549d8-268b-4dfe-adaf-a4ae5c8510d5": TestResource.datastore = "create" return MockResponse( 200, @@ -633,8 +622,7 @@ class MockSession: def post(url, data, headers, files, allow_redirects, auth=None): if isinstance(data, dict): datadict = { - k.decode("utf8"): v.decode("utf8") - for k, v in data.items() + k.decode("utf8"): v.decode("utf8") for k, v in data.items() } else: datadict = json.loads(data.decode("utf-8")) @@ -657,19 +645,13 @@ def post(url, data, headers, files, allow_redirects, auth=None): Configuration.read().remoteckan().session = MockSession() def test_read_from_hdx(self, configuration, read): - resource = Resource.read_from_hdx( - "74b74ae1-df0c-4716-829f-4f939a046811" - ) + resource = Resource.read_from_hdx("74b74ae1-df0c-4716-829f-4f939a046811") assert resource["id"] == "de6549d8-268b-4dfe-adaf-a4ae5c8510d5" assert resource["name"] == "MyResource1" assert resource["package_id"] == "6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d" - resource = Resource.read_from_hdx( - "74b74ae1-df0c-4716-829f-4f939a046812" - ) + resource = Resource.read_from_hdx("74b74ae1-df0c-4716-829f-4f939a046812") assert resource is None - resource = Resource.read_from_hdx( - "74b74ae1-df0c-4716-829f-4f939a046813" - ) + resource = Resource.read_from_hdx("74b74ae1-df0c-4716-829f-4f939a046813") assert resource is None with pytest.raises(HDXError): Resource.read_from_hdx("ABC") @@ -694,9 +676,7 @@ def test_check_url_filetoupload(self, configuration): resource.check_url_filetoupload() def test_get_set_date_of_resource(self): - resource = Resource( - {"daterange_for_data": "[2020-01-07T00:00:00 TO *]"} - ) + resource = Resource({"daterange_for_data": "[2020-01-07T00:00:00 TO *]"}) result = resource.get_date_of_resource(today=datetime(2020, 11, 17)) assert result == { "startdate": datetime(2020, 1, 7, 0, 0, tzinfo=timezone.utc), @@ -779,9 +759,7 @@ def test_update_in_hdx(self, configuration, date_pattern, post_update): with pytest.raises(HDXError): resource.update_in_hdx() - resource = Resource.read_from_hdx( - "74b74ae1-df0c-4716-829f-4f939a046811" - ) + resource = Resource.read_from_hdx("74b74ae1-df0c-4716-829f-4f939a046811") assert resource["id"] == "de6549d8-268b-4dfe-adaf-a4ae5c8510d5" assert resource.get_format() == "csv" @@ -806,9 +784,7 @@ def test_update_in_hdx(self, configuration, date_pattern, post_update): assert match filetoupload = join("tests", "fixtures", "test_data.csv") - resource.set_file_to_upload( - filetoupload, guess_format_from_suffix=True - ) + resource.set_file_to_upload(filetoupload, guess_format_from_suffix=True) assert resource["format"] == "csv" resource.update_in_hdx() assert resource["url_type"] == "upload" @@ -853,9 +829,7 @@ def test_update_in_hdx(self, configuration, date_pattern, post_update): resource.update_in_hdx() def test_delete_from_hdx(self, configuration, post_delete): - resource = Resource.read_from_hdx( - "74b74ae1-df0c-4716-829f-4f939a046811" - ) + resource = Resource.read_from_hdx("74b74ae1-df0c-4716-829f-4f939a046811") resource.delete_from_hdx() del resource["id"] with pytest.raises(HDXError): @@ -905,12 +879,8 @@ def test_search_in_hdx(self, configuration, search): Resource.search_in_hdx("fail") def test_download(self, configuration, read): - resource = Resource.read_from_hdx( - "74b74ae1-df0c-4716-829f-4f939a046811" - ) - resource2 = Resource.read_from_hdx( - "74b74ae1-df0c-4716-829f-4f939a046814" - ) + resource = Resource.read_from_hdx("74b74ae1-df0c-4716-829f-4f939a046811") + resource2 = Resource.read_from_hdx("74b74ae1-df0c-4716-829f-4f939a046814") url, path = resource.download() remove(path) assert ( @@ -924,9 +894,7 @@ def test_download(self, configuration, read): with pytest.raises(DownloadError): resource2.download() - def test_datastore( - self, configuration, post_datastore, topline_yaml, topline_json - ): + def test_datastore(self, configuration, post_datastore, topline_yaml, topline_json): resource_ids = Resource.get_all_resource_ids_in_datastore() assert resource_ids == [ "f9228459-d808-4b51-948f-68a5850abfde", @@ -937,12 +905,8 @@ def test_datastore( "b9d2eb36-e65c-417a-bc28-f4dadb149302", "ca6a0891-8395-4d58-9168-6c44e17e0193", ] - resource = Resource.read_from_hdx( - "74b74ae1-df0c-4716-829f-4f939a046811" - ) - resource2 = Resource.read_from_hdx( - "74b74ae1-df0c-4716-829f-4f939a046815" - ) + resource = Resource.read_from_hdx("74b74ae1-df0c-4716-829f-4f939a046811") + resource2 = Resource.read_from_hdx("74b74ae1-df0c-4716-829f-4f939a046815") TestResource.datastore = None assert resource.has_datastore() is True assert TestResource.datastore == "create" @@ -964,12 +928,8 @@ def test_resource_views(self, configuration, post_resourceview): resource.add_update_resource_views("123") resource.add_update_resource_views([resource_view]) resource_views = resource.get_resource_views() - assert ( - resource_views[0]["id"] == "d80301b5-4abd-49bd-bf94-fa4af7b6e7a4" - ) - assert ( - resource_views[1]["id"] == "c06b5a0d-1d41-4a74-a196-41c251c76023" - ) + assert resource_views[0]["id"] == "d80301b5-4abd-49bd-bf94-fa4af7b6e7a4" + assert resource_views[1]["id"] == "c06b5a0d-1d41-4a74-a196-41c251c76023" with pytest.raises(HDXError): resource.delete_resource_view("123") resource.delete_resource_view("d80301b5-4abd-49bd-bf94-fa4af7b6e7a4") @@ -989,14 +949,10 @@ def test_resource_views(self, configuration, post_resourceview): resource_view = copy.deepcopy(resource_view_list[0]) resource_view["id"] = "123" with pytest.raises(HDXError): - resource.reorder_resource_views( - [resource_view_list[1], resource_view] - ) + resource.reorder_resource_views([resource_view_list[1], resource_view]) def test_mark_broken(self, configuration, post_broken): - resource = Resource.read_from_hdx( - "74b74ae1-df0c-4716-829f-4f939a046811" - ) + resource = Resource.read_from_hdx("74b74ae1-df0c-4716-829f-4f939a046811") assert resource["id"] == "de6549d8-268b-4dfe-adaf-a4ae5c8510d5" assert resource["name"] == "MyResource1" assert resource["package_id"] == "6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d" @@ -1004,9 +960,7 @@ def test_mark_broken(self, configuration, post_broken): assert resource.is_broken() is True def test_date_data_updated(self, configuration, read): - resource = Resource.read_from_hdx( - "74b74ae1-df0c-4716-829f-4f939a046811" - ) + resource = Resource.read_from_hdx("74b74ae1-df0c-4716-829f-4f939a046811") assert resource["id"] == "de6549d8-268b-4dfe-adaf-a4ae5c8510d5" assert resource["name"] == "MyResource1" assert resource["package_id"] == "6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d" @@ -1017,34 +971,26 @@ def test_date_data_updated(self, configuration, read): resource.get_date_data_updated().isoformat(timespec="microseconds") == expected ) - resource.set_date_data_updated( - parse_date(date, include_microseconds=True) - ) + resource.set_date_data_updated(parse_date(date, include_microseconds=True)) assert ( resource.get_date_data_updated().isoformat(timespec="microseconds") == expected ) def test_get_hdx_url(self, configuration, read): - resource = Resource.read_from_hdx( - "74b74ae1-df0c-4716-829f-4f939a046811" - ) + resource = Resource.read_from_hdx("74b74ae1-df0c-4716-829f-4f939a046811") assert ( resource.get_hdx_url() == "https://data.humdata.org/dataset/6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d/resource/de6549d8-268b-4dfe-adaf-a4ae5c8510d5" ) del resource["id"] assert resource.get_hdx_url() is None - resource = Resource.read_from_hdx( - "74b74ae1-df0c-4716-829f-4f939a046811" - ) + resource = Resource.read_from_hdx("74b74ae1-df0c-4716-829f-4f939a046811") del resource["package_id"] assert resource.get_hdx_url() is None def test_get_api_url(self, configuration, read): - resource = Resource.read_from_hdx( - "74b74ae1-df0c-4716-829f-4f939a046811" - ) + resource = Resource.read_from_hdx("74b74ae1-df0c-4716-829f-4f939a046811") assert ( resource.get_api_url() == "https://data.humdata.org/api/3/action/resource_show?id=de6549d8-268b-4dfe-adaf-a4ae5c8510d5" diff --git a/tests/hdx/data/test_resource_view.py b/tests/hdx/data/test_resource_view.py index a6653add..ab5914ee 100755 --- a/tests/hdx/data/test_resource_view.py +++ b/tests/hdx/data/test_resource_view.py @@ -134,15 +134,11 @@ class TestResourceView: @pytest.fixture(scope="class") def static_yaml(self): - return join( - "tests", "fixtures", "config", "hdx_resource_view_static.yaml" - ) + return join("tests", "fixtures", "config", "hdx_resource_view_static.yaml") @pytest.fixture(scope="class") def static_json(self): - return join( - "tests", "fixtures", "config", "hdx_resource_view_static.json" - ) + return join("tests", "fixtures", "config", "hdx_resource_view_static.json") @pytest.fixture(scope="function") def read(self): @@ -323,10 +319,7 @@ def test_update_in_hdx(self, configuration, post_update): resource_view.update_in_hdx() assert resource_view["id"] == "NOTEXIST" assert resource_view["view_type"] == "hdx_hxl_preview" - assert ( - resource_view["resource_id"] - == "25982d1c-f45a-45e1-b14e-87d367413045" - ) + assert resource_view["resource_id"] == "25982d1c-f45a-45e1-b14e-87d367413045" del resource_view["id"] resource_view["title"] = "NOTEXIST" @@ -365,10 +358,7 @@ def test_update_yaml(self, configuration, static_yaml): assert resource_view["view_type"] == "hdx_hxl_preview" assert resource_view["title"] == "Quick Charts" assert resource_view["description"] == "lala" - assert ( - resource_view["resource_id"] - == "25982d1c-f45a-45e1-b14e-87d367413045" - ) + assert resource_view["resource_id"] == "25982d1c-f45a-45e1-b14e-87d367413045" def test_update_json(self, configuration, static_json): data = copy.deepcopy(self.resource_view_data) @@ -380,28 +370,19 @@ def test_update_json(self, configuration, static_json): assert resource_view["view_type"] == "recline_view" assert resource_view["title"] == "Data Explorer" assert resource_view["description"] == "haha" - assert ( - resource_view["resource_id"] - == "25982d1c-f45a-45e1-b14e-87d367413045" - ) + assert resource_view["resource_id"] == "25982d1c-f45a-45e1-b14e-87d367413045" def test_copy(self, configuration, read): data = copy.deepcopy(self.resource_view_data) resource_view = ResourceView(data) resource_view.copy(resultdict) - assert ( - resource_view["resource_id"] - == self.resource_view_data["resource_id"] - ) + assert resource_view["resource_id"] == self.resource_view_data["resource_id"] assert resource_view["view_type"] == "hdx_hxl_preview" assert resource_view["hxl_preview_config"] == hxl_preview_config data = copy.deepcopy(self.resource_view_data) resource_view = ResourceView(data) resource_view.copy("c06b5a0d-1d41-4a74-a196-41c251c76023") - assert ( - resource_view["resource_id"] - == self.resource_view_data["resource_id"] - ) + assert resource_view["resource_id"] == self.resource_view_data["resource_id"] assert resource_view["view_type"] == "hdx_hxl_preview" assert resource_view["hxl_preview_config"] == hxl_preview_config with pytest.raises(HDXError): @@ -413,9 +394,5 @@ def test_get_all_for_resource(self, configuration, post_list): resource_views = ResourceView.get_all_for_resource( "25982d1c-f45a-45e1-b14e-87d367413045" ) - assert ( - resource_views[0]["id"] == "d80301b5-4abd-49bd-bf94-fa4af7b6e7a4" - ) - assert ( - resource_views[1]["id"] == "c06b5a0d-1d41-4a74-a196-41c251c76023" - ) + assert resource_views[0]["id"] == "d80301b5-4abd-49bd-bf94-fa4af7b6e7a4" + assert resource_views[1]["id"] == "c06b5a0d-1d41-4a74-a196-41c251c76023" diff --git a/tests/hdx/data/test_showcase.py b/tests/hdx/data/test_showcase.py index 0d5c24ff..e2e06ed8 100755 --- a/tests/hdx/data/test_showcase.py +++ b/tests/hdx/data/test_showcase.py @@ -57,12 +57,8 @@ "name": "showcase-1", } -datasetsdict = load_yaml( - join("tests", "fixtures", "dataset_search_results.yaml") -) -allsearchdict = load_yaml( - join("tests", "fixtures", "showcase_all_search_results.yaml") -) +datasetsdict = load_yaml(join("tests", "fixtures", "dataset_search_results.yaml")) +allsearchdict = load_yaml(join("tests", "fixtures", "showcase_all_search_results.yaml")) def mockshow(url, datadict): @@ -143,9 +139,7 @@ def mockallsearch(url, datadict): ) if datadict["q"] == "*:*": newsearchdict = copy.deepcopy(allsearchdict) - newsearchdict["results"].extend( - copy.deepcopy(newsearchdict["results"]) - ) + newsearchdict["results"].extend(copy.deepcopy(newsearchdict["results"])) for i, x in enumerate(newsearchdict["results"]): x["id"] = f"{x['id']}{i}" return MockResponse( @@ -342,9 +336,7 @@ def post(url, data, headers, files, allow_redirects, auth=None): Configuration.read().remoteckan().session = MockSession() def test_read_from_hdx(self, configuration, read): - showcase = Showcase.read_from_hdx( - "05e392bf-04e0-4ca6-848c-4e87bba10746" - ) + showcase = Showcase.read_from_hdx("05e392bf-04e0-4ca6-848c-4e87bba10746") assert showcase["id"] == "05e392bf-04e0-4ca6-848c-4e87bba10746" assert showcase["title"] == "MyShowcase1" showcase = Showcase.read_from_hdx("TEST2") @@ -386,9 +378,7 @@ def test_update_in_hdx(self, configuration, post_update): with pytest.raises(HDXError): showcase.update_in_hdx() - showcase = Showcase.read_from_hdx( - "05e392bf-04e0-4ca6-848c-4e87bba10746" - ) + showcase = Showcase.read_from_hdx("05e392bf-04e0-4ca6-848c-4e87bba10746") assert showcase["id"] == "05e392bf-04e0-4ca6-848c-4e87bba10746" assert showcase["title"] == "MyShowcase1" @@ -431,9 +421,7 @@ def test_update_in_hdx(self, configuration, post_update): assert showcase["state"] == "active" def test_delete_from_hdx(self, configuration, post_delete): - showcase = Showcase.read_from_hdx( - "05e392bf-04e0-4ca6-848c-4e87bba10746" - ) + showcase = Showcase.read_from_hdx("05e392bf-04e0-4ca6-848c-4e87bba10746") showcase.delete_from_hdx() del showcase["id"] with pytest.raises(HDXError): @@ -482,9 +470,7 @@ def test_tags(self, configuration, post_update): assert result is False def test_datasets(self, configuration, read): - showcase = Showcase.read_from_hdx( - "05e392bf-04e0-4ca6-848c-4e87bba10746" - ) + showcase = Showcase.read_from_hdx("05e392bf-04e0-4ca6-848c-4e87bba10746") datasets = showcase.get_datasets() assert len(datasets) == 10 assert datasets[0].data == datasetsdict["results"][0] @@ -495,10 +481,7 @@ def test_datasets(self, configuration, read): showcase.remove_dataset(datasets[0]) assert TestShowcase.association == "delete" TestShowcase.association = None - assert ( - showcase.add_dataset("a2f32edd-bac2-4940-aa58-49e565041055") - is True - ) + assert showcase.add_dataset("a2f32edd-bac2-4940-aa58-49e565041055") is True assert TestShowcase.association == "create" TestShowcase.association = None assert ( diff --git a/tests/hdx/data/test_update_dataset_resources.py b/tests/hdx/data/test_update_dataset_resources.py index 0e3fb674..438d45f0 100644 --- a/tests/hdx/data/test_update_dataset_resources.py +++ b/tests/hdx/data/test_update_dataset_resources.py @@ -145,33 +145,21 @@ def test_dataset_update_resources( test=True, ) assert results["files_to_upload"] == { - "update__resources__0__upload": join( - fixture_path, "sdg_data_zwe.csv" - ), + "update__resources__0__upload": join(fixture_path, "sdg_data_zwe.csv"), "update__resources__1__upload": join( fixture_path, "sdg_indicatorlist_zwe.csv" ), - "update__resources__2__upload": join( - fixture_path, "sdg_metadata_zwe.csv" - ), - "update__resources__3__upload": join( - fixture_path, "dem_data_zwe.csv" - ), + "update__resources__2__upload": join(fixture_path, "sdg_metadata_zwe.csv"), + "update__resources__3__upload": join(fixture_path, "dem_data_zwe.csv"), "update__resources__4__upload": join( fixture_path, "dem_indicatorlist_zwe.csv" ), - "update__resources__5__upload": join( - fixture_path, "opri_data_zwe.csv" - ), + "update__resources__5__upload": join(fixture_path, "opri_data_zwe.csv"), "update__resources__6__upload": join( fixture_path, "opri_indicatorlist_zwe.csv" ), - "update__resources__7__upload": join( - fixture_path, "opri_metadata_zwe.csv" - ), - "update__resources__8__upload": join( - fixture_path, "qc_sdg_data_zwe.csv" - ), + "update__resources__7__upload": join(fixture_path, "opri_metadata_zwe.csv"), + "update__resources__8__upload": join(fixture_path, "qc_sdg_data_zwe.csv"), } resources = results["update"]["resources"] cutdown_resources = [] diff --git a/tests/hdx/data/test_update_logic.py b/tests/hdx/data/test_update_logic.py index 98fbfcf7..5a65b39b 100644 --- a/tests/hdx/data/test_update_logic.py +++ b/tests/hdx/data/test_update_logic.py @@ -103,9 +103,7 @@ def add_dataset_resources(dataset, yaml_path, include=None, exclude=None): dataset.add_update_resource(resource) @staticmethod - def add_new_dataset_resources( - new_dataset, yaml_path, include=None, exclude=None - ): + def add_new_dataset_resources(new_dataset, yaml_path, include=None, exclude=None): resources_uploads = load_yaml(yaml_path) if include is None: include = range(len(resources_uploads)) @@ -154,9 +152,7 @@ def test_update_logic_1( new_resources_yaml, resources_yaml, ): - self.add_new_dataset_resources( - new_dataset, new_resources_yaml, exclude=[13] - ) + self.add_new_dataset_resources(new_dataset, new_resources_yaml, exclude=[13]) self.add_dataset_resources(dataset, resources_yaml) dataset.old_data = new_dataset.data dataset.old_data["resources"] = new_dataset.resources @@ -632,9 +628,7 @@ def test_update_logic_2( new_resources_yaml, resources_yaml, ): - self.add_new_dataset_resources( - new_dataset, new_resources_yaml, exclude=[52] - ) + self.add_new_dataset_resources(new_dataset, new_resources_yaml, exclude=[52]) self.add_dataset_resources(dataset, resources_yaml) dataset.old_data = new_dataset.data dataset.old_data["resources"] = new_dataset.resources @@ -1110,9 +1104,7 @@ def test_update_logic_3( new_resources_yaml, resources_yaml, ): - self.add_new_dataset_resources( - new_dataset, new_resources_yaml, include=[0, 3] - ) + self.add_new_dataset_resources(new_dataset, new_resources_yaml, include=[0, 3]) self.add_dataset_resources(dataset, resources_yaml, include=[0, 1, 2]) dataset.old_data = new_dataset.data dataset.old_data["resources"] = new_dataset.resources diff --git a/tests/hdx/data/test_user.py b/tests/hdx/data/test_user.py index 066ce0c9..31a0d9c5 100755 --- a/tests/hdx/data/test_user.py +++ b/tests/hdx/data/test_user.py @@ -30,9 +30,7 @@ "id": "9f3e9973-7dbe-4c65-8820-f48578e3ffea", "number_created_packages": 0, } -orgdict = load_yaml( - join("tests", "fixtures", "organization_show_results.yaml") -) +orgdict = load_yaml(join("tests", "fixtures", "organization_show_results.yaml")) orgdict2 = copy.deepcopy(orgdict) del orgdict2["users"] del orgdict2["packages"] @@ -295,8 +293,7 @@ def post(url, data, headers, files, allow_redirects, auth=None): if "show" in url: result = json.dumps(orgdict) if ( - datadict["id"] - == "b67e6c74-c185-4f43-b561-0e114a736f19" + datadict["id"] == "b67e6c74-c185-4f43-b561-0e114a736f19" or datadict["id"] == "TEST1" ): return MockResponse( @@ -611,9 +608,7 @@ def test_get_organizations(self, configuration, post_listorgs): organizations = user.get_organizations() assert organizations[0]["id"] == "b67e6c74-c185-4f43-b561-0e114a736f19" assert len(organizations) == 1 - result = user.check_organization_access( - "b67e6c74-c185-4f43-b561-0e114a736f19" - ) + result = user.check_organization_access("b67e6c74-c185-4f43-b561-0e114a736f19") assert result is True result = user.check_organization_access("acled") assert result is True @@ -635,9 +630,7 @@ def test_get_organizations(self, configuration, post_listorgs): result = User.check_current_user_organization_access("lala") assert result is False - def test_get_organizations_invalid_user( - self, configuration, post_listorgs_invalid - ): + def test_get_organizations_invalid_user(self, configuration, post_listorgs_invalid): user = User.read_from_hdx("9f3e9973-7dbe-4c65-8820-f48578e3ffea") user["name"] = "lala" assert user.get_organization_dicts() == [] diff --git a/tests/hdx/data/test_vocabulary.py b/tests/hdx/data/test_vocabulary.py index a20f18a2..c526a793 100755 --- a/tests/hdx/data/test_vocabulary.py +++ b/tests/hdx/data/test_vocabulary.py @@ -1138,15 +1138,11 @@ class TestVocabulary: @pytest.fixture(scope="class") def static_yaml(self): - return join( - "tests", "fixtures", "config", "hdx_vocabulary_static.yaml" - ) + return join("tests", "fixtures", "config", "hdx_vocabulary_static.yaml") @pytest.fixture(scope="class") def static_json(self): - return join( - "tests", "fixtures", "config", "hdx_vocabulary_static.json" - ) + return join("tests", "fixtures", "config", "hdx_vocabulary_static.json") @pytest.fixture(scope="function") def read(self): @@ -1310,9 +1306,7 @@ def test_init(self, configuration): ] def test_read_from_hdx(self, configuration, read): - vocabulary = Vocabulary.read_from_hdx( - "1731e7fc-ff62-4551-8a70-2a5878e1142b" - ) + vocabulary = Vocabulary.read_from_hdx("1731e7fc-ff62-4551-8a70-2a5878e1142b") assert vocabulary["id"] == "1731e7fc-ff62-4551-8a70-2a5878e1142b" assert vocabulary["name"] == "miketest" vocabulary = Vocabulary.read_from_hdx("TEST2") @@ -1355,9 +1349,7 @@ def test_update_in_hdx(self, configuration, post_update): with pytest.raises(HDXError): vocabulary.update_in_hdx() - vocabulary = Vocabulary.read_from_hdx( - "1731e7fc-ff62-4551-8a70-2a5878e1142b" - ) + vocabulary = Vocabulary.read_from_hdx("1731e7fc-ff62-4551-8a70-2a5878e1142b") assert vocabulary["id"] == "1731e7fc-ff62-4551-8a70-2a5878e1142b" assert vocabulary["name"] == "miketest" @@ -1393,15 +1385,11 @@ def test_update_in_hdx(self, configuration, post_update): assert "state" not in vocabulary def test_delete_from_hdx(self, configuration, post_delete): - vocabulary = Vocabulary.read_from_hdx( - "1731e7fc-ff62-4551-8a70-2a5878e1142b" - ) + vocabulary = Vocabulary.read_from_hdx("1731e7fc-ff62-4551-8a70-2a5878e1142b") with pytest.raises(HDXError): vocabulary.delete_from_hdx(empty=False) vocabulary.delete_from_hdx() - vocabulary = Vocabulary.read_from_hdx( - "1731e7fc-ff62-4551-8a70-2a5878e1142b" - ) + vocabulary = Vocabulary.read_from_hdx("1731e7fc-ff62-4551-8a70-2a5878e1142b") del vocabulary["id"] with pytest.raises(HDXError): vocabulary.delete_from_hdx() @@ -1460,29 +1448,20 @@ def test_get_approved_vocabulary(self, configuration, read): Vocabulary._approved_vocabulary = None vocabulary = Vocabulary.get_approved_vocabulary() assert vocabulary["name"] == "Topics" - assert ( - vocabulary["tags"][0]["name"] - == "administrative boundaries-divisions" - ) + assert vocabulary["tags"][0]["name"] == "administrative boundaries-divisions" Vocabulary._approved_vocabulary = None def test_create_approved_vocabulary(self, configuration, post_create): vocabulary = Vocabulary.create_approved_vocabulary() assert vocabulary["name"] == "Topics" - assert ( - vocabulary["tags"][0]["name"] - == "administrative boundaries-divisions" - ) + assert vocabulary["tags"][0]["name"] == "administrative boundaries-divisions" Vocabulary._approved_vocabulary = None def test_update_approved_vocabulary(self, configuration, post_update): Vocabulary._approved_vocabulary = None vocabulary = Vocabulary.update_approved_vocabulary() assert vocabulary["name"] == "Topics" - assert ( - vocabulary["tags"][0]["name"] - == "administrative boundaries-divisions" - ) + assert vocabulary["tags"][0]["name"] == "administrative boundaries-divisions" vocabulary["tags"] = [ { "vocabulary_id": "b891512e-9516-4bf5-962a-7a289772a2a1", @@ -1493,10 +1472,7 @@ def test_update_approved_vocabulary(self, configuration, post_update): ] vocabulary = Vocabulary.update_approved_vocabulary(replace=False) assert vocabulary["tags"][0]["name"] == "blah" - assert ( - vocabulary["tags"][1]["name"] - == "administrative boundaries-divisions" - ) + assert vocabulary["tags"][1]["name"] == "administrative boundaries-divisions" Vocabulary._approved_vocabulary = None def test_tag_mappings(self, configuration, read): diff --git a/tests/hdx/utilities/test_dataset_title_helper.py b/tests/hdx/utilities/test_dataset_title_helper.py index 45d936e3..ee3fa73f 100755 --- a/tests/hdx/utilities/test_dataset_title_helper.py +++ b/tests/hdx/utilities/test_dataset_title_helper.py @@ -43,9 +43,7 @@ def test_get_date_from_title(self, expected_ranges_2019): "Myanmar Self Administered Regions Boundaries MIMU v9.2.1", [], ) - assert DatasetTitleHelper.get_dates_from_title( - "Myanmar Town 2019 July" - ) == ( + assert DatasetTitleHelper.get_dates_from_title("Myanmar Town 2019 July") == ( "Myanmar Town", expected_ranges_2019, ) @@ -255,9 +253,7 @@ def test_get_date_from_title(self, expected_ranges_2019): ), ], ) - assert DatasetTitleHelper.get_dates_from_title( - "Risk, 2020/19-2014" - ) == ( + assert DatasetTitleHelper.get_dates_from_title("Risk, 2020/19-2014") == ( "Risk, 2020/19", [ ( diff --git a/tests/hdx/utilities/test_hdx_error_handler.py b/tests/hdx/utilities/test_hdx_error_handler.py index 2cb22c4f..027eb274 100644 --- a/tests/hdx/utilities/test_hdx_error_handler.py +++ b/tests/hdx/utilities/test_hdx_error_handler.py @@ -39,12 +39,7 @@ def test_hdx_error_handler(self, caplog): err_to_hdx=True, ) assert ( - len( - errors.shared_errors["error"][ - "pipeline1 - dataset1" - ] - ) - == 3 + len(errors.shared_errors["error"]["pipeline1 - dataset1"]) == 3 ) assert ( len(