Skip to content

Commit c22544d

Browse files
mcaransmcarans
committed
Make path aware
1 parent 272d329 commit c22544d

26 files changed

Lines changed: 218 additions & 223 deletions

‎src/hdx/api/configuration.py‎

Lines changed: 27 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@
44
import os
55
from base64 import b64decode
66
from collections import UserDict
7-
from os.path import expanduser, isfile, join
7+
from os.path import expanduser, isfile
8+
from pathlib import Path
89
from typing import Any, Optional
910

1011
import ckanapi
@@ -38,22 +39,22 @@ class Configuration(UserDict):
3839
hdx_read_only (bool): Whether to access HDX in read only mode. Defaults to False.
3940
hdx_key (str): Your HDX key. Ignored if hdx_read_only = True.
4041
hdx_config_dict (dict): HDX configuration dictionary to use instead of above 3 parameters OR
41-
hdx_config_json (str): Path to JSON HDX configuration OR
42-
hdx_config_yaml (str): Path to YAML HDX configuration
42+
hdx_config_json (Path | str): Path to JSON HDX configuration OR
43+
hdx_config_yaml (Path | str): Path to YAML HDX configuration
4344
project_config_dict (dict): Project configuration dictionary OR
44-
project_config_json (str): Path to JSON Project configuration OR
45-
project_config_yaml (str): Path to YAML Project configuration
45+
project_config_json (Path | str): Path to JSON Project configuration OR
46+
project_config_yaml (Path | str): Path to YAML Project configuration
4647
hdx_base_config_dict (dict): HDX base configuration dictionary OR
47-
hdx_base_config_json (str): Path to JSON HDX base configuration OR
48-
hdx_base_config_yaml (str): Path to YAML HDX base configuration. Defaults to library's internal hdx_base_configuration.yaml.
48+
hdx_base_config_json (Path | str): Path to JSON HDX base configuration OR
49+
hdx_base_config_yaml (Path | str): Path to YAML HDX base configuration. Defaults to library's internal hdx_base_configuration.yaml.
4950
"""
5051

5152
_configuration = None
5253
home_folder = expanduser("~")
5354
default_hdx_base_config_yaml = script_dir_plus_file(
5455
"hdx_base_configuration.yaml", ConfigurationError
5556
)
56-
default_hdx_config_yaml = join(home_folder, ".hdx_configuration.yaml")
57+
default_hdx_config_yaml = Path(home_folder, ".hdx_configuration.yaml")
5758

5859
prefix = f"HDXPythonLibrary/{__version__}"
5960

@@ -509,14 +510,14 @@ def setup(
509510
hdx_read_only (bool): Whether to access HDX in read only mode. Defaults to False.
510511
hdx_key (str): Your HDX key. Ignored if hdx_read_only = True.
511512
hdx_config_dict (dict): HDX configuration dictionary to use instead of above 3 parameters OR
512-
hdx_config_json (str): Path to JSON HDX configuration OR
513-
hdx_config_yaml (str): Path to YAML HDX configuration
513+
hdx_config_json (Path | str): Path to JSON HDX configuration OR
514+
hdx_config_yaml (Path | str): Path to YAML HDX configuration
514515
project_config_dict (dict): Project configuration dictionary OR
515-
project_config_json (str): Path to JSON Project configuration OR
516-
project_config_yaml (str): Path to YAML Project configuration
516+
project_config_json (Path | str): Path to JSON Project configuration OR
517+
project_config_yaml (Path | str): Path to YAML Project configuration
517518
hdx_base_config_dict (dict): HDX base configuration dictionary OR
518-
hdx_base_config_json (str): Path to JSON HDX base configuration OR
519-
hdx_base_config_yaml (str): Path to YAML HDX base configuration. Defaults to library's internal hdx_base_configuration.yaml.
519+
hdx_base_config_json (Path | str): Path to JSON HDX base configuration OR
520+
hdx_base_config_yaml (Path | str): Path to YAML HDX base configuration. Defaults to library's internal hdx_base_configuration.yaml.
520521
521522
Returns:
522523
None
@@ -549,14 +550,14 @@ def _create(
549550
hdx_read_only (bool): Whether to access HDX in read only mode. Defaults to False.
550551
hdx_key (str): Your HDX key. Ignored if hdx_read_only = True.
551552
hdx_config_dict (dict): HDX configuration dictionary to use instead of above 3 parameters OR
552-
hdx_config_json (str): Path to JSON HDX configuration OR
553-
hdx_config_yaml (str): Path to YAML HDX configuration
553+
hdx_config_json (Path | str): Path to JSON HDX configuration OR
554+
hdx_config_yaml (Path | str): Path to YAML HDX configuration
554555
project_config_dict (dict): Project configuration dictionary OR
555-
project_config_json (str): Path to JSON Project configuration OR
556-
project_config_yaml (str): Path to YAML Project configuration
556+
project_config_json (Path | str): Path to JSON Project configuration OR
557+
project_config_yaml (Path | str): Path to YAML Project configuration
557558
hdx_base_config_dict (dict): HDX base configuration dictionary OR
558-
hdx_base_config_json (str): Path to JSON HDX base configuration OR
559-
hdx_base_config_yaml (str): Path to YAML HDX base configuration. Defaults to library's internal hdx_base_configuration.yaml.
559+
hdx_base_config_json (Path | str): Path to JSON HDX base configuration OR
560+
hdx_base_config_yaml (Path | str): Path to YAML HDX base configuration. Defaults to library's internal hdx_base_configuration.yaml.
560561
561562
Returns:
562563
HDX site url
@@ -588,14 +589,14 @@ def create(
588589
hdx_read_only (bool): Whether to access HDX in read only mode. Defaults to False.
589590
hdx_key (str): Your HDX key. Ignored if hdx_read_only = True.
590591
hdx_config_dict (dict): HDX configuration dictionary to use instead of above 3 parameters OR
591-
hdx_config_json (str): Path to JSON HDX configuration OR
592-
hdx_config_yaml (str): Path to YAML HDX configuration
592+
hdx_config_json (Path | str): Path to JSON HDX configuration OR
593+
hdx_config_yaml (Path | str): Path to YAML HDX configuration
593594
project_config_dict (dict): Project configuration dictionary OR
594-
project_config_json (str): Path to JSON Project configuration OR
595-
project_config_yaml (str): Path to YAML Project configuration
595+
project_config_json (Path | str): Path to JSON Project configuration OR
596+
project_config_yaml (Path | str): Path to YAML Project configuration
596597
hdx_base_config_dict (dict): HDX base configuration dictionary OR
597-
hdx_base_config_json (str): Path to JSON HDX base configuration OR
598-
hdx_base_config_yaml (str): Path to YAML HDX base configuration. Defaults to library's internal hdx_base_configuration.yaml.
598+
hdx_base_config_json (Path | str): Path to JSON HDX base configuration OR
599+
hdx_base_config_yaml (Path | str): Path to YAML HDX base configuration. Defaults to library's internal hdx_base_configuration.yaml.
599600
600601
Returns:
601602
HDX site url

‎src/hdx/api/utilities/hdx_state.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
import logging
44
from collections.abc import Callable
5-
from os.path import join
5+
from pathlib import Path
66
from typing import Any
77

88
from hdx.utilities.loader import load_text
@@ -32,7 +32,7 @@ class HDXState(State):
3232
def __init__(
3333
self,
3434
dataset_name_or_id: str,
35-
path: str,
35+
path: Path | str,
3636
read_fn: Callable[[str], Any] = lambda x: x,
3737
write_fn: Callable[[Any], str] = lambda x: x,
3838
configuration: Configuration | None = None,
@@ -65,7 +65,7 @@ def write(self) -> None:
6565
"""
6666
logger.info(f"State written to {self._dataset_name_or_id} = {self.state}")
6767
filename = self._resource["name"]
68-
file_to_upload = join(self.path, filename)
68+
file_to_upload = self.path / filename
6969
save_text(self.write_fn(self.state), file_to_upload)
7070
self._resource.set_file_to_upload(file_to_upload)
7171
self._resource.update_in_hdx()

‎src/hdx/data/dataset.py‎

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@
77
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
88
from copy import deepcopy
99
from datetime import datetime
10-
from os.path import isfile, join
10+
from os.path import isfile
11+
from pathlib import Path
1112
from typing import (
1213
TYPE_CHECKING,
1314
Any,
@@ -198,7 +199,7 @@ def get_dataset_dict(self) -> dict:
198199
package["resources"] = self._convert_hdxobjects(self._resources)
199200
return package
200201

201-
def save_to_json(self, path: str, follow_urls: bool = False):
202+
def save_to_json(self, path: Path | str, follow_urls: bool = False):
202203
"""Save dataset to JSON. If follow_urls is True, resource urls that point to
203204
datasets, HXL proxy urls etc. are followed to retrieve final urls.
204205
@@ -219,7 +220,7 @@ def save_to_json(self, path: str, follow_urls: bool = False):
219220
save_json(dataset_dict, path)
220221

221222
@staticmethod
222-
def load_from_json(path: str) -> Optional["Dataset"]:
223+
def load_from_json(path: Path | str) -> Optional["Dataset"]:
223224
"""Load dataset from JSON
224225
225226
Args:
@@ -457,7 +458,7 @@ def move_resource(
457458
return resource
458459

459460
def update_from_yaml(
460-
self, path: str = join("config", "hdx_dataset_static.yaml")
461+
self, path: Path | str = Path("config", "hdx_dataset_static.yaml")
461462
) -> None:
462463
"""Update dataset metadata with static metadata from YAML file
463464
@@ -471,7 +472,7 @@ def update_from_yaml(
471472
self.separate_resources()
472473

473474
def update_from_json(
474-
self, path: str = join("config", "hdx_dataset_static.json")
475+
self, path: Path | str = Path("config", "hdx_dataset_static.json")
475476
) -> None:
476477
"""Update dataset metadata with static metadata from JSON file
477478
@@ -2330,7 +2331,7 @@ def _create_preview_resourceview(self) -> None:
23302331
def _generate_resource_view(
23312332
self,
23322333
resource: Union["Resource", dict, str, int] = 0,
2333-
path: str | None = None,
2334+
path: Path | str | None = None,
23342335
bites_disabled: Sequence[bool] | None = None,
23352336
indicators: Sequence[dict] | None = None,
23362337
findreplace: dict | None = None,
@@ -2374,9 +2375,9 @@ def _generate_resource_view(
23742375
resourceview = resource_view.ResourceView(resourceview_data)
23752376
if path is None:
23762377
if indicators is None:
2377-
path = join("config", "hdx_resource_view_static.yaml")
2378+
path = Path("config", "hdx_resource_view_static.yaml")
23782379
if not isfile(path):
2379-
path = path.replace(".yaml", ".yml")
2380+
path = path.with_suffix(".yml")
23802381
else:
23812382
path = script_dir_plus_file(
23822383
"indicator_resource_view_template.yaml",
@@ -2488,7 +2489,7 @@ def replace_indicator(qc_config, index):
24882489
def generate_quickcharts(
24892490
self,
24902491
resource: Union["Resource", dict, str, int] = 0,
2491-
path: str | None = None,
2492+
path: Path | str | None = None,
24922493
bites_disabled: Sequence[bool] | None = None,
24932494
indicators: Sequence[dict] | None = None,
24942495
findreplace: dict | None = None,
@@ -2607,7 +2608,7 @@ def remove_dates_from_title(
26072608

26082609
def generate_resource(
26092610
self,
2610-
folder: str,
2611+
folder: Path | str,
26112612
filename: str,
26122613
rows: Iterable[Sequence | Mapping],
26132614
resourcedata: dict,
@@ -2703,7 +2704,7 @@ def process_row(row: Sequence | Mapping) -> Sequence | Mapping | None:
27032704
dates[1] = enddate
27042705
return row
27052706

2706-
filepath = join(folder, filename)
2707+
filepath = Path(folder) / filename
27072708
rows = save_iterable(
27082709
filepath,
27092710
rows,
@@ -2739,7 +2740,7 @@ def process_row(row: Sequence | Mapping) -> Sequence | Mapping | None:
27392740

27402741
def generate_resource_from_rows(
27412742
self,
2742-
folder: str,
2743+
folder: Path | str,
27432744
filename: str,
27442745
rows: Iterable[Sequence | Mapping],
27452746
resourcedata: dict,
@@ -2776,7 +2777,7 @@ def generate_resource_from_iterable(
27762777
headers: Sequence[str],
27772778
iterable: Iterable[Sequence | dict],
27782779
hxltags: dict[str, str],
2779-
folder: str,
2780+
folder: Path | str,
27802781
filename: str,
27812782
resourcedata: dict,
27822783
datecol: int | str | None = None,
@@ -2988,7 +2989,7 @@ def generate_resource_from_iterator(
29882989
headers: Sequence[str],
29892990
iterator: Iterator[Sequence | dict],
29902991
hxltags: dict[str, str],
2991-
folder: str,
2992+
folder: Path | str,
29922993
filename: str,
29932994
resourcedata: dict,
29942995
datecol: int | str | None = None,
@@ -3019,7 +3020,7 @@ def download_generate_resource(
30193020
self,
30203021
downloader: BaseDownload,
30213022
url: str,
3022-
folder: str,
3023+
folder: Path | str,
30233024
filename: str,
30243025
resourcedata: dict,
30253026
header_insertions: Sequence[tuple[int, str]] | None = None,
@@ -3122,7 +3123,7 @@ def download_and_generate_resource(
31223123
downloader: BaseDownload,
31233124
url: str,
31243125
hxltags: dict[str, str],
3125-
folder: str,
3126+
folder: Path | str,
31263127
filename: str,
31273128
resourcedata: dict,
31283129
header_insertions: Sequence[tuple[int, str]] | None = None,

‎src/hdx/data/hdxobject.py‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
from collections import UserDict
99
from collections.abc import Sequence
1010
from os.path import isfile
11+
from pathlib import Path
1112
from typing import Any, Optional, Union
1213

1314
from ckanapi.errors import NotFound
@@ -62,7 +63,7 @@ def get_old_data_dict(self) -> dict:
6263
"""
6364
return self._old_data
6465

65-
def update_from_yaml(self, path: str) -> None:
66+
def update_from_yaml(self, path: Path | str) -> None:
6667
"""Update metadata with static metadata from YAML file
6768
6869
Args:
@@ -72,10 +73,10 @@ def update_from_yaml(self, path: str) -> None:
7273
None
7374
"""
7475
if not isfile(path):
75-
path = path.replace(".yaml", ".yml")
76-
self.data = load_yaml_into_existing_dict(self.data, path)
76+
path = Path(path).with_suffix(".yml")
77+
load_yaml_into_existing_dict(self.data, path)
7778

78-
def update_from_json(self, path: str) -> None:
79+
def update_from_json(self, path: Path | str) -> None:
7980
"""Update metadata with static metadata from JSON file
8081
8182
Args:
@@ -84,7 +85,7 @@ def update_from_json(self, path: str) -> None:
8485
Returns:
8586
None
8687
"""
87-
self.data = load_json_into_existing_dict(self.data, path)
88+
load_json_into_existing_dict(self.data, path)
8889

8990
def _read_from_hdx(
9091
self,

‎src/hdx/data/organization.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
import logging
44
from collections.abc import Sequence
5-
from os.path import join
5+
from pathlib import Path
66
from typing import TYPE_CHECKING, Any, Optional, Union
77

88
import hdx.data.dataset
@@ -50,7 +50,7 @@ def actions() -> dict[str, str]:
5050
}
5151

5252
def update_from_yaml(
53-
self, path: str = join("config", "hdx_organization_static.yaml")
53+
self, path: Path | str = Path("config", "hdx_organization_static.yaml")
5454
) -> None:
5555
"""Update organization metadata with static metadata from YAML file
5656
@@ -63,7 +63,7 @@ def update_from_yaml(
6363
super().update_from_yaml(path)
6464

6565
def update_from_json(
66-
self, path: str = join("config", "hdx_organization_static.json")
66+
self, path: Path | str = Path("config", "hdx_organization_static.json")
6767
) -> None:
6868
"""Update organization metadata with static metadata from JSON file
6969

‎src/hdx/data/resource.py‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
import warnings
55
from collections.abc import Sequence
66
from datetime import datetime
7-
from os.path import join
87
from pathlib import Path
98
from typing import Any, Optional
109

@@ -67,7 +66,7 @@ def actions() -> dict[str, str]:
6766
}
6867

6968
def update_from_yaml(
70-
self, path: str = join("config", "hdx_resource_static.yaml")
69+
self, path: Path | str = Path("config", "hdx_resource_static.yaml")
7170
) -> None:
7271
"""Update resource metadata with static metadata from YAML file
7372
@@ -80,7 +79,7 @@ def update_from_yaml(
8079
super().update_from_yaml(path)
8180

8281
def update_from_json(
83-
self, path: str = join("config", "hdx_resource_static.json")
82+
self, path: Path | str = Path("config", "hdx_resource_static.json")
8483
) -> None:
8584
"""Update resource metadata with static metadata from JSON file
8685
@@ -300,7 +299,7 @@ def get_file_to_upload(self) -> str | None:
300299
return self._file_to_upload
301300

302301
def set_file_to_upload(
303-
self, file_to_upload: str, guess_format_from_suffix: bool = False
302+
self, file_to_upload: Path | str, guess_format_from_suffix: bool = False
304303
) -> str:
305304
"""Delete any existing url and set the file uploaded to the local path provided
306305
@@ -647,8 +646,8 @@ def search_in_hdx(
647646
return resources
648647

649648
def download(
650-
self, folder: str | None = None, retriever: Retrieve | None = None
651-
) -> tuple[str, str]:
649+
self, folder: Path | str | None = None, retriever: Retrieve | None = None
650+
) -> tuple[str, Path]:
652651
"""Download resource store to provided folder or temporary folder if no folder
653652
supplied
654653

0 commit comments

Comments
 (0)