From 5bc12412ee1be3169364aece6e8dccf03aaf91b6 Mon Sep 17 00:00:00 2001 From: dkazanc Date: Tue, 20 Jan 2026 11:37:21 +0000 Subject: [PATCH 1/3] a fix for the preview expansion to work in parallel --- httomo/ui_layer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/httomo/ui_layer.py b/httomo/ui_layer.py index 0eeeb77c3..e21b578f2 100644 --- a/httomo/ui_layer.py +++ b/httomo/ui_layer.py @@ -174,8 +174,7 @@ def fix_preview_y_if_smaller_than_padding( ) -> None: vertical_preview_length = ( loader.preview.detector_y.stop - loader.preview.detector_y.start - ) - # we need to know the largest padding value over all methods in the pipeline + ) // loader.comm.size max_pad_value = 0 for _, m in enumerate(methods_list): if m.padding: From f50e603e05f293624a597784126e2a33b0cab154 Mon Sep 17 00:00:00 2001 From: dkazanc Date: Tue, 20 Jan 2026 12:13:31 +0000 Subject: [PATCH 2/3] black corrections for 26.* version that is installed with the CI job --- httomo/cli.py | 1 - httomo/data/dataset_store.py | 2 +- httomo/data/mpiutil.py | 1 - httomo/data/padding.py | 1 - httomo/loaders/standard_tomo_loader.py | 2 +- httomo/monitors/__init__.py | 1 - httomo/runner/task_runner.py | 2 +- httomo/types.py | 1 - httomo/ui_layer.py | 6 +++--- httomo/yaml_checker.py | 4 ++-- tests/scripts/create_numpy_from_hdf5.py | 1 + tests/test_cli.py | 1 - tests/test_parallel_pipeline_big.py | 10 +++++----- tests/test_pipeline_big.py | 12 ++++++------ tests/test_transform_loader_params.py | 2 +- 15 files changed, 21 insertions(+), 26 deletions(-) diff --git a/httomo/cli.py b/httomo/cli.py index 6d527078c..c80ba286f 100644 --- a/httomo/cli.py +++ b/httomo/cli.py @@ -21,7 +21,6 @@ from httomo.runner.task_runner import TaskRunner from httomo.ui_layer import UiLayer, PipelineFormat - try: from . import __version__ except: diff --git a/httomo/data/dataset_store.py b/httomo/data/dataset_store.py index 29898782e..45b22c38f 100644 --- a/httomo/data/dataset_store.py +++ b/httomo/data/dataset_store.py @@ -355,7 +355,7 @@ def _reslice( array, newdim, startidx = reslice( data, old_slicing_dim + 1, new_slicing_dim + 1, self._comm ) - self._chunk_shape = array.shape # type: ignore + self._chunk_shape = array.shape # type: ignore assert newdim == new_slicing_dim + 1 idx = [0, 0, 0] idx[new_slicing_dim] = startidx diff --git a/httomo/data/mpiutil.py b/httomo/data/mpiutil.py index 6194ccc5a..a36deab24 100644 --- a/httomo/data/mpiutil.py +++ b/httomo/data/mpiutil.py @@ -7,7 +7,6 @@ from httomo.utils import log_once - __all__ = ["alltoall", "alltoall_ring"] diff --git a/httomo/data/padding.py b/httomo/data/padding.py index fe3e28532..dd5162c67 100644 --- a/httomo/data/padding.py +++ b/httomo/data/padding.py @@ -5,7 +5,6 @@ from httomo.preview import PreviewConfig - DIMS = [0, 1, 2] diff --git a/httomo/loaders/standard_tomo_loader.py b/httomo/loaders/standard_tomo_loader.py index e33aab662..fc0e1bcd3 100644 --- a/httomo/loaders/standard_tomo_loader.py +++ b/httomo/loaders/standard_tomo_loader.py @@ -428,7 +428,7 @@ def make_data_source(self, padding: Tuple[int, int] = (0, 0)) -> DataSetSource: comm=self.comm, padding=padding, ) - (self._angles_total, self._detector_y, self._detector_x) = loader.global_shape + self._angles_total, self._detector_y, self._detector_x = loader.global_shape return loader @property diff --git a/httomo/monitors/__init__.py b/httomo/monitors/__init__.py index 8d848b337..a8cf7e288 100644 --- a/httomo/monitors/__init__.py +++ b/httomo/monitors/__init__.py @@ -7,7 +7,6 @@ from httomo.monitors.summary import SummaryMonitor from httomo.runner.monitoring_interface import MonitoringInterface - MONITORS_MAP = {"bench": BenchmarkMonitoring, "summary": SummaryMonitor} diff --git a/httomo/runner/task_runner.py b/httomo/runner/task_runner.py index fbb65eb6b..b6384536d 100644 --- a/httomo/runner/task_runner.py +++ b/httomo/runner/task_runner.py @@ -395,7 +395,7 @@ def determine_max_slices(self, section: Section, slicing_dim: int): continue output_dims = m.calculate_output_dims(non_slice_dims_shape) - (slices_estimated, available_memory) = m.calculate_max_slices( + slices_estimated, available_memory = m.calculate_max_slices( SOURCE_DTYPE, # self.source.dtype, non_slice_dims_shape, available_memory, diff --git a/httomo/types.py b/httomo/types.py index 25aa7a8ae..31a5a29e6 100644 --- a/httomo/types.py +++ b/httomo/types.py @@ -4,5 +4,4 @@ from httomo.utils import xp - generic_array: TypeAlias = Union[np.ndarray, xp.ndarray] diff --git a/httomo/ui_layer.py b/httomo/ui_layer.py index e21b578f2..46e0816ff 100644 --- a/httomo/ui_layer.py +++ b/httomo/ui_layer.py @@ -138,8 +138,8 @@ def _setup_loader(self) -> LoaderInterface: "There is no loader in the pipeline. Please add the loader as the first method" ) parameters = task_conf.get("parameters", dict()) - (data_config, image_key_path, angles, darks_config, flats_config) = ( - parse_config(self.in_data_file, parameters) + data_config, image_key_path, angles, darks_config, flats_config = parse_config( + self.in_data_file, parameters ) with h5py.File(data_config.in_file, "r") as f: @@ -229,7 +229,7 @@ def update_side_output_references( pattern = get_regex_pattern() # check if there is a reference to side_outputs to cross-link for param_name, param_value in valid_refs.items(): - (ref_id, side_str, ref_arg) = get_ref_split(param_value, pattern) + ref_id, side_str, ref_arg = get_ref_split(param_value, pattern) if ref_id is None: continue method = method_id_map.get(ref_id, None) diff --git a/httomo/yaml_checker.py b/httomo/yaml_checker.py index a0a773534..fe12b2e48 100644 --- a/httomo/yaml_checker.py +++ b/httomo/yaml_checker.py @@ -329,7 +329,7 @@ def check_ref_id_valid(conf: PipelineConfig) -> bool: for k, v in get_valid_ref_str(m.get("parameters", dict())).items() } for k, v in ref_strs.items(): - (ref_id, side_str, ref_arg) = get_ref_split(v, pattern) + ref_id, side_str, ref_arg = get_ref_split(v, pattern) if ref_id not in method_ids: _print_with_colour( f"The reference id: {ref_id} was not found to have a matching method id." @@ -347,7 +347,7 @@ def check_side_out_matches_ref_arg(conf: PipelineConfig) -> bool: for k, v in get_valid_ref_str(m.get("parameters", dict())).items() } for k, v in ref_strs.items(): - (ref_id, side_str, ref_arg) = get_ref_split(v, pattern) + ref_id, side_str, ref_arg = get_ref_split(v, pattern) side_dicts = [ m.get(side_str) for m in conf diff --git a/tests/scripts/create_numpy_from_hdf5.py b/tests/scripts/create_numpy_from_hdf5.py index eed9d252a..dadeb9f61 100644 --- a/tests/scripts/create_numpy_from_hdf5.py +++ b/tests/scripts/create_numpy_from_hdf5.py @@ -25,6 +25,7 @@ Please run the generator as: python -m create_numpy_from_hdf5 -i /path/to/file.hdf5 -o /path/to/output/file.npz """ + import argparse import os import h5py diff --git a/tests/test_cli.py b/tests/test_cli.py index 093172818..5776fbbb7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -16,7 +16,6 @@ ) from httomo.ui_layer import PipelineFormat - # Sample JSON pipeline data for testing SAMPLE_JSON_PIPELINE = [ { diff --git a/tests/test_parallel_pipeline_big.py b/tests/test_parallel_pipeline_big.py index 2e8d9b0fb..c9935de82 100644 --- a/tests/test_parallel_pipeline_big.py +++ b/tests/test_parallel_pipeline_big.py @@ -72,7 +72,7 @@ def test_pipe_parallel_FBP3d_tomobar_k11_38730_in_disk_preview( # load the pre-saved numpy array for comparison bellow data_gt = FBP3d_tomobar_k11_38730_npz["data"] axis_slice = FBP3d_tomobar_k11_38730_npz["axis_slice"] - (slices, sizeX, sizeY) = np.shape(data_gt) + slices, sizeX, sizeY = np.shape(data_gt) step = axis_slice // (slices + 2) # store for the result @@ -151,7 +151,7 @@ def test_pipe_parallel_FBP3d_tomobar_k11_38730_in_memory_preview( # load the pre-saved numpy array for comparison bellow data_gt = FBP3d_tomobar_k11_38730_npz["data"] axis_slice = FBP3d_tomobar_k11_38730_npz["axis_slice"] - (slices, sizeX, sizeY) = np.shape(data_gt) + slices, sizeX, sizeY = np.shape(data_gt) step = axis_slice // (slices + 2) # store for the result @@ -221,7 +221,7 @@ def test_parallel_pipe_LPRec3d_tomobar_i12_119647_preview( # load the pre-saved numpy array for comparison bellow data_gt = LPRec3d_tomobar_i12_119647_npz["data"] axis_slice = LPRec3d_tomobar_i12_119647_npz["axis_slice"] - (slices, sizeX, sizeY) = np.shape(data_gt) + slices, sizeX, sizeY = np.shape(data_gt) step = axis_slice // (slices + 2) # store for the result @@ -302,7 +302,7 @@ def test_parallel_pipe_360deg_distortion_FBP3d_tomobar_i13_179623_preview( # load the pre-saved numpy array for comparison bellow data_gt = FBP3d_tomobar_distortion_i13_179623_npz["data"] axis_slice = FBP3d_tomobar_distortion_i13_179623_npz["axis_slice"] - (slices, sizeX, sizeY) = np.shape(data_gt) + slices, sizeX, sizeY = np.shape(data_gt) step = axis_slice // (slices + 2) # store for the result @@ -442,7 +442,7 @@ def test_parallel_pipe_FISTA3d_tomobar_k11_38731( # load the pre-saved numpy array for comparison bellow data_gt = FISTA3d_tomobar_k11_38731_npz["data"] axis_slice = FISTA3d_tomobar_k11_38731_npz["axis_slice"] - (slices, sizeX, sizeY) = np.shape(data_gt) + slices, sizeX, sizeY = np.shape(data_gt) step = axis_slice // (slices + 2) # store for the result diff --git a/tests/test_pipeline_big.py b/tests/test_pipeline_big.py index 8fee669ef..be24b5fd9 100644 --- a/tests/test_pipeline_big.py +++ b/tests/test_pipeline_big.py @@ -59,7 +59,7 @@ def test_pipe_FBP3d_tomobar_k11_38731_in_disk( # load the pre-saved numpy array for comparison bellow data_gt = FBP3d_tomobar_k11_38731_npz["data"] axis_slice = FBP3d_tomobar_k11_38731_npz["axis_slice"] - (slices, sizeX, sizeY) = np.shape(data_gt) + slices, sizeX, sizeY = np.shape(data_gt) step = axis_slice // (slices + 2) # store for the result @@ -124,7 +124,7 @@ def test_pipe_FBP3d_tomobar_i12_119647_preview( # load the pre-saved numpy array for comparison bellow data_gt = FBP3d_tomobar_i12_119647_npz["data"] axis_slice = FBP3d_tomobar_i12_119647_npz["axis_slice"] - (slices, sizeX, sizeY) = np.shape(data_gt) + slices, sizeX, sizeY = np.shape(data_gt) step = axis_slice // (slices + 2) # store for the result @@ -191,7 +191,7 @@ def test_pipe_LPRec3d_tomobar_i12_119647_preview( # load the pre-saved numpy array for comparison bellow data_gt = LPRec3d_tomobar_i12_119647_npz["data"] axis_slice = LPRec3d_tomobar_i12_119647_npz["axis_slice"] - (slices, sizeX, sizeY) = np.shape(data_gt) + slices, sizeX, sizeY = np.shape(data_gt) step = axis_slice // (slices + 2) # store for the result @@ -258,7 +258,7 @@ def test_pipe_FBP2d_astra_i12_119647_preview( # load the pre-saved numpy array for comparison bellow data_gt = FBP2d_astra_i12_119647_npz["data"] axis_slice = FBP2d_astra_i12_119647_npz["axis_slice"] - (slices, sizeX, sizeY) = np.shape(data_gt) + slices, sizeX, sizeY = np.shape(data_gt) step = axis_slice // (slices + 2) # store for the result @@ -370,7 +370,7 @@ def test_pipe_FBP3d_tomobar_denoising_i13_177906_preview( # load the pre-saved numpy array for comparison bellow data_gt_tv = FBP3d_tomobar_TVdenoising_i13_177906_npz["data"] axis_slice = FBP3d_tomobar_TVdenoising_i13_177906_npz["axis_slice"] - (slices, sizeX, sizeY) = np.shape(data_gt_tv) + slices, sizeX, sizeY = np.shape(data_gt_tv) step = axis_slice // (slices + 2) # store for the result @@ -445,7 +445,7 @@ def test_pipe_360deg_paganin_FBP3d_tomobar_i13_179623_preview( # load the pre-saved numpy array for comparison bellow data_gt = FBP3d_tomobar_paganin_i13_179623_npz["data"] axis_slice = FBP3d_tomobar_paganin_i13_179623_npz["axis_slice"] - (slices, sizeX, sizeY) = np.shape(data_gt) + slices, sizeX, sizeY = np.shape(data_gt) step = axis_slice // (slices + 2) # store for the result diff --git a/tests/test_transform_loader_params.py b/tests/test_transform_loader_params.py index f6a5bceaa..44e01951f 100644 --- a/tests/test_transform_loader_params.py +++ b/tests/test_transform_loader_params.py @@ -674,7 +674,7 @@ def test_parse_loader_config( expected_darks_config: DarksFlatsFileConfig, expected_flats_config: DarksFlatsFileConfig, ): - (data_config, image_key_path, angles_config, darks_config, flats_config) = ( + data_config, image_key_path, angles_config, darks_config, flats_config = ( parse_config(input_file, config) ) assert data_config == expected_data_config From fad6fe23dced039ec128ea4129fcc656c2574538 Mon Sep 17 00:00:00 2001 From: dkazanc Date: Tue, 20 Jan 2026 14:00:40 +0000 Subject: [PATCH 3/3] fixing tests --- tests/test_ui_layer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_ui_layer.py b/tests/test_ui_layer.py index cd89e57cf..082577661 100644 --- a/tests/test_ui_layer.py +++ b/tests/test_ui_layer.py @@ -368,6 +368,7 @@ def total_variation_PD(data: np.ndarray, regularisation_parameter: float, iterat method_name="testloader", reslice=False, preview=preview, + comm=MPI.COMM_WORLD, ) def mock_make_data_source(padding) -> DataSetSource: