a fix for the preview expansion to work in parallel

httomo/cli.py

@@ -21,7 +21,6 @@
 from httomo.runner.task_runner import TaskRunner
 from httomo.ui_layer import UiLayer, PipelineFormat
 
-
 try:
     from . import __version__
 except:

httomo/data/dataset_store.py

@@ -355,7 +355,7 @@ def _reslice(
                 array, newdim, startidx = reslice(
                     data, old_slicing_dim + 1, new_slicing_dim + 1, self._comm
                 )
-                self._chunk_shape = array.shape  #  type: ignore
+                self._chunk_shape = array.shape  # type: ignore
                 assert newdim == new_slicing_dim + 1
                 idx = [0, 0, 0]
                 idx[new_slicing_dim] = startidx

httomo/data/mpiutil.py

@@ -7,7 +7,6 @@
 
 from httomo.utils import log_once
 
-
 __all__ = ["alltoall", "alltoall_ring"]
 
 

httomo/data/padding.py

@@ -5,7 +5,6 @@
 
 from httomo.preview import PreviewConfig
 
-
 DIMS = [0, 1, 2]
 
 

httomo/loaders/standard_tomo_loader.py

@@ -428,7 +428,7 @@ def make_data_source(self, padding: Tuple[int, int] = (0, 0)) -> DataSetSource:
             comm=self.comm,
             padding=padding,
         )
-        (self._angles_total, self._detector_y, self._detector_x) = loader.global_shape
+        self._angles_total, self._detector_y, self._detector_x = loader.global_shape
         return loader
 
     @property

httomo/monitors/__init__.py

@@ -7,7 +7,6 @@
 from httomo.monitors.summary import SummaryMonitor
 from httomo.runner.monitoring_interface import MonitoringInterface
 
-
 MONITORS_MAP = {"bench": BenchmarkMonitoring, "summary": SummaryMonitor}
 
 

httomo/runner/task_runner.py

@@ -395,7 +395,7 @@ def determine_max_slices(self, section: Section, slicing_dim: int):
                 continue
 
             output_dims = m.calculate_output_dims(non_slice_dims_shape)
-            (slices_estimated, available_memory) = m.calculate_max_slices(
+            slices_estimated, available_memory = m.calculate_max_slices(
                 SOURCE_DTYPE,  # self.source.dtype,
                 non_slice_dims_shape,
                 available_memory,

httomo/types.py

@@ -4,5 +4,4 @@
 
 from httomo.utils import xp
 
-
 generic_array: TypeAlias = Union[np.ndarray, xp.ndarray]

httomo/ui_layer.py

@@ -138,8 +138,8 @@ def _setup_loader(self) -> LoaderInterface:
                 "There is no loader in the pipeline. Please add the loader as the first method"
             )
         parameters = task_conf.get("parameters", dict())
-        (data_config, image_key_path, angles, darks_config, flats_config) = (
-            parse_config(self.in_data_file, parameters)
+        data_config, image_key_path, angles, darks_config, flats_config = parse_config(
+            self.in_data_file, parameters
         )
 
         with h5py.File(data_config.in_file, "r") as f:
@@ -174,8 +174,7 @@ def fix_preview_y_if_smaller_than_padding(
 ) -> None:
     vertical_preview_length = (
         loader.preview.detector_y.stop - loader.preview.detector_y.start
-    )
-    # we need to know the largest padding value over all methods in the pipeline
+    ) // loader.comm.size
     max_pad_value = 0
     for _, m in enumerate(methods_list):
         if m.padding:
@@ -230,7 +229,7 @@ def update_side_output_references(
     pattern = get_regex_pattern()
     # check if there is a reference to side_outputs to cross-link
     for param_name, param_value in valid_refs.items():
-        (ref_id, side_str, ref_arg) = get_ref_split(param_value, pattern)
+        ref_id, side_str, ref_arg = get_ref_split(param_value, pattern)
         if ref_id is None:
             continue
         method = method_id_map.get(ref_id, None)

httomo/yaml_checker.py

@@ -329,7 +329,7 @@ def check_ref_id_valid(conf: PipelineConfig) -> bool:
         for k, v in get_valid_ref_str(m.get("parameters", dict())).items()
     }
     for k, v in ref_strs.items():
-        (ref_id, side_str, ref_arg) = get_ref_split(v, pattern)
+        ref_id, side_str, ref_arg = get_ref_split(v, pattern)
         if ref_id not in method_ids:
             _print_with_colour(
                 f"The reference id: {ref_id} was not found to have a matching method id."
@@ -347,7 +347,7 @@ def check_side_out_matches_ref_arg(conf: PipelineConfig) -> bool:
         for k, v in get_valid_ref_str(m.get("parameters", dict())).items()
     }
     for k, v in ref_strs.items():
-        (ref_id, side_str, ref_arg) = get_ref_split(v, pattern)
+        ref_id, side_str, ref_arg = get_ref_split(v, pattern)
         side_dicts = [
             m.get(side_str)
             for m in conf

tests/scripts/create_numpy_from_hdf5.py

@@ -25,6 +25,7 @@
 Please run the generator as:
     python -m create_numpy_from_hdf5 -i /path/to/file.hdf5 -o /path/to/output/file.npz
 """
+
 import argparse
 import os
 import h5py

tests/test_cli.py

@@ -16,7 +16,6 @@
 )
 from httomo.ui_layer import PipelineFormat
 
-
 # Sample JSON pipeline data for testing
 SAMPLE_JSON_PIPELINE = [
     {

tests/test_parallel_pipeline_big.py

@@ -72,7 +72,7 @@ def test_pipe_parallel_FBP3d_tomobar_k11_38730_in_disk_preview(
     # load the pre-saved numpy array for comparison bellow
     data_gt = FBP3d_tomobar_k11_38730_npz["data"]
     axis_slice = FBP3d_tomobar_k11_38730_npz["axis_slice"]
-    (slices, sizeX, sizeY) = np.shape(data_gt)
+    slices, sizeX, sizeY = np.shape(data_gt)
 
     step = axis_slice // (slices + 2)
     # store for the result
@@ -151,7 +151,7 @@ def test_pipe_parallel_FBP3d_tomobar_k11_38730_in_memory_preview(
     # load the pre-saved numpy array for comparison bellow
     data_gt = FBP3d_tomobar_k11_38730_npz["data"]
     axis_slice = FBP3d_tomobar_k11_38730_npz["axis_slice"]
-    (slices, sizeX, sizeY) = np.shape(data_gt)
+    slices, sizeX, sizeY = np.shape(data_gt)
 
     step = axis_slice // (slices + 2)
     # store for the result
@@ -221,7 +221,7 @@ def test_parallel_pipe_LPRec3d_tomobar_i12_119647_preview(
     # load the pre-saved numpy array for comparison bellow
     data_gt = LPRec3d_tomobar_i12_119647_npz["data"]
     axis_slice = LPRec3d_tomobar_i12_119647_npz["axis_slice"]
-    (slices, sizeX, sizeY) = np.shape(data_gt)
+    slices, sizeX, sizeY = np.shape(data_gt)
 
     step = axis_slice // (slices + 2)
     # store for the result
@@ -302,7 +302,7 @@ def test_parallel_pipe_360deg_distortion_FBP3d_tomobar_i13_179623_preview(
     # load the pre-saved numpy array for comparison bellow
     data_gt = FBP3d_tomobar_distortion_i13_179623_npz["data"]
     axis_slice = FBP3d_tomobar_distortion_i13_179623_npz["axis_slice"]
-    (slices, sizeX, sizeY) = np.shape(data_gt)
+    slices, sizeX, sizeY = np.shape(data_gt)
 
     step = axis_slice // (slices + 2)
     # store for the result
@@ -442,7 +442,7 @@ def test_parallel_pipe_FISTA3d_tomobar_k11_38731(
     # load the pre-saved numpy array for comparison bellow
     data_gt = FISTA3d_tomobar_k11_38731_npz["data"]
     axis_slice = FISTA3d_tomobar_k11_38731_npz["axis_slice"]
-    (slices, sizeX, sizeY) = np.shape(data_gt)
+    slices, sizeX, sizeY = np.shape(data_gt)
 
     step = axis_slice // (slices + 2)
     # store for the result

tests/test_pipeline_big.py

@@ -59,7 +59,7 @@ def test_pipe_FBP3d_tomobar_k11_38731_in_disk(
     # load the pre-saved numpy array for comparison bellow
     data_gt = FBP3d_tomobar_k11_38731_npz["data"]
     axis_slice = FBP3d_tomobar_k11_38731_npz["axis_slice"]
-    (slices, sizeX, sizeY) = np.shape(data_gt)
+    slices, sizeX, sizeY = np.shape(data_gt)
 
     step = axis_slice // (slices + 2)
     # store for the result
@@ -124,7 +124,7 @@ def test_pipe_FBP3d_tomobar_i12_119647_preview(
     # load the pre-saved numpy array for comparison bellow
     data_gt = FBP3d_tomobar_i12_119647_npz["data"]
     axis_slice = FBP3d_tomobar_i12_119647_npz["axis_slice"]
-    (slices, sizeX, sizeY) = np.shape(data_gt)
+    slices, sizeX, sizeY = np.shape(data_gt)
 
     step = axis_slice // (slices + 2)
     # store for the result
@@ -191,7 +191,7 @@ def test_pipe_LPRec3d_tomobar_i12_119647_preview(
     # load the pre-saved numpy array for comparison bellow
     data_gt = LPRec3d_tomobar_i12_119647_npz["data"]
     axis_slice = LPRec3d_tomobar_i12_119647_npz["axis_slice"]
-    (slices, sizeX, sizeY) = np.shape(data_gt)
+    slices, sizeX, sizeY = np.shape(data_gt)
 
     step = axis_slice // (slices + 2)
     # store for the result
@@ -258,7 +258,7 @@ def test_pipe_FBP2d_astra_i12_119647_preview(
     # load the pre-saved numpy array for comparison bellow
     data_gt = FBP2d_astra_i12_119647_npz["data"]
     axis_slice = FBP2d_astra_i12_119647_npz["axis_slice"]
-    (slices, sizeX, sizeY) = np.shape(data_gt)
+    slices, sizeX, sizeY = np.shape(data_gt)
 
     step = axis_slice // (slices + 2)
     # store for the result
@@ -370,7 +370,7 @@ def test_pipe_FBP3d_tomobar_denoising_i13_177906_preview(
     # load the pre-saved numpy array for comparison bellow
     data_gt_tv = FBP3d_tomobar_TVdenoising_i13_177906_npz["data"]
     axis_slice = FBP3d_tomobar_TVdenoising_i13_177906_npz["axis_slice"]
-    (slices, sizeX, sizeY) = np.shape(data_gt_tv)
+    slices, sizeX, sizeY = np.shape(data_gt_tv)
 
     step = axis_slice // (slices + 2)
     # store for the result
@@ -445,7 +445,7 @@ def test_pipe_360deg_paganin_FBP3d_tomobar_i13_179623_preview(
     # load the pre-saved numpy array for comparison bellow
     data_gt = FBP3d_tomobar_paganin_i13_179623_npz["data"]
     axis_slice = FBP3d_tomobar_paganin_i13_179623_npz["axis_slice"]
-    (slices, sizeX, sizeY) = np.shape(data_gt)
+    slices, sizeX, sizeY = np.shape(data_gt)
 
     step = axis_slice // (slices + 2)
     # store for the result

tests/test_transform_loader_params.py

@@ -674,7 +674,7 @@ def test_parse_loader_config(
     expected_darks_config: DarksFlatsFileConfig,
     expected_flats_config: DarksFlatsFileConfig,
 ):
-    (data_config, image_key_path, angles_config, darks_config, flats_config) = (
+    data_config, image_key_path, angles_config, darks_config, flats_config = (
         parse_config(input_file, config)
     )
     assert data_config == expected_data_config

tests/test_ui_layer.py

@@ -368,6 +368,7 @@ def total_variation_PD(data: np.ndarray, regularisation_parameter: float, iterat
         method_name="testloader",
         reslice=False,
         preview=preview,
+        comm=MPI.COMM_WORLD,
     )
 
     def mock_make_data_source(padding) -> DataSetSource: