Merge pull request #419 from DiamondLightSource/sweep_updates

A feature for Sweep to insert save_to_images methods automatically

docs/source/howto/httomo_features/parameter_sweeping.rst

@@ -77,6 +77,8 @@ pipeline to sweep over the :code:`size` parameter of a median filter:
 .. literalinclude:: ../../../../tests/samples/pipeline_template_examples/testing/sweep_manual.yaml
   :language: yaml
 
+.. note:: There is no need to add image saving method after the `sweep` method. The result of the `sweep` method will be saved into images automatically. 
+
 How big should the input data be?
 =================================
 

docs/source/pipelines/yaml.rst

@@ -58,9 +58,16 @@ Parameter Sweeps templates
 These templates demonstrate how to perform a sweep across multiple values of a
 single parameter (see :ref:`parameter_sweeping` for more details).
 
-.. dropdown:: Parameter sweep over 10 CoR values (`center` param) in recon
-   method, and saving the result as tiffs
+.. dropdown:: Parameter sweep over 6 CoR values (`center` param) in recon
+   method, and saving the result as tiffs. Note that there is need to add image saving plugin in this case. It is also preferable to keep `preview` small. 
 
    .. literalinclude:: ../../../tests/samples/pipeline_template_examples/parameter-sweep-cor.yaml
        :language: yaml
-       :emphasize-lines: 9-11,37-40
+       :emphasize-lines: 30-33
+
+.. dropdown:: Parameter sweep over 50 (`alpha` param) values of Paganin filter
+   method, and saving the result as tiffs for both Paganin filter and the reconstruction module.
+
+   .. literalinclude:: ../../../tests/samples/pipeline_template_examples/parameter-sweep-paganin.yaml
+       :language: yaml
+       :emphasize-lines: 25-28       

httomo/method_wrappers/generic.py

@@ -103,6 +103,11 @@ def __init__(
 
         # check if the given kwargs are actually supported by the method
         self._config_params = kwargs
+        # check if a tuple present in parameters to make the method a sweep method
+        self._sweep = False
+        for value in self._config_params.values():
+            if type(value) is tuple:
+                self._sweep = True
         self._output_mapping = output_mapping
         self._check_config_params()
 
@@ -222,6 +227,10 @@ def recon_algorithm(self) -> Optional[str]:
     def padding(self) -> bool:
         return self._padding
 
+    @property
+    def sweep(self) -> bool:
+        return self._sweep
+
     def _build_kwargs(
         self,
         dict_params: MethodParameterDictType,

httomo/runner/method_wrapper.py

@@ -129,7 +129,12 @@ def recon_algorithm(self) -> Optional[str]:
 
     @property
     def padding(self) -> bool:
-        """Deterime if the method needs padding"""
+        """Determine if the method needs padding"""
+        ...  # pragma: nocover
+
+    @property
+    def sweep(self) -> bool:
+        """Determine if the method performs sweep"""
         ...  # pragma: nocover
 
     # Methods

httomo/transform_layer.py

@@ -24,6 +24,9 @@ def __init__(
     def transform(self, pipeline: Pipeline) -> Pipeline:
         pipeline = self.insert_save_methods(pipeline)
         pipeline = self.insert_data_reducer(pipeline)
+        pipeline = self.insert_save_images_after_sweep(
+            pipeline
+        )  # will be applied to sweep methods only
         return pipeline
 
     def insert_save_methods(self, pipeline: Pipeline) -> Pipeline:
@@ -62,9 +65,36 @@ def insert_data_reducer(self, pipeline: Pipeline) -> Pipeline:
                 "data_reducer",
                 comm=self._comm,
                 save_result=False,
-                task_id=f"task_{0}",
+                task_id="reducer_0",
             ),
         )
         for m in pipeline:
             methods.append(m)
         return Pipeline(loader, methods)
+
+    def insert_save_images_after_sweep(self, pipeline: Pipeline) -> Pipeline:
+        """For sweep methods we add image saving method after.
+        In addition we also add saving the results of the reconstruction,
+        if the module is present"""
+        loader = pipeline.loader
+        methods = []
+        sweep_before = False
+        for m in pipeline:
+            methods.append(m)
+            if m.sweep or "recon" in m.module_path and sweep_before:
+                methods.append(
+                    make_method_wrapper(
+                        self._repo,
+                        "httomolib.misc.images",
+                        "save_to_images",
+                        comm=self._comm,
+                        save_result=False,
+                        task_id=f"saveimage_sweep_{m.task_id}",
+                        subfolder_name="images_sweep_" + str(m.method_name),
+                        axis=1,
+                        perc_range_min=5,
+                        perc_range_max=95,
+                    ),
+                )
+                sweep_before = True
+        return Pipeline(loader, methods)

httomo/yaml_checker.py

@@ -229,6 +229,29 @@ def check_no_required_parameter_values(conf: PipelineConfig) -> bool:
     return True
 
 
+def check_no_imagesaver_after_sweep_method(f: Path) -> bool:
+    """check that there shouldn't be image saver present after the sweep method"""
+    loader = UniqueKeyLoader
+    loader.add_constructor("!Sweep", ParamSweepYamlLoader.sweep_manual)
+    loader.add_constructor("!SweepRange", ParamSweepYamlLoader.sweep_range)
+    pipeline = yaml_loader(f, loader=loader)
+
+    method_is_sweep = False
+    for m in pipeline:
+        for value in m["parameters"].values():
+            if type(value) is tuple:
+                method_is_sweep = True
+                sweep_method_name = m["method"]
+        if method_is_sweep and m["method"] == "save_to_images":
+            _print_with_colour(
+                f"This pipeline contains a sweep method ({sweep_method_name}) and also save_to_images method(s)."
+                " Please note that the result of the sweep method will be automatically saved as images."
+                " Therefore there is no need to add save_to_images after any sweep method, please remove."
+            )
+            return False
+    return True
+
+
 def check_no_duplicated_keys(f: Path) -> bool:
     """there should be no duplicate keys in yaml file
     Parameters

tests/conftest.py

@@ -293,6 +293,16 @@ def yaml_gpu_pipeline360_2():
     return "tests/samples/pipeline_template_examples/pipeline_360deg_gpu2.yaml"
 
 
+@pytest.fixture
+def yaml_gpu_pipeline_sweep_cor():
+    return "tests/samples/pipeline_template_examples/parameter-sweep-cor.yaml"
+
+
+@pytest.fixture
+def yaml_gpu_pipeline_sweep_paganin():
+    return "tests/samples/pipeline_template_examples/parameter-sweep-paganin.yaml"
+
+
 @pytest.fixture(scope="session")
 def distortion_correction_path(test_data_path):
     return os.path.join(test_data_path, "distortion-correction")

tests/samples/pipeline_template_examples/parameter-sweep-cor.yaml

@@ -7,48 +7,30 @@
       data_path: /entry1/tomo_entry/data/rotation_angle
     preview:
       detector_y:
-        start: 100
-        stop: 107
+        start: 60
+        stop: 67
 - method: normalize
   module_path: httomolibgpu.prep.normalize
   parameters:
     cutoff: 10.0
-    minus_log: true
+    minus_log: false
     nonnegativity: false
-    remove_nans: false
+    remove_nans: true
 - method: paganin_filter_tomopy
   module_path: httomolibgpu.prep.phase
   parameters:
     pixel_size: 0.0001
     dist: 50.0
     energy: 53.0
     alpha: 0.001
-- method: remove_all_stripe
-  module_path: httomolibgpu.prep.stripe
-  parameters:
-    snr: 3.0
-    la_size: 61
-    sm_size: 21
-    dim: 1
 - method: FBP
   module_path: httomolibgpu.recon.algorithm
   save_result: False
   parameters:
     center: !SweepRange
-      start: 1000
-      stop: 1010
-      step: 1
-    filter_freq_cutoff: 0.6
+      start: 60
+      stop: 120
+      step: 10
+    filter_freq_cutoff: 1.1
     recon_size: null
-    recon_mask_radius: null
-- method: save_to_images
-  module_path: httomolib.misc.images
-  parameters:
-    subfolder_name: images
-    axis: 1
-    file_format: tif
-    bits: 8
-    perc_range_min: 0.0
-    perc_range_max: 100.0
-    jpeg_quality: 95
-    asynchronous: true
+    recon_mask_radius: null

tests/samples/pipeline_template_examples/parameter-sweep-paganin.yaml

@@ -0,0 +1,35 @@
+- method: standard_tomo
+  module_path: httomo.data.hdf.loaders
+  parameters:
+    data_path: entry1/tomo_entry/data/data
+    image_key_path: entry1/tomo_entry/instrument/detector/image_key
+    rotation_angles:
+      data_path: /entry1/tomo_entry/data/rotation_angle
+    preview:
+      detector_y:
+        start: 60
+        stop: 67
+- method: normalize
+  module_path: httomolibgpu.prep.normalize
+  parameters:
+    cutoff: 10.0
+    minus_log: false
+    nonnegativity: false
+    remove_nans: true
+- method: paganin_filter_tomopy
+  module_path: httomolibgpu.prep.phase
+  parameters:
+    pixel_size: 0.0004
+    dist: 50.0
+    energy: 53.0
+    alpha: !SweepRange
+      start: 0.001
+      stop: 0.5
+      step: 0.01 
+- method: FBP
+  module_path: httomolibgpu.recon.algorithm
+  parameters:
+    center: 80
+    filter_freq_cutoff: 1.1
+    recon_size: null
+    recon_mask_radius: null

tests/samples/pipeline_template_examples/testing/imagesave_after_sweep.yaml

@@ -0,0 +1,33 @@
+- method: standard_tomo
+  module_path: httomo.data.hdf.loaders
+  parameters:
+    data_path: entry1/tomo_entry/data/data
+    image_key_path: entry1/tomo_entry/instrument/detector/image_key
+    rotation_angles:
+      data_path: /entry1/tomo_entry/data/rotation_angle
+    preview:
+      detector_y:
+        start: 10
+        stop: 17
+- method: normalize
+  module_path: tomopy.prep.normalize
+  parameters:
+    cutoff: 10.0
+    averaging: mean
+- method: median_filter
+  module_path: tomopy.misc.corr
+  parameters:
+    size: !Sweep
+      - 3
+      - 5
+    axis: 0
+- method: save_to_images
+  module_path: httomolib.misc.images
+  parameters:
+    subfolder_name: images
+    axis: auto
+    file_format: tif
+    bits: 8
+    perc_range_min: 0.0
+    perc_range_max: 100.0
+    jpeg_quality: 95    

tests/test_pipeline.py

@@ -701,3 +701,64 @@ def test_run_pipeline_360deg_gpu2(
     assert "Global min -0.002859" in verbose_log_contents
     assert "Global max 0.00561" in verbose_log_contents
     assert "Global mean 0.000889" in verbose_log_contents
+
+
+@pytest.mark.cupy
+def test_run_gpu_pipeline_sweep_cor(
+    get_files: Callable, cmd, standard_data, yaml_gpu_pipeline_sweep_cor, output_folder
+):
+    cmd.pop(4)  #: don't save all
+    cmd.insert(6, standard_data)
+    cmd.insert(7, yaml_gpu_pipeline_sweep_cor)
+    cmd.insert(8, output_folder)
+    subprocess.check_output(cmd)
+
+    # recurse through output_dir and check that all files are there
+    files = get_files("output_dir/")
+    assert len(files) == 9
+
+    #: check the generated h5 files
+    h5_files = list(filter(lambda x: ".h5" in x, files))
+    assert len(h5_files) == 0
+
+    log_files = list(filter(lambda x: ".log" in x, files))
+    assert len(log_files) == 2
+    verbose_log_file = list(filter(lambda x: "debug.log" in x, files))
+    verbose_log_contents = _get_log_contents(verbose_log_file[0])
+
+    assert "Data shape is (180, 7, 160) of type uint16" in verbose_log_contents
+    assert "Total number of values across all processes: 6" in verbose_log_contents
+    assert "Values executed in this process: 6" in verbose_log_contents
+
+
+@pytest.mark.cupy
+def test_run_gpu_pipeline_sweep_paganin(
+    get_files: Callable,
+    cmd,
+    standard_data,
+    yaml_gpu_pipeline_sweep_paganin,
+    output_folder,
+):
+    cmd.pop(4)  #: don't save all
+    cmd.insert(6, standard_data)
+    cmd.insert(7, yaml_gpu_pipeline_sweep_paganin)
+    cmd.insert(8, output_folder)
+    subprocess.check_output(cmd)
+
+    # recurse through output_dir and check that all files are there
+    files = get_files("output_dir/")
+    assert len(files) == 104
+
+    #: check the generated h5 files
+    h5_files = list(filter(lambda x: ".h5" in x, files))
+    assert len(h5_files) == 1
+
+    log_files = list(filter(lambda x: ".log" in x, files))
+    assert len(log_files) == 2
+    verbose_log_file = list(filter(lambda x: "debug.log" in x, files))
+    verbose_log_contents = _get_log_contents(verbose_log_file[0])
+
+    assert "Data shape is (180, 7, 160) of type uint16" in verbose_log_contents
+    assert "Total number of values across all processes: 50" in verbose_log_contents
+    assert "Values executed in this process: 50" in verbose_log_contents
+    assert "Parameter name: alpha" in verbose_log_contents