DiamondLightSource · dkazanc · Sep 9, 2024 · Aug 22, 2024 · Aug 23, 2024 · Aug 23, 2024
diff --git a/.github/workflows/doc_conda.yml → .github/workflows/httomo_docs.yml b/.github/workflows/doc_conda.yml → .github/workflows/httomo_docs.yml
@@ -34,16 +34,6 @@ jobs:
         - name: Build api docs
           run: sphinx-apidoc -feT -t=./docs/source/_templates -o ./docs/source/api ./httomo
 
-        - name: Generate yaml templates
-          run: |
-            python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/tomopy/tomopy_modules.yaml -o ./docs/build/yaml_templates/tomopy
-            python ./scripts/yaml_unsupported_tomopy_remove.py -t ./docs/build/yaml_templates/tomopy -l ./httomo/methods_database/packages/external/tomopy/tomopy.yaml
-            python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/httomolib/httomolib_modules.yaml -o ./docs/build/yaml_templates/httomolib
-            python ./scripts/yaml_templates_generator.py -i ./httomo/methods_database/packages/external/httomolibgpu/httomolibgpu_modules.yaml -o ./docs/build/yaml_templates/httomolibgpu
-
-        - name: Generate yml docs
-          run: python ./docs/source/yaml_doc_generator.py
-
         - name: Build html
           run: sphinx-build -a -E -b html ./docs/source/ ./docs/build/
 

diff --git a/.github/workflows/conda_upload.yml → .github/workflows/httomo_version_tag.yml b/.github/workflows/conda_upload.yml → .github/workflows/httomo_version_tag.yml
@@ -1,4 +1,4 @@
-name: HTTomo version build
+name: version tag pushed
 
 # Run the workflow whenever a tag beginning with `v` is pushed to any branch
 on:

diff --git a/.github/workflows/main_conda_package_test.yml b/.github/workflows/main_conda_package_test.yml
@@ -34,7 +34,7 @@ jobs:
       - name: Install httomo
         run: |
           micromamba activate httomo
-          micromamba install "httomo/linux-64::httomo * py310_openmpi_regular*" -c conda-forge -c astra-toolbox -c rapidsai -y
+          micromamba install "httomo/linux-64::httomo * py310_openmpi_regular*" -c conda-forge -y
           micromamba list
 
       - name: Generate yaml templates

diff --git a/.github/workflows/run_tests_iris.yml b/.github/workflows/run_tests_iris.yml
@@ -36,6 +36,7 @@ jobs:
         run: |
           micromamba activate httomo
           pip install .[dev]
+          pip install --upgrade --force-reinstall pillow
           micromamba list
 
       - name: Generate yaml templates

diff --git a/.scripts/conda_upload.sh b/.scripts/conda_upload.sh
@@ -9,7 +9,7 @@ mkdir ~/conda-bld
 conda config --set anaconda_upload no
 export CONDA_BLD_PATH=~/conda-bld
 
-$CONDA/bin/conda build . -c conda-forge -c https://conda.anaconda.org/httomo/ -c rapidsai --no-test
+$CONDA/bin/conda build . -c conda-forge -c https://conda.anaconda.org/httomo/ --no-test
 
 # upload packages to conda
 find $CONDA_BLD_PATH/$OS -name *.tar.bz2 | while read file

diff --git a/conda/environment-cpu.yml b/conda/environment-cpu.yml
@@ -1,20 +1,19 @@
 name: httomo
 channels:
   - conda-forge
-  - anaconda
   - httomo
 dependencies:
   - conda-forge::tomopy==1.15
-  - httomo::httomolib==2.0
+  - httomo::httomolib==2.1
   - conda-forge::click>=8.1.3
   - conda-forge::mpi4py>=3.1
   - conda-forge::h5py=*=*mpi_openmpi*
   - conda-forge::pyyaml
   - conda-forge::numpy<=1.25
   - conda-forge::python
   - conda-forge::numexpr>=2.8.4
-  - anaconda::ipython
-  - anaconda::loguru
+  - conda-forge::ipython
+  - conda-forge::loguru
   - conda-forge::graypy
   - conda-forge::plumbum
   - conda-forge::tqdm

diff --git a/conda/environment.yml b/conda/environment.yml
@@ -1,15 +1,12 @@
 name: httomo
 channels:
-  - astra-toolbox
   - conda-forge
   - httomo
-  - anaconda
-  - rapidsai
 dependencies:
-  - astra-toolbox::astra-toolbox
+  - conda-forge::astra-toolbox
   - conda-forge::click>=8.1.3
-  - conda-forge::cupy<=12.3.0
-  - conda-forge::cudatoolkit
+  - conda-forge::cupy=12.3.0
+  - conda-forge::cudatoolkit>=11.2
   - conda-forge::graypy
   - conda-forge::h5py=*=*mpi_openmpi*
   - conda-forge::hdf5plugin
@@ -22,9 +19,8 @@ dependencies:
   - conda-forge::plumbum
   - conda-forge::tqdm
   - conda-forge::typing_extensions
-  - anaconda::ipython
-  - conda-forge::tomopy==1.15
-  - httomo::httomolib==2.0
-  - httomo::httomolibgpu==2.1
-  - httomo::tomobar
-  - rapidsai::cucim
+  - conda-forge::ipython
+  - conda-forge::tomopy=1.15
+  - httomo::httomolib=2.1
+  - httomo::httomolibgpu=2.1
+  - httomo::tomobar
diff --git a/conda/recipe/meta.yaml b/conda/recipe/meta.yaml
@@ -31,8 +31,9 @@ test:
 requirements:
   build:
     - h5py=*=*mpi_openmpi*
-    - httomolib==2.0
-    - httomolibgpu==2.0
+    - httomolib==2.1
+    - httomolibgpu==2.1
+    - tomopy==1.15    
     - loguru
     - mpi4py
     - numpy<=1.25
@@ -41,19 +42,18 @@ requirements:
     - pyyaml
     - setuptools
     - setuptools-git-versioning
-    - tomopy==1.15
     - typing_extensions
   run:
     - astra-toolbox
     - click>=8.1.3
-    - cucim
-    - cupy<=12.3.0
+    - cupy=12.3.0
     - cudatoolkit
     - graypy
     - h5py=*=*mpi_openmpi*
     - hdf5plugin
-    - httomolib==2.0
-    - httomolibgpu==2.0
+    - httomolib==2.1
+    - httomolibgpu==2.1
+    - tomopy==1.15    
     - ipython
     - loguru
     - mpi4py
@@ -68,7 +68,6 @@ requirements:
     - python
     - plumbum
     - tomobar
-    - tomopy==1.15
     - tqdm
     - typing_extensions
 

diff --git a/docs/source/backends/list.rst b/docs/source/backends/list.rst
@@ -1,20 +1,19 @@
 .. _backends_list:
 
-Supported backends
-==============================
+Supported libraries
+===================
 
 HTTomo currently supports several software packages that are used as
-backends to perform data processing and reconstruction. The list of
+`backends` to perform data processing and reconstruction. The list of
 the external packages will be growing in future to meet users' needs.
 
 If the package has a modular structure with an easy access to every
-method, for example as `TomoPy <https://tomopy.readthedocs.io>`_
-software or `scikit <https://scikit-image.org/>`_ library, then the
+method, for example as in the `TomoPy <https://tomopy.readthedocs.io>`_
+software or in the `scikit <https://scikit-image.org/>`_ library, then the
 integration process is straightforward.
-The required YAML template can be generated by using :ref:`utilities_yamlgenerator`,
-or manually, and used with HTTomo. More complicated in structure packages would need
-an additional wrapping, see for instance the reconstruction methods in
-the `HTTomolibgpu <https://github.com/DiamondLightSource/httomolibgpu/blob/main/httomolibgpu/recon/algorithm.py>`_ library.
+
+A YAML template can be generated by using `YAML generator <https://diamondlightsource.github.io/httomo-backends/utilities/yaml_generator.html>`_,
+or manually. More complicated in structure packages would need an additional wrapping, see :ref:`info_wrappers`.
 
 Please see the provided list of :ref:`reference_templates`.
 
@@ -42,5 +41,3 @@ by Data Analysis Group to work together with the HTTomo software.
 HTTomolib library (CPU)
 --------------------------
 `HTTomolib <https://github.com/DiamondLightSource/httomolib>`_ library is similar to HTTomolibgpu, but contains mostly CPU modules.
-
-.. note:: The HTTomo release usually supports a specific version of the backend package. Please check that the YAML templates are linked to the version advertised in the release.
diff --git a/docs/source/backends/templates.rst b/docs/source/backends/templates.rst
@@ -4,42 +4,13 @@
 Methods YAML Templates
 ======================
 
-This section contains YAML templates from :ref:`backends_list`. These are ready-to-use templates can be either copy-pasted or
-downloaded in order to build your pipeline.
+This section refers to YAML templates from :ref:`backends_list` which are distributed separately through `httomo-backends`. 
+You will be redirected to the `httomo-backends` page where those templates can be either copy-pasted or
+downloaded in order to help you to construct your pipeline.
 
-.. note:: When you click on a module you can find a link to the API of that module. This is where the description of the method and its arguments can be found.
+`TomoPy Modules <https://diamondlightsource.github.io/httomo-backends/backends/templates.html#tomopy-modules>`_
 
+`HTTomolibgpu Modules <https://diamondlightsource.github.io/httomo-backends/backends/templates.html#httomolibgpu-modules>`_
 
-TomoPy Modules
----------------
-v.1.15
-''''''
+`HTTomolib Modules <https://diamondlightsource.github.io/httomo-backends/backends/templates.html#httomolib-modules>`_
 
-.. toctree::
-   :glob:
-
-   ../api/tomopy*
-
-.. _reference_templates_httomolibgpu:
-
-HTTomolibgpu Modules
---------------------
-v.2.1
-'''''
-
-.. toctree::
-   :glob:
-
-   ../api/httomolibgpu*
-
-.. _reference_templates_httomolib:
-
-HTTomolib Modules
-------------------
-v.2.0
-'''''
-
-.. toctree::
-   :glob:
-
-   ../api/httomolib.*
diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -52,6 +52,25 @@
 for mod_name in MOCK_MODULES:
     sys.modules[mod_name] = mock.Mock()
 
+
+class CustomMock(mock.Mock):
+    def __repr__(self):
+        return "<cp.ndarray>"
+
+
+sys.modules["cupy"] = CustomMock()
+sys.modules["numpy"] = CustomMock()
+sys.modules["cupyx.scipy.interpolate"] = CustomMock()
+sys.modules["cupyx.scipy.ndimage"] = CustomMock()
+sys.modules["cupyx.scipy.fft"] = CustomMock()
+sys.modules["cupyx.scipy.fftpack"] = CustomMock()
+sys.modules["scipy.fftpack"] = CustomMock()
+sys.modules["tomobar.methodsDIR_CuPy"] = CustomMock()
+sys.modules["tomobar.methodsIR_CuPy "] = CustomMock()
+sys.modules["skimage.registration"] = CustomMock()
+sys.modules["httomolibgpu.cuda_kernels"] = CustomMock()
+
+
 # ------------------------------------------------------------------------------
 
 project = "HTTomo"

diff --git a/docs/source/developers/how_to_contribute.rst b/docs/source/developers/how_to_contribute.rst
@@ -29,7 +29,7 @@ the issues section on `HTTomo's Github page <https://github.com/DiamondLightSour
 4. Generate the Yaml template
 
    HTTomo's UI requires :ref:`reference_templates` to execute the created method. One can either construct that YAML template manually or employ
-   :ref:`utilities_yamlgenerator`.
+   `YAML generator <https://diamondlightsource.github.io/httomo-backends/utilities/yaml_generator.html>`_.
 
 
 

diff --git a/docs/source/doc-conda-requirements.yml b/docs/source/doc-conda-requirements.yml
@@ -1,15 +1,9 @@
 name: httomo-docs
 channels:
-  - conda-forge
-  - defaults
-  - httomo
-  - rapidsai
+  - conda-forge 
 dependencies:
   - python>=3.10
   - numpy
-  - tomopy==1.15
-  - httomolibgpu==2.1
-  - httomolib==2.0
   - sphinx
   - sphinx-book-theme
   - nbsphinx

diff --git a/docs/source/explanation/templates.rst b/docs/source/explanation/templates.rst
@@ -26,6 +26,6 @@ by importing the method as:
 
 The set of parameter values for that method is given in the *parameters* field.
 
-.. note:: Please note that in the original TomoPy's method, there is also the :code:`arr` parameter. This parameter is not exposed in the template because HTTomo will deal with all I/O aspects behind the scenes by using special wrappers. More on that in :ref:`detailed_about`. :ref:`utilities_yamlgenerator` will automatically generate the ready-to-be-used templates.
+.. note:: Please note that in the original TomoPy's method, there is also the :code:`arr` parameter. This parameter is not exposed in the template because HTTomo will deal with all I/O aspects behind the scenes by using special wrappers. More on that in :ref:`detailed_about`. `YAML generator <https://diamondlightsource.github.io/httomo-backends/utilities/yaml_generator.html>`_ will automatically generate the ready-to-be-used templates.
 
 Please see the list of all supported :ref:`reference_templates`.
diff --git a/docs/source/howto/installation.rst b/docs/source/howto/installation.rst
@@ -9,7 +9,7 @@ This installation is preferable as it should take care all of dependencies inclu
 .. code-block:: console
 
    $ conda env create --name httomo # create a fresh conda environment
-   $ conda install "httomo/linux-64::httomo * py310_openmpi_regular*" -c conda-forge -c astra-toolbox -c rapidsai
+   $ conda install "httomo/linux-64::httomo * py310_openmpi_regular*" -c conda-forge
 
 Install as a Python module
 ===========================
@@ -35,4 +35,4 @@ Build HTTomo as a conda Python package
 ======================================================
 .. code-block:: console
 
-   $ conda build conda/recipe/ -c conda-forge -c https://conda.anaconda.org/httomo/ -c rapidsai --no-test
+   $ conda build conda/recipe/ -c conda-forge -c https://conda.anaconda.org/httomo/ --no-test
diff --git a/docs/source/howto/process_lists/side_outputs/side_out.rst b/docs/source/howto/process_lists/side_outputs/side_out.rst
@@ -66,7 +66,7 @@ information of :ref:`centering` is stored in the reference
 There could be various configurations when this reference is required from other methods as well. We present more verbose :ref:`side_output_example` below.
 
 .. note:: Side outputs and references to them are generated automatically with the
-   :ref:`utilities_yamlgenerator`. Usually there is no need to modify them when
+   `YAML generator <https://diamondlightsource.github.io/httomo-backends/utilities/yaml_generator.html>`_. Usually there is no need to modify them when
    editing a process list.
 
 .. _side_output_example:

diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -36,7 +36,7 @@
 .. _backends_content:
 
 .. toctree::
-    :caption: Templates for backends
+    :caption: Data processing
     :maxdepth: 2
 
     backends/list
@@ -58,7 +58,6 @@
     :maxdepth: 2
 
     utilities/yaml_checker
-    utilities/yaml_generator
 
 .. _developers_content:
 

diff --git a/docs/source/utilities/yaml_generator.rst b/docs/source/utilities/yaml_generator.rst