Merge branch 'master' into dataexample_update

Signed-off-by: Hannah Robarts <[email protected]>

.github/workflows/README.md

@@ -23,12 +23,16 @@ Details on some of these jobs are given below.
 
 ## conda
 
-When opening or modifying a pull request to `master`, a single variant is built and tested. This variant is for linux with `python=3.11` and `numpy=1.25`.
+When opening or modifying a pull request to `master`, two variants are built and tested (for linux with minimum & maximum supported `python` & `numpy` versions).
 
 > [!NOTE]
-> The action does not publish to conda, instead this is done by jenkins. We will eventually move from jenkins to conda-forge instead.
 > When pushing to `master` or creating an [annotated tag](https://git-scm.com/book/en/v2/Git-Basics-Tagging), *all* variants are built and tested.
 
+<!-- <br/> -->
+
+> [!NOTE]
+> The action publishes `ccpi` as well as `https://tomography.stfc.ac.uk/conda/` conda channels. We will eventually move to conda-forge instead.
+
 It looks for conda-build dependencies in the channels listed [here](./build.yml#L118). If you add any new dependencies, the appropriate channels need to be added to this line.
 
 > [!TIP]

.github/workflows/build.yml

@@ -1,16 +1,3 @@
-# Copyright 2021 United Kingdom Research and Innovation
-# Copyright 2021 The University of Manchester
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#     http://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# Authors:
-# CIL Developers, listed at: https://github.com/TomographicImaging/CIL/blob/master/NOTICE.txt
 name: build
 on:
   release:
@@ -100,13 +87,32 @@ jobs:
         cmake --build ./build --target install
     - name: test
       run: python -m unittest discover -v ./Wrappers/Python/test
+  conda-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      python-version: ${{ steps.matrix.outputs.python-version }}
+      numpy-version: ${{ steps.matrix.outputs.numpy-version }}
+    steps:
+    - id: matrix
+      run: |
+        if ${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags') }}; then
+          echo "python-version=['3.10', 3.11]" >> $GITHUB_OUTPUT
+          echo "numpy-version=[1.23, 1.24, 1.25, 1.26]" >> $GITHUB_OUTPUT
+        else
+          echo "python-version=['3.10']" >> $GITHUB_OUTPUT
+          echo "numpy-version=[1.23]" >> $GITHUB_OUTPUT
+        fi
   conda:
     defaults: {run: {shell: 'bash -el {0}'}}
     runs-on: ubuntu-latest
+    needs: conda-matrix
     strategy:
       matrix:
-        python-version: [3.11]
-        numpy-version: [1.25]
+        python-version: ${{ fromJson(needs.conda-matrix.outputs.python-version) }}
+        numpy-version: ${{ fromJson(needs.conda-matrix.outputs.numpy-version) }}
+        include:
+        - python-version: 3.12
+          numpy-version: 1.26
     steps:
     - uses: actions/checkout@v4
       with:
@@ -122,12 +128,34 @@ jobs:
       working-directory: recipe
       run: |
         conda install boa
-        conda mambabuild . -c conda-forge -c https://software.repos.intel.com/python/conda -c ccpi --python=${{ matrix.python-version }} --numpy=${{ matrix.numpy-version }} --output-folder .
-    - name: Upload artifact of the conda package
-      uses: actions/upload-artifact@v4
+        conda mambabuild . -c conda-forge -c https://tomography.stfc.ac.uk/conda --override-channels --python=${{ matrix.python-version }} --numpy=${{ matrix.numpy-version }} --output-folder .
+    - uses: actions/upload-artifact@v4
       with:
-        name: cil-package
+        name: cil-package-py${{ matrix.python-version }}-np${{ matrix.numpy-version }}
         path: recipe/linux-64/cil*
+    - name: anaconda upload -c ccpi
+      if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags')
+      run: |
+        conda install anaconda-client
+        anaconda -v -t ${{ secrets.CCPI_CONDA_TOKEN }} upload --force --label ${{ startsWith(github.ref, 'refs/tags') && 'main' || 'dev' }} recipe/linux-64/cil*
+    - name: conda upload -c tomography.stfc.ac.uk/conda
+      if: startsWith(github.ref, 'refs/tags')
+      run: |
+        echo '${{ secrets.STFC_SSH_KEY }}' > ./key
+        chmod 600 ./key
+        rsync -e 'ssh -o StrictHostKeyChecking=no -i ./key' -P recipe/linux-64/cil* \
+          '${{ secrets.STFC_SSH_HOST }}:${{ secrets.STFC_SSH_CONDA_DIR }}/linux-64/'
+  conda-reindex:
+    if: startsWith(github.ref, 'refs/tags')
+    needs: conda
+    runs-on: ubuntu-latest
+    steps:
+    - name: conda index tomography.stfc.ac.uk/conda
+      run: |
+        echo '${{ secrets.STFC_SSH_KEY }}' > ./key
+        chmod 600 ./key
+        ssh -o StrictHostKeyChecking=no -i ./key ${{ secrets.STFC_SSH_HOST }} \
+          'conda index --bz2 --zst --run-exports --channeldata --rss -n ccpi ${{ secrets.STFC_SSH_CONDA_DIR }}'
   docs:
     defaults: {run: {shell: 'bash -el {0}', working-directory: docs}}
     runs-on: ubuntu-latest

.gitignore

@@ -1,6 +1,7 @@
 /build/
+__pycache__/
 /Wrappers/Python/build/
-/Wrappers/Python/*.egg*/
 /Wrappers/Python/cil/version.py
-/Wrappers/Python/setup.py
-__pycache__/
+/build*
+*.vscode*
+*.egg*

CHANGELOG.md

@@ -1,3 +1,30 @@
+* 24.x.x
+  - New Features:
+    - Added SVRG and LSVRG stochastic functions (#1625)
+    - Added SAG and SAGA stochastic functions (#1624)
+    - Allow `SumFunction` with 1 item (#1857)
+    - Added  PD3O algorithm (#1834)
+    - Added Barzilai-Borwein step size rule to work with GD, ISTA, FISTA (#1859)
+    - Added callback `optimisation.utilities.callbacks.EarlyStoppingObjectiveValue` which stops iterations if an algorithm objective changes less than a provided threshold (#1892)
+    - Added callback `optimisation.utilities.callbacks.CGLSEarlyStopping` which replicates the automatic behaviour of CGLS in CIL versions <=24. (#1892)
+    - Added `labels` module with `ImageDimension`, `AcquisitionDimension`, `AcquisitionType`, `AngleUnit`, `FillType` (#1692)
+  - Enhancements:
+    - Use ravel instead of flat in KullbackLeibler numba backend (#1874)
+    - Upgrade Python wrapper (#1873, #1875)
+    - Updated the documentation for the algorithm base class (#1809)
+    - Add checks on out argument passed to processors to ensure corrrect dtype and size (#1805)
+    - Internal refactor: Replaced string-based label checks with enum-based checks for improved type safety and consistency (#1692)
+    - Internal refactor: Separate framework into multiple files (#1692)
+    - Allow the SIRT algorithm to take `initial=None` (#1906)
+  - Testing:
+    - New unit tests for operators and functions to check for in place errors and the behaviour of `out` (#1805)
+    - Updates in SPDHG vs PDHG unit test to reduce test time and adjustments to parameters (#1898)
+    - Drop Jenkins in favour of GHA for conda builds (#1914)
+  - Bug fixes:
+    - `ImageData` removes dimensions of size 1 from the input array. This fixes an issue where single slice reconstructions from 3D data would fail due to shape mismatches (#1885)
+    - Make Binner accept accelerated=False (#1887)
+  - Changes that break backwards compatibility:
+    - CGLS will no longer automatically stop iterations once a default tolerance is reached. The option to pass `tolerance` will be deprecated to be replaced by `optimisation.utilities.callbacks` (#1892)
 
 * 24.1.0
   - New Features:
@@ -15,7 +42,7 @@
     - The CIL function class now has a `__neg__` function, so you can write `-YourFunction(x)` rather than `-1*YourFunction(x)` (#1808)
     - Added documentation for the Partitioner to `framework.rst` (#1828)
     - Added CIL vs SIRF tests comparing preconditioned ISTA in CIL and MLEM in SIRF (#1823)
-    - Update to CCPi-Regularisation toolkit v24.0.1 (#1868) 
+    - Update to CCPi-Regularisation toolkit v24.0.1 (#1868)
   - Bug fixes:
     - gradient descent `update_objective` called twice on the initial point.(#1789)
     - ProjectionMap operator bug fix in adjoint and added documentation (#1743)
@@ -25,10 +52,7 @@
     - Update dataexample remote data download to work with windows (#1774)
   - Changes that break backwards compatibility:
     - Merged the files `BlockGeometry.py` and `BlockDataContainer.py` in `framework` to one file `block.py`. Please use `from cil.framework import BlockGeometry, BlockDataContainer` as before (#1799)
-    - Bug fix in `FGP_TV` function to set the default behaviour not to enforce non-negativity (#1826). 
-
-
-
+    - Bug fix in `FGP_TV` function to set the default behaviour not to enforce non-negativity (#1826).
 
 * 24.0.0
   - Update to new CCPi-Regularisation toolkit v24.0.0. This is a backward incompatible release of the toolkit.
@@ -75,7 +99,7 @@
   - Bug fix for missing factor of 1/2 in SIRT update objective and catch in place errors in the SIRT constraint
   - Bug fix to allow safe in place calculation for the soft shrinkage algorithm
   - Allow Masker to take integer arrays in addition to boolean
-  - Add remote data class to example data to enable download of relevant datasets from remote repositories 
+  - Add remote data class to example data to enable download of relevant datasets from remote repositories
   - Improved import error/warning messages
   - New adjoint operator
   - Bug fix for complex matrix adjoint

CMakeLists.txt

@@ -23,7 +23,7 @@ else()
   cmake_minimum_required(VERSION 3.4)
 endif()
 
-project (cil LANGUAGES CXX)
+project(cil LANGUAGES CXX)
 
 if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.27)
   cmake_policy(SET CMP0148 OLD)
@@ -44,37 +44,8 @@ if(NOT DEFINED CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE "RelWithDebInfo")
 endif()
 
-
-# CIL version TAG is determined by git describe as for the conda package
-# variable GIT_DESCRIBE_TAG
-# https://docs.conda.io/projects/conda-build/en/stable/resources/define-metadata.html#templating-with-jinja
-# https://stackoverflow.com/questions/38919840/get-package-version-for-conda-meta-yaml-from-source-file
-
 find_package(Git REQUIRED)
-
 find_package(Python COMPONENTS Interpreter REQUIRED)
 
-if (Python_FOUND)
-  execute_process(COMMAND "${Python_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/Wrappers/Python/CMake/parse_git_describe.py "${GIT_EXECUTABLE}"
-                RESULT_VARIABLE worked
-                OUTPUT_VARIABLE CIL_VERSION
-                WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-                OUTPUT_STRIP_TRAILING_WHITESPACE)
-  if (NOT worked EQUAL 0)
-    message(FATAL_ERROR "git describe returned ${worked}")
-  else()
-    string(REPLACE " " ";" CIL_VERSION ${CIL_VERSION})
-    list(LENGTH CIL_VERSION CIL_version_len)
-
-    list(GET CIL_VERSION 0 CIL_VERSION_MAJOR)
-    list(GET CIL_VERSION 1 CIL_VERSION_MINOR)
-    list(GET CIL_VERSION 2 CIL_VERSION_PATCH)
-    list(GET CIL_VERSION 3 CIL_NUM_COMMIT)
-    list(GET CIL_VERSION 4 CIL_COMMIT_HASH)
-
-  endif()
-endif()
-
-
 add_subdirectory(src/Core)
 add_subdirectory(Wrappers/Python)

NOTICE.txt

@@ -65,6 +65,8 @@ Ashley Gillman (2024) -12
 Zeljko Kereta (2024) - 5
 Evgueni Ovtchinnikov (2024) -1 
 Georg Schramm (2024) - 13
+Joshua Hellier (2024) - 3
+Nicholas Whyatt (2024) - 1
 
 CIL Advisory Board:
 Llion Evans - 9

README.md

@@ -1,8 +1,6 @@
 # CIL - Core Imaging Library
 
-Master | Development | Conda binaries
--|-|-
-[![CI-master](https://anvil.softeng-support.ac.uk/jenkins/buildStatus/icon?job=CILsingle/CCPi-Framework)](https://anvil.softeng-support.ac.uk/jenkins/job/CILsingle/job/CCPi-Framework) | [![CI-dev](https://anvil.softeng-support.ac.uk/jenkins/buildStatus/icon?job=CILsingle/CCPi-Framework-dev)](https://anvil.softeng-support.ac.uk/jenkins/job/CILsingle/job/CCPi-Framework-dev) | ![conda-ver](https://anaconda.org/ccpi/cil/badges/version.svg) ![conda-date](https://anaconda.org/ccpi/cil/badges/latest_release_date.svg) [![conda-plat](https://anaconda.org/ccpi/cil/badges/platforms.svg) ![conda-dl](https://anaconda.org/ccpi/cil/badges/downloads.svg)](https://anaconda.org/ccpi/cil)
+[![CI-master](https://github.com/TomographicImaging/CIL/actions/workflows/build.yml/badge.svg)](https://github.com/TomographicImaging/CIL/actions/workflows/build.yml) ![conda-ver](https://anaconda.org/ccpi/cil/badges/version.svg) ![conda-date](https://anaconda.org/ccpi/cil/badges/latest_release_date.svg) [![conda-plat](https://anaconda.org/ccpi/cil/badges/platforms.svg) ![conda-dl](https://anaconda.org/ccpi/cil/badges/downloads.svg)](https://anaconda.org/ccpi/cil)
 
 [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/TomographicImaging/CIL-Demos/HEAD?urlpath=lab/tree/binder%2Findex.ipynb)