Merge pull request #385 from DiamondLightSource/precommit

CI changes, precommit hook

.github/pull_request_template.md

@@ -0,0 +1,6 @@
+## Checklist
+
+- [ ] I have opened PR to `dev` branch
+- [ ] I have added tests that prove my fix is effective or that my feature works
+- [ ] New and existing unit tests pass locally with my changes
+- [ ] I have made corresponding changes to the documentation

.github/workflows/checker_action.yml

@@ -0,0 +1,12 @@
+name: 'PR Tasks Completed Check'
+on: 
+  pull_request:
+    types: [opened, edited]
+
+jobs:
+  task-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: kentaro-m/[email protected]
+        with:
+          repo-token: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/conda_upload.yml

@@ -1,4 +1,4 @@
-name: httomo conda upload
+name: HTTomo version build
 
 # Run the workflow whenever a tag beginning with `v` is pushed to any branch
 on:
@@ -7,7 +7,7 @@ on:
       - v*
 
 jobs:
-  build-linux:
+  conda-build-upload:
     runs-on: ubuntu-20.04
 
     defaults:
@@ -35,7 +35,7 @@ jobs:
           $CONDA/bin/conda install -c conda-forge conda-build
           $CONDA/bin/conda install -c conda-forge anaconda-client
           $CONDA/bin/conda update conda
-          $CONDA/bin/conda update conda-build          
+          $CONDA/bin/conda update conda-build
           $CONDA/bin/conda list
 
       - name: Decrypt a secret

.github/workflows/dev_build_test.yml

@@ -0,0 +1,87 @@
+name: HTTomo dev build
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  build-conda-upload:
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    steps:
+      - name: Checkout repository code
+        uses: actions/checkout@v3
+        with:
+          ref: "dev"
+          fetch-depth: 0
+
+
+        # setup Python 3.10
+      - name: Setup Python 3.10
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies with Conda
+        run: |
+          $CONDA/bin/conda install -c conda-forge conda-build
+          $CONDA/bin/conda install -c conda-forge anaconda-client
+          $CONDA/bin/conda update conda
+          $CONDA/bin/conda update conda-build
+          $CONDA/bin/conda list
+
+      - name: Decrypt a secret
+        run: ./.scripts/decrypt_secret.sh
+        env:
+          LARGE_SECRET_PASSPHRASE: ${{ secrets.LARGE_SECRET_PASSPHRASE }}
+
+      - name: Build and upload the package (dev) to httomo conda cloud
+        env:
+          LABEL: dev
+        run: |
+          chmod +x ./.scripts/conda_upload.sh
+          ./.scripts/conda_upload.sh
+          run: exit 0
+
+  install-run-tests:
+    runs-on: ubuntu-latest
+    needs: build-conda-upload
+
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    steps:
+      - name: Checkout repository code
+        uses: actions/checkout@v3
+
+        # setup Python 3.10
+      - name: Setup Python 3.10
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.10'
+
+      - name: Create conda environment
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: conda/environment.yml
+          environment-name: httomo
+          post-cleanup: 'all'
+          init-shell: bash
+
+      - name: Install httomo
+        run: |
+          conda install "httomo/linux-64::httomo * py310_openmpi_regular*"
+          micromamba list
+
+      - name: Run tests
+        run: |
+          pytest tests/
+          run: exit 0
+
+          

.github/workflows/doc_conda.yml

@@ -1,16 +1,16 @@
-name: HTTomo doc test
+name: HTTomo docs
 
 on:
   workflow_dispatch:
   pull_request:
     branches:
-      - main
+      - dev
   push:
     branches:
-      - main
+      - dev
 
 jobs:
-  build-linux:
+  build-docs-publish:
     runs-on: ubuntu-latest
     defaults:
       run:

.github/workflows/enforcer.yaml

@@ -0,0 +1,14 @@
+name: 'Check Branch'
+
+on:
+  pull_request:
+
+jobs:
+  check_branch:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check branch
+        if: github.base_ref == 'main' && github.head_ref != 'dev'
+        run: |
+          echo "ERROR: You can only merge to main from dev branch."
+          exit 1

.github/workflows/lint.yml

@@ -3,10 +3,10 @@ name: Python linters
 on:
   push:
     branches:
-      - main
+      - dev
   pull_request:
     branches:
-      - main
+      - dev
 
 jobs:
   run-linters:
@@ -17,11 +17,11 @@ jobs:
       - name: Checkout repository code
         uses: actions/checkout@v2
 
-      # setup Python 3.9
-      - name: Setup Python 3.9
+      # setup Python 3.10
+      - name: Setup Python 3.10
         uses: actions/setup-python@v2
         with:
-          python-version: 3.9
+          python-version: '3.10'
 
       - name: Install Python dependencies
         run: pip install black

.github/workflows/nightly_build.yml → .github/workflows/weekly_main_build.yml

@@ -1,13 +1,12 @@
-name: httomo nightly dev conda package build + upload
+name: HTTomo weekly conda-build (main)
 
 on:
   schedule:
-    # Run at midnight every day
-    - cron: '0 0 * * *'
+    - cron: '0 0 * * 4' # At 00:00 on Thursday
 
 jobs:
-  build-linux:
-    runs-on: ubuntu-20.04
+  notest-conda-upload:
+    runs-on: ubuntu-latest
 
     defaults:
       run:
@@ -20,19 +19,19 @@ jobs:
           ref: "main"
           fetch-depth: 0
 
-      # setup Python 3.9
-      - name: Setup Python 3.9
+      # setup Python 3.10
+      - name: Setup Python 3.10
         uses: actions/setup-python@v2
         with:
-          python-version: 3.9
-          
+          python-version: '3.10'
+
       - name: Install dependencies with Conda
         run: |
           $CONDA/bin/conda install -c conda-forge conda-build
           $CONDA/bin/conda install -c conda-forge anaconda-client
           $CONDA/bin/conda update conda
           $CONDA/bin/conda update conda-build
-          $CONDA/bin/conda list          
+          $CONDA/bin/conda list
 
       - name: Decrypt a secret
         run: ./.scripts/decrypt_secret.sh
@@ -41,7 +40,7 @@ jobs:
 
       - name: Build and upload the package to httomo conda cloud
         env:
-          LABEL: dev
+          LABEL: main
         run: |
           chmod +x ./.scripts/conda_upload.sh
           ./.scripts/conda_upload.sh

.github/workflows/weekly_main_tests_iris.yml

@@ -0,0 +1,41 @@
+name: Weekly main tests
+
+on:
+  schedule:
+    - cron: '55 0 * * 4' # At 00:55 every Thursday
+
+jobs:
+  iris-gpu:
+    runs-on: iris-gpu
+    container:
+      image: nvidia/cuda:11.6.2-devel-ubi8
+      env:
+        NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
+
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    steps:
+      - name: Checkout repository code
+        uses: actions/checkout@v3
+        with:
+          ref: "main"
+          fetch-depth: 0
+
+      - name: Create conda environment
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: conda/environment.yml
+          environment-name: httomo
+          post-cleanup: 'all'
+          init-shell: bash
+
+      - name: Install httomo
+        run: |
+          conda install "httomo/linux-64::httomo * py310_openmpi_regular*"
+          micromamba list
+
+      - name: Run tests
+        run: |
+          pytest tests/

.pre-commit-config.yaml

@@ -0,0 +1,33 @@
+repos:
+- repo: https://github.com/psf/black
+  rev: 23.12.1
+  hooks:
+    - id: black
+      language_version: python3
+      exclude: ^docs/source/examples/
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.5.0
+  hooks:
+    - id: trailing-whitespace  # This hook trims trailing whitespace.
+    - id: check-docstring-first  # Checks a common error of defining a docstring after code.
+    - id: check-merge-conflict  # Check for files that contain merge conflict strings.
+    - id: detect-private-key  # Detects the presence of private keys.
+    - id: check-symlinks
+    - id: check-toml
+- repo: https://github.com/pre-commit/pygrep-hooks
+  rev: v1.10.0
+  hooks:
+    - id: python-use-type-annotations
+- repo: https://github.com/kynan/nbstripout
+  rev: 0.6.1
+  hooks:
+    - id: nbstripout
+      exclude: ^docs/source/examples/
+-   repo: https://github.com/asottile/blacken-docs
+    rev: 1.16.0
+    hooks:
+    - id: blacken-docs
+- repo: https://github.com/nbQA-dev/nbQA
+  rev: 1.7.1
+  hooks:
+    - id: nbqa-black

README.rst

@@ -1,13 +1,13 @@
 HTTomo (High Throughput Tomography pipeline)
 *******************************************************
 
-HTTomo is a user interface (UI) written in Python for fast big data processing using MPI protocols. 
+HTTomo is a user interface (UI) written in Python for fast big data processing using MPI protocols.
 It orchestrates I/O data operations and enables processing on a CPU and/or a GPU. HTTomo utilises other libraries, such as `TomoPy <https://tomopy.readthedocs.io>`_ and `HTTomolibgpu <https://github.com/DiamondLightSource/httomolibgpu>`_
 as backends for data processing. The methods from the libraries are exposed through YAML templates to enable fast task programming.
 
 Installation
 ============
-See detailed instructions for `installation <https://diamondlightsource.github.io/httomo/howto/installation.html>`_ . 
+See detailed instructions for `installation <https://diamondlightsource.github.io/httomo/howto/installation.html>`_ .
 
 Documentation
 ==============