Merge branch 'v4-dev' into bugfix/2239-oob-spatialhash

Author: fluidnumerics-joe

.github/actions/install-parcels/action.yml

@@ -14,8 +14,6 @@ dependencies:
   # (see https://github.com/Unidata/netcdf4-python/issues/1090)
   - netcdf4=1.6
   - numpy=1.23
-  - psutil=5.9
-  - pymbolic=2022.1
   - pytest=7.1
   - scipy=1.9
   - tqdm=4.64

.github/ci/min-core-deps.yml

@@ -35,8 +35,6 @@ requirements:
     - netcdf4 >=1.1.9
     - numpy >=1.11
     - platformdirs
-    - psutil
-    - pymbolic
     - pytest
     - scipy >=0.16.0
     - trajan

.github/ci/recipe.yaml

@@ -29,16 +29,14 @@ jobs:
           python .github/ci/min_deps_check.py .github/ci/min-core-deps.yml
 
   linkcheck:
-    name: Sphinx linkcheck
+    name: pixi run docs-linkcheck
     runs-on: "ubuntu-latest"
     defaults:
       run:
         shell: bash -l {0}
 
     steps:
       - uses: actions/checkout@v4
-      - name: Setup parcels
-        uses: ./.github/actions/install-parcels
-        with:
-          environment-file: environment.yml
-      - run: sphinx-build -b linkcheck docs/ _build/linkcheck
+      - uses: actions/checkout@v4
+      - uses: prefix-dev/[email protected]
+      - run: pixi run docs-linkcheck

.github/workflows/additional.yml

@@ -0,0 +1,49 @@
+name: Generate and cache Pixi lockfile
+
+on:
+  workflow_call:
+    outputs:
+      cache-id:
+        description: "The lock file contents"
+        value: ${{ jobs.cache-pixi-lock.outputs.cache-id }}
+
+jobs:
+  cache-pixi-lock:
+    name: Generate output
+    runs-on: ubuntu-latest
+    outputs:
+      cache-id: ${{ steps.restore.outputs.cache-primary-key }}
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+          sparse-checkout: pixi.toml
+      - name: Get current date
+        id: date
+        run: echo "date=$(date +'%Y-%m-%d')" >> "$GITHUB_OUTPUT"
+      - uses: actions/cache/restore@v4
+        id: restore
+        with:
+          path: |
+            pixi.lock
+          key: ${{ steps.date.outputs.date }}_${{hashFiles('pixi.toml')}}
+      - uses: prefix-dev/[email protected]
+        if: ${{ !steps.restore.outputs.cache-hit }}
+        with:
+          pixi-version: v0.49.0
+          run-install: false
+      - name: Run pixi lock
+        if: ${{ !steps.restore.outputs.cache-hit }}
+        run: pixi lock
+      - uses: actions/cache/save@v4
+        if: ${{ !steps.restore.outputs.cache-hit }}
+        id: cache
+        with:
+          path: |
+            pixi.lock
+          key: ${{ steps.restore.outputs.cache-primary-key }}
+      - name: Upload pixi.lock
+        uses: actions/upload-artifact@v4
+        with:
+          name: pixi-lock
+          path: pixi.lock

.github/workflows/cache-pixi-lock.yml

@@ -17,29 +17,38 @@ defaults:
     shell: bash -el {0}
 
 jobs:
+  cache-pixi-lock:
+    uses: ./.github/workflows/cache-pixi-lock.yml
   unit-test:
-    name: "py${{ matrix.python-version }} | ${{ matrix.os }} | unit tests"
+    name: "Unit tests: ${{ matrix.os }} | pixi run -e ${{ matrix.pixi-environment }} tests"
     runs-on: ${{ matrix.os }}-latest
+    needs: [cache-pixi-lock]
+    env:
+      COVERAGE_REPORT: "${{ matrix.os }}_${{ matrix.pixi-environment }}_unit_test_report.html"
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu] #, mac, windows] # TODO v4: Re-enable windows and mac
-        python-version: ["3.12"]
+        pixi-environment: [test-latest]
         include:
           - os: ubuntu
-            python-version: "3.11"
+            pixi-environment: "test-py311"
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Setup Conda and parcels
-        uses: ./.github/actions/install-parcels
+      - uses: actions/checkout@v4
+      - name: Restore cached pixi lockfile
+        uses: actions/cache/restore@v4
+        id: restore-pixi-lock
+        with:
+          path: |
+            pixi.lock
+          key: ${{ needs.cache-pixi-lock.outputs.cache-id }}
+      - uses: prefix-dev/[email protected]
         with:
-          environment-file: environment.yml
-          python-version: ${{ matrix.python-version }}
+          cache: true
+          cache-write: ${{ github.event_name == 'push' && github.ref_name == 'v4-dev' }} # TODO: Update v4-dev to main when v4 is released
       - name: Unit test
         run: |
-          coverage run -m pytest -v -s --html=${{ matrix.os }}_${{ matrix.python-version }}_unit_test_report.html --self-contained-html
-          coverage xml
+          pixi run -e ${{ matrix.pixi-environment }} tests -v -s --cov=parcels --cov-report=xml --html="${{ env.COVERAGE_REPORT }}" --self-contained-html
       - name: Codecov
         uses: codecov/[email protected]
         env:
@@ -50,13 +59,16 @@ jobs:
         if: ${{ always() }} # Always run this step, even if tests fail
         uses: actions/upload-artifact@v4
         with:
-          name: Unittest report ${{ matrix.os }}-${{ matrix.python-version }}
-          path: ${{ matrix.os }}_${{ matrix.python-version }}_unit_test_report.html
+          name: Unittest report ${{ matrix.os }}-${{ matrix.pixi-environment }}
+          path: ${{ env.COVERAGE_REPORT }}
   integration-test:
     # TODO v4: Re-enable the workflow once development has stabilized and we want to run integration tests again
     if: false
-    name: "py${{ matrix.python-version }} | ${{ matrix.os }} | integration tests"
+    name: "Integration: ${{ matrix.os }} | pixi run -e ${{ matrix.pixi-environment }} tests-notebooks"
     runs-on: ${{ matrix.os }}-latest
+    needs: [cache-pixi-lock]
+    env:
+      COVERAGE_REPORT: "${{ matrix.os }}_${{ matrix.pixi-environment }}_integration_test_report.html"
     strategy:
       fail-fast: false
       matrix:
@@ -66,22 +78,21 @@ jobs:
           - os: ubuntu
             python-version: "3.11"
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Setup Conda and parcels
-        uses: ./.github/actions/install-parcels
+      - uses: actions/checkout@v4
+      - name: Restore cached pixi lockfile
+        uses: actions/cache/restore@v4
+        id: restore-pixi-lock
         with:
-          environment-file: environment.yml
+          path: |
+            pixi.lock
+          key: ${{ needs.cache-pixi-lock.outputs.cache-id }}
+      - uses: prefix-dev/[email protected]
+        with:
+          cache: true
+          cache-write: ${{ github.event_name == 'push' && github.ref_name == 'v4-dev' }} # TODO: Update v4-dev to main when v4 is released
       - name: Integration test
-        # TODO v4: Re-enable `tutorial_periodic_boundaries`
-        # TODO v4: Re-enable `tutorial_timevaryingdepthdimensions`
-        # TODO v4: Re-enable `tutorial_particle_field_interaction`
-        # TODO v4: Re-enable `tutorial_croco_3D`
-        # TODO v4: Re-enable `tutorial_nemo_3D` (https://github.com/OceanParcels/Parcels/pull/1936#issuecomment-2717666705)
-        # TODO v4: Re-enable `tutorial_analyticaladvection`
         run: |
-          coverage run -m pytest -v -s --nbval-lax -k "not documentation and not tutorial_periodic_boundaries and not tutorial_timevaryingdepthdimensions and not tutorial_particle_field_interaction and not tutorial_croco_3D and not tutorial_nemo_3D and not tutorial_analyticaladvection" --html="${{ matrix.os }}_${{ matrix.python-version }}_integration_test_report.html" --self-contained-html docs/examples
-          coverage xml
+          pixi run test-notebooks -v -s --html="${{ env.COVERAGE_REPORT }}" --self-contained-html --cov=parcels --cov-report=xml
       - name: Codecov
         uses: codecov/[email protected]
         env:
@@ -92,8 +103,8 @@ jobs:
         if: ${{ always() }} # Always run this step, even if tests fail
         uses: actions/upload-artifact@v4
         with:
-          name: Integration test report ${{ matrix.os }}-${{ matrix.python-version }}
-          path: ${{ matrix.os }}_${{ matrix.python-version }}_integration_test_report.html
+          name: Integration test report ${{ matrix.os }}-${{ matrix.pixi-environment }}
+          path: ${{ env.COVERAGE_REPORT }}
   merge-test-artifacts:
     runs-on: ubuntu-latest
     needs:
@@ -107,21 +118,28 @@ jobs:
           name: Testing reports
           pattern: "* report *"
   typechecking:
-    name: mypy
+    name: "TypeChecking: pixi run typing"
     # TODO v4: Enable typechecking again
     if: false
     runs-on: ubuntu-latest
+    needs: [cache-pixi-lock]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Setup Conda and parcels
-        uses: ./.github/actions/install-parcels
+      - name: Restore cached pixi lockfile
+        uses: actions/cache/restore@v4
+        id: restore-pixi-lock
+        with:
+          path: |
+            pixi.lock
+          key: ${{ needs.cache-pixi-lock.outputs.cache-id }}
+      - uses: prefix-dev/[email protected]
         with:
-          environment-file: environment.yml
-      - run: conda install lxml # dep for report generation
+          cache: true
+          cache-write: ${{ github.event_name == 'push' && github.ref_name == 'v4-dev' }} # TODO: Update v4-dev to main when v4 is released
       - name: Typechecking
         run: |
-          mypy --install-types --non-interactive parcels --html-report mypy-report
+          pixi run typing --non-interactive --html-report mypy-report
       - name: Upload test results
         if: ${{ always() }} # Upload even on mypy error
         uses: actions/upload-artifact@v4

.github/workflows/ci.yml

@@ -43,3 +43,14 @@ repos:
     rev: 0.8.1
     hooks:
       - id: nbstripout
+  - repo: https://github.com/ComPWA/taplo-pre-commit
+    rev: v0.9.3
+    hooks:
+      - id: taplo-format
+        args:
+          [
+            "--option",
+            "array_auto_collapse=false",
+            "--option",
+            "align_comments=false",
+          ]

.pre-commit-config.yaml

@@ -1,13 +1,18 @@
 version: 2
 build:
-  os: ubuntu-22.04
+  os: ubuntu-lts-latest
   tools:
-    python: mambaforge-22.9
+    # just so RTD stops complaining
+    python: "latest"
   jobs:
-    post_create_environment:
-      - pip install -e .
+    create_environment:
+      - asdf plugin add pixi
+      - asdf install pixi latest
+      - asdf global pixi latest
+    install:
+      - pixi install -e docs
+    build:
+      html:
+        - pixi run -e docs sphinx-build -T -b html docs $READTHEDOCS_OUTPUT/html
 sphinx:
   configuration: docs/conf.py
-
-conda:
-  environment: environment.yml

.readthedocs.yaml

@@ -1,5 +1,6 @@
 ## Parcels
 
+[![Pixi Badge](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/prefix-dev/pixi/main/assets/badge/v0.json)](https://pixi.sh)
 [![Anaconda-release](https://anaconda.org/conda-forge/parcels/badges/version.svg)](https://anaconda.org/conda-forge/parcels/)
 [![Anaconda-date](https://anaconda.org/conda-forge/parcels/badges/latest_release_date.svg)](https://anaconda.org/conda-forge/parcels/)
 [![Zenodo](https://zenodo.org/badge/DOI/10.5281/zenodo.823561.svg)](https://doi.org/10.5281/zenodo.823561)

README.md

@@ -19,6 +19,10 @@ Version 4 of Parcels is unreleased at the moment. The information in this migrat
 
 - `interp_method` has to be an Interpolation function, instead of a string.
 
+## Particle
+
+- `Particle.add_variables()` has been replaced by `Particle.add_variable()`, which now also takes a list of `Variables`.
+
 ## ParticleSet
 
 - `repeatdt` and `lonlatdepth_dtype` have been removed from the ParticleSet.
@@ -28,3 +32,4 @@ Version 4 of Parcels is unreleased at the moment. The information in this migrat
 ## ParticleFile
 
 - Particlefiles should be created by `ParticleFile(...)` instead of `pset.ParticleFile(...)`
+- The `name` argument in `ParticleFile` has been replaced by `store` and can now be a string, a Path or a zarr store.