Merge branch 'main' into climate_patterns_only

.circleci/config.yml

@@ -2,9 +2,10 @@
 version: 2.1
 
 orbs:
-  coverage-reporter: codacy/coverage-reporter@11.9.2
+  coverage-reporter: codacy/coverage-reporter@13.13.0
 
 commands:
+
   check_changes:
     steps:
       - run:
@@ -19,20 +20,33 @@ commands:
               echo Skipping installation tests
               circleci step halt
             fi
+
+  log_versions:
+    steps:
+      - run:
+          name: Log versions
+          command: |
+            . /opt/conda/etc/profile.d/conda.sh
+            conda activate esmvaltool
+            esmvaltool version
+            dpkg -l > /logs/versions.txt
+            conda env export > /logs/environment.yml
+            pip freeze > /logs/requirements.txt
+
   test_and_report:
     steps:
       - run:
           name: Run tests
           command: |
             . /opt/conda/etc/profile.d/conda.sh
             conda activate esmvaltool
-            pytest -n 4 -m 'not flake8' --junitxml=test-reports/report.xml
+            pytest -n 4 --junitxml=test-reports/report.xml
             esmvaltool version
             esmvaltool -- --help
             ncl -V
             cdo --version
       - store_test_results:
-          path: test-reports/
+          path: test-reports/report.xml
       - store_artifacts:
           path: /logs
       - run:
@@ -47,6 +61,7 @@ commands:
           when: always
       - store_artifacts:
           path: test-reports.tar.gz
+
   test_installation_from_source:
     parameters:
       extra:
@@ -81,20 +96,13 @@ commands:
             then
               rm -r esmvaltool
             fi
-      - run:
-          name: Log versions
-          command: |
-            . /opt/conda/etc/profile.d/conda.sh
-            conda activate esmvaltool
-            dpkg -l > /logs/versions.txt
-            conda env export > /logs/environment.yml
-            pip freeze > /logs/requirements.txt
+      - log_versions
       - run:
           name: Lint source code
           command: |
             . /opt/conda/etc/profile.d/conda.sh
             conda activate esmvaltool
-            pytest -n 4 -m flake8
+            flake8 -j 4
       - test_and_report
       - save_cache:
           key: install-<< parameters.extra >>-{{ .Branch }}-{{ checksum "cache_key.txt" }}
@@ -119,16 +127,21 @@ jobs:
       - run:
           name: Install dependencies
           command: |
+            set -x
             . /opt/conda/etc/profile.d/conda.sh
-            mkdir /logs
             conda activate esmvaltool
+            mkdir /logs
             pip install .[test] > /logs/install.txt 2>&1
-            # Run flake8 check with source code (it doesn't work without it)
-            pytest -n 4 -m flake8
-            # Update/install Julia dependencies
             esmvaltool install Julia > /logs/install_julia.txt 2>&1
-            # Remove source to test installed software
-            rm -r esmvaltool
+      - run:
+          name: Check Python code style and mistakes
+          command: |
+            . /opt/conda/etc/profile.d/conda.sh
+            conda activate esmvaltool
+            flake8 -j 4
+      - run:
+          name: Remove source code to test the installed software
+          command: rm -r esmvaltool
       - test_and_report
       - save_cache:
           key: test-{{ .Branch }}-{{ checksum "cache_key.txt" }}
@@ -143,7 +156,7 @@ jobs:
   test_installation_from_source_test_mode:
     # Test installation from source
     docker:
-      - image: condaforge/mambaforge
+      - image: condaforge/mambaforge:latest
     resource_class: large
     steps:
       - test_installation_from_source:
@@ -152,17 +165,73 @@ jobs:
   test_installation_from_source_develop_mode:
     # Test development installation
     docker:
-      - image: condaforge/mambaforge
+      - image: condaforge/mambaforge:latest
     resource_class: large
     steps:
       - test_installation_from_source:
           extra: develop
           flags: "--editable"
 
+  test_upstream_development:
+    # Test running recipes with the development version of ESMValCore. The
+    # purpose of this test to discover backward-incompatible changes early on in
+    # the development cycle.
+    docker:
+      - image: condaforge/mambaforge:latest
+    resource_class: large
+    steps:
+      - run:
+          name: Install git and ssh
+          environment:
+            DEBIAN_FRONTEND: noninteractive  # needed to install tzdata
+          command: apt update && apt install -y git ssh
+      - checkout
+      - run:
+          name: Generate cache key
+          command: echo $(date '+%Y')-$(expr $(date '+%V') / 2) | tee cache_key.txt
+      - restore_cache:
+          key: test-upstream-{{ .Branch }}-{{ checksum "cache_key.txt" }}
+      - run:
+          name: Install
+          command: |
+            # Install according to instructions on readthedocs with the
+            # development version of ESMValTool and ESMValCore:
+            # https://docs.esmvaltool.org/en/latest/quickstart/installation.html#install-from-source
+            . /opt/conda/etc/profile.d/conda.sh
+            mkdir /logs
+            mamba env create >> /logs/conda.txt 2>&1
+            conda activate esmvaltool
+            pip install --editable .[develop]
+            esmvaltool install Julia > /logs/install_julia.txt 2>&1
+            git clone https://github.com/ESMValGroup/ESMValCore $HOME/ESMValCore
+            pip install --editable $HOME/ESMValCore[develop]
+      - log_versions
+      - test_and_report
+      - run:
+          name: Run recipes
+          command: |
+            . /opt/conda/etc/profile.d/conda.sh
+            conda activate esmvaltool
+            mkdir -p ~/climate_data
+            esmvaltool config get_config_user
+            echo "offline: false" >> ~/.esmvaltool/config-user.yml
+            cat ~/.esmvaltool/config-user.yml
+            for recipe in esmvaltool/recipes/testing/recipe_*.yml; do
+              esmvaltool run "$recipe"
+            done
+      - store_artifacts:
+          path: /root/esmvaltool_output
+      - save_cache:
+          key: test-upstream-{{ .Branch }}-{{ checksum "cache_key.txt" }}
+          paths:
+            - /opt/conda/pkgs
+            - /root/.cache/pip
+            - /root/climate_data
+
   build_documentation:
     # Test building documentation
     docker:
-      - image: condaforge/mambaforge
+      - image: condaforge/mambaforge:latest
     resource_class: small
     steps:
       - checkout
@@ -186,7 +255,7 @@ jobs:
   test_installation_from_conda:
     # Test conda package installation
     docker:
-      - image: condaforge/mambaforge
+      - image: condaforge/mambaforge:latest
     resource_class: medium
     steps:
       - run:
@@ -229,5 +298,6 @@ workflows:
       - run_tests
       - test_installation_from_source_test_mode
       - test_installation_from_source_develop_mode
+      - test_upstream_development
       - build_documentation
       - test_installation_from_conda

.github/workflows/create-condalock-file.yml

@@ -22,12 +22,14 @@ jobs:
         with:
           auto-update-conda: true
           activate-environment: esmvaltool-fromlock
-          python-version: "3.10"
-          miniconda-version: "latest"
-          channels: conda-forge
+          python-version: "3.11"
+          miniforge-version: "latest"
+          miniforge-variant: Mambaforge
+          use-mamba: true
       - name: Show conda config
         shell: bash -l {0}
         run: |
+          conda update -n base -c conda-forge conda
           conda info
           conda list
           conda config --show-sources
@@ -39,7 +41,10 @@ jobs:
           python --version
       - name: Install conda-lock
         shell: bash -l {0}
-        run: conda install -y conda-lock
+        run: mamba install -y conda-lock
+      - name: Check version of conda-lock
+        shell: bash -l {0}
+        run: conda-lock --version
       - name: Create conda lock file for linux-64
         shell: bash -l {0}
         run: conda-lock lock --platform linux-64 -f environment.yml --mamba --kind explicit
@@ -52,7 +57,7 @@ jobs:
       - shell: bash -l {0}
         run: conda create --name esmvaltool-fromlock --file conda-linux-64.lock
       - shell: bash -l {0}
-        run: conda install -y -c conda-forge pip
+        run: mamba install -y pip
       - shell: bash -l {0}
         run: which python
       - shell: bash -l {0}
@@ -63,6 +68,8 @@ jobs:
         run: esmvaltool --help
       - shell: bash -l {0}
         run: esmvaltool version
+      - shell: bash -l {0}
+        run: flake8
       - shell: bash -l {0}
         run: pytest -n 2 -m "not installation"
       # Automated PR

.github/workflows/install-from-conda.yml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11"]
     name: Linux Python ${{ matrix.python-version }}
     steps:
       - uses: conda-incubator/setup-miniconda@v2
@@ -55,7 +55,7 @@ jobs:
 #    runs-on: "macos-latest"
 #    strategy:
 #      matrix:
-#        python-version: ["3.8", "3.9", "3.10"]
+#        python-version: ["3.9", "3.10", "3.11"]
 #      fail-fast: false
 #    name: OSX Python ${{ matrix.python-version }}
 #    steps:

.github/workflows/install-from-condalock-file.yml

@@ -11,7 +11,7 @@ on:
   push:
     branches:
       - main
-  #     - condalock-update
+  #    - condalock-update
   # run the test only if the PR is to main
   # turn it on if required
   #pull_request:
@@ -25,7 +25,7 @@ jobs:
     runs-on: "ubuntu-latest"
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11"]
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:
@@ -58,6 +58,8 @@ jobs:
         run: esmvaltool --help
       - shell: bash -l {0}
         run: esmvaltool version 2>&1 | tee source_install_linux_artifacts_python_${{ matrix.python-version }}/version.txt
+      - shell: bash -l {0}
+        run: flake8
       - shell: bash -l {0}
         run: pytest -n 2 -m "not installation"
       - name: Upload artifacts
@@ -71,7 +73,7 @@ jobs:
 #    runs-on: "macos-latest"
 #    strategy:
 #      matrix:
-#        python-version: ["3.8", "3.9", "3.10"]
+#        python-version: ["3.9", "3.10", "3.11"]
 #      fail-fast: false
 #    name: OSX Python ${{ matrix.python-version }}
 #    steps:

.github/workflows/install-from-source.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: "ubuntu-latest"
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11"]
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:
@@ -58,7 +58,7 @@ jobs:
 #    runs-on: "macos-latest"
 #    strategy:
 #      matrix:
-#        python-version: ["3.8", "3.9", "3.10"]
+#        python-version: ["3.9", "3.10", "3.11"]
 #      fail-fast: false
 #    name: OSX Python ${{ matrix.python-version }}
 #    steps:

.github/workflows/pypi-build-and-deploy.yml

@@ -17,10 +17,10 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - name: Set up Python 3.10
+      - name: Set up Python 3.11
         uses: actions/setup-python@v1
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       - name: Install pep517
         run: >-
           python -m

.github/workflows/run-tests-monitor.yml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11"]
     name: Linux Python ${{ matrix.python-version }}
     steps:
       - uses: actions/checkout@v3
@@ -67,7 +67,7 @@ jobs:
     runs-on: "macos-latest"
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11"]
       fail-fast: false
     name: OSX Python ${{ matrix.python-version }}
     steps:

.github/workflows/test-development.yml

@@ -21,7 +21,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11"]
     name: Linux Python ${{ matrix.python-version }}
     steps:
       - uses: actions/checkout@v3
@@ -55,6 +55,9 @@ jobs:
           git clone https://github.com/ESMValGroup/ESMValCore.git
           cd ESMValCore
           pip install -e .[develop]
+      - name: Run flake8
+        shell: bash -l {0}
+        run: flake8
       - name: Run tests
         shell: bash -l {0}
         run: pytest -n 2 -m "not installation" 2>&1 | tee develop_test_linux_artifacts_python_${{ matrix.python-version }}/test_report.txt