Modernize Makefile and CI workflow

.github/workflows/ci.yml

@@ -1,50 +1,31 @@
 # CI Workflow Optimization Summary
 # =====================================
-# This workflow has been optimized for speed and maintainability while maintaining
-# full matrix testing coverage. Key improvements:
+# This workflow is designed as a 3-stage pipeline to balance fast feedback
+# with comprehensive cross-platform validation.
 #
-# 1. QUICK VALIDATION JOB (2-3 minutes feedback):
-#    - Fast linting and formatting checks on Ubuntu with Python 3.12
-#    - Parallel execution of black and flake8
-#    - Immediate feedback for common issues
+# STAGE 1: QUICK VALIDATION (Parallel with Stage 2)
+# - Goal: 2-3 minute feedback on common PR issues.
+# - Actions: Linting, formatting, and Jupyter Notebook cleanliness checks.
+# - Optimization: Uses environment caching to minimize setup time.
 #
-# 2. ENHANCED FULL MATRIX TESTING:
-#    - Maintains all 6 combinations (Ubuntu/macOS × Python 3.11/3.12/3.13)
-#    - Smart coverage strategy: Ubuntu + Python 3.12 runs coverage without --runslow
-#    - All other jobs run full test suite with --runslow
-#    - All jobs run integration tests on the full matrix
+# STAGE 2: FULL COMPATIBILITY MATRIX
+# - Goal: Verify Firecrown across all supported environments.
+# - Actions: Comprehensive unit and integration testing.
+# - Matrix: Covers OS (Linux/macOS) and Python versions defined in the job below.
+# - Optimization: Ubuntu + Python 3.12 performs coverage analysis.
 #
-# 3. OPTIMIZED CACHING STRATEGY:
-#    - Enhanced conda caching with both /envs and /pkgs paths
-#    - Better cache keys with restore fallbacks
-#    - Uses global CACHE_VERSION environment variable
+# STAGE 3: DOWNSTREAM & DOCUMENTATION (Sequential - depends on Stage 2)
+# - Goal: Verify ecosystem integration and build stability.
+# - Actions: Tests with Smokescreen and Augur; full documentation verification.
+# - Optimization: Only runs if the main matrix succeeds to save resources.
 #
-# 4. PARALLEL EXECUTION OPTIMIZATIONS:
-#    - Linting: All tools run in parallel (black, flake8, mypy, pylint)
-#    - Dependencies: CosmoSIS and Cobaya setup run in parallel
-#    - Tests: Uses -n auto for pytest parallelization
+# MAINTENANCE NOTE:
+# When updating supported Python versions, only the matrix configuration
+# in Stage 2 needs to be changed. This header is intentionally abstract.
 #
-# 5. PATH-BASED OPTIMIZATION:
-#    - Skips CI for documentation-only changes (docs/**/*.md, *.md)
-#    - Reduces unnecessary CI runs
-#
-# 6. STREAMLINED EXTERNAL DEPENDENCIES:
-#    - Moved Smokescreen, Augur, and documentation to separate job
-#    - Only runs after main matrix succeeds
-#    - Reduces resource usage for failed builds
-#
-# Performance Improvements:
-# - Quick feedback: 25-30 min → 2-3 min (90% faster)
-# - Coverage upload: 25-30 min → 8-12 min (60% faster)
-# - Failed PRs: 25-30 min → 3-5 min (85% faster)
-# - Successful PRs: 25-30 min → 20-25 min (20% faster)
-#
-# Maintained Requirements:
-# ✓ Full matrix testing (6 combinations)
-# ✓ Integration tests on all combinations
-# ✓ All existing test coverage
-# ✓ Coverage upload from single job
-# ✓ All external dependency testing
+# MacOS Note:
+# Stage 2 include a workaround for RPATH issues in 'isitgr' on macOS to
+# ensure libraries are correctly loaded during integration tests.
 
 name: firecrown-ci
 on:
@@ -65,7 +46,7 @@ on:
 
 env:
   CONDA_ENV: firecrown_developer
-  CACHE_VERSION: 4
+  CACHE_VERSION: 0
 
 jobs:
   quick-validation:
@@ -78,23 +59,43 @@ jobs:
         with:
           miniforge-version: latest
           python-version: "3.12"
-          show-channel-urls: true
-          conda-remove-defaults: true
-          environment-file: environment.yml
           activate-environment: ${{ env.CONDA_ENV }}
+      - name: Cache date
+        id: get-date
+        run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT
+        shell: bash
+      - name: Compute environment.yml hash
+        id: env-hash
+        run: |
+          if command -v sha256sum &>/dev/null 2>&1; then
+            h=$(sha256sum environment.yml | cut -d' ' -f1)
+          else
+            h=$(shasum -a 256 environment.yml | cut -d' ' -f1)
+          fi
+          echo "env_hash=${h}" >> $GITHUB_OUTPUT
+        shell: bash
+      - name: Cache Conda env
+        uses: actions/cache/restore@v4
+        id: cache
+        with:
+          path: ${{ env.CONDA }}/envs
+          key: conda-Linux-X64-py3.12-${{ steps.get-date.outputs.today }}-${{ steps.env-hash.outputs.env_hash }}-v${{ env.CACHE_VERSION }}
+      - name: Update environment
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          python .github/update_ci.py 3.12
+          conda env update -n ${{ env.CONDA_ENV }} -f env_tmp.yml --prune
+      - name: Save conda environment
+        if: steps.cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: ${{ env.CONDA }}/envs
+          key: ${{ steps.cache.outputs.cache-primary-key }}
       - name: Quick setup and linting
         shell: bash -l {0}
         run: |
-          set -e
           pip install --no-deps -e .
-          # Fast parallel linting for immediate feedback
-          black --check firecrown tests examples &
-          BLACK_PID=$!
-          flake8 firecrown examples tests &
-          FLAKE8_PID=$!
-
-          wait $BLACK_PID
-          wait $FLAKE8_PID
+          make lint
       - name: Ensure clear Jupyter Notebooks
         uses: ResearchSoftwareActions/EnsureCleanNotebooksAction@1.1
 
@@ -176,42 +177,15 @@ jobs:
           conda list
       - name: Code quality checks
         shell: bash -l {0}
-        run: |
-          set -e
-          # Parallel linting and type checking
-          black --check firecrown tests examples &
-          BLACK_PID=$!
-          flake8 firecrown examples tests &
-          FLAKE8_PID=$!
-          mypy -p firecrown -p examples -p tests &
-          MYPY_PID=$!
-          pylint firecrown &
-          PYLINT1_PID=$!
-          pylint --rcfile tests/pylintrc tests &
-          PYLINT2_PID=$!
-          pylint --rcfile examples/pylintrc examples &
-          PYLINT3_PID=$!
-
-          wait $BLACK_PID
-          wait $FLAKE8_PID
-          wait $MYPY_PID
-          wait $PYLINT1_PID
-          wait $PYLINT2_PID
-          wait $PYLINT3_PID
-      - name: Run unit tests with smart execution
+        run: make lint
+      - name: Run CI tests
         shell: bash -l {0}
         run: |
-          # matrix.coverage might be an empty string, so we need the or condition
           if [[ "${{ matrix.coverage || false }}" == "true" ]]; then
-            # Coverage job: fast execution without --runslow
-            python -m pytest -vv --cov firecrown --cov-report xml --cov-branch -n auto
+            make test-ci
           else
-            # All other jobs: full test suite including slow tests
-            python -m pytest -vv --runslow -n auto
+            make test-all
           fi
-      - name: Running example tests
-        shell: bash -l {0}
-        run: python -m pytest -vv -s --example tests/example -n auto
       - name: Upload coverage reports to Codecov
         if: ${{ matrix.coverage == true }}
         uses: codecov/codecov-action@v5
@@ -230,6 +204,7 @@ jobs:
 
   external-dependencies:
     name: External dependencies and documentation
+    needs: [firecrown-miniforge]
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@v5
@@ -306,12 +281,9 @@ jobs:
           repository: "lsstdesc/augur"
           path: "augur"
           ref: "b59cfaf3dec90aa606a4add453a5a77e0c8ea942"
-      - name: Build tutorials and documentation and check links
+      - name: Build and verify tutorials and documentation
         shell: bash -l {0}
-        run: |
-          quarto render tutorial --output-dir=../docs/_static
-          make -C docs html
-          firecrown-link-checker docs/_build/html -v
+        run: make docs-verify
       - name: Pip-install Augur and test it
         shell: bash -l {0}
         run: |

.gitignore

@@ -20,7 +20,6 @@ examples/srd_sn/sndata/
 examples/*/output_*
 examples/output_*
 examples/desc_srd_v1/srd_data/LSST_DESC_SRD_v1_release
-cosmosis-standard-library/
 
 *.pytest_cache
 *.ipynb_checkpoints
@@ -169,3 +168,4 @@ tutorial/theoretical_predictions_tutorial.html
 coverage.json
 
 **/*.quarto_ipynb
+.ruff_cache