Refactor ECCC functionality and create Preprocess module

.cruft.json

@@ -1,6 +1,6 @@
 {
   "template": "https://github.com/Ouranosinc/cookiecutter-pypackage",
-  "commit": "1d9ee5f08d3e8e4f78a4aabb75e2ce4eff8750bf",
+  "commit": "63f44fcbfe2e16118a4fa6b09fe847aa44e0715a",
   "checkout": null,
   "context": {
     "cookiecutter": {

.github/workflows/cache-cleaner.yml

@@ -26,7 +26,9 @@ jobs:
             objects.githubusercontent.com:443
 
       - name: Checkout Repository
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: false
 
       - name: Cleanup
         run: |

.github/workflows/codeql-analysis.yml

@@ -13,7 +13,8 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [ "main" ]
+    branches:
+      - main
     paths-ignore:
       - ../../CHANGELOG.rst
       - pyproject.toml
@@ -43,6 +44,8 @@ jobs:
     steps:
     - name: Checkout repository
       uses: actions/checkout@v4
+      with:
+        persist-credentials: false
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/dependency-review.yml

@@ -28,7 +28,9 @@ jobs:
             github.com:443
 
       - name: Checkout Repository
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: false
 
       - name: Dependency Review
         uses: actions/dependency-review-action@ce3cf9537a52e8119d91fd484ab5b8a807627bf8 # v4.6.0

.github/workflows/main.yml

@@ -39,7 +39,9 @@ jobs:
           disable-sudo: true
           egress-policy: audit
       - name: Checkout Repository
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: false
       - name: Set up Python${{ matrix.python-version }}
         uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0
         with:
@@ -61,15 +63,22 @@ jobs:
     strategy:
       matrix:
         os: [ 'ubuntu-latest' ]
-        python-version: [ "3.9", "3.10", "3.11", "3.12" ] # "3.13"
+        python-version:
+          - "3.9"
+          - "3.10"
+          - "3.11"
+          - "3.12"
+          # - "3.13"
     steps:
       - name: Harden Runner
         uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1
         with:
           disable-sudo: true
           egress-policy: audit
       - name: Checkout Repository
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: false
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0
         with:
@@ -104,7 +113,12 @@ jobs:
     strategy:
       matrix:
         os: [ 'ubuntu-latest' ]
-        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
+        python-version:
+          - "3.9"
+          - "3.10"
+          - "3.11"
+          - "3.12"
+          # - "3.13"
     defaults:
       run:
         shell: bash -l {0}
@@ -115,15 +129,16 @@ jobs:
           disable-sudo: true
           egress-policy: audit
       - name: Checkout Repository
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: false
       - name: Setup Conda (Micromamba) with Python${{ matrix.python-version }}
         uses: mamba-org/setup-micromamba@0dea6379afdaffa5d528b3d1dabc45da37f443fc # v2.0.4
         with:
           cache-downloads: true
           environment-file: environment-dev.yml
           create-args: >-
             python=${{ matrix.python-version }}
-          micromamba-version: 1.5.10-0 # Pin micromamba version because of following issue: https://github.com/mamba-org/setup-micromamba/issues/225
       - name: Install miranda
         run: |
           python -m pip install --no-deps .

.github/workflows/publish-pypi.yml

@@ -28,7 +28,9 @@ jobs:
             pypi.org:443
             upload.pypi.org:443
       - name: Checkout Repository
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: false
       - name: Set up Python3
         uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0
         with:

.github/workflows/scorecard.yml

@@ -16,7 +16,9 @@ on:
       - main
 
 # Declare default permissions as read only.
-permissions: read-all
+# Read-all permission is not technically needed for this workflow.
+permissions:
+  contents: read
 
 jobs:
   analysis:
@@ -47,7 +49,7 @@ jobs:
             www.bestpractices.dev:443
 
       - name: Checkout Repository
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           persist-credentials: false
 

.github/workflows/tag-testpypi.yml

@@ -21,7 +21,9 @@ jobs:
         with:
           egress-policy: audit
       - name: Checkout Repository
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: false
       - name: Create Release
         uses: softprops/action-gh-release@c95fe1489396fe8a9eb87c0abf8aa5b2ef267fda # 2.2.1
         env:
@@ -52,7 +54,9 @@ jobs:
             pypi.org:443
             test.pypi.org:443
       - name: Checkout Repository
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: false
       - name: Set up Python3
         uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0
         with:

.gitignore

@@ -113,3 +113,7 @@ ENV/
 # IDE settings
 .vscode/
 .idea/
+
+# Manually excluded folders
+excluded/
+templates/downloaded/

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
     rev: v3.19.1
     hooks:
       - id: pyupgrade
-        args: [ '--py38-plus' ]
+        args: [ '--py39-plus' ]
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v5.0.0
     hooks:
@@ -41,7 +41,7 @@ repos:
     hooks:
       - id: isort
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.4
+    rev: v0.11.6
     hooks:
       - id: ruff
         args: [ '--fix' ]
@@ -74,10 +74,15 @@ repos:
       - id: detect-secrets
         args: [ '--baseline=.secrets.baseline' ]
   - repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: 0.32.1
+    rev: 0.33.0
     hooks:
       - id: check-github-workflows
       - id: check-readthedocs
+  - repo: https://github.com/woodruffw/zizmor-pre-commit
+    rev: v1.5.2
+    hooks:
+      - id: zizmor
+        args: [ '--config=.zizmor.yml' ]
   - repo: meta
     hooks:
       - id: check-hooks-apply

.readthedocs.yml

@@ -11,9 +11,9 @@ sphinx:
   fail_on_warning: true
 
 build:
-  os: ubuntu-22.04
+  os: ubuntu-24.04
   tools:
-    python: "mambaforge-22.9"
+    python: "mambaforge-23.11"
   jobs:
     pre_build:
       - sphinx-apidoc -o docs/apidoc --private --module-first src/miranda

.zizmor.yml

@@ -0,0 +1,6 @@
+rules:
+  dangerous-triggers:
+    ignore:
+      - label.yml:9
+      - first-pull-request.yml:3
+      - workflow-warning.yml:3

CHANGELOG.rst

@@ -4,28 +4,37 @@
 Changelog
 =========
 
+.. _changes_0.6.0:
+
 v0.6.0 (unreleased)
 -------------------
-Contributors to this version: Travis Logan (:user:`tlogan2000`), Trevor James Smith (:user:`Zeitsperre`).
+Contributors to this version: Travis Logan (:user:`tlogan2000`), Trevor James Smith (:user:`Zeitsperre`), Aslı Beşe (:user:`aslibese`).
 
 Announcements
 ^^^^^^^^^^^^^
 * `miranda` boilerplate code is now versioned with `cruft <https://cruft.github.io/cruft>`_ and the `Ouranosinc/cookiecutter-pypackage <https://github.com/Ouranosinc/cookiecutter-pypackage>`_ template.
 * Python3.8 support has been dropped in favor of Python3.9+.
+* The `miranda` library has undergone a significant refactoring to remove unused code and modules that were out of scope for the library.
+    * The modules `miranda.archive` and `miranda.remote` have been removed and will be moved to a separate project.
+    * Data treatment functions have been moved to `miranda.treatments`.
+    * Pre-processing logic and functions have been moved to `miranda.preprocess`.
 
 New features
 ^^^^^^^^^^^^
-* weather station GHCN-D dataset conversion is now supported.
+* Conversions for variables in the `GHCN-D` weather station network dataset are now supported.
+* Conversion support has been added for the `ORRC`, `CaSR v3.1`, and `RDRS v2.1` datasets.
+* ECMWF: Added support for the `era5-single-levels-monthly-means` project.
+* ECMWF: Added support for ocean variables (`sst`, `siconc`), convective precipitation variables (`cp`, `cape`), and wind speed (`u`, `v`).
 * Aggregation operations now support more variables (`hur`, `hurs`, `huss`, `rlds`, `ta`, `tdp`, `ua`, `uas`, `va`, `vas`)
-* `RDRSv21` has been added as a dataset to be converted.
 * Minimum values of `0 kg m2 s-1` has been set for both `tp` and `sf` variables in ERA5 and ERA5-Land projects.
 * Project user and developer documentation has been greatly expanded. All public functions and modules now have `numpy`-based docstrings.
 * The `miranda` library now uses a `src` layout for better packaging and distribution.
 * `ruff` checks and formatting standards have been adopted for the entire codebase.
+* Added a new configuration for converting the NRCAN gridded climate dataset (`NRCANmet`).
 
 Breaking changes
 ^^^^^^^^^^^^^^^^
-* Removed modules ``miranda.archive`` and ``miranda.remote`` (split into separate project to be announced).
+* Removed modules ``miranda.archive`` and ``miranda.remote`` (split into a separate project yet to be published).
 
 Bug fixes
 ^^^^^^^^^
@@ -38,6 +47,7 @@ Internal changes
 * GitHub Workflows now use commit hashes for both running GitHub Actions and installing Python dependencies from PyPI.
 * `miranda` now has a `CODE_OF_CONDUCT.md` file for setting community standards and expectations.
 * Now using the GitHub Ouranos bot for automatic version bumping via `bump-version.yml`.
+* Adjusted calls using `os.path` to use `pathlib` for better cross-platform compatibility.
 
 .. _changes_0.5.0:
 

environment-dev.yml

@@ -30,20 +30,20 @@ dependencies:
   - xesmf
   - zarr
   # Dev tools and testing
-  - pip >=24.2.0
-  - bump-my-version >=0.25.1
-  - watchdog >=4.0.0
+  - pip >=24.3.1
+  - black ==24.10.0
+  - blackdoc ==0.3.9
+  - bump-my-version >=0.28.0
+  - coverage >=7.5.0
+  - coveralls >=4.0.1
   - flake8 >=7.1.1
   - flake8-rst-docstrings >=0.3.0
   - flit >=3.9.0,<4.0
-  - tox >=4.17.1
-  - coverage >=7.5.0
-  - coveralls >=4.0.1
-  - pytest >=8.3.2
-  - pytest-cov >=5.0.0
-  - black ==24.8.0
-  - blackdoc ==0.3.9
   - isort ==5.13.2
   - numpydoc >=1.8.0
   - pre-commit >=3.5.0
-  - ruff >=0.5.7
+  - pytest >=8.3.2
+  - pytest-cov >=5.0.0
+  - ruff >=0.8.2
+  - tox >=4.23.2
+  - watchdog >=4.0.0

environment-docs.yml

@@ -2,15 +2,14 @@ name: miranda
 channels:
   - conda-forge
 dependencies:
-  - python >=3.12,<3.13
-  - sphinx >=7.0.0
-  - pandoc
+  - python >=3.9,<3.13
+  # Docs
   - furo >=2023.07.26
   - ipython
   - ipykernel
   - nbsphinx
+  - sphinx >=7.1.0
   - sphinx-codeautolink
   - sphinx-copybutton
   - sphinx-intl
   - sphinx-mdinclude
-  - sphinxcontrib-napoleon

pyproject.toml

@@ -54,21 +54,20 @@ dev = [
   # Dev tools and testing
   "black ==25.1.0",
   "blackdoc ==0.3.9",
-  "bump-my-version >=0.26.0",
+  "bump-my-version >=0.28.0",
   "coverage >=7.5.0",
-  "coveralls >=4.0.1",
   "flake8 >=7.1.1",
   "flake8-rst-docstrings >=0.3.0",
   "flit >=3.9.0,<4.0",
   "isort ==6.0.1",
   "mypy",
   "numpydoc >=1.8.0",
-  "pip >=24.2.0",
+  "pip >=25.0",
   "pre-commit >=3.5.0",
   "pytest >=8.3.2",
   "pytest-cov >=5.0.0",
-  "ruff >=0.5.7",
-  "tox >=4.18.0",
+  "ruff >=0.9.0",
+  "tox >=4.24.1",
   "watchdog >=4.0.0"
 ]
 docs = [
@@ -78,7 +77,6 @@ docs = [
   "ipython",
   "jupyter_client",
   "nbsphinx",
-  "pygments",
   "sphinx >=7.0.0,<8.2.0",
   "sphinx-intl",
   "sphinx-mdinclude",
@@ -230,6 +228,7 @@ exclude = [
   ".pre-commit-config.yaml",
   ".readthedocs.yml",
   ".yamllint.yaml",
+  ".zizmor.yml",
   "docs/_*",
   "docs/apidoc/modules.rst",
   "docs/apidoc/miranda*.rst",