diff --git a/.github/workflows/docker_pygem.yml b/.github/workflows/docker_pygem.yml new file mode 100644 index 00000000..cfa3a618 --- /dev/null +++ b/.github/workflows/docker_pygem.yml @@ -0,0 +1,64 @@ +name: 'Build bespoke PyGEM Docker container' + +on: + # Trigger when these files change in an open PR + pull_request: + paths: + - '.github/workflows/docker_pygem.yml' + - 'docker/Dockerfile' + + # Trigger when these files change on the master or dev branches + push: + branches: + - master + - dev + paths: + - '.github/workflows/docker_pygem.yml' + - 'docker/Dockerfile' + + # Trigger every Saturday at 12AM GMT + schedule: + - cron: '0 0 * * 6' + + # Manually trigger the workflow + workflow_dispatch: + +# Stop the workflow if a new one is started +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + packages: write + +jobs: + docker: + name: 'Build and push Docker container' + runs-on: ubuntu-latest + + steps: + - name: 'Check out the repo' + uses: actions/checkout@v4 + + - name: 'Set up Docker buildx' + uses: docker/setup-buildx-action@v3 + + - name: 'Log into GitHub Container Repository' + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + logout: true + + - name: 'Build and Push Docker Container' + uses: docker/build-push-action@v5 + with: + push: ${{ github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev' }} + no-cache: true + file: 'docker/Dockerfile' + build-args: | + PYGEM_BRANCH=${{ github.ref == 'refs/heads/master' && 'master' || 'dev' }} + tags: | + ghcr.io/pygem-community/pygem:${{ github.ref == 'refs/heads/master' && 'latest' || github.ref == 'refs/heads/dev' && 'dev' }} \ No newline at end of file diff --git a/.github/workflows/test_suite.yml b/.github/workflows/test_suite.yml new file mode 100644 index 00000000..2a9f2bf0 --- /dev/null +++ b/.github/workflows/test_suite.yml @@ -0,0 +1,62 @@ +name: 'Install PyGEM and Run Test Suite' + +on: + push: + branches: + - master + - dev + paths: + - '**.py' + - '.github/workflows/test_suite.yml' + - 'pyproject.toml' + + pull_request: + paths: + - '**.py' + - '.github/workflows/test_suite.yml' + - 'pyproject.toml' + + # Run test suite every Saturday at 1AM GMT (1 hour after the Docker image is updated) + schedule: + - cron: '0 1 * * 6' + +# Stop the workflow if a new one is started +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test_suite: + name: 'Test suite' + runs-on: ubuntu-latest + container: + # Use pygem:latest for master branch and pygem:dev otherwise + image: ghcr.io/pygem-community/pygem:${{ github.ref == 'refs/heads/master' && 'latest' || 'dev' }} + options: --user root + env: + # Since we are root we need to set PYTHONPATH to be able to find the installed packages + PYTHONPATH: /home/ubuntu/.local/lib/python3.12/site-packages + + steps: + - name: 'Checkout the PyGEM repo' + id: checkout + uses: actions/checkout@v4 + + - name: 'Reinstall PyGEM' + run: pip install --break-system-packages -e . + + - name: 'Initialize PyGEM' + run: initialize + + - name: 'Clone the PyGEM-notebooks repo' + run: | + # Use PyGEM-notebook:main for master branch and PyGEM-notebooks:dev otherwise + BRANCH=${GITHUB_REF#refs/heads/} + git clone --depth 1 --branch $([[ "$BRANCH" == "master" ]] && echo "main" || echo "dev") \ + https://github.com/pygem-community/PyGEM-notebooks.git + echo "PYGEM_NOTEBOOKS_DIRPATH=$(pwd)/PyGEM-notebooks" >> $GITHUB_ENV + + - name: 'Run tests' + run: | + python3 -m coverage erase + python3 -m pytest --cov=pygem -v --durations=20 pygem/tests diff --git a/README.md b/README.md index 44f3bf85..94efdbc4 100755 --- a/README.md +++ b/README.md @@ -1,53 +1,10 @@ ## Python Glacier Evolution Model (PyGEM) -Overview: Python Glacier Evolution Model (PyGEM) is an open-source glacier evolution model coded in Python that models the transient evolution of glaciers. Each glacier is modeled independently using a monthly timestep. PyGEM has a modular framework that allows different schemes to be used for model calibration or model physics (e.g., climatic mass balance, glacier dynamics). +The Python Glacier Evolution Model (PyGEM) is an open-source glacier evolution model coded in Python that models the transient evolution of glaciers. Each glacier is modeled independently using a monthly timestep. PyGEM has a modular framework that allows different schemes to be used for model calibration or model physics (e.g., climatic mass balance, glacier dynamics). -Manual: Details concerning the model physics, installation, and running the model may be found [here](https://pygem.readthedocs.io/en/latest/). - -Usage: PyGEM is meant for large-scale glacier evolution modeling. PyGEM<1.0.0 are no longer being actively being supported. - -*** - -### Installation -PyGEM can be downloaded from the Python Package Index ([PyPI](https://pypi.org/project/pygem/)). We recommend creating a dedicated [Anaconda](https://anaconda.org/) environment to house PyGEM. -``` -conda create --name python=3.12 -conda activate -pip install pygem -``` -This will install all PyGEM dependencies within your conda environment, and set up PyGEM command line tools to run core model scripts. - -*** - -### Setup -Following installation, an initialization script should to be executed. - -The initialization script accomplishes two things: -1. Initializes the PyGEM configuration file *~/PyGEM/config.yaml*. If this file already exists, an overwrite prompt will appear. -2. Downloads and unzips a set of sample data files to *~/PyGEM/*, which can also be manually downloaded [here](https://drive.google.com/file/d/1Wu4ZqpOKxnc4EYhcRHQbwGq95FoOxMfZ/view?usp=drive_link). - -Run the initialization script by entering the following in the terminal: -``` -initialize -``` - -*** - -### Development -Please report any bugs [here](https://github.com/PyGEM-Community/PyGEM/issues). - -If you are interested in contributing to further development of PyGEM, we recommend forking [PyGEM](https://github.com/PyGEM-Community/PyGEM) and then [cloning](https://docs.github.com/en/repositories/creating-and-managing-repositories/cloning-a-repository) onto your local machine. - -Note, if PyGEM was already installed via PyPI, first uninstall: -``` -pip uninstall pygem -```` - -You can then use pip to install your locally cloned fork of PyGEM in 'editable' mode to easily facilitate development like so: -``` -pip install -e /path/to/your/cloned/pygem/fork/ -``` +Details concerning the model installation, physics, and more may be found at [pygem.readthedocs.io](https://pygem.readthedocs.io/en/latest/). +PyGEM versions prior to 1.0.0 are no longer actively supported. *** diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..11b4dd60 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,26 @@ +FROM ubuntu:latest + +# Install system dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + sudo curl vim git tree python3-pip python3-venv python3-dev build-essential \ + && rm -rf /var/lib/apt/lists/* + +# Add non-root user 'ubuntu' to sudo group +RUN usermod -aG sudo ubuntu && \ + echo "ubuntu ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/ubuntu + +# Switch to non-root user +USER ubuntu +WORKDIR /home/ubuntu + +# Add .local/bin to PATH +ENV PATH="/home/ubuntu/.local/bin:${PATH}" + +# What PyGEM branch to clone (either master or dev; see docker_pygem.yml) +ARG PYGEM_BRANCH=master + +RUN git clone --branch ${PYGEM_BRANCH} https://github.com/PyGEM-Community/PyGEM.git && \ + pip install --break-system-packages -e PyGEM + +# Clone the PyGEM notebooks repository, which are used for testing +RUN git clone https://github.com/PyGEM-Community/PyGEM-notebooks.git \ No newline at end of file diff --git a/docs/contributing.md b/docs/contributing.md new file mode 100644 index 00000000..054c356a --- /dev/null +++ b/docs/contributing.md @@ -0,0 +1,58 @@ +(contributing_pygem_target)= +# PyGEM Contribution Guide + +Before contributing to PyGEM, it is recommended that you either clone [PyGEM's GitHub repository](https://github.com/PyGEM-Community/PyGEM) directly, or initiate your own fork (as described [here](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/fork-a-repo)) to then clone. + +If PyGEM was already installed in your conda environment (as outlined [here](install_pygem_target)), it is recommended that you first uninstall: +``` +pip uninstall pygem +``` + +Next, clone PyGEM. This will place the code at your current directory, so you may wish to navigate to a desired location in your terminal before cloning: +``` +git clone https://github.com/PyGEM-Community/PyGEM.git +``` +If you opted to create your own fork, clone using appropriate repo URL: `git clone https://github.com/YOUR-USERNAME/PyGEM.git` + +Navigate to root project directory: +``` +cd PyGEM +``` + +Install PyGEM in 'editable' mode: +``` +pip install -e . +``` + +Installing a package in editable mode creates a symbolic link to your source code directory (*/path/to/your/PyGEM/clone*), rather than copying the package files into the site-packages directory. This allows you to modify the package code without reinstalling it.
+ +## General +- The `dev` branch is the repository's working branch and should almost always be the base branch for Pull Requests (PRs). Exceptions include hotfixes that need to be pushed to the `master` branch immediately, or updates to the `README`. +- Do not push to other people's branches. Instead create a new branch and open a PR that merges your new branch into the branch you want to modify. + +## Issues +- Check whether an issue describing your problem already exists [here](https://github.com/PyGEM-Community/PyGEM/issues). +- Keep issues simple: try to describe only one problem per issue. Open multiple issues or sub-issues when appropriate. +- Label the issue with the appropriate label (e.g., bug, documentation, etc.). +- If you start working on an issue, assign it to yourself. There is no need to ask for permission unless someone is already assigned to it. + +## Pull requests (PRs) +- PRs should be submitted [here](https://github.com/PyGEM-Community/PyGEM/pulls). +- PRs should be linked to issues they address (unless it's a minor fix that doesn't warrant a new issue). Think of Issues like a ticketing system. +- PRs should generally address only one issue. This helps PRs stay shorter, which in turn makes the review process easier. +- Concisely describe what your PR does. Avoid repeating what was already said in the issue. +- Assign the PR to yourself. +- First, open a Draft PR. Then consider: + - Have you finished making changes? + - Have you added tests for all new functionalities you introduced? + - Have all tests passed in the CI? (Check the progress in the Checks tab of the PR.) + + If the answer to all of the above is "yes", mark the PR as "Ready for review" and request a review from an appropriate reviewer. If in doubt of which reviewer to assign, assign [drounce](https://github.com/drounce). +- You will not be able to merge into `master` and `dev` branches without a reviewer's approval. + +### Reviewing PRs and responding to a review +- Reviewers should leave comments on appropriate lines. Then: + - The original author of the PR should address all comments, specifying what was done and in which commit. For example, a short response like "Fixed in [link to commit]." is often sufficient. + - After responding to a reviewer's comment, do not mark it as resolved. + - Once all comments are addressed, request a new review from the same reviewer. The reviewer should then resolve the comments they are satisfied with. +- After approving someone else's PR, do not merge it. Let the original author of the PR merge it when they are ready, as they might notice necessary last-minute changes. diff --git a/docs/dev.md b/docs/dev.md deleted file mode 100644 index f83e8534..00000000 --- a/docs/dev.md +++ /dev/null @@ -1,16 +0,0 @@ -(dev_pygem_target)= -# Development -Are you interested in contributing to the development of PyGEM? If so, we recommend forking the [PyGEM's GitHub repository](https://github.com/PyGEM-Community/PyGEM) and then [cloning the GitHub repository](https://docs.github.com/en/repositories/creating-and-managing-repositories/cloning-a-repository) onto your local machine. - -Note, if PyGEM was already installed via PyPI, first uninstall: -``` -pip uninstall pygem -```` - -You can then use pip to install your locally cloned fork of PyGEM in 'editable' mode like so: -``` -pip install -e /path/to/your/PyGEM/clone -``` - -Installing a package in editable mode (also called development mode) creates a symbolic link to your source code directory (*/path/to/your/PyGEM/clone*), rather than copying the package files into the site-packages directory. This allows you to modify the package code without reinstalling it. Changes to the source code take effect immediately without needing to reinstall the package, thus efficiently facilitating development.

-Pull requests can be made to [PyGEM's GitHub repository](https://github.com/PyGEM-Community/PyGEM). diff --git a/docs/index.rst b/docs/index.rst index 19fa98f2..7fb88910 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -56,7 +56,7 @@ with respect to modeling glacier dynamics and ice thickness inversions. :maxdepth: 1 :caption: Contributing: - dev + contributing citing .. Indices and tables diff --git a/docs/install_pygem.md b/docs/install_pygem.md index 95cfe856..ac1c4cb1 100644 --- a/docs/install_pygem.md +++ b/docs/install_pygem.md @@ -1,30 +1,35 @@ (install_pygem_target)= -# Installing PyGEM -The Python Glacier Evolution Model has been packaged using Poetry and is hosted on the Python Package Index ([PyPI](https://pypi.org/project/pygem/)), such that all dependencies should install seamlessly. It is recommended that users create a [Anaconda](https://anaconda.org/) environment from which to install the model dependencies and core code. +# Installation +The Python Glacier Evolution Model has been packaged using Poetry and is hosted on the Python Package Index ([PyPI](https://pypi.org/project/pygem/)), to ensure that all dependencies install seamlessly. It is recommended that users create a [conda](https://docs.conda.io/projects/conda/en/latest/user-guide/index.html) environment from which to install the model dependencies and core code. If you do not yet have conda installed, see [conda's documentation](https://docs.conda.io/projects/conda/en/latest/user-guide/install) for instructions. -### Setup Conda Environment -Anaconda is a Python dependency management tool. An Anaconda (conda) environment is essentially a directory that contains a specific collection of installed packages. The use of environments reduces issues caused by package dependencies. It is recommended that users first create conda environment from which to install PyGEM and its dependencies (if you do not yet have conda installed, see [conda's documentation](https://docs.conda.io/projects/conda/en/latest/user-guide/install) for instructions). We recommend a conda environment with python >=3.10, <3.13. +Next, choose your preferred PyGEM installation option:
+- [**stable**](stable_install_target): this is the latest version that has been officially released to PyPI, with a fixed version number (e.g. v1.0.1). It is intended for general use. +- [**development**](dev_install_target): this is the development version of PyGEM hosted on [GitHub](https://github.com/PyGEM-Community/PyGEM/tree/dev). It might contain new features and bug fixes, but is also likely to continue to change until a new release is made. This is the recommended option if you want to work with the latest changes to the code. Note, this installation options require [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) software to be installed on your computer. -A new conda environment can be created from the command line such as: -``` -conda create --name python=3.12 -``` +**Copyright note**: PyGEM's installation instructions are modified from that of [OGGM](https://docs.oggm.org/en/stable/installing-oggm.html) -### PyPI installation -Ensure you've activated your PyGEM environment -``` -conda activate +(stable_install_target)= +## Stable install +The simplest **stable** installation method is to use an environment file. Right-click and save PyGEM's recommended environment file from [this link](https://github.com/PyGEM-Community/PyGEM/tree/master/docs/pygem_env.yml). + +From the folder where you saved the file, run `conda env create -f pygem_environment.yml`. +```{note} +By default the environment will be named `pygem`. A different name can be specified in the environment file. ``` -Next, install PyGEM via [PyPI](https://pypi.org/project/pygem/): +(dev_install_target)= +## Development install +Install the [development version](https://github.com/PyGEM-Community/PyGEM/tree/dev) of PyGEM in your conda environment using pip: ``` -pip install pygem +pip uninstall pygem +pip install git+https://github.com/PyGEM-Community/pygem/@dev ``` -This will install all PyGEM dependencies within your conda environment, and set up PyGEM command line tools to run core model scripts. +If you intend to access and make your own edits to the model's source code, see the [contribution guide](contributing_pygem_target). -### Setup -Following installation, an initialization script should to be executed. +(setup_target)= +# Setup +Following installation, an initialization script should be executed. The initialization script accomplishes two things: 1. Initializes the PyGEM configuration file *~/PyGEM/config.yaml*. If this file already exists, an overwrite prompt will appear. @@ -35,5 +40,5 @@ Run the initialization script by entering the following in the terminal: initialize ``` -### Demonstration Notebooks -A series of accompanying Jupyter notebooks have been produces for demonstrating the functionality of PyGEM. These can be acquired and installed from [GitHub](https://github.com/PyGEM-Community/PyGEM-notebooks). \ No newline at end of file +# Demonstration Notebooks +A series of accompanying Jupyter notebooks has been produced for demonstrating the functionality of PyGEM. These are hosted in the [PyGEM-notebooks repository](https://github.com/PyGEM-Community/PyGEM-notebooks). \ No newline at end of file diff --git a/docs/pygem_environment.yml b/docs/pygem_environment.yml new file mode 100644 index 00000000..67d54697 --- /dev/null +++ b/docs/pygem_environment.yml @@ -0,0 +1,6 @@ +name: pygem +dependencies: + - python>=3.10,<3.13 + - pip + - pip: + - pygem \ No newline at end of file diff --git a/pygem/bin/op/duplicate_gdirs.py b/pygem/bin/op/duplicate_gdirs.py index 016f8c29..a84a2f4b 100644 --- a/pygem/bin/op/duplicate_gdirs.py +++ b/pygem/bin/op/duplicate_gdirs.py @@ -11,11 +11,11 @@ import os import shutil # pygem imports -import pygem.setup.config as config -# check for config -config.ensure_config() +from pygem.setup.config import ConfigManager +# instantiate ConfigManager +config_manager = ConfigManager() # read the config -pygem_prms = config.read_config() +pygem_prms = config_manager.read_config() def main(): parser = argparse.ArgumentParser(description="Script to make duplicate oggm glacier directories - primarily to avoid corruption if parellelizing runs on a single glacier") diff --git a/pygem/bin/op/initialize.py b/pygem/bin/op/initialize.py index 21685a20..ead82679 100644 --- a/pygem/bin/op/initialize.py +++ b/pygem/bin/op/initialize.py @@ -11,25 +11,11 @@ import zipfile import os,sys import shutil -from ruamel.yaml import YAML -# set up config.yaml -import pygem.setup.config as config -config.ensure_config(overwrite=True) - -def update_config_root(conf_path, datapath): - yaml = YAML() - yaml.preserve_quotes = True # Preserve quotes around string values - - # Read the YAML file - with open(conf_path, 'r') as file: - config = yaml.load(file) - - # Update the key with the new value - config['root'] = datapath - - # Save the updated configuration back to the file - with open(conf_path, 'w') as file: - yaml.dump(config, file) +from pygem.setup.config import ConfigManager +# instantiate ConfigManager +config_manager = ConfigManager(overwrite=True) +# read the config +pygem_prms = config_manager.read_config() def print_file_tree(start_path, indent=""): # Loop through all files and directories in the current directory @@ -136,7 +122,7 @@ def main(): # update root path in config.yaml try: - update_config_root(config.config_file, out+'/sample_data/') + config_manager.update_config(updates={'root':f'{out}/sample_data'}) except: pass diff --git a/pygem/bin/op/list_failed_simulations.py b/pygem/bin/op/list_failed_simulations.py index 66f83c15..707b1f12 100644 --- a/pygem/bin/op/list_failed_simulations.py +++ b/pygem/bin/op/list_failed_simulations.py @@ -15,11 +15,11 @@ import argparse import numpy as np # pygem imports -import pygem.setup.config as config -# check for config -config.ensure_config() +from pygem.setup.config import ConfigManager +# instantiate ConfigManager +config_manager = ConfigManager() # read the config -pygem_prms = config.read_config() +pygem_prms = config_manager.read_config() import pygem.pygem_modelsetup as modelsetup def run(reg, simpath, gcm, scenario, calib_opt, bias_adj, gcm_startyear, gcm_endyear): diff --git a/pygem/bin/postproc/postproc_binned_monthly_mass.py b/pygem/bin/postproc/postproc_binned_monthly_mass.py index 3fea6ba1..c72961ff 100644 --- a/pygem/bin/postproc/postproc_binned_monthly_mass.py +++ b/pygem/bin/postproc/postproc_binned_monthly_mass.py @@ -21,9 +21,11 @@ import numpy as np import xarray as xr # pygem imports -import pygem.setup.config as config -# read config -pygem_prms = config.read_config() +from pygem.setup.config import ConfigManager +# instantiate ConfigManager +config_manager = ConfigManager() +# read the config +pygem_prms = config_manager.read_config() # ----- FUNCTIONS ----- def getparser(): diff --git a/pygem/bin/postproc/postproc_compile_simulations.py b/pygem/bin/postproc/postproc_compile_simulations.py index 2f097dca..8b998626 100644 --- a/pygem/bin/postproc/postproc_compile_simulations.py +++ b/pygem/bin/postproc/postproc_compile_simulations.py @@ -21,11 +21,11 @@ # pygem imports import pygem -import pygem.setup.config as config -# check for config -config.ensure_config() +from pygem.setup.config import ConfigManager +# instantiate ConfigManager +config_manager = ConfigManager() # read the config -pygem_prms = config.read_config() +pygem_prms = config_manager.read_config() import pygem.pygem_modelsetup as modelsetup rgi_reg_dict = {'all':'Global', diff --git a/pygem/bin/postproc/postproc_distribute_ice.py b/pygem/bin/postproc/postproc_distribute_ice.py index 892c335a..cf350f8f 100644 --- a/pygem/bin/postproc/postproc_distribute_ice.py +++ b/pygem/bin/postproc/postproc_distribute_ice.py @@ -24,9 +24,11 @@ from oggm import workflow, tasks, cfg from oggm.sandbox import distribute_2d # pygem imports -import pygem.setup.config as config -# read config -pygem_prms = config.read_config() +from pygem.setup.config import ConfigManager +# instantiate ConfigManager +config_manager = ConfigManager() +# read the config +pygem_prms = config_manager.read_config() import pygem import pygem.pygem_modelsetup as modelsetup from pygem.oggm_compat import single_flowline_glacier_directory diff --git a/pygem/bin/postproc/postproc_monthly_mass.py b/pygem/bin/postproc/postproc_monthly_mass.py index ea18f641..972bb570 100644 --- a/pygem/bin/postproc/postproc_monthly_mass.py +++ b/pygem/bin/postproc/postproc_monthly_mass.py @@ -25,9 +25,11 @@ import xarray as xr # pygem imports import pygem -import pygem.setup.config as config -# read config -pygem_prms = config.read_config() +from pygem.setup.config import ConfigManager +# instantiate ConfigManager +config_manager = ConfigManager() +# read the config +pygem_prms = config_manager.read_config() import pygem.pygem_modelsetup as modelsetup diff --git a/pygem/bin/preproc/preproc_fetch_mbdata.py b/pygem/bin/preproc/preproc_fetch_mbdata.py index d25cf9ef..c2282c22 100644 --- a/pygem/bin/preproc/preproc_fetch_mbdata.py +++ b/pygem/bin/preproc/preproc_fetch_mbdata.py @@ -19,11 +19,11 @@ # oggm from oggm import utils # pygem imports -import pygem.setup.config as config -# check for config -config.ensure_config() +from pygem.setup.config import ConfigManager +# instantiate ConfigManager +config_manager = ConfigManager() # read the config -pygem_prms = config.read_config() +pygem_prms = config_manager.read_config() import pygem.pygem_modelsetup as modelsetup diff --git a/pygem/bin/preproc/preproc_wgms_estimate_kp.py b/pygem/bin/preproc/preproc_wgms_estimate_kp.py index 684258a7..0cabd84c 100644 --- a/pygem/bin/preproc/preproc_wgms_estimate_kp.py +++ b/pygem/bin/preproc/preproc_wgms_estimate_kp.py @@ -22,11 +22,11 @@ from scipy.stats import median_abs_deviation # pygem imports from pygem import class_climate -import pygem.setup.config as config -# check for config -config.ensure_config() +from pygem.setup.config import ConfigManager +# instantiate ConfigManager +config_manager = ConfigManager() # read the config -pygem_prms = config.read_config() +pygem_prms = config_manager.read_config() import pygem.pygem_modelsetup as modelsetup diff --git a/pygem/bin/run/run_calibration.py b/pygem/bin/run/run_calibration.py index 424780ad..d78378bb 100755 --- a/pygem/bin/run/run_calibration.py +++ b/pygem/bin/run/run_calibration.py @@ -31,11 +31,11 @@ import sklearn.model_selection # pygem imports -import pygem.setup.config as config -# check for config -config.ensure_config() +from pygem.setup.config import ConfigManager +# instantiate ConfigManager +config_manager = ConfigManager() # read the config -pygem_prms = config.read_config() +pygem_prms = config_manager.read_config() from pygem import mcmc from pygem import class_climate from pygem.massbalance import PyGEMMassBalance diff --git a/pygem/bin/run/run_calibration_frontalablation.py b/pygem/bin/run/run_calibration_frontalablation.py index cd8b0c3d..6b3a5fb3 100644 --- a/pygem/bin/run/run_calibration_frontalablation.py +++ b/pygem/bin/run/run_calibration_frontalablation.py @@ -24,11 +24,11 @@ from scipy.stats import linregress import xarray as xr # pygem imports -import pygem.setup.config as config -# check for config -config.ensure_config() +from pygem.setup.config import ConfigManager +# instantiate ConfigManager +config_manager = ConfigManager() # read the config -pygem_prms = config.read_config() +pygem_prms = config_manager.read_config() import pygem.pygem_modelsetup as modelsetup from pygem.massbalance import PyGEMMassBalance from pygem.glacierdynamics import MassRedistributionCurveModel diff --git a/pygem/bin/run/run_calibration_reg_glena.py b/pygem/bin/run/run_calibration_reg_glena.py index fb0d1bb5..18e5ca16 100644 --- a/pygem/bin/run/run_calibration_reg_glena.py +++ b/pygem/bin/run/run_calibration_reg_glena.py @@ -22,11 +22,11 @@ from scipy.optimize import brentq # pygem imports import pygem -import pygem.setup.config as config -# Check for config -config.ensure_config() # This will ensure the config file is created -# Read the config -pygem_prms = config.read_config() # This reads the configuration file +from pygem.setup.config import ConfigManager +# instantiate ConfigManager +config_manager = ConfigManager() +# read the config +pygem_prms = config_manager.read_config() from pygem import class_climate from pygem.massbalance import PyGEMMassBalance from pygem.oggm_compat import single_flowline_glacier_directory diff --git a/pygem/bin/run/run_mcmc_priors.py b/pygem/bin/run/run_mcmc_priors.py index b196a7f1..04fc088f 100644 --- a/pygem/bin/run/run_mcmc_priors.py +++ b/pygem/bin/run/run_mcmc_priors.py @@ -13,11 +13,11 @@ from scipy import stats # pygem imports -import pygem.setup.config as config -# check for config -config.ensure_config() +from pygem.setup.config import ConfigManager +# instantiate ConfigManager +config_manager = ConfigManager() # read the config -pygem_prms = config.read_config() +pygem_prms = config_manager.read_config() import pygem.pygem_modelsetup as modelsetup # Region dictionary for titles diff --git a/pygem/bin/run/run_simulation.py b/pygem/bin/run/run_simulation.py index fb6069c7..3436797c 100755 --- a/pygem/bin/run/run_simulation.py +++ b/pygem/bin/run/run_simulation.py @@ -34,12 +34,11 @@ import xarray as xr # pygem imports -import pygem -import pygem.setup.config as config -# check for config -config.ensure_config() +from pygem.setup.config import ConfigManager +# instantiate ConfigManager +config_manager = ConfigManager() # read the config -pygem_prms = config.read_config() +pygem_prms = config_manager.read_config() import pygem.gcmbiasadj as gcmbiasadj import pygem.pygem_modelsetup as modelsetup from pygem.massbalance import PyGEMMassBalance @@ -162,6 +161,8 @@ def getparser(): help='number of simulations (note, defaults to 1 if `option_calibration` != `MCMC`)') parser.add_argument('-modelprms_fp', action='store', type=str, default=None, help='model parameters filepath') + parser.add_argument('-outputfn_sfix', action='store', type=str, default='', + help='append custom filename suffix to simulation output') # flags parser.add_argument('-export_all_simiters', action='store_true', help='Flag to export data from all simulations', default=pygem_prms['sim']['out']['export_all_simiters']) @@ -1057,7 +1058,6 @@ def run(list_packed_vars): output_stats = output.glacierwide_stats(glacier_rgi_table=glacier_rgi_table, dates_table=dates_table, nsims=1, - pygem_version=pygem.__version__, gcm_name = gcm_name, scenario = scenario, realization=realization, @@ -1100,13 +1100,13 @@ def run(list_packed_vars): output_ds_all_stats['offglac_snowpack_monthly'].values[0,:] = output_offglac_snowpack_monthly[:,n_iter] # export glacierwide stats for iteration - output_stats.save_xr_ds(output_stats.get_fn().replace('SETS',f'set{n_iter}') + 'all.nc') + output_stats.set_fn(output_stats.get_fn().replace('SETS',f'set{n_iter}') + args.outputfn_sfix + 'all.nc') + output_stats.save_xr_ds() # instantiate dataset for merged simulations output_stats = output.glacierwide_stats(glacier_rgi_table=glacier_rgi_table, dates_table=dates_table, nsims=nsims, - pygem_version=pygem.__version__, gcm_name = gcm_name, scenario = scenario, realization=realization, @@ -1195,7 +1195,8 @@ def run(list_packed_vars): output_ds_all_stats['offglac_snowpack_monthly_mad'].values[0,:] = output_offglac_snowpack_monthly_stats[:,1] # export merged netcdf glacierwide stats - output_stats.save_xr_ds(output_stats.get_fn().replace('SETS',f'{nsims}sets') + 'all.nc') + output_stats.set_fn(output_stats.get_fn().replace('SETS',f'{nsims}sets') + args.outputfn_sfix + 'all.nc') + output_stats.save_xr_ds() # ----- DECADAL ICE THICKNESS STATS FOR OVERDEEPENINGS ----- if args.export_binned_data and glacier_rgi_table.Area > pygem_prms['sim']['out']['export_binned_area_threshold']: @@ -1210,7 +1211,6 @@ def run(list_packed_vars): nsims=1, nbins = surface_h_initial.shape[0], binned_components = args.export_binned_components, - pygem_version=pygem.__version__, gcm_name = gcm_name, scenario = scenario, realization=realization, @@ -1241,7 +1241,8 @@ def run(list_packed_vars): output_ds_binned_stats['bin_refreeze_monthly'].values[0,:,:] = output_glac_bin_refreeze_monthly[:,:,n_iter] # export binned stats for iteration - output_binned.save_xr_ds(output_binned.get_fn().replace('SETS',f'set{n_iter}') + 'binned.nc') + output_binned.set_fn(output_binned.get_fn().replace('SETS',f'set{n_iter}') + args.outputfn_sfix + 'binned.nc') + output_binned.save_xr_ds() # instantiate dataset for merged simulations output_binned = output.binned_stats(glacier_rgi_table=glacier_rgi_table, @@ -1249,7 +1250,6 @@ def run(list_packed_vars): nsims=nsims, nbins = surface_h_initial.shape[0], binned_components = args.export_binned_components, - pygem_version=pygem.__version__, gcm_name = gcm_name, scenario = scenario, realization=realization, @@ -1293,7 +1293,8 @@ def run(list_packed_vars): median_abs_deviation(output_glac_bin_massbalclim_annual, axis=2)[np.newaxis,:,:]) # export merged netcdf glacierwide stats - output_binned.save_xr_ds(output_binned.get_fn().replace('SETS',f'{nsims}sets') + 'binned.nc') + output_binned.set_fn(output_binned.get_fn().replace('SETS',f'{nsims}sets') + args.outputfn_sfix + 'binned.nc') + output_binned.save_xr_ds() except Exception as err: # LOG FAILURE diff --git a/pygem/class_climate.py b/pygem/class_climate.py index 10c28935..bd701d39 100755 --- a/pygem/class_climate.py +++ b/pygem/class_climate.py @@ -12,10 +12,11 @@ class of climate data and functions associated with manipulating the dataset to import pandas as pd import numpy as np import xarray as xr -# Local libraries -import pygem.setup.config as config -# Read the config -pygem_prms = config.read_config() # This reads the configuration file +from pygem.setup.config import ConfigManager +# instantiate ConfigManager +config_manager = ConfigManager() +# read the config +pygem_prms = config_manager.read_config() class GCM(): """ diff --git a/pygem/gcmbiasadj.py b/pygem/gcmbiasadj.py index 52ebce0f..e3cc8e81 100755 --- a/pygem/gcmbiasadj.py +++ b/pygem/gcmbiasadj.py @@ -16,10 +16,11 @@ import numpy as np from scipy.ndimage import uniform_filter from scipy.stats import percentileofscore -# load pygem config -import pygem.setup.config as config -# Read the config -pygem_prms = config.read_config() +from pygem.setup.config import ConfigManager +# instantiate ConfigManager +config_manager = ConfigManager() +# read the config +pygem_prms = config_manager.read_config() #%% FUNCTIONS def annual_avg_2darray(x): diff --git a/pygem/glacierdynamics.py b/pygem/glacierdynamics.py index 6b6b024f..03c22a26 100755 --- a/pygem/glacierdynamics.py +++ b/pygem/glacierdynamics.py @@ -17,10 +17,11 @@ from oggm.core.flowline import FlowlineModel from oggm.exceptions import InvalidParamsError from oggm import __version__ -# Local libraries -import pygem.setup.config as config -# Read the config -pygem_prms = config.read_config() # This reads the configuration file +from pygem.setup.config import ConfigManager +# instantiate ConfigManager +config_manager = ConfigManager() +# read the config +pygem_prms = config_manager.read_config() cfg.initialize() diff --git a/pygem/massbalance.py b/pygem/massbalance.py index 15532c2a..2b143162 100644 --- a/pygem/massbalance.py +++ b/pygem/massbalance.py @@ -10,10 +10,11 @@ # Local libraries from oggm.core.massbalance import MassBalanceModel from pygem.utils._funcs import annualweightedmean_array -# Local libraries -import pygem.setup.config as config -# Read the config -pygem_prms = config.read_config() # This reads the configuration file +from pygem.setup.config import ConfigManager +# instantiate ConfigManager +config_manager = ConfigManager() +# read the config +pygem_prms = config_manager.read_config() #%% class PyGEMMassBalance(MassBalanceModel): diff --git a/pygem/mcmc.py b/pygem/mcmc.py index 98420d0f..1f41c27c 100644 --- a/pygem/mcmc.py +++ b/pygem/mcmc.py @@ -14,10 +14,11 @@ from tqdm import tqdm import matplotlib.pyplot as plt import matplotlib.cm as cm -# Local libraries -import pygem.setup.config as config -# Read the config -pygem_prms = config.read_config() # This reads the configuration file +from pygem.setup.config import ConfigManager +# instantiate ConfigManager +config_manager = ConfigManager() +# read the config +pygem_prms = config_manager.read_config() torch.set_default_dtype(torch.float64) plt.rcParams["font.family"] = "arial" diff --git a/pygem/oggm_compat.py b/pygem/oggm_compat.py index b3bebdb4..7a98e844 100755 --- a/pygem/oggm_compat.py +++ b/pygem/oggm_compat.py @@ -20,10 +20,11 @@ from oggm.core.massbalance import MassBalanceModel #from oggm.shop import rgitopo from pygem.shop import debris, mbdata, icethickness -# Local libraries -import pygem.setup.config as config -# read config -pygem_prms = config.read_config() +from pygem.setup.config import ConfigManager +# instantiate ConfigManager +config_manager = ConfigManager() +# read the config +pygem_prms = config_manager.read_config() class CompatGlacDir: diff --git a/pygem/output.py b/pygem/output.py index a04e7ec9..579947dd 100644 --- a/pygem/output.py +++ b/pygem/output.py @@ -18,19 +18,53 @@ import pandas as pd import xarray as xr import os, types, json, cftime, collections -import pygem.setup.config as config -# Read the config -pygem_prms = config.read_config() # This reads the configuration file +import pygem +from pygem.setup.config import ConfigManager +# instantiate ConfigManager +config_manager = ConfigManager() +# read the config +pygem_prms = config_manager.read_config() + +__all__ = ["single_glacier", "glacierwide_stats", "binned_stats", "set_fn", "get_fn", + "set_modelprms", "create_xr_ds", "get_xr_ds", "save_xr_ds", "calc_stats_array"] -### single glacier output parent class ### @dataclass class single_glacier: """ Single glacier output dataset class for the Python Glacier Evolution Model. + This serves as the parent class to both `output.glacierwide_stats` and `output.binned_stats`. + + Attributes + ---------- + glacier_rgi_table : pd.DataFrame + DataFrame containing metadata and characteristics of the glacier from the Randolph Glacier Inventory. + dates_table : pd.DataFrame + DataFrame containing the time series of dates associated with the model output. + gcm_name : str + Name of the General Circulation Model (GCM) used for climate forcing. + scenario : str + Emission or climate scenario under which the simulation is run. + realization : str + Specific realization or ensemble member of the GCM simulation. + nsims : int + Number of simulation runs performed. + modelprms : dict + Dictionary containing model parameters used in the simulation. + ref_startyear : int + Start year of the reference period for model calibration or comparison. + ref_endyear : int + End year of the reference period for model calibration or comparison. + gcm_startyear : int + Start year of the GCM forcing data used in the simulation. + gcm_endyear : int + End year of the GCM forcing data used in the simulation. + option_calibration : str + Model calibration method. + option_bias_adjustment : int + Bias adjustment method applied to the climate input data """ glacier_rgi_table : pd.DataFrame dates_table : pd.DataFrame - pygem_version : float gcm_name : str scenario : str realization : str @@ -44,47 +78,77 @@ class single_glacier: option_bias_adjustment: str def __post_init__(self): + """ + Initializes additional attributes after the dataclass fields are set. + + This method: + - Retrieves and stores the PyGEM version. + - Extracts field names in RGI glacier table. + - Formats the glacier RGI ID as a string with five decimal places. + - Extracts and zero-pads the primary region code from the RGI table. + - Defines the output directory path for storing simulation results. + - Calls setup functions to initialize and store filenames, time values, model parameters, and dictionaries. + """ + self.pygem_version = pygem.__version__ self.glac_values = np.array([self.glacier_rgi_table.name]) self.glacier_str = '{0:0.5f}'.format(self.glacier_rgi_table['RGIId_float']) self.reg_str = str(self.glacier_rgi_table.O1Region).zfill(2) self.outdir = pygem_prms['root'] + '/Output/simulations/' self.set_fn() - self.set_time_vals() - self.model_params_record() - self.init_dicts() - - # set output dataset filename - def set_fn(self): - self.outfn = self.glacier_str + '_' + self.gcm_name + '_' - if self.scenario: - self.outfn += f'{self.scenario}_' - if self.realization: - self.outfn += f'{self.realization}_' - if self.option_calibration: - self.outfn += f'{self.option_calibration}_' - else: - self.outfn += f'kp{self.modelprms["kp"]}_ddfsnow{self.modelprms["ddfsnow"]}_tbias{self.modelprms["tbias"]}_' - if self.gcm_name not in ['ERA-Interim', 'ERA5', 'COAWST']: - self.outfn += f'ba{self.option_bias_adjustment}_' + self._set_time_vals() + self._model_params_record() + self._init_dicts() + + def set_fn(self, outfn=None): + """Set the dataset output file name. + Parameters + ---------- + outfn : str + Output filename string. + """ + if outfn: + self.outfn = outfn else: - self.outfn += 'ba0_' - if self.option_calibration: - self.outfn += 'SETS_' - self.outfn += f'{self.gcm_startyear}_' - self.outfn += f'{self.gcm_endyear}_' + self.outfn = self.glacier_str + '_' + self.gcm_name + '_' + if self.scenario: + self.outfn += f'{self.scenario}_' + if self.realization: + self.outfn += f'{self.realization}_' + if self.option_calibration: + self.outfn += f'{self.option_calibration}_' + else: + self.outfn += f'kp{self.modelprms["kp"]}_ddfsnow{self.modelprms["ddfsnow"]}_tbias{self.modelprms["tbias"]}_' + if self.gcm_name not in ['ERA-Interim', 'ERA5', 'COAWST']: + self.outfn += f'ba{self.option_bias_adjustment}_' + else: + self.outfn += 'ba0_' + if self.option_calibration: + self.outfn += 'SETS_' + self.outfn += f'{self.gcm_startyear}_' + self.outfn += f'{self.gcm_endyear}_' - # return output dataset filename def get_fn(self): + """Return the output dataset filename.""" return self.outfn - # set modelprms def set_modelprms(self, modelprms): + """ + Set the model parameters and update the dataset record. + + Parameters + ---------- + modelprms : dict + Dictionary containing model parameters used in the simulation. + + This method updates the `modelprms` attribute with the provided dictionary and esnures that + the model parameter record is updated accordingly by calling `self._update_modelparams_record()`. + """ self.modelprms = modelprms # update model_params_record - self.update_modelparams_record() + self._update_modelparams_record() - # set dataset time value coordiantes - def set_time_vals(self): + def _set_time_vals(self): + """Set output dataset time and year values from dates_table.""" if pygem_prms['climate']['gcm_wateryear'] == 'hydro': self.year_type = 'water year' self.annual_columns = np.unique(self.dates_table['wateryear'].values)[0:int(self.dates_table.shape[0]/12)] @@ -99,8 +163,8 @@ def set_time_vals(self): self.year_values = self.annual_columns[pygem_prms['climate']['gcm_spinupyears']:self.annual_columns.shape[0]] self.year_values = np.concatenate((self.year_values, np.array([self.annual_columns[-1] + 1]))) - # record all model parameters from run_simualtion and pygem_input - def model_params_record(self): + def _model_params_record(self): + """Build model parameters attribute dictionary to be saved to output dataset.""" # get all locally defined variables from the pygem_prms, excluding imports, functions, and classes self.mdl_params_dict = {} # overwrite variables that are possibly different from pygem_input @@ -115,15 +179,15 @@ def model_params_record(self): self.mdl_params_dict['option_bias_adjustment'] = self.option_bias_adjustment # record manually defined modelprms if calibration option is None if not self.option_calibration: - self.update_modelparams_record() + self._update_modelparams_record() - # update model_params_record - def update_modelparams_record(self): + def _update_modelparams_record(self): + """Update the values in the output dataset's model parameters dictionary.""" for key, value in self.modelprms.items(): self.mdl_params_dict[key] = value - # initialize boilerplate coordinate and attribute dictionaries - these will be the same for both glacier-wide and binned outputs - def init_dicts(self): + def _init_dicts(self): + """Initialize output coordinate and attribute dictionaries.""" self.output_coords_dict = collections.OrderedDict() self.output_coords_dict['RGIId'] = collections.OrderedDict([('glac', self.glac_values)]) self.output_coords_dict['CenLon'] = collections.OrderedDict([('glac', self.glac_values)]) @@ -166,8 +230,8 @@ def init_dicts(self): 'comment': 'value from RGIv6.0'} } - # create dataset def create_xr_ds(self): + """Create an xarrray dataset with placeholders for data arrays.""" # Add variables to empty dataset and merge together count_vn = 0 self.encoding = {} @@ -208,14 +272,14 @@ def create_xr_ds(self): 'references': 'doi:10.1126/science.abo1324', 'model_parameters':json.dumps(self.mdl_params_dict)} - # return dataset def get_xr_ds(self): + """Return the xarray dataset.""" return self.output_xr_ds - # save dataset - def save_xr_ds(self, netcdf_fn): + def save_xr_ds(self): + """Save the xarray dataset.""" # export netcdf - self.output_xr_ds.to_netcdf(self.outdir + netcdf_fn, encoding=self.encoding) + self.output_xr_ds.to_netcdf(self.outdir + self.outfn, encoding=self.encoding) # close datasets self.output_xr_ds.close() @@ -223,16 +287,27 @@ def save_xr_ds(self, netcdf_fn): @dataclass class glacierwide_stats(single_glacier): """ - Single glacier-wide statistics dataset + Single glacier-wide statistics dataset. + + This class extends `single_glacier` to store and manage glacier-wide statistical outputs. """ def __post_init__(self): - super().__post_init__() # call parent class __post_init__ (get glacier values, time stamps, and instantiate output dictionaries that will form netcdf file output) - self.set_outdir() - self.update_dicts() # add required fields to output dictionary - - # set output directory - def set_outdir(self): + """ + Initializes additional attributes after the dataclass fields are set. + + This method: + - Calls the parent class `__post_init__` to initialize glacier values, + time stamps, and instantiate output dataset dictionarie. + - Sets the output directory specific to glacier-wide statistics. + - Updates the output dictionaries with required fields. + """ + super().__post_init__() + self._set_outdir() + self._update_dicts() + + def _set_outdir(self): + """Set the output directory path. Create if it does not already exist.""" self.outdir += self.reg_str + '/' + self.gcm_name + '/' if self.gcm_name not in ['ERA-Interim', 'ERA5', 'COAWST']: self.outdir += self.scenario + '/' @@ -240,8 +315,8 @@ def set_outdir(self): # Create filepath if it does not exist os.makedirs(self.outdir, exist_ok=True) - # update coordinate and attribute dictionaries - def update_dicts(self): + def _update_dicts(self): + """Update coordinate and attribute dictionaries specific to glacierwide_stats outputs""" self.output_coords_dict['glac_runoff_monthly'] = collections.OrderedDict([('glac', self.glac_values), ('time', self.time_values)]) self.output_attrs_dict['glac_runoff_monthly'] = { @@ -284,6 +359,8 @@ def update_dicts(self): 'units': 'm3', 'temporal_resolution': 'monthly', 'comment': 'off-glacier runoff from area where glacier no longer exists'} + + # if nsims > 1, store median-absolute deviation metrics if self.nsims > 1: self.output_coords_dict['glac_runoff_monthly_mad'] = collections.OrderedDict([('glac', self.glac_values), ('time', self.time_values)]) @@ -327,7 +404,8 @@ def update_dicts(self): 'units': 'm3', 'temporal_resolution': 'monthly', 'comment': 'off-glacier runoff from area where glacier no longer exists'} - + + # optionally store extra variables if pygem_prms['sim']['out']['export_extra_vars']: self.output_coords_dict['glac_prec_monthly'] = collections.OrderedDict([('glac', self.glac_values), ('time', self.time_values)]) @@ -422,6 +500,7 @@ def update_dicts(self): 'temporal_resolution': 'monthly', 'comment': 'snow remaining accounting for new accumulation, melt, and refreeze'} + # if nsims > 1, store median-absolute deviation metrics if self.nsims > 1: self.output_coords_dict['glac_prec_monthly_mad'] = collections.OrderedDict([('glac', self.glac_values), ('time', self.time_values)]) @@ -521,19 +600,39 @@ def update_dicts(self): @dataclass class binned_stats(single_glacier): """ - Single glacier binned dataset - """ + Single glacier binned dataset. + + This class extends `single_glacier` to store and manage binned glacier output data. + + Attributes + ---------- nbins : int + Number of bins used to segment the glacier dataset. binned_components : bool + Flag indicating whether additional binned components are included in the dataset. + """ - def __post_init__(self): - super().__post_init__() # call parent class __post_init__ (get glacier values, time stamps, and instantiate output dictionaries that will form netcdf file output) - self.bin_values = np.arange(self.nbins) # bin indices - self.set_outdir() - self.update_dicts() # add required fields to output dictionary + nbins: int + binned_components: bool - # set output directory - def set_outdir(self): + def __post_init__(self): + """ + Initializes additional attributes after the dataclass fields are set. + + This method: + - Calls the parent class `__post_init__` to initialize glacier values, + time stamps, and instantiate output dataset dictionaries. + - Creates an array of bin indices based on the number of bins. + - Sets the output directory specific to binned statistics. + - Updates the output dictionaries with required fields. + """ + super().__post_init__() + self.bin_values = np.arange(self.nbins) + self._set_outdir() + self._update_dicts() + + def _set_outdir(self): + """Set the output directory path. Create if it does not already exist.""" self.outdir += self.reg_str + '/' + self.gcm_name + '/' if self.gcm_name not in ['ERA-Interim', 'ERA5', 'COAWST']: self.outdir += self.scenario + '/' @@ -541,8 +640,8 @@ def set_outdir(self): # Create filepath if it does not exist os.makedirs(self.outdir, exist_ok=True) - # update coordinate and attribute dictionaries - def update_dicts(self): + def _update_dicts(self): + """Update coordinate and attribute dictionaries specific to glacierwide_stats outputs""" self.output_coords_dict['bin_distance'] = collections.OrderedDict([('glac', self.glac_values), ('bin', self.bin_values)]) self.output_attrs_dict['bin_distance'] = { 'long_name': 'distance downglacier', @@ -587,6 +686,8 @@ def update_dicts(self): 'units': 'm', 'temporal_resolution': 'monthly', 'comment': 'monthly climatic mass balance from the PyGEM mass balance module'} + + # optionally store binned mass balance components if self.binned_components: self.output_coords_dict['bin_accumulation_monthly'] = ( collections.OrderedDict([('glac', self.glac_values), ('bin', self.bin_values), ('time', self.time_values)])) @@ -609,7 +710,8 @@ def update_dicts(self): 'units': 'm', 'temporal_resolution': 'monthly', 'comment': 'monthly refreeze from the PyGEM mass balance module'} - + + # if nsims > 1, store median-absolute deviation metrics if self.nsims > 1: self.output_coords_dict['bin_mass_annual_mad'] = ( collections.OrderedDict([('glac', self.glac_values), ('bin', self.bin_values), ('year', self.year_values)])) @@ -632,39 +734,6 @@ def update_dicts(self): 'units': 'm', 'temporal_resolution': 'annual', 'comment': 'climatic mass balance is computed before dynamics so can theoretically exceed ice thickness'} - - -### compiled regional output parent class ### -@dataclass -class compiled_regional: - """ - Compiled regional output dataset for the Python Glacier Evolution Model. - """ - -@dataclass -class regional_annual_mass(compiled_regional): - """ - compiled regional annual mass - """ - -@dataclass -class regional_annual_area(compiled_regional): - """ - compiled regional annual area - """ - -@dataclass -class regional_monthly_runoff(compiled_regional): - """ - compiled regional monthly runoff - """ - -@dataclass -class regional_monthly_massbal(compiled_regional): - """ - compiled regional monthly climatic mass balance - """ - def calc_stats_array(data, stats_cns=pygem_prms['sim']['out']['sim_stats']): """ diff --git a/pygem/pygem_modelsetup.py b/pygem/pygem_modelsetup.py index a602decc..3182669e 100755 --- a/pygem/pygem_modelsetup.py +++ b/pygem/pygem_modelsetup.py @@ -13,10 +13,12 @@ import pandas as pd import numpy as np from datetime import datetime -# Local libraries -import pygem.setup.config as config -# Read the config -pygem_prms = config.read_config() # This reads the configuration file +from pygem.setup.config import ConfigManager +# instantiate ConfigManager +config_manager = ConfigManager() +# read the config +pygem_prms = config_manager.read_config() + def datesmodelrun(startyear=pygem_prms['climate']['ref_startyear'], endyear=pygem_prms['climate']['ref_endyear'], spinupyears=pygem_prms['climate']['ref_spinupyears'], option_wateryear=pygem_prms['climate']['ref_wateryear']): diff --git a/pygem/setup/config.py b/pygem/setup/config.py index 4645af90..0eeb3869 100644 --- a/pygem/setup/config.py +++ b/pygem/setup/config.py @@ -7,42 +7,355 @@ """ import os import shutil -import sys -import yaml - -# pygem_params file name -config_fn = 'config.yaml' - -# Define the base directory and the path to the configuration file -basedir = os.path.join(os.path.expanduser('~'), 'PyGEM') -config_file = os.path.join(basedir, config_fn) # Path where you want the config file - -# Get the source configuration file path from your package -package_dir = os.path.dirname(__file__) # Get the directory of the current script -source_config_file = os.path.join(package_dir, config_fn) # Path to params.py - -def ensure_config(overwrite=False): - isfile = os.path.isfile(config_file) - if isfile and overwrite: - overwrite = None - while overwrite is None: - # Ask the user for a y/n response - response = input(f"PyGEM configuration.yaml file already exist ({config_file}), do you wish to overwrite (y/n):").strip().lower() - # Check if the response is valid - if response == 'y' or response == 'yes': - overwrite = True - elif response == 'n' or response == 'no': - overwrite = False - else: - print("Invalid input. Please enter 'y' or 'n'.") +import ruamel.yaml + +__all__ = ["ConfigManager"] + +class ConfigManager: + """Manages PyGEMs configuration file, ensuring it exists, reading, updating, and validating its contents.""" + def __init__(self, config_filename='config.yaml', base_dir=None, overwrite=False): + """ + Initialize the ConfigManager class. + + Parameters: + config_filename (str, optional): Name of the configuration file. Defaults to 'config.yaml'. + base_dir (str, optional): Directory where the configuration file is stored. Defaults to '~/PyGEM'. + overwrite (bool, optional): Whether to overwrite an existing configuration file. Defaults to False. + """ + self.config_filename = config_filename + self.base_dir = base_dir or os.path.join(os.path.expanduser('~'), 'PyGEM') + self.config_path = os.path.join(self.base_dir, self.config_filename) + self.source_config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "config.yaml") + self.overwrite = overwrite + self._ensure_config() + + def _ensure_config(self): + """Ensure the configuration file exists, creating or overwriting it if necessary""" + if not os.path.isfile(self.config_path) or self.overwrite: + self._copy_source_config() + + def _copy_source_config(self): + """Copy the default configuration file to the expected location""" + + os.makedirs(self.base_dir, exist_ok=True) + shutil.copy(self.source_config_path, self.config_path) + print(f"Copied default configuration to {self.config_path}") + + def read_config(self, validate=True): + """Read the configuration file and return its contents as a dictionary while preserving formatting. + Parameters: + validate (bool): Whether to validate the configuration file contents. Defaults to True. + """ + ryaml = ruamel.yaml.YAML() + with open(self.config_path, 'r') as f: + user_config = ryaml.load(f) + + if validate: + self._validate_config(user_config) + + return user_config + + def _write_config(self, config): + """Write the configuration dictionary to the file while preserving quotes. + + Parameters: + config (dict): configuration dictionary object + """ + ryaml = ruamel.yaml.YAML() + ryaml.preserve_quotes = True + with open(self.config_path, 'w') as file: + ryaml.dump(config, file) + + def update_config(self, updates): + """Update multiple keys in the YAML configuration file while preserving quotes and original types. + + Parameters: + updates (dict): Key-Value pairs to be updated + """ + config = self.read_config(validate=False) + + for key, value in updates.items(): + if key not in self.EXPECTED_TYPES: + raise KeyError(f"Unrecognized configuration key: {key}") + keys = key.split('.') + d = config + for sub_key in keys[:-1]: + d = d[sub_key] + + d[keys[-1]] = value + + self._validate_config(config) + self._write_config(config) - if (not isfile) or (overwrite): - os.makedirs(basedir, exist_ok=True) # Ensure the base directory exists - shutil.copy(source_config_file, config_file) # Copy the file - print(f"Copied default configuration to {config_file}") + def _validate_config(self, config): + """Validate the configuration dictionary against expected types and required keys. + + Parameters: + config (dict): The configuration dictionary to be validated. + """ + for key, expected_type in self.EXPECTED_TYPES.items(): + keys = key.split(".") + sub_data = config + for sub_key in keys: + if isinstance(sub_data, dict) and sub_key in sub_data: + sub_data = sub_data[sub_key] + else: + raise KeyError(f"Missing required key in configuration: {key}") + + if not isinstance(sub_data, expected_type): + raise TypeError(f"Invalid type for '{key}': expected {expected_type}, not {type(sub_data)}") + + # Check elements inside lists (if defined) + if key in self.LIST_ELEMENT_TYPES and isinstance(sub_data, list): + elem_type = self.LIST_ELEMENT_TYPES[key] + if not all(isinstance(item, elem_type) for item in sub_data): + raise TypeError(f"Invalid type for elements in '{key}': expected all elements to be {elem_type}, but got {sub_data}") -def read_config(): - """Read the configuration file and return its contents as a dictionary.""" - with open(config_file, 'r') as f: - config = yaml.safe_load(f) # Use safe_load to avoid arbitrary code execution - return config \ No newline at end of file + + # expected config types + EXPECTED_TYPES = { + "root": str, + "user": dict, + "user.name": (str, type(None)), + "user.institution": (str, type(None)), + "user.email": (str, type(None)), + "setup": dict, + "setup.rgi_region01": list, + "setup.rgi_region02": str, + "setup.glac_no_skip": (list, type(None)), + "setup.glac_no": (list, type(None)), + "setup.min_glac_area_km2": int, + "setup.include_landterm": bool, + "setup.include_laketerm": bool, + "setup.include_tidewater": bool, + "setup.include_frontalablation": bool, + "oggm": dict, + "oggm.base_url": str, + "oggm.logging_level": str, + "oggm.border": int, + "oggm.oggm_gdir_relpath": str, + "oggm.overwrite_gdirs": bool, + "oggm.has_internet": bool, + "climate": dict, + "climate.ref_gcm_name": str, + "climate.ref_startyear": int, + "climate.ref_endyear": int, + "climate.ref_wateryear": str, + "climate.ref_spinupyears": int, + "climate.gcm_name": str, + "climate.scenario": (str, type(None)), + "climate.gcm_startyear": int, + "climate.gcm_endyear": int, + "climate.gcm_wateryear": str, + "climate.constantarea_years": int, + "climate.gcm_spinupyears": int, + "climate.hindcast": bool, + "climate.paths": dict, + "climate.paths.era5_relpath": str, + "climate.paths.era5_temp_fn": str, + "climate.paths.era5_tempstd_fn": str, + "climate.paths.era5_prec_fn": str, + "climate.paths.era5_elev_fn": str, + "climate.paths.era5_pressureleveltemp_fn": str, + "climate.paths.era5_lr_fn": str, + "climate.paths.cmip5_relpath": str, + "climate.paths.cmip5_fp_var_ending": str, + "climate.paths.cmip5_fp_fx_ending": str, + "climate.paths.cmip6_relpath": str, + "climate.paths.cesm2_relpath": str, + "climate.paths.cesm2_fp_var_ending": str, + "climate.paths.cesm2_fp_fx_ending": str, + "climate.paths.gfdl_relpath": str, + "climate.paths.gfdl_fp_var_ending": str, + "climate.paths.gfdl_fp_fx_ending": str, + "calib": dict, + "calib.option_calibration": str, + "calib.priors_reg_fn": str, + "calib.HH2015_params": dict, + "calib.HH2015_params.tbias_init": int, + "calib.HH2015_params.tbias_step": int, + "calib.HH2015_params.kp_init": float, + "calib.HH2015_params.kp_bndlow": float, + "calib.HH2015_params.kp_bndhigh": int, + "calib.HH2015_params.ddfsnow_init": float, + "calib.HH2015_params.ddfsnow_bndlow": float, + "calib.HH2015_params.ddfsnow_bndhigh": float, + "calib.HH2015mod_params": dict, + "calib.HH2015mod_params.tbias_init": int, + "calib.HH2015mod_params.tbias_step": float, + "calib.HH2015mod_params.kp_init": int, + "calib.HH2015mod_params.kp_bndlow": float, + "calib.HH2015mod_params.kp_bndhigh": int, + "calib.HH2015mod_params.ddfsnow_init": float, + "calib.HH2015mod_params.method_opt": str, + "calib.HH2015mod_params.params2opt": list, + "calib.HH2015mod_params.ftol_opt": float, + "calib.HH2015mod_params.eps_opt": float, + "calib.emulator_params": dict, + "calib.emulator_params.emulator_sims": int, + "calib.emulator_params.overwrite_em_sims": bool, + "calib.emulator_params.opt_hh2015_mod": bool, + "calib.emulator_params.tbias_step": float, + "calib.emulator_params.tbias_init": int, + "calib.emulator_params.kp_init": int, + "calib.emulator_params.kp_bndlow": float, + "calib.emulator_params.kp_bndhigh": int, + "calib.emulator_params.ddfsnow_init": float, + "calib.emulator_params.option_areaconstant": bool, + "calib.emulator_params.tbias_disttype": str, + "calib.emulator_params.tbias_sigma": int, + "calib.emulator_params.kp_gamma_alpha": int, + "calib.emulator_params.kp_gamma_beta": int, + "calib.emulator_params.ddfsnow_disttype": str, + "calib.emulator_params.ddfsnow_mu": float, + "calib.emulator_params.ddfsnow_sigma": float, + "calib.emulator_params.ddfsnow_bndlow": int, + "calib.emulator_params.ddfsnow_bndhigh": float, + "calib.emulator_params.method_opt": str, + "calib.emulator_params.params2opt": list, + "calib.emulator_params.ftol_opt": float, + "calib.emulator_params.eps_opt": float, + "calib.MCMC_params": dict, + "calib.MCMC_params.option_use_emulator": bool, + "calib.MCMC_params.emulator_sims": int, + "calib.MCMC_params.tbias_step": float, + "calib.MCMC_params.tbias_stepsmall": float, + "calib.MCMC_params.option_areaconstant": bool, + "calib.MCMC_params.mcmc_step": float, + "calib.MCMC_params.n_chains": int, + "calib.MCMC_params.mcmc_sample_no": int, + "calib.MCMC_params.mcmc_burn_pct": int, + "calib.MCMC_params.thin_interval": int, + "calib.MCMC_params.ddfsnow_disttype": str, + "calib.MCMC_params.ddfsnow_mu": float, + "calib.MCMC_params.ddfsnow_sigma": float, + "calib.MCMC_params.ddfsnow_bndlow": int, + "calib.MCMC_params.ddfsnow_bndhigh": float, + "calib.MCMC_params.kp_disttype": str, + "calib.MCMC_params.tbias_disttype": str, + "calib.MCMC_params.tbias_mu": int, + "calib.MCMC_params.tbias_sigma": int, + "calib.MCMC_params.tbias_bndlow": int, + "calib.MCMC_params.tbias_bndhigh": int, + "calib.MCMC_params.kp_gamma_alpha": int, + "calib.MCMC_params.kp_gamma_beta": int, + "calib.MCMC_params.kp_lognorm_mu": int, + "calib.MCMC_params.kp_lognorm_tau": int, + "calib.MCMC_params.kp_mu": int, + "calib.MCMC_params.kp_sigma": float, + "calib.MCMC_params.kp_bndlow": float, + "calib.MCMC_params.kp_bndhigh": float, + "calib.data": dict, + "calib.data.massbalance": dict, + "calib.data.massbalance.hugonnet2021_relpath": str, + "calib.data.massbalance.hugonnet2021_fn": str, + "calib.data.massbalance.hugonnet2021_facorrected_fn": str, + "calib.data.frontalablation": dict, + "calib.data.frontalablation.frontalablation_relpath": str, + "calib.data.frontalablation.frontalablation_cal_fn": str, + "calib.data.icethickness": dict, + "calib.data.icethickness.h_consensus_relpath": str, + "calib.icethickness_cal_frac_byarea": float, + "sim": dict, + "sim.option_dynamics": (str, type(None)), + "sim.option_bias_adjustment": int, + "sim.nsims": int, + "sim.out": dict, + "sim.out.sim_stats": list, + "sim.out.export_all_simiters": bool, + "sim.out.export_extra_vars": bool, + "sim.out.export_binned_data": bool, + "sim.out.export_binned_components": bool, + "sim.out.export_binned_area_threshold": int, + "sim.oggm_dynamics": dict, + "sim.oggm_dynamics.cfl_number": float, + "sim.oggm_dynamics.cfl_number_calving": float, + "sim.oggm_dynamics.glena_reg_relpath": str, + "sim.oggm_dynamics.use_reg_glena": bool, + "sim.oggm_dynamics.fs": int, + "sim.oggm_dynamics.glen_a_multiplier": int, + "sim.icethickness_advancethreshold": int, + "sim.terminus_percentage": int, + "sim.params": dict, + "sim.params.use_constant_lapserate": bool, + "sim.params.kp": int, + "sim.params.tbias": int, + "sim.params.ddfsnow": float, + "sim.params.ddfsnow_iceratio": float, + "sim.params.precgrad": float, + "sim.params.lapserate": float, + "sim.params.tsnow_threshold": int, + "sim.params.calving_k": float, + "mb": dict, + "mb.option_surfacetype_initial": int, + "mb.include_firn": bool, + "mb.include_debris": bool, + "mb.debris_relpath": str, + "mb.option_elev_ref_downscale": str, + "mb.option_temp2bins": int, + "mb.option_adjusttemp_surfelev": int, + "mb.option_prec2bins": int, + "mb.option_preclimit": int, + "mb.option_accumulation": int, + "mb.option_ablation": int, + "mb.option_ddf_firn": int, + "mb.option_refreezing": str, + "mb.Woodard_rf_opts": dict, + "mb.Woodard_rf_opts.rf_month": int, + "mb.HH2015_rf_opts": dict, + "mb.HH2015_rf_opts.rf_layers": int, + "mb.HH2015_rf_opts.rf_dz": int, + "mb.HH2015_rf_opts.rf_dsc": int, + "mb.HH2015_rf_opts.rf_meltcrit": float, + "mb.HH2015_rf_opts.pp": float, + "mb.HH2015_rf_opts.rf_dens_top": int, + "mb.HH2015_rf_opts.rf_dens_bot": int, + "mb.HH2015_rf_opts.option_rf_limit_meltsnow": int, + "rgi": dict, + "rgi.rgi_relpath": str, + "rgi.rgi_lat_colname": str, + "rgi.rgi_lon_colname": str, + "rgi.elev_colname": str, + "rgi.indexname": str, + "rgi.rgi_O1Id_colname": str, + "rgi.rgi_glacno_float_colname": str, + "rgi.rgi_cols_drop": list, + "time": dict, + "time.option_leapyear": int, + "time.startmonthday": str, + "time.endmonthday": str, + "time.wateryear_month_start": int, + "time.winter_month_start": int, + "time.summer_month_start": int, + "time.option_dates": int, + "time.timestep": str, + "constants": dict, + "constants.density_ice": int, + "constants.density_water": int, + "constants.area_ocean": float, + "constants.k_ice": float, + "constants.k_air": float, + "constants.ch_ice": int, + "constants.ch_air": int, + "constants.Lh_rf": int, + "constants.tolerance": float, + "constants.gravity": float, + "constants.pressure_std": int, + "constants.temp_std": float, + "constants.R_gas": float, + "constants.molarmass_air": float, + "debug": dict, + "debug.refreeze": bool, + "debug.mb": bool, + } + + # expected types of elements in lists + LIST_ELEMENT_TYPES = { + "setup.rgi_region01": int, + "setup.glac_no_skip": float, + "setup.glac_no": float, + "calib.HH2015mod_params.params2opt": str, + "calib.emulator_params.params2opt": str, + "sim.out.sim_stats": str, + "rgi.rgi_cols_drop": str, + } diff --git a/pygem/setup/config.yaml b/pygem/setup/config.yaml index 58e1a061..b86044c6 100644 --- a/pygem/setup/config.yaml +++ b/pygem/setup/config.yaml @@ -194,11 +194,6 @@ calib: hugonnet2021_relpath: /DEMs/Hugonnet2021/ # relative to main data path hugonnet2021_fn: df_pergla_global_20yr-filled.csv # this file is 'raw', filled geodetic mass balance from Hugonnet et al. (2021) - pulled by prerproc_fetch_mbdata.py hugonnet2021_facorrected_fn: df_pergla_global_20yr-filled-frontalablation-corrected.csv # frontal ablation corrected geodetic mass balance (produced by run_calibration_frontalablation.py) - # operation icebridge altimetric surface elevations - oib: - oib_relpath: /OIB/lidar_cop30_deltas/ # relative to main data path - oib_rebin: 100 # elevation rebinning in meters - oib_filter_pctl: 10 # pixel count percentile filter - for each survey # frontal ablation frontalablation: frontalablation_relpath: /frontalablation_data/ # relative to main data path diff --git a/pygem/shop/debris.py b/pygem/shop/debris.py index 191f256a..13b3f933 100755 --- a/pygem/shop/debris.py +++ b/pygem/shop/debris.py @@ -16,10 +16,12 @@ from oggm.utils import entity_task from oggm.core.gis import rasterio_to_gdir from oggm.utils import ncDataset -# Local libraries -import pygem.setup.config as config -# Read the config -pygem_prms = config.read_config() # This reads the configuration file +# pygem imports +from pygem.setup.config import ConfigManager +# instantiate ConfigManager +config_manager = ConfigManager() +# read the config +pygem_prms = config_manager.read_config() """ To-do list: diff --git a/pygem/shop/icethickness.py b/pygem/shop/icethickness.py index dcdc4656..6d7660d3 100755 --- a/pygem/shop/icethickness.py +++ b/pygem/shop/icethickness.py @@ -18,10 +18,12 @@ from oggm.utils import entity_task from oggm.core.gis import rasterio_to_gdir from oggm.utils import ncDataset -# Local libraries -import pygem.setup.config as config -# Read the config -pygem_prms = config.read_config() # This reads the configuration file +# pygem imports +from pygem.setup.config import ConfigManager +# instantiate ConfigManager +config_manager = ConfigManager() +# read the config +pygem_prms = config_manager.read_config() if not 'consensus_mass' in cfg.BASENAMES: cfg.BASENAMES['consensus_mass'] = ('consensus_mass.pkl', 'Glacier mass from consensus ice thickness data') diff --git a/pygem/shop/mbdata.py b/pygem/shop/mbdata.py index f73ebdc2..ba87ffcd 100755 --- a/pygem/shop/mbdata.py +++ b/pygem/shop/mbdata.py @@ -23,11 +23,11 @@ #from oggm.core.gis import rasterio_to_gdir #from oggm.utils import ncDataset # pygem imports -import pygem.setup.config as config -# check for config -config.ensure_config() +from pygem.setup.config import ConfigManager +# instantiate ConfigManager +config_manager = ConfigManager() # read the config -pygem_prms = config.read_config() +pygem_prms = config_manager.read_config() import pygem.pygem_modelsetup as modelsetup diff --git a/pygem/shop/oib.py b/pygem/shop/oib.py index cbc79edc..70f1f9fc 100644 --- a/pygem/shop/oib.py +++ b/pygem/shop/oib.py @@ -12,10 +12,11 @@ import pandas as pd from scipy import signal, stats import matplotlib.pyplot as plt -# Local libraries -import pygem.setup.config as config -# Read the config -pygem_prms = config.read_config() # This reads the configuration file +from pygem.setup.config import ConfigManager +# instantiate ConfigManager +config_manager = ConfigManager() +# read the config +pygem_prms = config_manager.read_config() class oib: def __init__(self, rgi6id='', rgi7id=''): diff --git a/pygem/tests/test_config.py b/pygem/tests/test_config.py new file mode 100644 index 00000000..8e1a727c --- /dev/null +++ b/pygem/tests/test_config.py @@ -0,0 +1,129 @@ +import pathlib +import pytest +import yaml +from pygem.setup.config import ConfigManager + + +class TestConfigManager: + """Tests for the ConfigManager class.""" + + @pytest.fixture(autouse=True) + def setup(self, tmp_path): + """Setup a ConfigManager instance for each test.""" + self.config_manager = ConfigManager( + config_filename='config.yaml', + base_dir=tmp_path, + overwrite=True + ) + + def test_config_created(self, tmp_path): + config_path = pathlib.Path(tmp_path) / 'config.yaml' + assert config_path.is_file() + + def test_read_config(self): + config = self.config_manager.read_config() + assert isinstance(config, dict) + assert "sim" in config + assert "nsims" in config["sim"] + + def test_update_config_unrecognized_key_error(self): + """Test that a KeyError is raised when updating a value with an unrecognized key.""" + with pytest.raises(KeyError, match="Unrecognized configuration key: invalid_key"): + self.config_manager.update_config({"invalid_key": None}) + + @pytest.mark.parametrize("key, invalid_value, expected_type, invalid_type", [ + ("sim.nsims", [1, 2, 3], "int", "list"), + ("calib.HH2015_params.kp_init", "0.5", "float", "str"), + ("setup.include_landterm", -999, "bool", "int"), + ("rgi.rgi_cols_drop", "not-a-list", "list", "str"), + ]) + def test_update_config_type_error(self, key, invalid_value, expected_type, invalid_type): + """ + Test that a TypeError is raised when updating a value with a new value of a + wrong type. + """ + with pytest.raises( + TypeError, + match=f"Invalid type for '{key.replace('.', '\\.')}':" + f" expected.*{expected_type}.*, not.*{invalid_type}.*" + ): + self.config_manager.update_config({key: invalid_value}) + + def test_update_config_list_element_type_error(self): + """ + Test that a TypeError is raised when updating a value with a new list value + containing elements of a different type than expected. + """ + key = "rgi.rgi_cols_drop" + invalid_value = ["a", "b", 100] + expected_type = "str" + + with pytest.raises( + TypeError, + match=f"Invalid type for elements in '{key.replace('.', '\\.')}':" + f" expected all elements to be .*{expected_type}.*, but got.*{invalid_value}.*" + ): + self.config_manager.update_config({key: invalid_value}) + + def test_compare_with_source(self): + """Test that compare_with_source detects missing keys.""" + # Remove a key from the config file + with open(self.config_manager.config_path, 'r') as f: + config = yaml.safe_load(f) + del config['sim']['nsims'] + with open(self.config_manager.config_path, 'w') as f: + yaml.dump(config, f) + + with pytest.raises(KeyError, match=r"Missing required key in configuration: sim\.nsims"): + self.config_manager.read_config(validate=True) + + def test_update_config(self): + """Test that update_config updates the config file for all data types.""" + updates = { + "sim.nsims": 5, # int + "calib.HH2015_params.kp_init": 0.5, # float + "user.email": "updated@example.com", # str + "setup.include_landterm": False, # bool + "rgi.rgi_cols_drop": ['Item1', 'Item2'], # list + } + + # Values before updating + config = self.config_manager.read_config() + assert config["sim"]["nsims"] == 1 + assert config["calib"]["HH2015_params"]["kp_init"] == 1.5 + assert config["user"]["email"] == "drounce@cmu.edu" + assert config["setup"]["include_landterm"] == True + assert config["rgi"]["rgi_cols_drop"] == ["GLIMSId", "BgnDate", "EndDate", "Status", "Linkages", "Name"] + + self.config_manager.update_config(updates) + config = self.config_manager.read_config() + + # Values after updating + assert config["sim"]["nsims"] == 5 + assert config["calib"]["HH2015_params"]["kp_init"] == 0.5 + assert config["setup"]["include_landterm"] == False + assert config["user"]["email"] == "updated@example.com" + assert config["rgi"]["rgi_cols_drop"] == ["Item1", "Item2"] + + def test_update_config_dict(self): + """Test that update_config updates the config file for nested dictionaries.""" + # Values before updating + config = self.config_manager.read_config() + assert config["user"]["name"] == "David Rounce" + assert config["user"]["email"] == "drounce@cmu.edu" + assert config["user"]["institution"] == "Carnegie Mellon University, Pittsburgh PA" + + updates = { + "user": { + "name": "New Name", + "email": "New email", + "institution": "New Institution", + } + } + self.config_manager.update_config(updates) + + # Values after updating + config = self.config_manager.read_config() + assert config["user"]["name"] == "New Name" + assert config["user"]["email"] == "New email" + assert config["user"]["institution"] == "New Institution" diff --git a/pygem/tests/test_notebooks.py b/pygem/tests/test_notebooks.py new file mode 100644 index 00000000..0a6de02a --- /dev/null +++ b/pygem/tests/test_notebooks.py @@ -0,0 +1,21 @@ +import os +import subprocess + +import pytest + +# Get all notebooks in the PyGEM-notebooks repository +nb_dir = os.environ.get("PYGEM_NOTEBOOKS_DIRPATH") or os.path.join( + os.path.expanduser("~"), "PyGEM-notebooks" +) +notebooks = [f for f in os.listdir(nb_dir) if f.endswith(".ipynb")] + + +@pytest.mark.parametrize("notebook", notebooks) +def test_notebook(notebook): + # TODO #54: Test all notebooks + + if notebook not in ("simple_test.ipynb", "advanced_test.ipynb", "advanced_test_tw.ipynb"): + pytest.skip() + subprocess.check_call( + ["pytest", "--nbmake", os.path.join(nb_dir, notebook)] + ) \ No newline at end of file diff --git a/pygem/tests/test_oggm_compat.py b/pygem/tests/test_oggm_compat.py deleted file mode 100755 index 28c8afff..00000000 --- a/pygem/tests/test_oggm_compat.py +++ /dev/null @@ -1,78 +0,0 @@ -from pygem import oggm_compat -import numpy as np - -do_plot = False - - -def test_single_flowline_glacier_directory(): - - rid = 'RGI60-15.03473' - gdir = oggm_compat.single_flowline_glacier_directory(rid) - assert gdir.rgi_area_km2 == 61.054 - - if do_plot: - from oggm import graphics - import matplotlib.pyplot as plt - f, (ax1, ax2) = plt.subplots(1, 2) - graphics.plot_googlemap(gdir, ax=ax1) - graphics.plot_inversion(gdir, ax=ax2) - plt.show() - - -def test_get_glacier_zwh(): - - rid = 'RGI60-15.03473' - gdir = oggm_compat.single_flowline_glacier_directory(rid) - df = oggm_compat.get_glacier_zwh(gdir) - - # Ref area km2 - ref_area = gdir.rgi_area_km2 - ref_area_m2 = ref_area * 1e6 - - # Check that glacier area is conserved at 0.1% - np.testing.assert_allclose((df.w * df.dx).sum(), ref_area_m2, rtol=0.001) - - # Check that volume is within VAS at 25% - vas_vol = 0.034 * ref_area**1.375 - vas_vol_m3 = vas_vol * 1e9 - np.testing.assert_allclose((df.w * df.dx * df.h).sum(), vas_vol_m3, - rtol=0.25) - - -def test_random_mb_run(): - - rid = 'RGI60-15.03473' - gdir = oggm_compat.single_flowline_glacier_directory(rid, prepro_border=80) - - # This initializes the mass balance model, but does not run it - mbmod = oggm_compat.RandomLinearMassBalance(gdir, seed=1, sigma_ela=300, - h_perc=55) - # HERE CAN BE THE LOOP SUCH THAT EVERYTHING IS ALREADY LOADED - for i in [1,2,3,4]: - # Change the model parameter - mbmod.param1 = i - # Run the mass balance model with fixed geometry - ts_mb = mbmod.get_specific_mb(years=[2000,2001,2002]) - - # Run the glacier flowline model with a mass balance model - from oggm.core.flowline import robust_model_run - flmodel = robust_model_run(gdir, mb_model=mbmod, ys=0, ye=700) - - # Check that "something" is computed - import xarray as xr - ds = xr.open_dataset(gdir.get_filepath('model_diagnostics')) - assert ds.isel(time=-1).volume_m3 > 0 - - if do_plot: - import matplotlib.pyplot as plt - from oggm import graphics - graphics.plot_modeloutput_section(flmodel) - f, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(12, 4)) - (ds.volume_m3 * 1e-9).plot(ax=ax1) - ax1.set_ylabel('Glacier volume (km$^{3}$)') - (ds.area_m2 * 1e-6).plot(ax=ax2) - ax2.set_ylabel('Glacier area (km$^{2}$)') - (ds.length_m * 1e3).plot(ax=ax3) - ax3.set_ylabel('Glacier length (km)') - plt.tight_layout() - plt.show() diff --git a/pygem/utils/_funcs.py b/pygem/utils/_funcs.py index 2ecb9b5d..11172bad 100755 --- a/pygem/utils/_funcs.py +++ b/pygem/utils/_funcs.py @@ -9,10 +9,11 @@ """ import numpy as np import json -# Local libraries -import pygem.setup.config as config -# Read the config -pygem_prms = config.read_config() # This reads the configuration file +from pygem.setup.config import ConfigManager +# instantiate ConfigManager +config_manager = ConfigManager() +# read the config +pygem_prms = config_manager.read_config() def annualweightedmean_array(var, dates_table): """ diff --git a/pyproject.toml b/pyproject.toml index b661a0b2..82eff614 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pygem" -version = "1.0.1" +version = "1.0.2" description = "Python Glacier Evolution Model (PyGEM)" authors = ["David Rounce ,Brandon Tober "] license = "MIT License" @@ -34,6 +34,9 @@ jupyter = "^1.1.1" arviz = "^0.20.0" oggm = "^1.6.2" ruamel-yaml = "^0.18.10" +pytest = ">=8.3.4" +pytest-cov = ">=6.0.0" +nbmake = ">=1.5.5" [tool.poetry.scripts] initialize = "pygem.bin.op.initialize:main" @@ -53,4 +56,9 @@ duplicate_gdirs = "pygem.bin.op.duplicate_gdirs:main" [build-system] requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" \ No newline at end of file +build-backend = "poetry.core.masonry.api" + +[tool.coverage.report] +omit = ["pygem/tests/*"] +show_missing = true +skip_empty = true \ No newline at end of file