Skip to content

Commit

Permalink
Update tests due to upgrade of hydromt artifact_data catalog (#302)
Browse files Browse the repository at this point in the history
* fix tests with latest artifact_data version

* skip tests with hydromt version 1

* add gdal to use pcraster in examples
  • Loading branch information
hboisgon authored Oct 15, 2024
1 parent 8158e1b commit b953441
Show file tree
Hide file tree
Showing 10 changed files with 108 additions and 106 deletions.
8 changes: 4 additions & 4 deletions .github/workflows/docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,10 @@ jobs:
export PATH=/usr/share/miniconda3/bin:$PATH
mamba env update -n hydromt_wflow -f environment.yml
- name: Install HydroMT-core dev
run: |
export PATH=/usr/share/miniconda3/bin:$PATH
mamba run -n hydromt_wflow python -m pip install git+https://github.com/Deltares/hydromt.git@main --user
# - name: Install HydroMT-core dev
# run: |
# export PATH=/usr/share/miniconda3/bin:$PATH
# mamba run -n hydromt_wflow python -m pip install git+https://github.com/Deltares/hydromt.git@main --user

- name: Generate dummy docs
if: ${{ github.event_name == 'pull_request' }}
Expand Down
148 changes: 74 additions & 74 deletions .github/workflows/test_dev.yml
Original file line number Diff line number Diff line change
@@ -1,87 +1,87 @@
name: Test dev
# name: Test dev

on:
schedule:
- cron: 0 0 * * * # run on main everyday at 00:00
push:
branches: [ main ]
pull_request:
branches: [ main ]
# on:
# schedule:
# - cron: 0 0 * * * # run on main everyday at 00:00
# push:
# branches: [ main ]
# pull_request:
# branches: [ main ]

jobs:
Test:
defaults:
run:
shell: bash -l {0}
# jobs:
# Test:
# defaults:
# run:
# shell: bash -l {0}

strategy:
fail-fast: false
matrix:
os: ["ubuntu-latest" ] #, "macos-latest", "windows-latest"]
python-version: ['3.10', '3.11']
include:
- os: ubuntu-latest
label: linux-64
prefix: /usr/share/miniconda3/envs/hydromt_wflow
# strategy:
# fail-fast: false
# matrix:
# os: ["ubuntu-latest" ] #, "macos-latest", "windows-latest"]
# python-version: ['3.10', '3.11']
# include:
# - os: ubuntu-latest
# label: linux-64
# prefix: /usr/share/miniconda3/envs/hydromt_wflow

name: ${{ matrix.label }} - py${{ matrix.python-version }}
runs-on: ${{ matrix.os }}
concurrency:
group: ${{ github.workflow }}-${{ matrix.python-version }}-${{ github.ref }}
cancel-in-progress: true
# name: ${{ matrix.label }} - py${{ matrix.python-version }}
# runs-on: ${{ matrix.os }}
# concurrency:
# group: ${{ github.workflow }}-${{ matrix.python-version }}-${{ github.ref }}
# cancel-in-progress: true

steps:
- name: Checkout source
uses: actions/checkout@v3
# steps:
# - name: Checkout source
# uses: actions/checkout@v3

- name: Generate env yaml
run: |
pip install tomli
python make_env.py test -p ${{ matrix.python-version}}.* -n hydromt_wflow
# - name: Generate env yaml
# run: |
# pip install tomli
# python make_env.py test -p ${{ matrix.python-version}}.* -n hydromt_wflow

- name: Load cache
id: cache
uses: actions/cache/restore@v3
with:
path: |
/usr/share/miniconda3
~/pycache
# the below two settings mean we'll alway srestore the cache
# but the cache hit output can tell us if we have to update afterwards
key: test-py${{ matrix.python-version }}-${{ hashFiles('environment.yml') }}
restore-keys: |
test-py${{ matrix.python-version }}
# - name: Load cache
# id: cache
# uses: actions/cache/restore@v3
# with:
# path: |
# /usr/share/miniconda3
# ~/pycache
# # the below two settings mean we'll alway srestore the cache
# # but the cache hit output can tell us if we have to update afterwards
# key: test-py${{ matrix.python-version }}-${{ hashFiles('environment.yml') }}
# restore-keys: |
# test-py${{ matrix.python-version }}

- name: Failed cache restore
if: steps.cache.outputs.cache-matched-key == ''
run: |
echo "Failed to restore any cache. exiting..."
exit 1
# - name: Failed cache restore
# if: steps.cache.outputs.cache-matched-key == ''
# run: |
# echo "Failed to restore any cache. exiting..."
# exit 1

# by avoiding the mamba setup stage by loading it from cache instead we save
# a lot of setup time, but we do have to do our own PATH management
# hence the exports
- name: Update environment
if: steps.cache.outputs.cache-hit != 'true'
run: |
export PATH=/usr/share/miniconda3/bin:$PATH
mamba env update -n hydromt_wflow -f environment.yml
# # by avoiding the mamba setup stage by loading it from cache instead we save
# # a lot of setup time, but we do have to do our own PATH management
# # hence the exports
# - name: Update environment
# if: steps.cache.outputs.cache-hit != 'true'
# run: |
# export PATH=/usr/share/miniconda3/bin:$PATH
# mamba env update -n hydromt_wflow -f environment.yml

- name: Install HydroMT-core dev
run: |
export PATH=/usr/share/miniconda3/bin:$PATH
mamba run -n hydromt_wflow python -m pip install git+https://github.com/Deltares/hydromt.git@main --user
# - name: Install HydroMT-core dev
# run: |
# export PATH=/usr/share/miniconda3/bin:$PATH
# mamba run -n hydromt_wflow python -m pip install git+https://github.com/Deltares/hydromt.git@main --user

- name: Conda info
run: |
export PATH=/usr/share/miniconda3/bin:$PATH
conda info
conda list -n hydromt_wflow
# - name: Conda info
# run: |
# export PATH=/usr/share/miniconda3/bin:$PATH
# conda info
# conda list -n hydromt_wflow

- name: Run tests
run: |
export PATH=/usr/share/miniconda3/bin:$PATH
PYTHONPYCACHEPREFIX=~/pycache mamba run -n hydromt_wflow python -m pytest --verbose --cov=hydromt_wflow --cov-report xml
# - name: Run tests
# run: |
# export PATH=/usr/share/miniconda3/bin:$PATH
# PYTHONPYCACHEPREFIX=~/pycache mamba run -n hydromt_wflow python -m pytest --verbose --cov=hydromt_wflow --cov-report xml

- name: Upload code coverage
uses: codecov/codecov-action@v3
# - name: Upload code coverage
# uses: codecov/codecov-action@v3
17 changes: 6 additions & 11 deletions examples/update_model_landuse.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 1,
"id": "adc6795e",
"metadata": {},
"outputs": [],
Expand Down Expand Up @@ -76,7 +76,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 2,
"id": "033f24ab",
"metadata": {},
"outputs": [],
Expand Down Expand Up @@ -104,9 +104,9 @@
"source": [
"Here we can see that we have five data sources in HydroMT related to landuse & landcover properties. Out of these, three are landuse classifications:\n",
"\n",
"- **globcover** (already used in our current model)\n",
"- **globcover_2009** (already used in our current model)\n",
"- **corine**\n",
"- **vito**\n",
"- **vito_2015**\n",
"\n",
"The other datasets include a Leaf Area Index dataset (modis_lai) and a canopy height dataset (simard).\n",
"\n",
Expand Down Expand Up @@ -270,7 +270,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "hydromt-dev",
"display_name": "hydromt-wflow",
"language": "python",
"name": "python3"
},
Expand All @@ -284,12 +284,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.9"
},
"vscode": {
"interpreter": {
"hash": "3808d5b5b54949c7a0a707a38b0a689040fa9c90ab139a050e41373880719ab1"
}
"version": "3.11.7"
}
},
"nbformat": 4,
Expand Down
5 changes: 3 additions & 2 deletions examples/wflow_build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ setup_basemaps:

setup_rivers:
hydrography_fn: merit_hydro # source hydrography data, should correspond to hydrography_fn in setup_basemaps
river_geom_fn: rivers_lin2019_v1 # river source data with river width and bankfull discharge
river_geom_fn: hydro_rivers_lin # river source data with river width and bankfull discharge
river_upa: 30 # minimum upstream area threshold for the river map [km2]
rivdph_method: powlaw # method to estimate depth {'powlaw', 'manning', 'gvf'}
min_rivdph: 1 # minimum river depth [m]
Expand Down Expand Up @@ -53,7 +53,8 @@ setup_glaciers:
min_area: 1.0 # minimum glacier area to consider [km2]

setup_lulcmaps:
lulc_fn : globcover # source for lulc maps: {globcover, vito, corine}
lulc_fn : globcover_2009 # source for lulc maps: {globcover, vito, corine}
lulc_mapping_fn: globcover_mapping_default # default mapping for lulc classes

setup_laimaps:
lai_fn: modis_lai # source for LAI: {modis_lai}
Expand Down
3 changes: 2 additions & 1 deletion examples/wflow_extend_sediment.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ setup_riverbedsed:
bedsed_mapping_fn: # path to a mapping csv file from streamorder to river bed particles characteristics if any, else default is used

setup_lulcmaps:
lulc_fn: globcover # source for lulc maps: {globcover, vito, corine}
lulc_fn: globcover_2009 # source for lulc maps: {globcover, vito, corine}
lulc_mapping_fn: globcover_mapping_default # default mapping for lulc classes

setup_canopymaps:
canopy_fn: simard # source for vegetation canopy height: {simard}
Expand Down
5 changes: 3 additions & 2 deletions examples/wflow_sediment_build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ setup_basemaps:

setup_rivers:
hydrography_fn: merit_hydro # source hydrography data, should correspond to hydrography_fn in setup_basemaps
river_geom_fn: rivers_lin2019_v1 # river source data with river width and bankfull discharge
river_geom_fn: hydro_rivers_lin # river source data with river width and bankfull discharge
river_upa: 30 # minimum upstream area threshold for the river map [km2]
rivdph_method: powlaw # method to estimate depth {'powlaw', 'manning', 'gvf'}
min_rivdph: 1 # minimum river depth [m]
Expand All @@ -31,7 +31,8 @@ setup_lakes:
setup_riverbedsed:

setup_lulcmaps:
lulc_fn: globcover # source for lulc maps: {globcover, vito, corine}
lulc_fn: globcover_2009 # source for lulc maps: {globcover, vito, corine}
lulc_mapping_fn: globcover_mapping_default # default mapping for lulc classes

setup_laimaps:
lai_fn: modis_lai # source for LAI: {modis_lai}
Expand Down
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ io = [
]
extra = [
"hydroengine",
"gdal",
"gwwapi",
"pcraster",
"pyet",
Expand Down
5 changes: 3 additions & 2 deletions tests/data/wflow_piave_build_subbasin.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ setup_basemaps:

setup_rivers:
hydrography_fn: merit_hydro
river_geom_fn: rivers_lin2019_v1
river_geom_fn: hydro_rivers_lin
river_upa: 30
rivdph_method: powlaw
min_rivdph: 1
Expand All @@ -31,7 +31,8 @@ setup_glaciers:
min_area: 0.0

setup_lulcmaps:
lulc_fn: globcover
lulc_fn: globcover_2009
lulc_mapping_fn: globcover_mapping_default

setup_laimaps:
lai_fn: modis_lai
Expand Down
5 changes: 3 additions & 2 deletions tests/data/wflow_sediment_piave_build_subbasin.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ setup_basemaps:

setup_rivers:
hydrography_fn: merit_hydro
river_geom_fn: rivers_lin2019_v1
river_geom_fn: hydro_rivers_lin
river_upa: 30
rivdph_method: powlaw
min_rivdph: 1
Expand All @@ -27,7 +27,8 @@ setup_lakes:
setup_riverbedsed:

setup_lulcmaps:
lulc_fn: globcover
lulc_fn: globcover_2009
lulc_mapping_fn: globcover_mapping_default

setup_laimaps:
lai_fn: modis_lai
Expand Down
17 changes: 9 additions & 8 deletions tests/test_model_methods.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ def test_setup_grid(example_wflow_model):
assert example_wflow_model.get_config("input.vertical.altitude") == "elevtn"

example_wflow_model.setup_grid_from_raster(
raster_fn="globcover",
raster_fn="globcover_2009",
reproject_method="mode",
wflow_variables=["input.vertical.landuse"],
)
Expand Down Expand Up @@ -153,7 +153,7 @@ def test_projected_crs(tmpdir):
)

# Add more data eg landuse
mod.setup_lulcmaps("globcover")
mod.setup_lulcmaps("globcover_2009", lulc_mapping_fn="globcover_mapping_default")

assert mod.grid.raster.crs == 3857
assert np.quantile(mod.grid["wflow_landuse"], 0.95) == 190.0 # urban
Expand Down Expand Up @@ -316,7 +316,7 @@ def test_setup_lai(tmpdir, example_wflow_model):
)
# Read landuse data
da_landuse = example_wflow_model.data_catalog.get_rasterdataset(
"vito", geom=example_wflow_model.region, buffer=2
"vito_2015", geom=example_wflow_model.region, buffer=2
)

# Derive mapping for using the method any
Expand Down Expand Up @@ -353,7 +353,7 @@ def test_setup_lai(tmpdir, example_wflow_model):

# Try to use the mapping tables to setup the LAI
example_wflow_model.setup_laimaps_from_lulc_mapping(
lulc_fn="vito",
lulc_fn="vito_2015",
lai_mapping_fn=df_lai_any,
)

Expand Down Expand Up @@ -619,7 +619,7 @@ def test_setup_gauges(example_wflow_model):
def test_setup_rivers(elevtn_map, floodplain1d_testdata, example_wflow_model):
example_wflow_model.setup_rivers(
hydrography_fn="merit_hydro",
river_geom_fn="rivers_lin2019_v1",
river_geom_fn="hydro_rivers_lin",
river_upa=30,
rivdph_method="powlaw",
min_rivdph=1,
Expand Down Expand Up @@ -652,7 +652,7 @@ def test_setup_floodplains_1d(example_wflow_model, floodplain1d_testdata):

example_wflow_model.setup_rivers(
hydrography_fn="merit_hydro",
river_geom_fn="rivers_lin2019_v1",
river_geom_fn="hydro_rivers_lin",
river_upa=30,
rivdph_method="powlaw",
min_rivdph=1,
Expand Down Expand Up @@ -687,7 +687,7 @@ def test_setup_floodplains_1d(example_wflow_model, floodplain1d_testdata):
def test_setup_floodplains_2d(elevtn_map, example_wflow_model, floodplain1d_testdata):
example_wflow_model.setup_rivers(
hydrography_fn="merit_hydro",
river_geom_fn="rivers_lin2019_v1",
river_geom_fn="hydro_rivers_lin",
river_upa=30,
rivdph_method="powlaw",
min_rivdph=1,
Expand Down Expand Up @@ -817,7 +817,8 @@ def test_skip_nodata_reservoir(clipped_wflow_model):

def test_setup_lulc_sed(example_sediment_model, planted_forest_testdata):
example_sediment_model.setup_lulcmaps(
lulc_fn="globcover",
lulc_fn="globcover_2009",
lulc_mapping_fn="globcover_mapping_default",
planted_forest_fn=planted_forest_testdata,
lulc_vars={"USLE_C": "input.vertical.usleC"},
planted_forest_c=0.0881,
Expand Down

0 comments on commit b953441

Please sign in to comment.