From b953441aeac59631a62b316b05a88b5fb7fd7e11 Mon Sep 17 00:00:00 2001 From: hboisgon <45457510+hboisgon@users.noreply.github.com> Date: Tue, 15 Oct 2024 17:39:49 +0800 Subject: [PATCH] Update tests due to upgrade of hydromt artifact_data catalog (#302) * fix tests with latest artifact_data version * skip tests with hydromt version 1 * add gdal to use pcraster in examples --- .github/workflows/docs.yml | 8 +- .github/workflows/test_dev.yml | 148 +++++++++--------- examples/update_model_landuse.ipynb | 17 +- examples/wflow_build.yml | 5 +- examples/wflow_extend_sediment.yml | 3 +- examples/wflow_sediment_build.yml | 5 +- pyproject.toml | 1 + tests/data/wflow_piave_build_subbasin.yml | 5 +- .../wflow_sediment_piave_build_subbasin.yml | 5 +- tests/test_model_methods.py | 17 +- 10 files changed, 108 insertions(+), 106 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index b83988da..b4c93139 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -78,10 +78,10 @@ jobs: export PATH=/usr/share/miniconda3/bin:$PATH mamba env update -n hydromt_wflow -f environment.yml - - name: Install HydroMT-core dev - run: | - export PATH=/usr/share/miniconda3/bin:$PATH - mamba run -n hydromt_wflow python -m pip install git+https://github.com/Deltares/hydromt.git@main --user + # - name: Install HydroMT-core dev + # run: | + # export PATH=/usr/share/miniconda3/bin:$PATH + # mamba run -n hydromt_wflow python -m pip install git+https://github.com/Deltares/hydromt.git@main --user - name: Generate dummy docs if: ${{ github.event_name == 'pull_request' }} diff --git a/.github/workflows/test_dev.yml b/.github/workflows/test_dev.yml index 3f5cda5e..585fdbc5 100644 --- a/.github/workflows/test_dev.yml +++ b/.github/workflows/test_dev.yml @@ -1,87 +1,87 @@ -name: Test dev +# name: Test dev -on: - schedule: - - cron: 0 0 * * * # run on main everyday at 00:00 - push: - branches: [ main ] - pull_request: - branches: [ main ] +# on: +# schedule: +# - cron: 0 0 * * * # run on main everyday at 00:00 +# push: +# branches: [ main ] +# pull_request: +# branches: [ main ] -jobs: - Test: - defaults: - run: - shell: bash -l {0} +# jobs: +# Test: +# defaults: +# run: +# shell: bash -l {0} - strategy: - fail-fast: false - matrix: - os: ["ubuntu-latest" ] #, "macos-latest", "windows-latest"] - python-version: ['3.10', '3.11'] - include: - - os: ubuntu-latest - label: linux-64 - prefix: /usr/share/miniconda3/envs/hydromt_wflow +# strategy: +# fail-fast: false +# matrix: +# os: ["ubuntu-latest" ] #, "macos-latest", "windows-latest"] +# python-version: ['3.10', '3.11'] +# include: +# - os: ubuntu-latest +# label: linux-64 +# prefix: /usr/share/miniconda3/envs/hydromt_wflow - name: ${{ matrix.label }} - py${{ matrix.python-version }} - runs-on: ${{ matrix.os }} - concurrency: - group: ${{ github.workflow }}-${{ matrix.python-version }}-${{ github.ref }} - cancel-in-progress: true +# name: ${{ matrix.label }} - py${{ matrix.python-version }} +# runs-on: ${{ matrix.os }} +# concurrency: +# group: ${{ github.workflow }}-${{ matrix.python-version }}-${{ github.ref }} +# cancel-in-progress: true - steps: - - name: Checkout source - uses: actions/checkout@v3 +# steps: +# - name: Checkout source +# uses: actions/checkout@v3 - - name: Generate env yaml - run: | - pip install tomli - python make_env.py test -p ${{ matrix.python-version}}.* -n hydromt_wflow +# - name: Generate env yaml +# run: | +# pip install tomli +# python make_env.py test -p ${{ matrix.python-version}}.* -n hydromt_wflow - - name: Load cache - id: cache - uses: actions/cache/restore@v3 - with: - path: | - /usr/share/miniconda3 - ~/pycache - # the below two settings mean we'll alway srestore the cache - # but the cache hit output can tell us if we have to update afterwards - key: test-py${{ matrix.python-version }}-${{ hashFiles('environment.yml') }} - restore-keys: | - test-py${{ matrix.python-version }} +# - name: Load cache +# id: cache +# uses: actions/cache/restore@v3 +# with: +# path: | +# /usr/share/miniconda3 +# ~/pycache +# # the below two settings mean we'll alway srestore the cache +# # but the cache hit output can tell us if we have to update afterwards +# key: test-py${{ matrix.python-version }}-${{ hashFiles('environment.yml') }} +# restore-keys: | +# test-py${{ matrix.python-version }} - - name: Failed cache restore - if: steps.cache.outputs.cache-matched-key == '' - run: | - echo "Failed to restore any cache. exiting..." - exit 1 +# - name: Failed cache restore +# if: steps.cache.outputs.cache-matched-key == '' +# run: | +# echo "Failed to restore any cache. exiting..." +# exit 1 - # by avoiding the mamba setup stage by loading it from cache instead we save - # a lot of setup time, but we do have to do our own PATH management - # hence the exports - - name: Update environment - if: steps.cache.outputs.cache-hit != 'true' - run: | - export PATH=/usr/share/miniconda3/bin:$PATH - mamba env update -n hydromt_wflow -f environment.yml +# # by avoiding the mamba setup stage by loading it from cache instead we save +# # a lot of setup time, but we do have to do our own PATH management +# # hence the exports +# - name: Update environment +# if: steps.cache.outputs.cache-hit != 'true' +# run: | +# export PATH=/usr/share/miniconda3/bin:$PATH +# mamba env update -n hydromt_wflow -f environment.yml - - name: Install HydroMT-core dev - run: | - export PATH=/usr/share/miniconda3/bin:$PATH - mamba run -n hydromt_wflow python -m pip install git+https://github.com/Deltares/hydromt.git@main --user +# - name: Install HydroMT-core dev +# run: | +# export PATH=/usr/share/miniconda3/bin:$PATH +# mamba run -n hydromt_wflow python -m pip install git+https://github.com/Deltares/hydromt.git@main --user - - name: Conda info - run: | - export PATH=/usr/share/miniconda3/bin:$PATH - conda info - conda list -n hydromt_wflow +# - name: Conda info +# run: | +# export PATH=/usr/share/miniconda3/bin:$PATH +# conda info +# conda list -n hydromt_wflow - - name: Run tests - run: | - export PATH=/usr/share/miniconda3/bin:$PATH - PYTHONPYCACHEPREFIX=~/pycache mamba run -n hydromt_wflow python -m pytest --verbose --cov=hydromt_wflow --cov-report xml +# - name: Run tests +# run: | +# export PATH=/usr/share/miniconda3/bin:$PATH +# PYTHONPYCACHEPREFIX=~/pycache mamba run -n hydromt_wflow python -m pytest --verbose --cov=hydromt_wflow --cov-report xml - - name: Upload code coverage - uses: codecov/codecov-action@v3 +# - name: Upload code coverage +# uses: codecov/codecov-action@v3 diff --git a/examples/update_model_landuse.ipynb b/examples/update_model_landuse.ipynb index 974de1d9..0f3f1908 100644 --- a/examples/update_model_landuse.ipynb +++ b/examples/update_model_landuse.ipynb @@ -38,7 +38,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "adc6795e", "metadata": {}, "outputs": [], @@ -76,7 +76,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "033f24ab", "metadata": {}, "outputs": [], @@ -104,9 +104,9 @@ "source": [ "Here we can see that we have five data sources in HydroMT related to landuse & landcover properties. Out of these, three are landuse classifications:\n", "\n", - "- **globcover** (already used in our current model)\n", + "- **globcover_2009** (already used in our current model)\n", "- **corine**\n", - "- **vito**\n", + "- **vito_2015**\n", "\n", "The other datasets include a Leaf Area Index dataset (modis_lai) and a canopy height dataset (simard).\n", "\n", @@ -270,7 +270,7 @@ ], "metadata": { "kernelspec": { - "display_name": "hydromt-dev", + "display_name": "hydromt-wflow", "language": "python", "name": "python3" }, @@ -284,12 +284,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.9" - }, - "vscode": { - "interpreter": { - "hash": "3808d5b5b54949c7a0a707a38b0a689040fa9c90ab139a050e41373880719ab1" - } + "version": "3.11.7" } }, "nbformat": 4, diff --git a/examples/wflow_build.yml b/examples/wflow_build.yml index ec13b587..cbadae8f 100644 --- a/examples/wflow_build.yml +++ b/examples/wflow_build.yml @@ -12,7 +12,7 @@ setup_basemaps: setup_rivers: hydrography_fn: merit_hydro # source hydrography data, should correspond to hydrography_fn in setup_basemaps - river_geom_fn: rivers_lin2019_v1 # river source data with river width and bankfull discharge + river_geom_fn: hydro_rivers_lin # river source data with river width and bankfull discharge river_upa: 30 # minimum upstream area threshold for the river map [km2] rivdph_method: powlaw # method to estimate depth {'powlaw', 'manning', 'gvf'} min_rivdph: 1 # minimum river depth [m] @@ -53,7 +53,8 @@ setup_glaciers: min_area: 1.0 # minimum glacier area to consider [km2] setup_lulcmaps: - lulc_fn : globcover # source for lulc maps: {globcover, vito, corine} + lulc_fn : globcover_2009 # source for lulc maps: {globcover, vito, corine} + lulc_mapping_fn: globcover_mapping_default # default mapping for lulc classes setup_laimaps: lai_fn: modis_lai # source for LAI: {modis_lai} diff --git a/examples/wflow_extend_sediment.yml b/examples/wflow_extend_sediment.yml index 0697d898..4091ddaf 100644 --- a/examples/wflow_extend_sediment.yml +++ b/examples/wflow_extend_sediment.yml @@ -10,7 +10,8 @@ setup_riverbedsed: bedsed_mapping_fn: # path to a mapping csv file from streamorder to river bed particles characteristics if any, else default is used setup_lulcmaps: - lulc_fn: globcover # source for lulc maps: {globcover, vito, corine} + lulc_fn: globcover_2009 # source for lulc maps: {globcover, vito, corine} + lulc_mapping_fn: globcover_mapping_default # default mapping for lulc classes setup_canopymaps: canopy_fn: simard # source for vegetation canopy height: {simard} diff --git a/examples/wflow_sediment_build.yml b/examples/wflow_sediment_build.yml index 26d0f35c..af2d2688 100644 --- a/examples/wflow_sediment_build.yml +++ b/examples/wflow_sediment_build.yml @@ -11,7 +11,7 @@ setup_basemaps: setup_rivers: hydrography_fn: merit_hydro # source hydrography data, should correspond to hydrography_fn in setup_basemaps - river_geom_fn: rivers_lin2019_v1 # river source data with river width and bankfull discharge + river_geom_fn: hydro_rivers_lin # river source data with river width and bankfull discharge river_upa: 30 # minimum upstream area threshold for the river map [km2] rivdph_method: powlaw # method to estimate depth {'powlaw', 'manning', 'gvf'} min_rivdph: 1 # minimum river depth [m] @@ -31,7 +31,8 @@ setup_lakes: setup_riverbedsed: setup_lulcmaps: - lulc_fn: globcover # source for lulc maps: {globcover, vito, corine} + lulc_fn: globcover_2009 # source for lulc maps: {globcover, vito, corine} + lulc_mapping_fn: globcover_mapping_default # default mapping for lulc classes setup_laimaps: lai_fn: modis_lai # source for LAI: {modis_lai} diff --git a/pyproject.toml b/pyproject.toml index 6ae08cd2..729eca1d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ io = [ ] extra = [ "hydroengine", + "gdal", "gwwapi", "pcraster", "pyet", diff --git a/tests/data/wflow_piave_build_subbasin.yml b/tests/data/wflow_piave_build_subbasin.yml index 1560e376..19dff109 100644 --- a/tests/data/wflow_piave_build_subbasin.yml +++ b/tests/data/wflow_piave_build_subbasin.yml @@ -10,7 +10,7 @@ setup_basemaps: setup_rivers: hydrography_fn: merit_hydro - river_geom_fn: rivers_lin2019_v1 + river_geom_fn: hydro_rivers_lin river_upa: 30 rivdph_method: powlaw min_rivdph: 1 @@ -31,7 +31,8 @@ setup_glaciers: min_area: 0.0 setup_lulcmaps: - lulc_fn: globcover + lulc_fn: globcover_2009 + lulc_mapping_fn: globcover_mapping_default setup_laimaps: lai_fn: modis_lai diff --git a/tests/data/wflow_sediment_piave_build_subbasin.yml b/tests/data/wflow_sediment_piave_build_subbasin.yml index 0ffcb4a9..e027f347 100644 --- a/tests/data/wflow_sediment_piave_build_subbasin.yml +++ b/tests/data/wflow_sediment_piave_build_subbasin.yml @@ -8,7 +8,7 @@ setup_basemaps: setup_rivers: hydrography_fn: merit_hydro - river_geom_fn: rivers_lin2019_v1 + river_geom_fn: hydro_rivers_lin river_upa: 30 rivdph_method: powlaw min_rivdph: 1 @@ -27,7 +27,8 @@ setup_lakes: setup_riverbedsed: setup_lulcmaps: - lulc_fn: globcover + lulc_fn: globcover_2009 + lulc_mapping_fn: globcover_mapping_default setup_laimaps: lai_fn: modis_lai diff --git a/tests/test_model_methods.py b/tests/test_model_methods.py index feb900a4..310f1b4c 100644 --- a/tests/test_model_methods.py +++ b/tests/test_model_methods.py @@ -82,7 +82,7 @@ def test_setup_grid(example_wflow_model): assert example_wflow_model.get_config("input.vertical.altitude") == "elevtn" example_wflow_model.setup_grid_from_raster( - raster_fn="globcover", + raster_fn="globcover_2009", reproject_method="mode", wflow_variables=["input.vertical.landuse"], ) @@ -153,7 +153,7 @@ def test_projected_crs(tmpdir): ) # Add more data eg landuse - mod.setup_lulcmaps("globcover") + mod.setup_lulcmaps("globcover_2009", lulc_mapping_fn="globcover_mapping_default") assert mod.grid.raster.crs == 3857 assert np.quantile(mod.grid["wflow_landuse"], 0.95) == 190.0 # urban @@ -316,7 +316,7 @@ def test_setup_lai(tmpdir, example_wflow_model): ) # Read landuse data da_landuse = example_wflow_model.data_catalog.get_rasterdataset( - "vito", geom=example_wflow_model.region, buffer=2 + "vito_2015", geom=example_wflow_model.region, buffer=2 ) # Derive mapping for using the method any @@ -353,7 +353,7 @@ def test_setup_lai(tmpdir, example_wflow_model): # Try to use the mapping tables to setup the LAI example_wflow_model.setup_laimaps_from_lulc_mapping( - lulc_fn="vito", + lulc_fn="vito_2015", lai_mapping_fn=df_lai_any, ) @@ -619,7 +619,7 @@ def test_setup_gauges(example_wflow_model): def test_setup_rivers(elevtn_map, floodplain1d_testdata, example_wflow_model): example_wflow_model.setup_rivers( hydrography_fn="merit_hydro", - river_geom_fn="rivers_lin2019_v1", + river_geom_fn="hydro_rivers_lin", river_upa=30, rivdph_method="powlaw", min_rivdph=1, @@ -652,7 +652,7 @@ def test_setup_floodplains_1d(example_wflow_model, floodplain1d_testdata): example_wflow_model.setup_rivers( hydrography_fn="merit_hydro", - river_geom_fn="rivers_lin2019_v1", + river_geom_fn="hydro_rivers_lin", river_upa=30, rivdph_method="powlaw", min_rivdph=1, @@ -687,7 +687,7 @@ def test_setup_floodplains_1d(example_wflow_model, floodplain1d_testdata): def test_setup_floodplains_2d(elevtn_map, example_wflow_model, floodplain1d_testdata): example_wflow_model.setup_rivers( hydrography_fn="merit_hydro", - river_geom_fn="rivers_lin2019_v1", + river_geom_fn="hydro_rivers_lin", river_upa=30, rivdph_method="powlaw", min_rivdph=1, @@ -817,7 +817,8 @@ def test_skip_nodata_reservoir(clipped_wflow_model): def test_setup_lulc_sed(example_sediment_model, planted_forest_testdata): example_sediment_model.setup_lulcmaps( - lulc_fn="globcover", + lulc_fn="globcover_2009", + lulc_mapping_fn="globcover_mapping_default", planted_forest_fn=planted_forest_testdata, lulc_vars={"USLE_C": "input.vertical.usleC"}, planted_forest_c=0.0881,