Merge pull request #241 from jmccreight/feat_prms_tests

Feat prms tests

.github/workflows/ci.yaml

@@ -9,6 +9,14 @@ on:
       - "*"
       - "!v[0-9]+.[0-9]+.[0-9]+*"
 
+  workflow_dispatch:
+    inputs:
+      debug_enabled:
+        type: boolean
+        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
+        required: false
+        default: false
+
 jobs:
 
   pyws_setup:
@@ -101,7 +109,7 @@ jobs:
           pylint --jobs=2 --errors-only --exit-zero ./pywatershed ./autotest
 
   test:
-    name: ${{ matrix.os}} py${{ matrix.python-version }}
+    name: ${{ matrix.os }} py${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
     defaults:
       run:
@@ -112,9 +120,14 @@ jobs:
         os: [ "ubuntu-latest", "macos-latest", "windows-latest" ]
         python-version: ["3.9", "3.10"]
     steps:
+
       - name: Checkout repo
         uses: actions/checkout@v3
 
+      - name: Setup tmate session
+        uses: mxschmitt/action-tmate@v3
+        if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
+
       - name: Set environment variables
         run: |
           echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV
@@ -138,7 +151,7 @@ jobs:
         with:
           compiler: gcc
           version: 11
-      
+
       - name: Link gfortran dylibs on Mac
         if: runner.os == 'macOS'
         run: .github/scripts/symlink_gfortran_mac.sh
@@ -174,40 +187,94 @@ jobs:
           pip -V
           pip list
 
-      - name: Run available domains with PRMS and convert csv output to NetCDF
+      - name: hru_1 - generate and manage test data domain, run PRMS and convert csv output to NetCDF
+        working-directory: test_data/generate
+        run: |
+          pytest -vv -n=2 --durations=0 run_prms_domains.py --domain=hru_1
+          pytest -vv -n=auto --durations=0 convert_prms_output_to_nc.py  --domain=hru_1
+          pytest -vv -n=auto --durations=0 remove_prms_csvs.py
+
+
+
+      - name: hru_1 - list netcdf input files
+        working-directory: test_data
+        run: |
+          find hru_1/output/ -name '*.nc'
+
+      - name: hru_1 - pywatershed tests
+        working-directory: autotest
+        run: pytest
+          -vv
+          -n=auto
+          --domain_yaml=../test_data/hru_1/hru_1.yaml
+          --durations=0
+          --cov=pywatershed
+          --cov-report=xml
+          --junitxml=pytest_hru_1.xml
+
+
+      - name: drb_2yr - generate and manage test data
+        working-directory: test_data/generate
+        run: |
+          pytest -vv remove_output_dirs.py --domain=hru_1
+          pytest -vv -n=2 run_prms_domains.py --domain=drb_2yr
+          pytest -vv -n=auto convert_prms_output_to_nc.py --domain=drb_2yr
+          pytest -vv -n=auto remove_prms_csvs.py
+
+      - name: drb_2yr - list netcdf input files
+        working-directory: test_data
+        run: |
+          find drb_2yr/output/ -name '*.nc'
+
+      - name: drb_2yr - pywatershed tests
+        working-directory: autotest
+        run: pytest
+          -vv
+          -n=auto
+          --domain_yaml=../test_data/drb_2yr/drb_2yr.yaml
+          --durations=0
+          --cov=pywatershed
+          --cov-report=xml
+          --junitxml=pytest_drb_2yr.xml
+
+      - name: ucb_2yr - generate and manage test data
         working-directory: test_data/generate
         run: |
-          pytest -v -n=auto --durations=0 run_prms_domains.py
-          pytest -v -n=auto --durations=0 convert_prms_output_to_nc.py
-          pytest -v -n=auto --durations=0 remove_prms_csvs.py
+          pytest -vv remove_output_dirs.py --domain=drb_2yr
+          pytest -vv -n=2 run_prms_domains.py --domain=ucb_2yr
+          pytest -vv -n=auto convert_prms_output_to_nc.py --domain=ucb_2yr
+          pytest -vv -n=auto remove_prms_csvs.py
 
-      - name: List all NetCDF files in test_data directory
+      - name: ucb_2yr - list netcdf input files
         working-directory: test_data
         run: |
-          find . -name "*.nc"
+          find ucb_2yr/output/ -name '*.nc'
 
-      - name: Run tests
+      - name: ucb_2yr - pywatershed tests
         working-directory: autotest
         run: pytest
-          -v
+          -vv
           -n=auto
+          --domain_yaml=../test_data/ucb_2yr/ucb_2yr.yaml
           --durations=0
-          --all_domains
           --cov=pywatershed
           --cov-report=xml
-          --junitxml=pytest.xml
+          --junitxml=pytest_ucb_2yr.xml
 
       - name: Upload test results
         if: always()
         uses: actions/upload-artifact@v3
         with:
           name: Test results for ${{ runner.os }}-${{ matrix.python-version }}
-          path: ./autotest/pytest.xml
+          path: |
+            ./autotest/pytest_hru_1.xml
+            ./autotest/pytest_drb_2yr.xml
+            ./autotest/pytest_ucb_2yr.xml
 
       - name: Upload code coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
-          file: ./autotest/coverage.xml
+          file: ./autotest/coverage.xml  # should be just the ucb result
           # flags: unittests
           env_vars: RUNNER_OS,PYTHON_VERSION
           # name: codecov-umbrella

autotest/test_data_model.py

@@ -167,6 +167,7 @@ def test_nc4_dd_nc4(tmp_path):
         "contiguous",
         "chunksizes",
         "coordinates",
+        "preferred_chunks",
     ]:
         for vv in sorted(ds1.variables):
             for dd in [ds1, ds2]:

autotest/test_model.py

@@ -12,6 +12,10 @@
 from pywatershed.base.model import Model
 from pywatershed.parameters import Parameters, PrmsParameters
 
+fortran_avail = getattr(
+    getattr(pywatershed.hydrology, "prms_canopy"), "has_prmscanopy_f"
+)
+
 compare_to_prms521 = False  # TODO TODO TODO
 failfast = True
 detailed = True
@@ -39,7 +43,10 @@ def control(domain):
     control = Control.load(domain["control_file"])
     control.options["verbose"] = 10
     control.options["budget_type"] = None
-    control.options["calc_method"] = "fortran"
+    if fortran_avail:
+        control.options["calc_method"] = "fortran"
+    else:
+        control.options["calc_method"] = "numba"
     control.options["load_n_time_batches"] = 1
     return control
 
@@ -141,21 +148,32 @@ def test_model(domain, model_args, tmp_path):
 
     control.options["input_dir"] = input_dir
 
-    model = Model(**model_args)
+    if control.options["calc_method"] == "fortran":
+        with pytest.warns(UserWarning):
+            model = Model(**model_args)
+    else:
+        model = Model(**model_args)
 
     # Test passing of control calc_method option
-    for proc in model.processes.keys():
-        if proc.lower() in ["prmssnow", "prmsrunoff", "prmssoilzone"]:
-            assert model.processes[proc]._calc_method == "numba"
-        elif proc.lower() in ["prmscanopy", "prmsgroundwater", "prmschannel"]:
-            # check if has fortran (has_f) because results depend on that
-            mod_name = "prms_" + proc.lower()[4:]
-            var_name = "has_" + proc.lower() + "_f"
-            has_f = getattr(getattr(pywatershed.hydrology, mod_name), var_name)
-            if has_f:
-                assert model.processes[proc]._calc_method == "fortran"
-            else:
+    if fortran_avail:
+        for proc in model.processes.keys():
+            if proc.lower() in ["prmssnow", "prmsrunoff", "prmssoilzone"]:
                 assert model.processes[proc]._calc_method == "numba"
+            elif proc.lower() in [
+                "prmscanopy",
+                "prmsgroundwater",
+                "prmschannel",
+            ]:
+                # check if has fortran (has_f) because results depend on that
+                mod_name = "prms_" + proc.lower()[4:]
+                var_name = "has_" + proc.lower() + "_f"
+                has_f = getattr(
+                    getattr(pywatershed.hydrology, mod_name), var_name
+                )
+                if has_f:
+                    assert model.processes[proc]._calc_method == "fortran"
+                else:
+                    assert model.processes[proc]._calc_method == "numba"
 
     # ---------------------------------
     # get the answer data against PRMS5.2.1

autotest/test_prms_channel.py

@@ -10,7 +10,8 @@
 from utils_compare import compare_in_memory, compare_netcdfs
 
 # compare in memory (faster) or full output files?
-compare_output_files = False
+do_compare_output_files = True
+do_compare_in_memory = True
 rtol = atol = 1.0e-7
 
 fail_fast = False
@@ -73,14 +74,14 @@ def test_compare_prms(
         calc_method=calc_method,
     )
 
-    if compare_output_files:
+    if do_compare_output_files:
         nc_parent = tmp_path / domain["domain_name"]
         channel.initialize_netcdf(nc_parent)
         # test that init netcdf twice raises a warning
         with pytest.warns(UserWarning):
             channel.initialize_netcdf(nc_parent)
 
-    else:
+    if do_compare_in_memory:
         answers = {}
         for var in PRMSChannel.get_variables():
             var_pth = output_dir / f"{var}.nc"
@@ -93,12 +94,12 @@ def test_compare_prms(
         channel.advance()
         channel.calculate(float(istep))
         channel.output()
-        if not compare_output_files:
+        if do_compare_in_memory:
             compare_in_memory(channel, answers, atol=atol, rtol=rtol)
 
     channel.finalize()
 
-    if compare_output_files:
+    if do_compare_output_files:
         compare_netcdfs(
             PRMSChannel.get_variables(),
             tmp_path / domain["domain_name"],

autotest/test_prms_groundwater.py

@@ -9,7 +9,8 @@
 from utils_compare import compare_in_memory, compare_netcdfs
 
 # compare in memory (faster) or full output files?
-compare_output_files = False
+do_compare_output_files = False
+do_compare_in_memory = True
 rtol = atol = 1.0e-13
 
 calc_methods = ("numpy", "numba", "fortran")
@@ -64,10 +65,11 @@ def test_compare_prms(
         calc_method=calc_method,
     )
 
-    if compare_output_files:
+    if do_compare_output_files:
         nc_parent = tmp_path / domain["domain_name"]
         gw.initialize_netcdf(nc_parent)
-    else:
+
+    if do_compare_in_memory:
         answers = {}
         for var in PRMSGroundwater.get_variables():
             var_pth = output_dir / f"{var}.nc"
@@ -81,12 +83,12 @@ def test_compare_prms(
         gw.calculate(float(istep))
         gw.output()
 
-        if not compare_output_files:
+        if do_compare_in_memory:
             compare_in_memory(gw, answers, atol=atol, rtol=rtol)
 
     gw.finalize()
 
-    if compare_output_files:
+    if do_compare_output_files:
         compare_netcdfs(
             PRMSGroundwater.get_variables(),
             tmp_path / domain["domain_name"],

autotest/test_prms_solar_geom.py

@@ -2,9 +2,14 @@
 import pytest
 
 from pywatershed.atmosphere.prms_solar_geometry import PRMSSolarGeometry
+from pywatershed.base.adapter import adapter_factory
 from pywatershed.base.control import Control
 from pywatershed.base.parameters import Parameters
 from pywatershed.parameters import PrmsParameters
+from utils_compare import compare_in_memory, compare_netcdfs
+
+# in this case we'll compare netcdf files and in memory
+atol = rtol = np.finfo(np.float32).resolution
 
 params = ("params_sep", "params_one")
 
@@ -37,6 +42,7 @@ def parameters(domain, request):
 def test_compare_prms(
     domain, control, discretization, parameters, tmp_path, from_prms_file
 ):
+    output_dir = domain["prms_output_dir"]
     prms_soltab_file = domain["prms_run_dir"] / "soltab_debug"
     if from_prms_file:
         from_prms_file = prms_soltab_file
@@ -50,48 +56,33 @@ def test_compare_prms(
         from_prms_file=from_prms_file,
         netcdf_output_dir=tmp_path,
     )
+    solar_geom.output()
     solar_geom.finalize()
 
-    ans = PRMSSolarGeometry(
-        control,
-        discretization=discretization,
-        parameters=parameters,
-        from_prms_file=prms_soltab_file,
+    compare_netcdfs(
+        PRMSSolarGeometry.get_variables(),
+        tmp_path,
+        output_dir,
+        atol=atol,
+        rtol=rtol,
     )
 
-    # check the shapes
-    for vv in solar_geom.variables:
-        assert ans[vv].data.shape == solar_geom[vv].data.shape
-
-    # check the 2D values
-    atol = np.finfo(np.float32).resolution
-    rtol = atol
-
-    for vv in solar_geom.variables:
-        assert np.allclose(
-            solar_geom[vv].data,
-            ans[vv].data,
-            atol=atol,
-            rtol=rtol,
+    answers = {}
+    for var in PRMSSolarGeometry.get_variables():
+        var_pth = output_dir / f"{var}.nc"
+        answers[var] = adapter_factory(
+            var_pth, variable_name=var, control=control
         )
 
     # check the advance/calculate the state
     sunhrs_id = id(solar_geom.soltab_sunhrs)
 
-    for ii in range(4):
+    for ii in range(control.n_times):
         control.advance()
         solar_geom.advance()
         solar_geom.calculate(1.0)
 
-        for vv in solar_geom.variables:
-            ans[vv].advance()
-
-            assert solar_geom[vv].current.shape == ans[vv].current.shape
-            assert np.allclose(solar_geom[vv].current, ans[vv].current)
-            assert np.allclose(
-                solar_geom[vv].current, ans[vv].current, atol=atol, rtol=rtol
-            )
-
+        compare_in_memory(solar_geom, answers, atol=atol, rtol=rtol)
         assert id(solar_geom.soltab_sunhrs) == sunhrs_id
 
     return