From 630c026dff4979d3e3dc7eef80dc964c34739564 Mon Sep 17 00:00:00 2001
From: James McCreight <jamesmcc@ucar.edu>
Date: Mon, 25 Sep 2023 17:26:58 -0600
Subject: [PATCH 1/8] refactor PRMSSolarGeom test to use new comparison utils

---
 autotest/test_prms_solar_geom.py | 50 ++++++++++++++------------------
 autotest/utils_compare.py        | 17 +++++++++--
 pywatershed/utils/prms5util.py   |  9 ++++--
 3 files changed, 41 insertions(+), 35 deletions(-)

diff --git a/autotest/test_prms_solar_geom.py b/autotest/test_prms_solar_geom.py
index f2a4560c..752dca21 100644
--- a/autotest/test_prms_solar_geom.py
+++ b/autotest/test_prms_solar_geom.py
@@ -2,10 +2,16 @@
 import pytest
 
 from pywatershed.atmosphere.prms_solar_geometry import PRMSSolarGeometry
+from pywatershed.base.adapter import adapter_factory
 from pywatershed.base.control import Control
 from pywatershed.base.parameters import Parameters
 from pywatershed.parameters import PrmsParameters
 
+from utils_compare import compare_in_memory, compare_netcdfs
+
+# in this case we'll compare netcdf files and in memory
+atol = rtol = np.finfo(np.float32).resolution
+
 params = ("params_sep", "params_one")
 
 
@@ -37,6 +43,7 @@ def parameters(domain, request):
 def test_compare_prms(
     domain, control, discretization, parameters, tmp_path, from_prms_file
 ):
+    output_dir = domain["prms_output_dir"]
     prms_soltab_file = domain["prms_run_dir"] / "soltab_debug"
     if from_prms_file:
         from_prms_file = prms_soltab_file
@@ -50,48 +57,33 @@ def test_compare_prms(
         from_prms_file=from_prms_file,
         netcdf_output_dir=tmp_path,
     )
+    solar_geom.output()
     solar_geom.finalize()
 
-    ans = PRMSSolarGeometry(
-        control,
-        discretization=discretization,
-        parameters=parameters,
-        from_prms_file=prms_soltab_file,
+    compare_netcdfs(
+        PRMSSolarGeometry.get_variables(),
+        tmp_path,
+        output_dir,
+        atol=atol,
+        rtol=rtol,
     )
 
-    # check the shapes
-    for vv in solar_geom.variables:
-        assert ans[vv].data.shape == solar_geom[vv].data.shape
-
-    # check the 2D values
-    atol = np.finfo(np.float32).resolution
-    rtol = atol
-
-    for vv in solar_geom.variables:
-        assert np.allclose(
-            solar_geom[vv].data,
-            ans[vv].data,
-            atol=atol,
-            rtol=rtol,
+    answers = {}
+    for var in PRMSSolarGeometry.get_variables():
+        var_pth = output_dir / f"{var}.nc"
+        answers[var] = adapter_factory(
+            var_pth, variable_name=var, control=control
         )
 
     # check the advance/calculate the state
     sunhrs_id = id(solar_geom.soltab_sunhrs)
 
-    for ii in range(4):
+    for ii in range(control.n_times):
         control.advance()
         solar_geom.advance()
         solar_geom.calculate(1.0)
 
-        for vv in solar_geom.variables:
-            ans[vv].advance()
-
-            assert solar_geom[vv].current.shape == ans[vv].current.shape
-            assert np.allclose(solar_geom[vv].current, ans[vv].current)
-            assert np.allclose(
-                solar_geom[vv].current, ans[vv].current, atol=atol, rtol=rtol
-            )
-
+        compare_in_memory(solar_geom, answers, atol=atol, rtol=rtol)
         assert id(solar_geom.soltab_sunhrs) == sunhrs_id
 
     return
diff --git a/autotest/utils_compare.py b/autotest/utils_compare.py
index dd379184..a3aebd5d 100644
--- a/autotest/utils_compare.py
+++ b/autotest/utils_compare.py
@@ -74,10 +74,17 @@ def compare_in_memory(
     error_message: str = None,
 ):
     # TODO: docstring
+
     for var in process.get_variables():
         answers[var].advance()
+
+        if isinstance(process[var], pws.base.timeseries.TimeseriesArray):
+            actual = process[var].current
+        else:
+            actual = process[var]
+
         assert_allclose(
-            process[var],
+            actual,
             answers[var].current.data,
             atol=atol,
             rtol=rtol,
@@ -103,8 +110,12 @@ def compare_netcdfs(
     # TODO: improve error message
     # TODO: collect failures in a try and report at end
     for var in var_list:
-        answer = xr.open_dataarray(answers_dir / f"{var}.nc")
-        result = xr.open_dataarray(results_dir / f"{var}.nc")
+        answer = xr.open_dataarray(
+            answers_dir / f"{var}.nc", decode_timedelta=False
+        )
+        result = xr.open_dataarray(
+            results_dir / f"{var}.nc", decode_timedelta=False
+        )
 
         if error_message is None:
             error_message = f"Comparison of variable '{var}' was unsuccessful"
diff --git a/pywatershed/utils/prms5util.py b/pywatershed/utils/prms5util.py
index a732651b..93b54ba4 100644
--- a/pywatershed/utils/prms5util.py
+++ b/pywatershed/utils/prms5util.py
@@ -6,6 +6,7 @@
 import numpy as np
 import pandas as pd
 
+from ..base import meta
 from ..base.accessor import Accessor
 
 fileish = Union[str, pl.Path]
@@ -304,11 +305,12 @@ def to_netcdf(
         variables = {}
         for vv in self.variables:
             out_file = self.output_dir / f"{vv}.nc"
+            var_meta = meta.find_variables(vv)[vv]
 
             ds = nc4.Dataset(out_file, "w", clobber=True)
             ds.setncattr("Description", "PRMS soltab data")
 
-            # time dime and coord
+            # time dim and coord
             ds.createDimension("doy", ndoy)
             doy = ds.createVariable("doy", "i4", ("doy",))
             doy.units = "Day of year"
@@ -326,14 +328,15 @@ def to_netcdf(
 
             variables[vv] = ds.createVariable(
                 vv,
-                "f4",
+                "f8",
                 dims,
-                fill_value=nc4.default_fillvals["f4"],
+                fill_value=nc4.default_fillvals["f8"],
                 zlib=zlib,
                 complevel=complevel,
                 chunksizes=tuple(chunk_sizes_var),
             )
             variables[vv][:] = self[vv]
+            variables[vv].setncatts(var_meta)
             ds.close()
             print(f"Wrote: {out_file}")
         return

From 9a5a21eaea78383a3598f31981728819bf2ebd41 Mon Sep 17 00:00:00 2001
From: James McCreight <jamesmcc@ucar.edu>
Date: Tue, 26 Sep 2023 15:18:56 -0600
Subject: [PATCH 2/8] 1) manage warnings due to calc_method in test_model, 2)
 refactor prms channel and groundwater to have one or both of
 compare_output_files and compare_in_memory

---
 autotest/test_model.py                 | 44 ++++++++++++++++++--------
 autotest/test_prms_channel.py          | 11 ++++---
 autotest/test_prms_groundwater.py      | 12 ++++---
 autotest/utils_compare.py              |  3 +-
 pywatershed/hydrology/prms_runoff.py   |  2 +-
 pywatershed/hydrology/prms_snow.py     |  2 +-
 pywatershed/hydrology/prms_soilzone.py |  2 +-
 7 files changed, 49 insertions(+), 27 deletions(-)

diff --git a/autotest/test_model.py b/autotest/test_model.py
index 8cc1bdd9..8e241429 100644
--- a/autotest/test_model.py
+++ b/autotest/test_model.py
@@ -12,6 +12,10 @@
 from pywatershed.base.model import Model
 from pywatershed.parameters import Parameters, PrmsParameters
 
+fortran_avail = getattr(
+    getattr(pywatershed.hydrology, "prms_canopy"), "has_prmscanopy_f"
+)
+
 compare_to_prms521 = False  # TODO TODO TODO
 failfast = True
 detailed = True
@@ -39,7 +43,10 @@ def control(domain):
     control = Control.load(domain["control_file"])
     control.options["verbose"] = 10
     control.options["budget_type"] = None
-    control.options["calc_method"] = "fortran"
+    if fortran_avail:
+        control.options["calc_method"] = "fortran"
+    else:
+        control.options["calc_method"] = "numba"
     control.options["load_n_time_batches"] = 1
     return control
 
@@ -141,21 +148,32 @@ def test_model(domain, model_args, tmp_path):
 
     control.options["input_dir"] = input_dir
 
-    model = Model(**model_args)
+    if control.options["calc_method"] == "fortran":
+        with pytest.warns(UserWarning):
+            model = Model(**model_args)
+    else:
+        model = Model(**model_args)
 
     # Test passing of control calc_method option
-    for proc in model.processes.keys():
-        if proc.lower() in ["prmssnow", "prmsrunoff", "prmssoilzone"]:
-            assert model.processes[proc]._calc_method == "numba"
-        elif proc.lower() in ["prmscanopy", "prmsgroundwater", "prmschannel"]:
-            # check if has fortran (has_f) because results depend on that
-            mod_name = "prms_" + proc.lower()[4:]
-            var_name = "has_" + proc.lower() + "_f"
-            has_f = getattr(getattr(pywatershed.hydrology, mod_name), var_name)
-            if has_f:
-                assert model.processes[proc]._calc_method == "fortran"
-            else:
+    if fortran_avail:
+        for proc in model.processes.keys():
+            if proc.lower() in ["prmssnow", "prmsrunoff", "prmssoilzone"]:
                 assert model.processes[proc]._calc_method == "numba"
+            elif proc.lower() in [
+                "prmscanopy",
+                "prmsgroundwater",
+                "prmschannel",
+            ]:
+                # check if has fortran (has_f) because results depend on that
+                mod_name = "prms_" + proc.lower()[4:]
+                var_name = "has_" + proc.lower() + "_f"
+                has_f = getattr(
+                    getattr(pywatershed.hydrology, mod_name), var_name
+                )
+                if has_f:
+                    assert model.processes[proc]._calc_method == "fortran"
+                else:
+                    assert model.processes[proc]._calc_method == "numba"
 
     # ---------------------------------
     # get the answer data against PRMS5.2.1
diff --git a/autotest/test_prms_channel.py b/autotest/test_prms_channel.py
index 5160e816..e21bdae1 100644
--- a/autotest/test_prms_channel.py
+++ b/autotest/test_prms_channel.py
@@ -10,7 +10,8 @@
 from utils_compare import compare_in_memory, compare_netcdfs
 
 # compare in memory (faster) or full output files?
-compare_output_files = False
+do_compare_output_files = True
+do_compare_in_memory = True
 rtol = atol = 1.0e-7
 
 fail_fast = False
@@ -73,14 +74,14 @@ def test_compare_prms(
         calc_method=calc_method,
     )
 
-    if compare_output_files:
+    if do_compare_output_files:
         nc_parent = tmp_path / domain["domain_name"]
         channel.initialize_netcdf(nc_parent)
         # test that init netcdf twice raises a warning
         with pytest.warns(UserWarning):
             channel.initialize_netcdf(nc_parent)
 
-    else:
+    if do_compare_in_memory:
         answers = {}
         for var in PRMSChannel.get_variables():
             var_pth = output_dir / f"{var}.nc"
@@ -93,12 +94,12 @@ def test_compare_prms(
         channel.advance()
         channel.calculate(float(istep))
         channel.output()
-        if not compare_output_files:
+        if do_compare_in_memory:
             compare_in_memory(channel, answers, atol=atol, rtol=rtol)
 
     channel.finalize()
 
-    if compare_output_files:
+    if do_compare_output_files:
         compare_netcdfs(
             PRMSChannel.get_variables(),
             tmp_path / domain["domain_name"],
diff --git a/autotest/test_prms_groundwater.py b/autotest/test_prms_groundwater.py
index f0209d8d..592e2d59 100644
--- a/autotest/test_prms_groundwater.py
+++ b/autotest/test_prms_groundwater.py
@@ -9,7 +9,8 @@
 from utils_compare import compare_in_memory, compare_netcdfs
 
 # compare in memory (faster) or full output files?
-compare_output_files = False
+do_compare_output_files = False
+do_compare_in_memory = True
 rtol = atol = 1.0e-13
 
 calc_methods = ("numpy", "numba", "fortran")
@@ -64,10 +65,11 @@ def test_compare_prms(
         calc_method=calc_method,
     )
 
-    if compare_output_files:
+    if do_compare_output_files:
         nc_parent = tmp_path / domain["domain_name"]
         gw.initialize_netcdf(nc_parent)
-    else:
+
+    if do_compare_in_memory:
         answers = {}
         for var in PRMSGroundwater.get_variables():
             var_pth = output_dir / f"{var}.nc"
@@ -81,12 +83,12 @@ def test_compare_prms(
         gw.calculate(float(istep))
         gw.output()
 
-        if not compare_output_files:
+        if do_compare_in_memory:
             compare_in_memory(gw, answers, atol=atol, rtol=rtol)
 
     gw.finalize()
 
-    if compare_output_files:
+    if do_compare_output_files:
         compare_netcdfs(
             PRMSGroundwater.get_variables(),
             tmp_path / domain["domain_name"],
diff --git a/autotest/utils_compare.py b/autotest/utils_compare.py
index a3aebd5d..bfe5e747 100644
--- a/autotest/utils_compare.py
+++ b/autotest/utils_compare.py
@@ -52,7 +52,8 @@ def assert_allclose(
         assert (actual_nan == desired_nan).all()
 
     abs_diff = abs(actual - desired)
-    rel_abs_diff = abs_diff / desired
+    with np.errstate(divide="ignore", invalid="ignore"):
+        rel_abs_diff = abs_diff / desired
 
     abs_close = abs_diff < atol
     rel_close = rel_abs_diff < rtol
diff --git a/pywatershed/hydrology/prms_runoff.py b/pywatershed/hydrology/prms_runoff.py
index 85304316..62565615 100644
--- a/pywatershed/hydrology/prms_runoff.py
+++ b/pywatershed/hydrology/prms_runoff.py
@@ -386,7 +386,7 @@ def _init_calc_method(self):
                 f"Invalid calc_method={self._calc_method} for {self.name}. "
                 f"Setting calc_method to 'numba' for {self.name}"
             )
-            warn(msg)
+            warn(msg, UserWarning)
             self._calc_method = "numba"
 
         if self._calc_method.lower() == "numba":
diff --git a/pywatershed/hydrology/prms_snow.py b/pywatershed/hydrology/prms_snow.py
index f46f970d..8a013c39 100644
--- a/pywatershed/hydrology/prms_snow.py
+++ b/pywatershed/hydrology/prms_snow.py
@@ -401,7 +401,7 @@ def _init_calc_method(self):
                 f"Invalid calc_method={self._calc_method} for {self.name}. "
                 f"Setting calc_method to 'numba' for {self.name}"
             )
-            warn(msg)
+            warn(msg, UserWarning)
             self._calc_method = "numba"
 
         if self._calc_method.lower() == "numba":
diff --git a/pywatershed/hydrology/prms_soilzone.py b/pywatershed/hydrology/prms_soilzone.py
index 4a77a017..b293f88e 100644
--- a/pywatershed/hydrology/prms_soilzone.py
+++ b/pywatershed/hydrology/prms_soilzone.py
@@ -395,7 +395,7 @@ def _init_calc_method(self):
                 f"Invalid calc_method={self._calc_method} for {self.name}. "
                 f"Setting calc_method to 'numba' for {self.name}"
             )
-            warn(msg)
+            warn(msg, UserWarning)
             self._calc_method = "numba"
 
         if self._calc_method.lower() == "numba":

From 27da969d76ea48758f9ca9c35fc5d79effeb89ff Mon Sep 17 00:00:00 2001
From: James McCreight <jamesmcc@ucar.edu>
Date: Fri, 29 Sep 2023 13:53:43 -0600
Subject: [PATCH 3/8] fix apparent encoding changes from a new xarray version

---
 autotest/test_data_model.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/autotest/test_data_model.py b/autotest/test_data_model.py
index 144643dd..da089911 100644
--- a/autotest/test_data_model.py
+++ b/autotest/test_data_model.py
@@ -167,6 +167,7 @@ def test_nc4_dd_nc4(tmp_path):
         "contiguous",
         "chunksizes",
         "coordinates",
+        "preferred_chunks",
     ]:
         for vv in sorted(ds1.variables):
             for dd in [ds1, ds2]:

From 0b8c01d0a4b82a8ac99590f9829503f41f6f9dfc Mon Sep 17 00:00:00 2001
From: James McCreight <jamesmcc@ucar.edu>
Date: Fri, 29 Sep 2023 14:42:46 -0600
Subject: [PATCH 4/8] matplotlib fixes with new version; linting

---
 autotest/test_prms_solar_geom.py       |  1 -
 examples/00_processes.ipynb            | 15 ++------
 examples/01_multi-process_models.ipynb | 48 ++++++++------------------
 examples/02_prms_legacy_models.ipynb   |  6 +++-
 pywatershed/analysis/process_plot.py   |  2 +-
 5 files changed, 23 insertions(+), 49 deletions(-)

diff --git a/autotest/test_prms_solar_geom.py b/autotest/test_prms_solar_geom.py
index 752dca21..cb2bd971 100644
--- a/autotest/test_prms_solar_geom.py
+++ b/autotest/test_prms_solar_geom.py
@@ -6,7 +6,6 @@
 from pywatershed.base.control import Control
 from pywatershed.base.parameters import Parameters
 from pywatershed.parameters import PrmsParameters
-
 from utils_compare import compare_in_memory, compare_netcdfs
 
 # in this case we'll compare netcdf files and in memory
diff --git a/examples/00_processes.ipynb b/examples/00_processes.ipynb
index 6ada3abb..58d17fc8 100644
--- a/examples/00_processes.ipynb
+++ b/examples/00_processes.ipynb
@@ -54,8 +54,7 @@
     "editable": true,
     "slideshow": {
      "slide_type": ""
-    },
-    "tags": []
+    }
    },
    "outputs": [],
    "source": [
@@ -545,22 +544,14 @@
   }
  ],
  "metadata": {
-  "kernelspec": {
-   "display_name": "Python [conda env:pws2] *",
-   "language": "python",
-   "name": "conda-env-pws2-py"
-  },
   "language_info": {
    "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
+    "name": "ipython"
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.12"
+   "nbconvert_exporter": "python"
   }
  },
  "nbformat": 4,
diff --git a/examples/01_multi-process_models.ipynb b/examples/01_multi-process_models.ipynb
index 51d5dae7..d97cdabb 100644
--- a/examples/01_multi-process_models.ipynb
+++ b/examples/01_multi-process_models.ipynb
@@ -7,8 +7,7 @@
     "editable": true,
     "slideshow": {
      "slide_type": ""
-    },
-    "tags": []
+    }
    },
    "source": [
     "# Multi-process models in pywatershed\n",
@@ -219,8 +218,7 @@
     "editable": true,
     "slideshow": {
      "slide_type": ""
-    },
-    "tags": []
+    }
    },
    "source": [
     "For the time being, `PRMSChannel` needs to know about both HRUs and segments, so `dis_both` is used. We plan to remove this requirement in the near future by implementing \"exchanges\" between processes into the model dictionary. Stay tuned.\n",
@@ -288,8 +286,7 @@
     "editable": true,
     "slideshow": {
      "slide_type": ""
-    },
-    "tags": []
+    }
    },
    "outputs": [],
    "source": [
@@ -372,8 +369,7 @@
     "editable": true,
     "slideshow": {
      "slide_type": ""
-    },
-    "tags": []
+    }
    },
    "source": [
     "We add the option `netcdf_output_dir` to the control since we assume we wont be able to do so at run time. Note that this option and the `input_dir` option are `pathlib.Path` objects. These are not what we want to write to file. We want their string version. We could do `str()` on each one by hand, but it will be more handy to write a small, recursive function to do this on a supplied dictionary since this will be a recurring task with the model dictionary we will create after the control YAML file."
@@ -600,8 +596,7 @@
     "editable": true,
     "slideshow": {
      "slide_type": ""
-    },
-    "tags": []
+    }
    },
    "outputs": [],
    "source": [
@@ -620,8 +615,7 @@
     "editable": true,
     "slideshow": {
      "slide_type": ""
-    },
-    "tags": []
+    }
    },
    "source": [
     "Now compare the values of all variables:"
@@ -670,8 +664,7 @@
     "editable": true,
     "slideshow": {
      "slide_type": ""
-    },
-    "tags": []
+    }
    },
    "outputs": [],
    "source": [
@@ -812,8 +805,7 @@
     "editable": true,
     "slideshow": {
      "slide_type": ""
-    },
-    "tags": []
+    }
    },
    "source": [
     "Reducing the output significantly reduced the time, in this case (on my machine) from 25s to 15s, or about 60%."
@@ -919,8 +911,7 @@
     "editable": true,
     "slideshow": {
      "slide_type": ""
-    },
-    "tags": []
+    }
    },
    "outputs": [],
    "source": [
@@ -938,8 +929,7 @@
     "editable": true,
     "slideshow": {
      "slide_type": ""
-    },
-    "tags": []
+    }
    },
    "source": [
     "Well, that was a lot of work. But, as alluded to above, the `Model` object does the above so you dont have to. You just learned something about how the flow of information between processes is enabled by the design and how one can query individual processes in `pywatershed`. But we could instantiate the submodel and plot this wiring up, just as we plotted the `ModelGraph` of the full model. We'll create the submodel in a new `run_dir` and we'll use outputs from the full model above as inputs to this submodel."
@@ -1043,8 +1033,7 @@
     "editable": true,
     "slideshow": {
      "slide_type": ""
-    },
-    "tags": []
+    }
    },
    "outputs": [],
    "source": [
@@ -1067,8 +1056,7 @@
     "editable": true,
     "slideshow": {
      "slide_type": ""
-    },
-    "tags": []
+    }
    },
    "source": [
     "Note that the required inputs to the submodel are quire different and rely on the existence of these files having already been output by the full model. \n",
@@ -1247,22 +1235,14 @@
   }
  ],
  "metadata": {
-  "kernelspec": {
-   "display_name": "Python [conda env:pws2] *",
-   "language": "python",
-   "name": "conda-env-pws2-py"
-  },
   "language_info": {
    "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
+    "name": "ipython"
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.12"
+   "nbconvert_exporter": "python"
   }
  },
  "nbformat": 4,
diff --git a/examples/02_prms_legacy_models.ipynb b/examples/02_prms_legacy_models.ipynb
index fe7767ac..85de45e0 100644
--- a/examples/02_prms_legacy_models.ipynb
+++ b/examples/02_prms_legacy_models.ipynb
@@ -68,7 +68,11 @@
   {
    "cell_type": "markdown",
    "id": "39abd69f-13a3-4f23-9678-a57c0b1f848d",
-   "metadata": {},
+   "metadata": {
+    "jupyter": {
+     "source_hidden": true
+    }
+   },
    "source": [
     "The domain directory is where we have all the required inputs to run this model (among others) and `nb_output_dir` is where this notebook will write its output. "
    ]
diff --git a/pywatershed/analysis/process_plot.py b/pywatershed/analysis/process_plot.py
index a4c98c0b..1ee3d050 100644
--- a/pywatershed/analysis/process_plot.py
+++ b/pywatershed/analysis/process_plot.py
@@ -122,7 +122,7 @@ def plot_seg_var(self, var_name: str, process: Process, cmap="cool"):
 
         metadata = meta.get_vars(var_name)[var_name]
         plt.title("Variable: {}".format(var_name))
-        plt.colorbar(mapper, shrink=0.6, label=metadata["units"])
+        plt.colorbar(mapper, shrink=0.6, label=metadata["units"], ax=ax)
 
         hru_poly = plot_polygon_collection(
             ax,

From b431b6ad2e89a90fad9c57b0ab9ba94f93f0f180 Mon Sep 17 00:00:00 2001
From: James McCreight <jamesmcc@ucar.edu>
Date: Tue, 3 Oct 2023 17:06:27 -0600
Subject: [PATCH 5/8] refactor CI to storage management

---
 .github/workflows/ci.yaml                | 78 ++++++++++++++++++++----
 test_data/generate/remove_output_dirs.py | 14 +++++
 test_data/generate/remove_prms_csvs.py   |  2 +-
 3 files changed, 80 insertions(+), 14 deletions(-)
 create mode 100644 test_data/generate/remove_output_dirs.py

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index a2af53e8..eee0f236 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -138,7 +138,7 @@ jobs:
         with:
           compiler: gcc
           version: 11
-      
+
       - name: Link gfortran dylibs on Mac
         if: runner.os == 'macOS'
         run: .github/scripts/symlink_gfortran_mac.sh
@@ -174,40 +174,92 @@ jobs:
           pip -V
           pip list
 
-      - name: Run available domains with PRMS and convert csv output to NetCDF
+      - name: hru_1 domain, run PRMS and convert csv output to NetCDF
+        working-directory: test_data/generate
+        run: |
+          pytest -vv -n=2 --durations=0 run_prms_domains.py --domain=hru_1
+          pytest -vv -n=auto --durations=0 convert_prms_output_to_nc.py  --domain=hru_1
+          pytest -vv -n=auto --durations=0 remove_prms_csvs.py
+
+      - name: hru_1 domain, list all NetCDF files in test_data directory
+        working-directory: test_data/hru_1/output
+        run: |
+          find . -name "*.nc"
+
+      - name: hru_1 domain , run tests
+        working-directory: autotest
+        run: pytest
+          -vv
+          -n=auto
+          --domain_yaml=../test_data/hru_1/hru_1.yaml
+          --durations=0
+          --cov=pywatershed
+          --cov-report=xml
+          --junitxml=pytest_hru_1.xml
+
+
+      - name: drb_2yr domain, run PRMS and convert csv output to NetCDF
+        working-directory: test_data/generate
+        run: |
+          pytest -vv remove_output_dirs.py --domain=hru_1
+          pytest -vv -n=2 run_prms_domains.py --domain=drb_2yr
+          pytest -vv -n=auto convert_prms_output_to_nc.py --domain=drb_2yr
+          pytest -vv -n=auto remove_prms_csvs.py
+
+      - name: drb_2yr domain, list all NetCDF files in test_data directory
+        working-directory: test_data/drb_2yr/output
+        run: |
+          find . -name "*.nc"
+
+      - name: drb_2yr domain , run tests
+        working-directory: autotest
+        run: pytest
+          -vv
+          -n=auto
+          --domain_yaml=../test_data/drb_2yr/drb_2yr.yaml
+          --durations=0
+          --cov=pywatershed
+          --cov-report=xml
+          --junitxml=pytest_drb_2yr.xml
+
+      - name: ucb_2yr domain, run PRMS and convert csv output to NetCDF
         working-directory: test_data/generate
         run: |
-          pytest -v -n=auto --durations=0 run_prms_domains.py
-          pytest -v -n=auto --durations=0 convert_prms_output_to_nc.py
-          pytest -v -n=auto --durations=0 remove_prms_csvs.py
+          pytest -vv remove_output_dirs.py --domain=drb_2yr
+          pytest -vv -n=2 run_prms_domains.py --domain=ucb_2yr
+          pytest -vv -n=auto convert_prms_output_to_nc.py --domain=ucb_2yr
+          pytest -vv -n=auto remove_prms_csvs.py
 
-      - name: List all NetCDF files in test_data directory
-        working-directory: test_data
+      - name: ucb_2yr domain, list all NetCDF files in test_data directory
+        working-directory: test_data/ucb_2yr/output
         run: |
           find . -name "*.nc"
 
-      - name: Run tests
+      - name: ucb_2yr domain , run tests
         working-directory: autotest
         run: pytest
-          -v
+          -vv
           -n=auto
+          --domain_yaml=../test_data/ucb_2yr/ucb_2yr.yaml
           --durations=0
-          --all_domains
           --cov=pywatershed
           --cov-report=xml
-          --junitxml=pytest.xml
+          --junitxml=pytest_ucb_2yr.xml
 
       - name: Upload test results
         if: always()
         uses: actions/upload-artifact@v3
         with:
           name: Test results for ${{ runner.os }}-${{ matrix.python-version }}
-          path: ./autotest/pytest.xml
+          path: |
+            ./autotest/pytest_hru_1.xml
+            ./autotest/pytest_drb_2yr.xml
+            ./autotest/pytest_ucb_2yr.xml
 
       - name: Upload code coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
-          file: ./autotest/coverage.xml
+          file: ./autotest/coverage.xml  # should be just the ucb result
           # flags: unittests
           env_vars: RUNNER_OS,PYTHON_VERSION
           # name: codecov-umbrella
diff --git a/test_data/generate/remove_output_dirs.py b/test_data/generate/remove_output_dirs.py
new file mode 100644
index 00000000..21d789c1
--- /dev/null
+++ b/test_data/generate/remove_output_dirs.py
@@ -0,0 +1,14 @@
+import pathlib as pl
+import shutil
+
+
+def test_remove_output_dirs(simulation):
+    ws = simulation["ws"]
+    output_dir = pl.Path(ws) / "output"
+    if output_dir.exists():
+        shutil.rmtree(output_dir)
+
+    # this shouldnt matter but it is necessary to exist for PRMS to write to it
+    output_dir.mkdir(parents=True)
+
+    return
diff --git a/test_data/generate/remove_prms_csvs.py b/test_data/generate/remove_prms_csvs.py
index 86a139e9..fac52c71 100644
--- a/test_data/generate/remove_prms_csvs.py
+++ b/test_data/generate/remove_prms_csvs.py
@@ -1,6 +1,6 @@
 csvs_to_keep = ["hru_ppt", "intcp_stor", "potet", "gwres_stor"]
 
 
-def test_csv_to_netcdf(csv_file):
+def test_remove_csv_files(csv_file):
     if csv_file.with_suffix("").name not in (csvs_to_keep):
         csv_file.unlink()

From 71ba40bb270369f69659d1feff84583da00ed045 Mon Sep 17 00:00:00 2001
From: James McCreight <jamesmcc@ucar.edu>
Date: Tue, 3 Oct 2023 17:43:48 -0600
Subject: [PATCH 6/8] minor CI workflow changes

---
 .github/workflows/ci.yaml | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index eee0f236..8ec92005 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -174,19 +174,19 @@ jobs:
           pip -V
           pip list
 
-      - name: hru_1 domain, run PRMS and convert csv output to NetCDF
+      - name: hru_1: generate and manage test data domain, run PRMS and convert csv output to NetCDF
         working-directory: test_data/generate
         run: |
           pytest -vv -n=2 --durations=0 run_prms_domains.py --domain=hru_1
           pytest -vv -n=auto --durations=0 convert_prms_output_to_nc.py  --domain=hru_1
           pytest -vv -n=auto --durations=0 remove_prms_csvs.py
 
-      - name: hru_1 domain, list all NetCDF files in test_data directory
+      - name: hru_1: list netcdf input files
         working-directory: test_data/hru_1/output
         run: |
-          find . -name "*.nc"
+          find . -name '*.nc'
 
-      - name: hru_1 domain , run tests
+      - name: hru_1: pywatershed tests
         working-directory: autotest
         run: pytest
           -vv
@@ -198,7 +198,7 @@ jobs:
           --junitxml=pytest_hru_1.xml
 
 
-      - name: drb_2yr domain, run PRMS and convert csv output to NetCDF
+      - name: drb_2yr: generate and manage test data
         working-directory: test_data/generate
         run: |
           pytest -vv remove_output_dirs.py --domain=hru_1
@@ -206,12 +206,12 @@ jobs:
           pytest -vv -n=auto convert_prms_output_to_nc.py --domain=drb_2yr
           pytest -vv -n=auto remove_prms_csvs.py
 
-      - name: drb_2yr domain, list all NetCDF files in test_data directory
+      - name: drb_2yr: list netcdf input files
         working-directory: test_data/drb_2yr/output
         run: |
-          find . -name "*.nc"
+          find . -name '*.nc'
 
-      - name: drb_2yr domain , run tests
+      - name: drb_2yr: pywatershed tests
         working-directory: autotest
         run: pytest
           -vv
@@ -222,7 +222,7 @@ jobs:
           --cov-report=xml
           --junitxml=pytest_drb_2yr.xml
 
-      - name: ucb_2yr domain, run PRMS and convert csv output to NetCDF
+      - name: ucb_2yr: generate and manage test data
         working-directory: test_data/generate
         run: |
           pytest -vv remove_output_dirs.py --domain=drb_2yr
@@ -230,12 +230,12 @@ jobs:
           pytest -vv -n=auto convert_prms_output_to_nc.py --domain=ucb_2yr
           pytest -vv -n=auto remove_prms_csvs.py
 
-      - name: ucb_2yr domain, list all NetCDF files in test_data directory
+      - name: ucb_2yr: list netcdf input files
         working-directory: test_data/ucb_2yr/output
         run: |
-          find . -name "*.nc"
+          find . -name '*.nc'
 
-      - name: ucb_2yr domain , run tests
+      - name: ucb_2yr: pywatershed tests
         working-directory: autotest
         run: pytest
           -vv

From 261e5b2a218bb128bbbfcec46f2f00967c2c8007 Mon Sep 17 00:00:00 2001
From: James McCreight <jamesmcc@ucar.edu>
Date: Tue, 3 Oct 2023 21:03:07 -0600
Subject: [PATCH 7/8] CI bring back tests

---
 .github/workflows/ci.yaml | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 8ec92005..d293185e 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -174,19 +174,19 @@ jobs:
           pip -V
           pip list
 
-      - name: hru_1: generate and manage test data domain, run PRMS and convert csv output to NetCDF
+      - name: hru_1 - generate and manage test data domain, run PRMS and convert csv output to NetCDF
         working-directory: test_data/generate
         run: |
           pytest -vv -n=2 --durations=0 run_prms_domains.py --domain=hru_1
           pytest -vv -n=auto --durations=0 convert_prms_output_to_nc.py  --domain=hru_1
           pytest -vv -n=auto --durations=0 remove_prms_csvs.py
 
-      - name: hru_1: list netcdf input files
+      - name: hru_1 - list netcdf input files
         working-directory: test_data/hru_1/output
         run: |
           find . -name '*.nc'
 
-      - name: hru_1: pywatershed tests
+      - name: hru_1 - pywatershed tests
         working-directory: autotest
         run: pytest
           -vv
@@ -198,7 +198,7 @@ jobs:
           --junitxml=pytest_hru_1.xml
 
 
-      - name: drb_2yr: generate and manage test data
+      - name: drb_2yr - generate and manage test data
         working-directory: test_data/generate
         run: |
           pytest -vv remove_output_dirs.py --domain=hru_1
@@ -206,12 +206,12 @@ jobs:
           pytest -vv -n=auto convert_prms_output_to_nc.py --domain=drb_2yr
           pytest -vv -n=auto remove_prms_csvs.py
 
-      - name: drb_2yr: list netcdf input files
+      - name: drb_2yr - list netcdf input files
         working-directory: test_data/drb_2yr/output
         run: |
           find . -name '*.nc'
 
-      - name: drb_2yr: pywatershed tests
+      - name: drb_2yr - pywatershed tests
         working-directory: autotest
         run: pytest
           -vv
@@ -222,7 +222,7 @@ jobs:
           --cov-report=xml
           --junitxml=pytest_drb_2yr.xml
 
-      - name: ucb_2yr: generate and manage test data
+      - name: ucb_2yr - generate and manage test data
         working-directory: test_data/generate
         run: |
           pytest -vv remove_output_dirs.py --domain=drb_2yr
@@ -230,12 +230,12 @@ jobs:
           pytest -vv -n=auto convert_prms_output_to_nc.py --domain=ucb_2yr
           pytest -vv -n=auto remove_prms_csvs.py
 
-      - name: ucb_2yr: list netcdf input files
+      - name: ucb_2yr - list netcdf input files
         working-directory: test_data/ucb_2yr/output
         run: |
           find . -name '*.nc'
 
-      - name: ucb_2yr: pywatershed tests
+      - name: ucb_2yr - pywatershed tests
         working-directory: autotest
         run: pytest
           -vv

From 48db5fd530c70fe419c4511515fe4eeaf28aae14 Mon Sep 17 00:00:00 2001
From: James McCreight <jamesmcc@ucar.edu>
Date: Wed, 4 Oct 2023 10:00:31 -0600
Subject: [PATCH 8/8] CI action-tmatev3 functionality

---
 .github/workflows/ci.yaml | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index d293185e..230d3f94 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -9,6 +9,14 @@ on:
       - "*"
       - "!v[0-9]+.[0-9]+.[0-9]+*"
 
+  workflow_dispatch:
+    inputs:
+      debug_enabled:
+        type: boolean
+        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
+        required: false
+        default: false
+
 jobs:
 
   pyws_setup:
@@ -101,7 +109,7 @@ jobs:
           pylint --jobs=2 --errors-only --exit-zero ./pywatershed ./autotest
 
   test:
-    name: ${{ matrix.os}} py${{ matrix.python-version }}
+    name: ${{ matrix.os }} py${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
     defaults:
       run:
@@ -112,9 +120,14 @@ jobs:
         os: [ "ubuntu-latest", "macos-latest", "windows-latest" ]
         python-version: ["3.9", "3.10"]
     steps:
+
       - name: Checkout repo
         uses: actions/checkout@v3
 
+      - name: Setup tmate session
+        uses: mxschmitt/action-tmate@v3
+        if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
+
       - name: Set environment variables
         run: |
           echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV
@@ -181,10 +194,12 @@ jobs:
           pytest -vv -n=auto --durations=0 convert_prms_output_to_nc.py  --domain=hru_1
           pytest -vv -n=auto --durations=0 remove_prms_csvs.py
 
+
+
       - name: hru_1 - list netcdf input files
-        working-directory: test_data/hru_1/output
+        working-directory: test_data
         run: |
-          find . -name '*.nc'
+          find hru_1/output/ -name '*.nc'
 
       - name: hru_1 - pywatershed tests
         working-directory: autotest
@@ -207,9 +222,9 @@ jobs:
           pytest -vv -n=auto remove_prms_csvs.py
 
       - name: drb_2yr - list netcdf input files
-        working-directory: test_data/drb_2yr/output
+        working-directory: test_data
         run: |
-          find . -name '*.nc'
+          find drb_2yr/output/ -name '*.nc'
 
       - name: drb_2yr - pywatershed tests
         working-directory: autotest
@@ -231,9 +246,9 @@ jobs:
           pytest -vv -n=auto remove_prms_csvs.py
 
       - name: ucb_2yr - list netcdf input files
-        working-directory: test_data/ucb_2yr/output
+        working-directory: test_data
         run: |
-          find . -name '*.nc'
+          find ucb_2yr/output/ -name '*.nc'
 
       - name: ucb_2yr - pywatershed tests
         working-directory: autotest