From c60d1da380900f49fe0a726950381cefbad62f94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=ADd=20Brakenhoff?= Date: Thu, 10 Oct 2024 12:29:16 +0200 Subject: [PATCH 1/9] up version --- pastastore/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pastastore/version.py b/pastastore/version.py index e64fc82..a4a2b2a 100644 --- a/pastastore/version.py +++ b/pastastore/version.py @@ -9,7 +9,7 @@ PASTAS_LEQ_022 = PASTAS_VERSION <= parse_version("0.22.0") PASTAS_GEQ_150 = PASTAS_VERSION >= parse_version("1.5.0") -__version__ = "1.7.1" +__version__ = "1.8.0.dev0" def show_versions(optional=False) -> None: From 312963afa329ba7a91b495a8a488e8f1ea1201fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=ADd=20Brakenhoff?= Date: Thu, 10 Oct 2024 12:29:41 +0200 Subject: [PATCH 2/9] fix for editable installs --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index faba810..ec2b645 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,6 +80,9 @@ docs = [ "nbsphinx_link", ] +[tool.setuptools.packages] +find = {} + [tool.setuptools.dynamic] version = { attr = "pastastore.version.__version__" } From 9c7fa364e6be36333d77be35e44a104a66f0f8f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=ADd=20Brakenhoff?= Date: Thu, 10 Oct 2024 12:30:00 +0200 Subject: [PATCH 3/9] fix for floating point issue --- pastastore/extensions/hpd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pastastore/extensions/hpd.py b/pastastore/extensions/hpd.py index 77a282f..309d4db 100644 --- a/pastastore/extensions/hpd.py +++ b/pastastore/extensions/hpd.py @@ -164,10 +164,10 @@ def add_observation( metadata.pop("name", None) metadata.pop("meta", None) unit = metadata.get("unit", None) - if unit == "m" and unit_multiplier == 1e3: + if unit == "m" and np.allclose(unit_multiplier, 1e-3): metadata["unit"] = "mm" elif unit_multiplier != 1.0: - metadata["unit"] = f"{unit_multiplier:e}*{unit}" + metadata["unit"] = f"{unit_multiplier:.1e}*{unit}" source = metadata.get("source", "") if len(source) > 0: From 32b823b5462d8bfb21618f8b57a8fe0f424a50ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=ADd=20Brakenhoff?= Date: Thu, 10 Oct 2024 12:35:43 +0200 Subject: [PATCH 4/9] add _get_tmin_tmax in separate function --- pastastore/extensions/hpd.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/pastastore/extensions/hpd.py b/pastastore/extensions/hpd.py index 309d4db..edb4112 100644 --- a/pastastore/extensions/hpd.py +++ b/pastastore/extensions/hpd.py @@ -199,6 +199,34 @@ def add_observation( else: raise ValueError("libname must be 'oseries' or 'stresses'.") + def _get_tmin_tmax(self, tmin, tmax, oseries=None): + """Get tmin and tmax from store if not specified. + + Parameters + ---------- + tmin : TimeType + start time + tmax : TimeType + end time + oseries : str, optional + name of the observation series to get tmin/tmax for, by default None + + Returns + ------- + tmin, tmax : TimeType, TimeType + tmin and tmax + """ + # get tmin/tmax if not specified + if tmin is None or tmax is None: + tmintmax = self._store.get_tmin_tmax( + "oseries", names=[oseries] if oseries else None + ) + if tmin is None: + tmin = tmintmax.loc[:, "tmin"].min() - Timedelta(days=10 * 365) + if tmax is None: + tmax = tmintmax.loc[:, "tmax"].max() + return tmin, tmax + def download_knmi_precipitation( self, stns: Optional[list[int]] = None, From 855b31e97316944822afe74accb915b422773c4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=ADd=20Brakenhoff?= Date: Thu, 10 Oct 2024 12:35:57 +0200 Subject: [PATCH 5/9] add _get_tmin_tmax in separate function --- pastastore/extensions/hpd.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/pastastore/extensions/hpd.py b/pastastore/extensions/hpd.py index edb4112..c493b34 100644 --- a/pastastore/extensions/hpd.py +++ b/pastastore/extensions/hpd.py @@ -331,7 +331,7 @@ def download_knmi_meteo( variable to download, by default "RH", valid options are e.g. ["RD", "RH", "EV24", "T", "Q"]. kind : str - kind identifier for observations, usually "prec" or "evap". + kind identifier for observations in pastastore, usually "prec" or "evap". stns : list of int/str, optional list of station numbers to download data for, by default None tmin : TimeType, optional @@ -348,12 +348,7 @@ def download_knmi_meteo( if True, normalize the datetime so stress value at midnight represents the daily total, by default True. """ - # get tmin/tmax if not specified - tmintmax = self._store.get_tmin_tmax("oseries") - if tmin is None: - tmin = tmintmax.loc[:, "tmin"].min() - Timedelta(days=10 * 365) - if tmax is None: - tmax = tmintmax.loc[:, "tmax"].max() + tmin, tmax = self._get_tmin_tmax(tmin, tmax) if stns is None: locations = self._store.oseries.loc[:, ["x", "y"]] From 88c659375f00d9d4603ad2bf2e973b5f45aec454 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=ADd=20Brakenhoff?= Date: Thu, 10 Oct 2024 12:36:16 +0200 Subject: [PATCH 6/9] move method --- pastastore/extensions/hpd.py | 52 ++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/pastastore/extensions/hpd.py b/pastastore/extensions/hpd.py index c493b34..d0a5c9a 100644 --- a/pastastore/extensions/hpd.py +++ b/pastastore/extensions/hpd.py @@ -227,6 +227,32 @@ def _get_tmin_tmax(self, tmin, tmax, oseries=None): tmax = tmintmax.loc[:, "tmax"].max() return tmin, tmax + @staticmethod + def _normalize_datetime_index(obs): + """Normalize observation datetime index (i.e. set observation time to midnight). + + Parameters + ---------- + obs : pandas.Series + observation series to normalize + + Returns + ------- + hpd.Obs + observation series with normalized datetime index + """ + if isinstance(obs, hpd.Obs): + metadata = {k: getattr(obs, k) for k in obs._metadata} + else: + metadata = {} + return obs.__class__( + timestep_weighted_resample( + obs, + obs.index.normalize(), + ).rename(obs.name), + **metadata, + ) + def download_knmi_precipitation( self, stns: Optional[list[int]] = None, @@ -520,32 +546,6 @@ def update_knmi_meteo( if raise_on_error: raise e - @staticmethod - def _normalize_datetime_index(obs): - """Normalize observation datetime index (i.e. set observation time to midnight). - - Parameters - ---------- - obs : pandas.Series - observation series to normalize - - Returns - ------- - hpd.Obs - observation series with normalized datetime index - """ - if isinstance(obs, hpd.Obs): - metadata = {k: getattr(obs, k) for k in obs._metadata} - else: - metadata = {} - return obs.__class__( - timestep_weighted_resample( - obs, - obs.index.normalize(), - ).rename(obs.name), - **metadata, - ) - def download_bro_gmw( self, extent: Optional[List[float]] = None, From 3ef1d81b69c8a62da61dc240bd88ab42f478aaac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=ADd=20Brakenhoff?= Date: Thu, 10 Oct 2024 12:37:23 +0200 Subject: [PATCH 7/9] add methods to download nearest meteo time series - general method - specific methods for prec/evap - should work when only providing name of stored oseries --- pastastore/extensions/hpd.py | 145 +++++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) diff --git a/pastastore/extensions/hpd.py b/pastastore/extensions/hpd.py index d0a5c9a..59962c7 100644 --- a/pastastore/extensions/hpd.py +++ b/pastastore/extensions/hpd.py @@ -403,6 +403,151 @@ def download_knmi_meteo( normalize_datetime_index=normalize_datetime_index, ) + def download_nearest_knmi_precipitation( + self, + oseries: str, + meteo_var: str = "RD", + tmin: Optional[TimeType] = None, + tmax: Optional[TimeType] = None, + unit_multiplier: float = 1e-3, + normalize_datetime_index: bool = True, + fill_missing_obs: bool = True, + **kwargs, + ): + """Download precipitation time series data from nearest KNMI station. + + Parameters + ---------- + oseries : str + download nearest precipitation information for this observation well + meteo_var : str, optional + variable to download, by default "RD", valid options are ["RD", "RH"]. + tmin : TimeType + start time + tmax : TimeType + end time + unit_multiplier : float, optional + multiply unit by this value before saving it in the store, + by default 1.0 (no conversion) + fill_missing_obs : bool, optional + if True, fill missing observations by getting observations from nearest + station with data. + fill_missing_obs : bool, optional + if True, fill missing observations by getting observations from nearest + station with data. + """ + self.download_nearest_knmi_meteo( + oseries=oseries, + meteo_var=meteo_var, + kind="prec", + tmin=tmin, + tmax=tmax, + unit_multiplier=unit_multiplier, + normalize_datetime_index=normalize_datetime_index, + fill_missing_obs=fill_missing_obs, + **kwargs, + ) + + def download_nearest_knmi_evaporation( + self, + oseries: str, + tmin: Optional[TimeType] = None, + tmax: Optional[TimeType] = None, + unit_multiplier: float = 1e-3, + normalize_datetime_index: bool = True, + fill_missing_obs: bool = True, + **kwargs, + ): + """Download evaporation time series data from nearest KNMI station. + + Parameters + ---------- + oseries : str + download nearest evaporation information for this observation well + tmin : TimeType + start time + tmax : TimeType + end time + unit_multiplier : float, optional + multiply unit by this value before saving it in the store, + by default 1.0 (no conversion) + fill_missing_obs : bool, optional + if True, fill missing observations by getting observations from nearest + station with data. + fill_missing_obs : bool, optional + if True, fill missing observations by getting observations from nearest + station with data. + """ + self.download_nearest_knmi_meteo( + oseries=oseries, + meteo_var="EV24", + kind="evap", + tmin=tmin, + tmax=tmax, + unit_multiplier=unit_multiplier, + normalize_datetime_index=normalize_datetime_index, + fill_missing_obs=fill_missing_obs, + **kwargs, + ) + + def download_nearest_knmi_meteo( + self, + oseries: str, + meteo_var: str, + kind: str, + tmin: Optional[TimeType] = None, + tmax: Optional[TimeType] = None, + unit_multiplier: float = 1.0, + normalize_datetime_index: bool = True, + fill_missing_obs: bool = True, + **kwargs, + ): + """Download meteorological data from nearest KNMI station. + + Parameters + ---------- + oseries : str + download nearest meteorological information for this observation well + meteo_var : str + meteorological variable to download, e.g. "RD", "RH", "EV24", "T", "Q" + kind : str + kind identifier for observations in pastastore, usually "prec" or "evap". + tmin : TimeType + start time + tmax : TimeType + end time + unit_multiplier : float, optional + multiply unit by this value before saving it in the store, + by default 1.0 (no conversion) + fill_missing_obs : bool, optional + if True, fill missing observations by getting observations from nearest + station with data. + fill_missing_obs : bool, optional + if True, fill missing observations by getting observations from nearest + station with data. + """ + xy = self._store.oseries.loc[[oseries], ["x", "y"]].to_numpy() + # download data + tmin, tmax = self._get_tmin_tmax(tmin, tmax, oseries=oseries) + knmi = hpd.read_knmi( + xy=xy, + meteo_vars=[meteo_var], + starts=tmin, + ends=tmax, + fill_missing_obs=fill_missing_obs, + **kwargs, + ) + # add to store + self.add_obscollection( + libname="stresses", + oc=knmi, + kind=kind, + data_column=meteo_var, + unit_multiplier=unit_multiplier, + update=False, + normalize_datetime_index=normalize_datetime_index, + ) + def update_knmi_meteo( self, names: Optional[List[str]] = None, From 1cf6332de164cbcd2471f7c88ed74a327a2c8d07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=ADd=20Brakenhoff?= Date: Thu, 10 Oct 2024 12:48:48 +0200 Subject: [PATCH 8/9] add test for nearest knmi --- tests/test_007_hpdextension.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_007_hpdextension.py b/tests/test_007_hpdextension.py index c543bc9..843bdbb 100644 --- a/tests/test_007_hpdextension.py +++ b/tests/test_007_hpdextension.py @@ -67,3 +67,19 @@ def test_update_stresses(): pstore.hpd.update_knmi_meteo(tmax="2024-01-31", normalize_datetime_index=False) tmintmax = pstore.get_tmin_tmax("stresses") assert (tmintmax["tmax"] >= Timestamp("2024-01-31")).all() + + +@pytest.mark.xfail(reason="KNMI is being flaky, so allow this test to xfail/xpass.") +@pytest.mark.pastas150 +def test_nearest_stresses(): + from pastastore.extensions import activate_hydropandas_extension + + activate_hydropandas_extension() + + pstore = pst.PastaStore.from_zip("tests/data/test_hpd_update.zip") + pstore.hpd.download_nearest_knmi_precipitation( + "GMW000000036319_1", tmin="2024-01-01" + ) + assert "RD_GROOT-AMMERS" in pstore.stresses_names + pstore.hpd.download_nearest_knmi_evaporation("GMW000000036319_1", tmin="2024-01-01") + assert "EV24_CABAUW-MAST" in pstore.stresses_names From a95cc1da1cc29b2b63ca76fb495c89e1a1b601fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dav=C3=ADd=20Brakenhoff?= Date: Thu, 10 Oct 2024 14:07:10 +0200 Subject: [PATCH 9/9] fix for empty stresses df --- pastastore/extensions/hpd.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pastastore/extensions/hpd.py b/pastastore/extensions/hpd.py index 59962c7..4a9cbfd 100644 --- a/pastastore/extensions/hpd.py +++ b/pastastore/extensions/hpd.py @@ -580,6 +580,17 @@ def update_knmi_meteo( **kwargs : dict, optional Additional keyword arguments to pass to `hpd.read_knmi()` """ + if "source" not in self._store.stresses.columns: + msg = ( + "Cannot update KNMI stresses! " + "KNMI stresses cannot be identified if 'source' column is not defined." + ) + logger.error(msg) + if raise_on_error: + raise ValueError(msg) + else: + return + if names is None: names = self._store.stresses.loc[ self._store.stresses["source"] == "KNMI"