From 0157cab623c25ca8a63a33cbe5fe5ea290662466 Mon Sep 17 00:00:00 2001 From: Eleanor Smith Date: Thu, 16 Dec 2021 13:31:43 +0000 Subject: [PATCH 01/60] add initial average_time functions --- clisops/core/average.py | 66 +++++++++++++++++++++++++++++++++-- clisops/ops/average.py | 72 +++++++++++++++++++++++++++++++++++++-- tests/ops/test_average.py | 23 ++++++++++++- 3 files changed, 155 insertions(+), 6 deletions(-) diff --git a/clisops/core/average.py b/clisops/core/average.py index 2a781a67..3a085ef8 100644 --- a/clisops/core/average.py +++ b/clisops/core/average.py @@ -5,9 +5,16 @@ import geopandas as gpd import xarray as xr from roocs_utils.exceptions import InvalidParameterValue -from roocs_utils.xarray_utils.xarray_utils import get_coord_type, known_coord_types +from roocs_utils.xarray_utils.xarray_utils import ( + get_coord_by_type, + get_coord_type, + known_coord_types, +) -__all__ = ["average_over_dims", "average_shape"] +__all__ = ["average_over_dims", "average_shape", "average_time"] + +# see https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#dateoffset-objects +freqs = {"month": "1MS", "year": "1AS"} def average_shape( @@ -198,3 +205,58 @@ def average_over_dims( if isinstance(ds, xr.Dataset): return xr.merge((ds_averaged_over_dims, untouched_ds)) return ds_averaged_over_dims + + +def average_time( + ds: Union[xr.DataArray, xr.Dataset], + freq: str, +) -> Union[xr.DataArray, xr.Dataset]: + """ + Average a DataArray or Dataset over the time frequency specified. + + Parameters + ---------- + ds : Union[xr.DataArray, xr.Dataset] + Input values. + freq: str + The frequency to average over. One of "month", "year". + + Returns + ------- + Union[xr.DataArray, xr.Dataset] + New Dataset or DataArray object averaged over the indicated time frequency. + + Examples + -------- + >>> from clisops.core.average import average_time # doctest: +SKIP + >>> pr = xr.open_dataset(path_to_pr_file).pr # doctest: +SKIP + ... + # Average data array over each month + >>> prAvg = average_time(pr, freq='month') # doctest: +SKIP + """ + + if not freq: + raise InvalidParameterValue( + "At least one frequency for averaging must be provided" + ) + + if freq not in list(freqs.keys()): + raise InvalidParameterValue( + f"Time frequency for averaging must be one of {list(freqs.keys())}" + ) + + # check time coordinate exists and get name + t = get_coord_by_type(ds, "time", ignore_aux_coords=False) + + if t is None: + raise Exception("Time dimension could not be found") + + # resample and average over time + ds_avg_over_time = ds.resample(indexer={t.name: freqs[freq]}).mean(dim=t.name) + + return ds_avg_over_time + + # questions: + # what do we want the label to be - start of the month/year/ end/ or the date that is already used e.g. 16th of the month for monthly datasets + # is MS/AS right or should we use A and M for datetime offsets + # look at other options in xarray resample - skipna? diff --git a/clisops/ops/average.py b/clisops/ops/average.py index bc644384..7b3574e1 100644 --- a/clisops/ops/average.py +++ b/clisops/ops/average.py @@ -2,6 +2,7 @@ from typing import List, Optional, Tuple, Union import xarray as xr +from roocs_utils.exceptions import InvalidParameterValue from roocs_utils.parameter.dimension_parameter import DimensionParameter from roocs_utils.xarray_utils.xarray_utils import ( convert_coord_to_axis, @@ -17,9 +18,7 @@ from clisops.utils.file_namers import get_file_namer from clisops.utils.output_utils import get_output, get_time_slices -__all__ = [ - "average_over_dims", -] +__all__ = ["average_over_dims", "average_time"] LOGGER = logging.getLogger(__file__) @@ -97,3 +96,70 @@ def average_over_dims( """ op = Average(**locals()) return op.process() + + +class AverageTime(Operation): + def _resolve_params(self, **params): + freq = params.get("freq", None) + + if freq not in list(average.freqs.keys()): + raise InvalidParameterValue( + f"Time frequency for averaging must be one of {list(average.freqs.keys())}" + ) + + self.params = {"freq": freq} + + def _get_file_namer(self): + extra = f"_avg-{self.params.get('freq')}" + namer = get_file_namer(self._file_namer)(extra=extra) + + return namer + + def _calculate(self): + avg_ds = average.average_time( + self.ds, + self.params.get("freq", None), + ) + + return avg_ds + + +def average_time( + ds, + freq: str, + output_dir: Optional[Union[str, Path]] = None, + output_type="netcdf", + split_method="time:auto", + file_namer="standard", +) -> List[Union[xr.Dataset, str]]: + """ + + Parameters + ---------- + ds: Union[xr.Dataset, str] + freq: str + The frequency to average over. One of "month", "year". + output_dir: Optional[Union[str, Path]] = None + output_type: {"netcdf", "nc", "zarr", "xarray"} + split_method: {"time:auto"} + file_namer: {"standard", "simple"} + + Returns + ------- + List[Union[xr.Dataset, str]] + A list of the outputs in the format selected; str corresponds to file paths if the + output format selected is a file. + + Examples + -------- + | ds: xarray Dataset or "cmip5.output1.MOHC.HadGEM2-ES.rcp85.mon.atmos.Amon.r1i1p1.latest.tas" + | dims: ['latitude', 'longitude'] + | ignore_undetected_dims: False + | output_dir: "/cache/wps/procs/req0111" + | output_type: "netcdf" + | split_method: "time:auto" + | file_namer: "standard" + + """ + op = AverageTime(**locals()) + return op.process() diff --git a/tests/ops/test_average.py b/tests/ops/test_average.py index 964c7a26..36696a5f 100644 --- a/tests/ops/test_average.py +++ b/tests/ops/test_average.py @@ -8,7 +8,7 @@ import clisops from clisops import CONFIG -from clisops.ops.average import average_over_dims +from clisops.ops.average import average_over_dims, average_time from .._common import CMIP5_TAS @@ -217,3 +217,24 @@ def test_aux_variables(): ) assert "do_i_get_written" in result[0].variables + + +def test_average_over_years(): + ds = _load_ds(CMIP5_TAS) # monthly dataset + + # check initial dataset + assert ds.time.shape == (3530,) + assert ds.time.values[0].isoformat() == "2005-12-16T00:00:00" + assert ds.time.values[-1].isoformat() == "2299-12-16T00:00:00" + + result = average_time( + CMIP5_TAS, + freq="year", + output_type="xarray", + ) + + time_length = ds.time.values[-1].year - ds.time.values[0].year + 1 + + assert result[0].time.shape == (time_length,) + assert result[0].time.values[0].isoformat() == "2005-01-01T00:00:00" + assert result[0].time.values[-1].isoformat() == "2299-01-01T00:00:00" From ffad88270e6686ed792269352e3a06e66c481c74 Mon Sep 17 00:00:00 2001 From: Eleanor Smith Date: Mon, 20 Dec 2021 12:09:23 +0000 Subject: [PATCH 02/60] add more tests for average time --- clisops/core/average.py | 13 +++--- clisops/ops/average.py | 7 ++- tests/core/test_average.py | 53 ++++++++++++++++++++++ tests/ops/test_average.py | 91 ++++++++++++++++++++++++++++++++++++-- 4 files changed, 152 insertions(+), 12 deletions(-) diff --git a/clisops/core/average.py b/clisops/core/average.py index 3a085ef8..1ffa7b1d 100644 --- a/clisops/core/average.py +++ b/clisops/core/average.py @@ -14,7 +14,7 @@ __all__ = ["average_over_dims", "average_shape", "average_time"] # see https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#dateoffset-objects -freqs = {"month": "1MS", "year": "1AS"} +freqs = {"day": "1D", "month": "1MS", "year": "1AS"} def average_shape( @@ -242,7 +242,7 @@ def average_time( if freq not in list(freqs.keys()): raise InvalidParameterValue( - f"Time frequency for averaging must be one of {list(freqs.keys())}" + "Time frequency for averaging must be one of 'month', 'year'." ) # check time coordinate exists and get name @@ -252,11 +252,8 @@ def average_time( raise Exception("Time dimension could not be found") # resample and average over time - ds_avg_over_time = ds.resample(indexer={t.name: freqs[freq]}).mean(dim=t.name) + ds_avg_over_time = ds.resample(indexer={t.name: freqs[freq]}).mean( + dim=t.name, skipna=True, keep_attrs=True + ) return ds_avg_over_time - - # questions: - # what do we want the label to be - start of the month/year/ end/ or the date that is already used e.g. 16th of the month for monthly datasets - # is MS/AS right or should we use A and M for datetime offsets - # look at other options in xarray resample - skipna? diff --git a/clisops/ops/average.py b/clisops/ops/average.py index 7b3574e1..4d1de1d9 100644 --- a/clisops/ops/average.py +++ b/clisops/ops/average.py @@ -102,9 +102,14 @@ class AverageTime(Operation): def _resolve_params(self, **params): freq = params.get("freq", None) + if not freq: + raise InvalidParameterValue( + "At least one frequency for averaging must be provided" + ) + if freq not in list(average.freqs.keys()): raise InvalidParameterValue( - f"Time frequency for averaging must be one of {list(average.freqs.keys())}" + "Time frequency for averaging must be one of 'month', 'year'." ) self.params = {"freq": freq} diff --git a/tests/core/test_average.py b/tests/core/test_average.py index 8448dc82..2e26b5b6 100644 --- a/tests/core/test_average.py +++ b/tests/core/test_average.py @@ -6,10 +6,12 @@ import xarray as xr from pkg_resources import parse_version from roocs_utils.exceptions import InvalidParameterValue +from roocs_utils.xarray_utils import xarray_utils as xu from clisops.core import average from clisops.utils import get_file +from .._common import CMIP5_RH, CMIP6_SICONC_DAY from .._common import XCLIM_TESTS_DATA as TESTS_DATA try: @@ -167,3 +169,54 @@ def test_average_wrong_format(self): str(exc.value) == "Dimensions for averaging must be one of ['time', 'level', 'latitude', 'longitude']" ) + + +class TestAverageTime: + month_ds = CMIP5_RH + day_ds = CMIP6_SICONC_DAY + + def test_average_month(self): + ds = xu.open_xr_dataset(self.day_ds) + assert ds.time.shape == (60225,) + + avg_ds = average.average_time(ds, freq="month") + assert avg_ds.time.shape == (1980,) + + def test_average_year(self): + ds = xu.open_xr_dataset(self.month_ds) + assert ds.time.shape == (1752,) + + avg_ds = average.average_time(ds, freq="year") + assert avg_ds.time.shape == (147,) + + def test_no_freq(self): + ds = xu.open_xr_dataset(self.month_ds) + + with pytest.raises(InvalidParameterValue) as exc: + average.average_time(ds, freq=None) + assert str(exc.value) == "At least one frequency for averaging must be provided" + + def test_incorrect_freq(self): + ds = xu.open_xr_dataset(self.month_ds) + + with pytest.raises(InvalidParameterValue) as exc: + average.average_time(ds, freq="wrong") + assert ( + str(exc.value) + == "Time frequency for averaging must be one of 'month', 'year'." + ) + + def test_freq_wrong_format(self): + ds = xu.open_xr_dataset(self.month_ds) + + with pytest.raises(InvalidParameterValue) as exc: + average.average_time(ds, freq=0) + assert str(exc.value) == "At least one frequency for averaging must be provided" + + def test_no_time(self): + ds = xu.open_xr_dataset(self.month_ds) + ds = ds.drop_dims("time") + + with pytest.raises(Exception) as exc: + average.average_time(ds, freq="year") + assert str(exc.value) == "Time dimension could not be found" diff --git a/tests/ops/test_average.py b/tests/ops/test_average.py index 36696a5f..ae71e158 100644 --- a/tests/ops/test_average.py +++ b/tests/ops/test_average.py @@ -10,7 +10,7 @@ from clisops import CONFIG from clisops.ops.average import average_over_dims, average_time -from .._common import CMIP5_TAS +from .._common import C3S_CORDEX_EUR_ZG500, CMIP5_TAS, CMIP6_SICONC_DAY def _check_output_nc(result, fname="output_001.nc"): @@ -18,7 +18,7 @@ def _check_output_nc(result, fname="output_001.nc"): def _load_ds(fpath): - return xr.open_mfdataset(fpath) + return xr.open_mfdataset(fpath, use_cftime=True) def test_average_basic_data_array(cmip5_tas_file): @@ -235,6 +235,91 @@ def test_average_over_years(): time_length = ds.time.values[-1].year - ds.time.values[0].year + 1 - assert result[0].time.shape == (time_length,) + assert result[0].time.shape == (time_length,) # get number of years assert result[0].time.values[0].isoformat() == "2005-01-01T00:00:00" assert result[0].time.values[-1].isoformat() == "2299-01-01T00:00:00" + + +def test_average_over_months(): + ds = _load_ds(CMIP6_SICONC_DAY) # monthly dataset + + # check initial dataset + assert ds.time.shape == (60225,) + assert ds.time.values[0].isoformat() == "1850-01-01T12:00:00" + assert ds.time.values[-1].isoformat() == "2014-12-31T12:00:00" + + # average over time + result = average_time( + CMIP6_SICONC_DAY, + freq="month", + output_type="xarray", + ) + + time_length = ( + ds.time.values[-1].year - ds.time.values[0].year + 1 + ) * 12 # get number of months + + assert result[0].time.shape == (time_length,) + assert result[0].time.values[0].isoformat() == "1850-01-01T00:00:00" + assert result[0].time.values[-1].isoformat() == "2014-12-01T00:00:00" + + +def test_average_time_no_freq(): + with pytest.raises(InvalidParameterValue) as exc: + # average over time + average_time( + CMIP6_SICONC_DAY, + freq=None, + output_type="xarray", + ) + assert str(exc.value) == "At least one frequency for averaging must be provided" + + +def test_average_time_incorrect_freq(): + with pytest.raises(InvalidParameterValue) as exc: + # average over time + average_time( + CMIP6_SICONC_DAY, + freq="week", + output_type="xarray", + ) + assert ( + str(exc.value) == "Time frequency for averaging must be one of 'month', 'year'." + ) + + +def test_average_time_file_name(tmpdir): + result = average_time( + CMIP5_TAS, + freq="year", + output_type="nc", + output_dir=tmpdir, + ) + + _check_output_nc( + result, fname="tas_mon_HadGEM2-ES_rcp85_r1i1p1_20050101-22990101_avg-year.nc" + ) + + +def test_average_time_cordex(): + ds = _load_ds(C3S_CORDEX_EUR_ZG500) + + # check initial dataset + assert ds.time.shape == (3653,) + assert ds.time.values[0].isoformat() == "2071-01-01T12:00:00" + assert ds.time.values[-1].isoformat() == "2080-12-31T12:00:00" + + # average over time + result = average_time( + C3S_CORDEX_EUR_ZG500, + freq="month", + output_type="xarray", + ) + + time_length = ( + ds.time.values[-1].year - ds.time.values[0].year + 1 + ) * 12 # get number of months + + assert result[0].time.shape == (time_length,) + assert result[0].time.values[0].isoformat() == "2071-01-01T00:00:00" + assert result[0].time.values[-1].isoformat() == "2080-12-01T00:00:00" From 645f4c199dbc391cc9937eeae691c857c1a144bd Mon Sep 17 00:00:00 2001 From: Eleanor Smith Date: Mon, 20 Dec 2021 12:47:10 +0000 Subject: [PATCH 03/60] update exception message --- clisops/core/average.py | 2 +- clisops/ops/average.py | 2 +- tests/core/test_average.py | 2 +- tests/ops/test_average.py | 3 ++- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/clisops/core/average.py b/clisops/core/average.py index b52062aa..5335725a 100644 --- a/clisops/core/average.py +++ b/clisops/core/average.py @@ -262,7 +262,7 @@ def average_time( if freq not in list(freqs.keys()): raise InvalidParameterValue( - "Time frequency for averaging must be one of 'month', 'year'." + f"Time frequency for averaging must be one of {list(freqs.keys())}." ) # check time coordinate exists and get name diff --git a/clisops/ops/average.py b/clisops/ops/average.py index 4d1de1d9..33a75d90 100644 --- a/clisops/ops/average.py +++ b/clisops/ops/average.py @@ -109,7 +109,7 @@ def _resolve_params(self, **params): if freq not in list(average.freqs.keys()): raise InvalidParameterValue( - "Time frequency for averaging must be one of 'month', 'year'." + f"Time frequency for averaging must be one of {list(average.freqs.keys())}." ) self.params = {"freq": freq} diff --git a/tests/core/test_average.py b/tests/core/test_average.py index 44ed86c9..535d5f30 100644 --- a/tests/core/test_average.py +++ b/tests/core/test_average.py @@ -209,7 +209,7 @@ def test_incorrect_freq(self): average.average_time(ds, freq="wrong") assert ( str(exc.value) - == "Time frequency for averaging must be one of 'month', 'year'." + == "Time frequency for averaging must be one of ['day', 'month', 'year']." ) def test_freq_wrong_format(self): diff --git a/tests/ops/test_average.py b/tests/ops/test_average.py index ae71e158..e7de03d3 100644 --- a/tests/ops/test_average.py +++ b/tests/ops/test_average.py @@ -284,7 +284,8 @@ def test_average_time_incorrect_freq(): output_type="xarray", ) assert ( - str(exc.value) == "Time frequency for averaging must be one of 'month', 'year'." + str(exc.value) + == "Time frequency for averaging must be one of ['day', 'month', 'year']." ) From a07ce689646ce0398a06d19781376ffc088a6d07 Mon Sep 17 00:00:00 2001 From: Eleanor Smith Date: Mon, 20 Dec 2021 13:07:38 +0000 Subject: [PATCH 04/60] update history and docs --- HISTORY.rst | 1 + docs/api.rst | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 642c9e69..b6862e92 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,6 +7,7 @@ v0.8.0 (unreleased) New Features ^^^^^^^^^^^^ * ``clisops.core.average.average_shape`` copies the global and variable attributes from the input data to the results. +* ``clisops.ops.average.average_time`` and ``clisops.core.average.average_time`` added. Allowing averaging over time frequencies of day, month and year. Bug fixes ^^^^^^^^^ diff --git a/docs/api.rst b/docs/api.rst index b84003b4..f44303e3 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -29,8 +29,8 @@ Subset operation :show-inheritance: -Average operation -================= +Average operations +================== .. automodule:: clisops.ops.average :noindex: From baf1b8157fc73092ee5c6a3f5782501176db3ee3 Mon Sep 17 00:00:00 2001 From: Eleanor Smith Date: Mon, 17 Jan 2022 11:03:41 +0000 Subject: [PATCH 05/60] add time bounds and test for them --- clisops/core/average.py | 43 +++++++++++++++++++++++++++++++++++++-- tests/ops/test_average.py | 31 +++++++++++++++++++++++++++- 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/clisops/core/average.py b/clisops/core/average.py index 5335725a..e345d2f6 100644 --- a/clisops/core/average.py +++ b/clisops/core/average.py @@ -3,6 +3,7 @@ from pathlib import Path from typing import Sequence, Tuple, Union +import cf_xarray import geopandas as gpd import numpy as np import xarray as xr @@ -272,8 +273,46 @@ def average_time( raise Exception("Time dimension could not be found") # resample and average over time - ds_avg_over_time = ds.resample(indexer={t.name: freqs[freq]}).mean( + ds_t_avg = ds.resample(indexer={t.name: freqs[freq]}).mean( dim=t.name, skipna=True, keep_attrs=True ) - return ds_avg_over_time + # add time_bounds to dataset + # get datetime class + dt_cls = ds_t_avg.time.values[0].__class__ + + # generate time_bouds depending on frequency + if freq == "month": + time_bounds = [ + [ + dt_cls(tm.year, tm.month, tm.day), + dt_cls(tm.year, tm.month, tm.daysinmonth), + ] + for tm in ds_t_avg.time.values + ] + + elif freq == "year": + # get number of days in december for calendar + dec_days = dt_cls(2000, 12, 1).daysinmonth + # generate time bounds + time_bounds = [ + [dt_cls(tm.year, 1, 1), dt_cls(tm.year, 12, dec_days)] + for tm in ds_t_avg.time.values + ] + + elif freq == "day": + time_bounds = [ + [ + dt_cls(tm.year, tm.month, tm.day, 0, 0, 0), + dt_cls(tm.year, tm.month, tm.day, 23, 59, 59), + ] + for tm in ds_t_avg.time.values + ] + + # get name of bounds dimension for time + bnds = ds.cf.get_bounds_dim_name("time") + + # add time bounds to dataset + ds_t_avg = ds_t_avg.assign({"time_bnds": ((t.name, bnds), np.asarray(time_bounds))}) + + return ds_t_avg diff --git a/tests/ops/test_average.py b/tests/ops/test_average.py index e7de03d3..4175fbe7 100644 --- a/tests/ops/test_average.py +++ b/tests/ops/test_average.py @@ -234,11 +234,20 @@ def test_average_over_years(): ) time_length = ds.time.values[-1].year - ds.time.values[0].year + 1 - assert result[0].time.shape == (time_length,) # get number of years assert result[0].time.values[0].isoformat() == "2005-01-01T00:00:00" assert result[0].time.values[-1].isoformat() == "2299-01-01T00:00:00" + # test time bounds + assert [t.isoformat() for t in result[0].time_bnds.values[0]] == [ + "2005-01-01T00:00:00", + "2005-12-30T00:00:00", + ] + assert [t.isoformat() for t in result[0].time_bnds.values[-1]] == [ + "2299-01-01T00:00:00", + "2299-12-30T00:00:00", + ] + def test_average_over_months(): ds = _load_ds(CMIP6_SICONC_DAY) # monthly dataset @@ -263,6 +272,16 @@ def test_average_over_months(): assert result[0].time.values[0].isoformat() == "1850-01-01T00:00:00" assert result[0].time.values[-1].isoformat() == "2014-12-01T00:00:00" + # test time bounds + assert [t.isoformat() for t in result[0].time_bnds.values[0]] == [ + "1850-01-01T00:00:00", + "1850-01-31T00:00:00", + ] + assert [t.isoformat() for t in result[0].time_bnds.values[-1]] == [ + "2014-12-01T00:00:00", + "2014-12-31T00:00:00", + ] + def test_average_time_no_freq(): with pytest.raises(InvalidParameterValue) as exc: @@ -324,3 +343,13 @@ def test_average_time_cordex(): assert result[0].time.shape == (time_length,) assert result[0].time.values[0].isoformat() == "2071-01-01T00:00:00" assert result[0].time.values[-1].isoformat() == "2080-12-01T00:00:00" + + # test time bounds + assert [t.isoformat() for t in result[0].time_bnds.values[0]] == [ + "2071-01-01T00:00:00", + "2071-01-31T00:00:00", + ] + assert [t.isoformat() for t in result[0].time_bnds.values[-1]] == [ + "2080-12-01T00:00:00", + "2080-12-31T00:00:00", + ] From c16e7872af59ceb01b8d7b691931c93abaab94e7 Mon Sep 17 00:00:00 2001 From: Eleanor Smith Date: Mon, 17 Jan 2022 12:02:48 +0000 Subject: [PATCH 06/60] move creation of time bounds to separate function --- HISTORY.rst | 1 + clisops/core/average.py | 35 ++++------------------------- clisops/utils/time_utils.py | 44 +++++++++++++++++++++++++++++++++++++ docs/api.rst | 10 +++++++++ 4 files changed, 59 insertions(+), 31 deletions(-) create mode 100644 clisops/utils/time_utils.py diff --git a/HISTORY.rst b/HISTORY.rst index b6862e92..a827c6c1 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,6 +8,7 @@ New Features ^^^^^^^^^^^^ * ``clisops.core.average.average_shape`` copies the global and variable attributes from the input data to the results. * ``clisops.ops.average.average_time`` and ``clisops.core.average.average_time`` added. Allowing averaging over time frequencies of day, month and year. +* New function ``create_time_bounds`` in ``clisops.utils.time_utils``, to generate time bounds for temporally averaged datasets. Bug fixes ^^^^^^^^^ diff --git a/clisops/core/average.py b/clisops/core/average.py index e345d2f6..5f3a0095 100644 --- a/clisops/core/average.py +++ b/clisops/core/average.py @@ -14,6 +14,8 @@ known_coord_types, ) +from clisops.utils.time_utils import create_time_bounds + from .subset import shape_bbox_indexer __all__ = ["average_over_dims", "average_shape", "average_time"] @@ -277,37 +279,8 @@ def average_time( dim=t.name, skipna=True, keep_attrs=True ) - # add time_bounds to dataset - # get datetime class - dt_cls = ds_t_avg.time.values[0].__class__ - - # generate time_bouds depending on frequency - if freq == "month": - time_bounds = [ - [ - dt_cls(tm.year, tm.month, tm.day), - dt_cls(tm.year, tm.month, tm.daysinmonth), - ] - for tm in ds_t_avg.time.values - ] - - elif freq == "year": - # get number of days in december for calendar - dec_days = dt_cls(2000, 12, 1).daysinmonth - # generate time bounds - time_bounds = [ - [dt_cls(tm.year, 1, 1), dt_cls(tm.year, 12, dec_days)] - for tm in ds_t_avg.time.values - ] - - elif freq == "day": - time_bounds = [ - [ - dt_cls(tm.year, tm.month, tm.day, 0, 0, 0), - dt_cls(tm.year, tm.month, tm.day, 23, 59, 59), - ] - for tm in ds_t_avg.time.values - ] + # generate time_bounds depending on frequency + time_bounds = create_time_bounds(ds_t_avg, freq) # get name of bounds dimension for time bnds = ds.cf.get_bounds_dim_name("time") diff --git a/clisops/utils/time_utils.py b/clisops/utils/time_utils.py new file mode 100644 index 00000000..ea6c6e9a --- /dev/null +++ b/clisops/utils/time_utils.py @@ -0,0 +1,44 @@ +from roocs_utils.exceptions import InvalidParameterValue + + +def create_time_bounds(ds, freq): + """ + Generate time bounds for datasets that have been temporally averaged. + Averaging frequencies supported are yearly, monthly and daily. + """ + # get datetime class + dt_cls = ds.time.values[0].__class__ + + if freq == "month": + time_bounds = [ + [ + dt_cls(tm.year, tm.month, tm.day), + dt_cls(tm.year, tm.month, tm.daysinmonth), + ] + for tm in ds.time.values + ] + + elif freq == "year": + # get number of days in december for calendar + dec_days = dt_cls(2000, 12, 1).daysinmonth + # generate time bounds + time_bounds = [ + [dt_cls(tm.year, 1, 1), dt_cls(tm.year, 12, dec_days)] + for tm in ds.time.values + ] + + elif freq == "day": + time_bounds = [ + [ + dt_cls(tm.year, tm.month, tm.day, 0, 0, 0), + dt_cls(tm.year, tm.month, tm.day, 23, 59, 59), + ] + for tm in ds.time.values + ] + + else: + raise InvalidParameterValue( + "Time frequency not supported for creation of time bounds." + ) + + return time_bounds diff --git a/docs/api.rst b/docs/api.rst index f44303e3..8f38233f 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -77,3 +77,13 @@ Dataset Utilities :members: :undoc-members: :show-inheritance: + + +Time Utilities +============== + +.. automodule:: clisops.utils.time_utils + :noindex: + :members: + :undoc-members: + :show-inheritance: From 53860d0cee8c4f31620a13fd073109a98d0a5e4a Mon Sep 17 00:00:00 2001 From: Eleanor Smith Date: Mon, 17 Jan 2022 12:04:31 +0000 Subject: [PATCH 07/60] change exception --- clisops/utils/time_utils.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/clisops/utils/time_utils.py b/clisops/utils/time_utils.py index ea6c6e9a..ae037967 100644 --- a/clisops/utils/time_utils.py +++ b/clisops/utils/time_utils.py @@ -1,6 +1,3 @@ -from roocs_utils.exceptions import InvalidParameterValue - - def create_time_bounds(ds, freq): """ Generate time bounds for datasets that have been temporally averaged. @@ -37,8 +34,6 @@ def create_time_bounds(ds, freq): ] else: - raise InvalidParameterValue( - "Time frequency not supported for creation of time bounds." - ) + raise Exception("Time frequency not supported for creation of time bounds.") return time_bounds From 8ae69352eea13c9b0b4751b30d780d366f9dab92 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 21 Jan 2022 11:31:20 -0500 Subject: [PATCH 08/60] update pre-commit and requirements --- .pre-commit-config.yaml | 12 ++++++------ environment.yml | 20 ++++++++++---------- requirements_dev.txt | 2 +- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 96ee5cbc..9c675961 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.4.0 + rev: v4.1.0 hooks: - id: trailing-whitespace language_version: python3 @@ -16,13 +16,13 @@ repos: - id: debug-statements language_version: python3 - repo: https://github.com/ambv/black - rev: 21.4b2 + rev: 21.12b0 hooks: - id: black language_version: python3 args: ["--target-version", "py36"] - repo: https://github.com/pycqa/flake8 - rev: 3.9.1 + rev: 4.0.1 hooks: - id: flake8 language_version: python3 @@ -33,7 +33,7 @@ repos: # - id: autopep8 # args: ['--global-config=setup.cfg','--in-place'] - repo: https://github.com/timothycrosley/isort - rev: 5.8.0 + rev: 5.10.1 hooks: - id: isort language_version: python3 @@ -44,7 +44,7 @@ repos: # - id: pydocstyle # args: ["--conventions=numpy"] - repo: https://github.com/asottile/pyupgrade - rev: v2.14.0 + rev: v2.31.0 hooks: - id: pyupgrade language_version: python3 @@ -53,7 +53,7 @@ repos: - id: check-hooks-apply - id: check-useless-excludes - repo: https://github.com/kynan/nbstripout - rev: 0.4.0 + rev: 0.5.0 hooks: - id: nbstripout language_version: python3 diff --git a/environment.yml b/environment.yml index 2a8114b2..bbe05cd9 100644 --- a/environment.yml +++ b/environment.yml @@ -4,21 +4,21 @@ channels: - defaults dependencies: - pip - - numpy>=1.16 - - xarray>=0.15 - - pandas>=1.0.3 + - bottleneck>=1.3.1,<1.4 + - cf_xarray>=0.5.1 - cftime>=1.4.1 + - dask>=2.6.0 + - geopandas>=0.7 - netCDF4>=1.4 + - numpy>=1.16 + - pandas>=1.0.3 - poppler>=0.67 - - shapely>=1.6 - - geopandas>=0.7 - - xesmf>=0.6.2 - - dask>=2.6.0 - - bottleneck>=1.3.1,<1.4 + - pygeos>=0.9 - pyproj>=2.5 - requests>=2.0 - roocs-utils>=0.5.0 - - cf_xarray>=0.5.1 - - pygeos>=0.9 + - shapely>=1.6 + - xarray>=0.15 + - xesmf>=0.6.2 # - pip: # - roocs-utils @ git+https://github.com/roocs/roocs-utils.git@master#egg=roocs-utils diff --git a/requirements_dev.txt b/requirements_dev.txt index a1567ef7..f24ba9aa 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -11,7 +11,7 @@ twine pytest pytest-runner pre-commit>=2.9.0 -black>=20.8b1 +black>=21.12b0 nbsphinx nbconvert ipython From d055589a6c4b853dfa118288b32c794e72506133 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 21 Jan 2022 18:30:23 -0500 Subject: [PATCH 09/60] first draft at fixing the logging - removed logging.conf, remove a clisops self-import, used relative location loggers, removed unused imports --- clisops/__init__.py | 26 +++++++++++++++++--------- clisops/core/average.py | 7 ++++--- clisops/core/subset.py | 2 +- clisops/etc/logging.conf | 28 ---------------------------- clisops/ops/average.py | 18 +++--------------- clisops/ops/base_operation.py | 6 +++--- clisops/ops/regrid.py | 0 clisops/ops/subset.py | 19 +++++++------------ clisops/utils/common.py | 2 -- clisops/utils/dataset_utils.py | 6 ------ clisops/utils/output_utils.py | 13 +++++++------ clisops/utils/tutorial.py | 10 +++++----- requirements.txt | 1 + 13 files changed, 48 insertions(+), 90 deletions(-) delete mode 100644 clisops/etc/logging.conf delete mode 100644 clisops/ops/regrid.py diff --git a/clisops/__init__.py b/clisops/__init__.py index 4432bf83..2e85909f 100644 --- a/clisops/__init__.py +++ b/clisops/__init__.py @@ -1,21 +1,29 @@ # -*- coding: utf-8 -*- """Top-level package for clisops.""" - -import logging.config +import logging import os from roocs_utils.config import get_config -import clisops - from .__version__ import __author__, __email__, __version__ -logging.config.fileConfig( - os.path.join(os.path.dirname(__file__), "etc", "logging.conf"), - disable_existing_loggers=False, -) -CONFIG = get_config(clisops) +logging.getLogger("clisops").addHandler(logging.NullHandler()) + +# logger.add( +# sys.stdout, +# format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", +# filter="clisops", +# ) +# clisops_logger = logger + + +# Workaround for roocs_utils to not re-import clisops +class Package: + __file__ = __file__ # noqa + +package = Package() +CONFIG = get_config(package) # Set the memory limit for each dask chunk chunk_memory_limit = CONFIG["clisops:read"].get("chunk_memory_limit", None) diff --git a/clisops/core/average.py b/clisops/core/average.py index add99dcd..a27ac391 100644 --- a/clisops/core/average.py +++ b/clisops/core/average.py @@ -1,12 +1,13 @@ """Average module.""" -from pathlib import Path -from typing import Tuple, Union, Sequence import warnings -import numpy as np +from pathlib import Path +from typing import Sequence, Tuple, Union + import geopandas as gpd import xarray as xr from roocs_utils.exceptions import InvalidParameterValue from roocs_utils.xarray_utils.xarray_utils import get_coord_type, known_coord_types + from .subset import shape_bbox_indexer __all__ = ["average_over_dims", "average_shape"] diff --git a/clisops/core/subset.py b/clisops/core/subset.py index 0a5790f2..c7bbd6ff 100644 --- a/clisops/core/subset.py +++ b/clisops/core/subset.py @@ -10,7 +10,7 @@ import geopandas as gpd import numpy as np import xarray -from pandas.api.types import is_integer_dtype +from pandas.api.types import is_integer_dtype # noqa from pyproj import Geod from pyproj.crs import CRS from pyproj.exceptions import CRSError diff --git a/clisops/etc/logging.conf b/clisops/etc/logging.conf deleted file mode 100644 index 3a0d6ceb..00000000 --- a/clisops/etc/logging.conf +++ /dev/null @@ -1,28 +0,0 @@ -[loggers] -keys=root,simpleExample - -[handlers] -keys=consoleHandler - -[formatters] -keys=simpleFormatter - -[logger_root] -level=INFO -handlers=consoleHandler - -[logger_simpleExample] -level=DEBUG -handlers=consoleHandler -qualname=simpleExample -propagate=0 - -[handler_consoleHandler] -class=StreamHandler -level=INFO -formatter=simpleFormatter -args=(sys.stdout,) - -[formatter_simpleFormatter] -format=%(asctime)s - %(name)s - %(levelname)s - %(message)s -datefmt= diff --git a/clisops/ops/average.py b/clisops/ops/average.py index bc644384..1cdd0318 100644 --- a/clisops/ops/average.py +++ b/clisops/ops/average.py @@ -3,25 +3,13 @@ import xarray as xr from roocs_utils.parameter.dimension_parameter import DimensionParameter -from roocs_utils.xarray_utils.xarray_utils import ( - convert_coord_to_axis, - get_coord_type, - known_coord_types, - open_xr_dataset, -) - -from clisops import logging, utils +from roocs_utils.xarray_utils.xarray_utils import convert_coord_to_axis + from clisops.core import average from clisops.ops.base_operation import Operation -from clisops.utils.common import expand_wildcards from clisops.utils.file_namers import get_file_namer -from clisops.utils.output_utils import get_output, get_time_slices - -__all__ = [ - "average_over_dims", -] -LOGGER = logging.getLogger(__file__) +__all__ = ["average_over_dims", "Average"] class Average(Operation): diff --git a/clisops/ops/base_operation.py b/clisops/ops/base_operation.py index f5bbb0b2..4e9d60fc 100644 --- a/clisops/ops/base_operation.py +++ b/clisops/ops/base_operation.py @@ -1,14 +1,14 @@ +import logging from pathlib import Path import xarray as xr from roocs_utils.xarray_utils.xarray_utils import get_main_variable, open_xr_dataset -from clisops import logging, utils from clisops.utils.common import expand_wildcards from clisops.utils.file_namers import get_file_namer from clisops.utils.output_utils import get_output, get_time_slices -LOGGER = logging.getLogger(__file__) +logger = logging.getLogger("clisops") class Operation(object): @@ -127,7 +127,7 @@ def process(self): else: result_ds = processed_ds.sel(time=slice(tslice[0], tslice[1])) - LOGGER.info(f"Processing {self.__class__.__name__} for times: {tslice}") + logger.info(f"Processing {self.__class__.__name__} for times: {tslice}") # Get the output (file or xarray Dataset) # When this is a file: xarray will read all the data and write the file diff --git a/clisops/ops/regrid.py b/clisops/ops/regrid.py deleted file mode 100644 index e69de29b..00000000 diff --git a/clisops/ops/subset.py b/clisops/ops/subset.py index b6504a35..80078985 100644 --- a/clisops/ops/subset.py +++ b/clisops/ops/subset.py @@ -1,3 +1,4 @@ +import logging from pathlib import Path from typing import Dict, List, Optional, Tuple, Union @@ -7,9 +8,7 @@ from roocs_utils.parameter.level_parameter import LevelParameter from roocs_utils.parameter.time_components_parameter import TimeComponentsParameter from roocs_utils.parameter.time_parameter import TimeParameter -from roocs_utils.xarray_utils.xarray_utils import open_xr_dataset -from clisops import logging from clisops.core import ( subset_bbox, subset_level, @@ -18,18 +17,13 @@ subset_time_by_components, subset_time_by_values, ) -from clisops.core.subset import assign_bounds, get_lat, get_lon +from clisops.core.subset import assign_bounds, get_lat, get_lon # noqa from clisops.ops.base_operation import Operation -from clisops.utils.common import expand_wildcards from clisops.utils.dataset_utils import check_lon_alignment -from clisops.utils.file_namers import get_file_namer -from clisops.utils.output_utils import get_output, get_time_slices -__all__ = [ - "Subset", -] +__all__ = ["Subset", "subset"] -LOGGER = logging.getLogger(__file__) +LOGGER = logging.getLogger("clisops") class Subset(Operation): @@ -109,8 +103,9 @@ def _calculate(self): lon_min, lon_max = lon.values.min(), lon.values.max() raise Exception( f"The requested longitude subset {self.params.get('lon_bnds')} is not within the longitude bounds " - f"of this dataset and the data could not be converted to this longitude frame successfully. " - f"Please re-run your request with longitudes within the bounds of the dataset: ({lon_min:.2f}, {lon_max:.2f})" + "of this dataset and the data could not be converted to this longitude frame successfully. " + "Please re-run your request with longitudes within the bounds of the dataset: " + f"({lon_min:.2f}, {lon_max:.2f})" ) else: kwargs = {} diff --git a/clisops/utils/common.py b/clisops/utils/common.py index 62efc6fb..acaad644 100644 --- a/clisops/utils/common.py +++ b/clisops/utils/common.py @@ -1,8 +1,6 @@ from pathlib import Path from typing import Union -from roocs_utils.parameter import parameterise - def expand_wildcards(paths: Union[str, Path]) -> list: """Expand the wildcards that may be present in Paths.""" diff --git a/clisops/utils/dataset_utils.py b/clisops/utils/dataset_utils.py index 73ce0075..ea805606 100644 --- a/clisops/utils/dataset_utils.py +++ b/clisops/utils/dataset_utils.py @@ -1,14 +1,8 @@ -import math - import cftime import numpy as np from roocs_utils.utils.time_utils import str_to_AnyCalendarDateTime from roocs_utils.xarray_utils.xarray_utils import get_coord_by_type -from clisops import logging - -LOGGER = logging.getLogger(__file__) - def calculate_offset(lon, first_element_value): """ diff --git a/clisops/utils/output_utils.py b/clisops/utils/output_utils.py index 1e5c78b1..c2c218ca 100644 --- a/clisops/utils/output_utils.py +++ b/clisops/utils/output_utils.py @@ -1,3 +1,4 @@ +import logging import math import os import shutil @@ -13,9 +14,9 @@ from roocs_utils.utils.common import parse_size from roocs_utils.xarray_utils import xarray_utils as xu -from clisops import CONFIG, chunk_memory_limit, logging +from clisops import CONFIG, chunk_memory_limit -LOGGER = logging.getLogger(__file__) +logger = logging.getLogger("clisops.utils") SUPPORTED_FORMATS = { "netcdf": {"method": "to_netcdf", "extension": "nc"}, @@ -200,11 +201,11 @@ def get_output(ds, output_type, output_dir, namer): the output format and chunking """ format_writer = get_format_writer(output_type) - LOGGER.info(f"format_writer={format_writer}, output_type={output_type}") + logger.info(f"format_writer={format_writer}, output_type={output_type}") # If there is no writer for this output type, just return the `ds` object if not format_writer: - LOGGER.info(f"Returning output as {type(ds)}") + logger.info(f"Returning output as {type(ds)}") return ds # Use the file namer to get the required file name @@ -235,7 +236,7 @@ def get_output(ds, output_type, output_dir, namer): tmp_dir = tempfile.TemporaryDirectory(dir=staging_dir) fname = os.path.basename(output_path) target_path = os.path.join(tmp_dir.name, fname) - LOGGER.info(f"Writing to temporary path: {target_path}") + logger.info(f"Writing to temporary path: {target_path}") else: target_path = output_path @@ -257,5 +258,5 @@ def get_output(ds, output_type, output_dir, namer): time.sleep(3) tmp_dir.cleanup() - LOGGER.info(f"Wrote output file: {output_path}") + logger.info(f"Wrote output file: {output_path}") return output_path diff --git a/clisops/utils/tutorial.py b/clisops/utils/tutorial.py index d37d3eb0..9a32a9f6 100644 --- a/clisops/utils/tutorial.py +++ b/clisops/utils/tutorial.py @@ -15,7 +15,7 @@ _default_cache_dir = Path.home() / ".clisops_testing_data" -LOGGER = logging.getLogger(__file__) +logger = logging.getLogger("clisops.utils") __all__ = ["get_file", "open_dataset", "query_folder"] @@ -45,12 +45,12 @@ def _get( local_file.parent.mkdir(parents=True, exist_ok=True) url = "/".join((github_url, "raw", branch, fullname.as_posix())) - LOGGER.info("Fetching remote file: %s" % fullname.as_posix()) + logger.info("Fetching remote file: %s" % fullname.as_posix()) urlretrieve(url, local_file) try: url = "/".join((github_url, "raw", branch, md5name.as_posix())) - LOGGER.info("Fetching remote file md5: %s" % md5name.as_posix()) + logger.info("Fetching remote file md5: %s" % md5name.as_posix()) urlretrieve(url, md5file) except HTTPError as e: msg = f"{md5name.as_posix()} not found. Aborting file retrieval." @@ -69,7 +69,7 @@ def _get( """ raise OSError(msg) except OSError as e: - LOGGER.error(e) + logger.error(e) return local_file @@ -220,7 +220,7 @@ def open_dataset( return ds except OSError: msg = "OPeNDAP file not read. Verify that service is available." - LOGGER.error(msg) + logger.error(msg) raise local_file = _get( diff --git a/requirements.txt b/requirements.txt index bec07c34..add235ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,4 @@ requests>=2.0 roocs-utils>=0.5.0 # roocs-utils @ git+https://github.com/roocs/roocs-utils.git@master#egg=roocs-utils cf-xarray>=0.5.1 +loguru>=0.5.0 From e469399491293341903cace3182520d66df1d442 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 26 Jan 2022 10:00:15 -0500 Subject: [PATCH 10/60] add a logging test, remove references to Travis CI --- tests/test_logging.py | 13 +++++++++++++ tox.ini | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 tests/test_logging.py diff --git a/tests/test_logging.py b/tests/test_logging.py new file mode 100644 index 00000000..3dd6db58 --- /dev/null +++ b/tests/test_logging.py @@ -0,0 +1,13 @@ +import logging + + +def test_logging_configuration(capsys): # noqa + logger = logging.getLogger("clisops") + logger.debug("1") + logger.info("2") + logger.warning("3") + logger.error("4") + logger.critical("5") + captured = capsys.readouterr() + + assert "DEBUG" not in captured.err diff --git a/tox.ini b/tox.ini index 978108eb..645dda50 100644 --- a/tox.ini +++ b/tox.ini @@ -31,7 +31,7 @@ setenv = PYTHONPATH = {toxinidir} GDAL_VERSION = 3.0.0 COV_CORE_SOURCE= -passenv = CI TRAVIS TRAVIS_* PROJ_DIR LD_LIBRARY_PATH GDAL_VERSION GDAL_DATA PATH +passenv = CI PROJ_DIR LD_LIBRARY_PATH GDAL_VERSION GDAL_DATA PATH extras = dev install_command = python -m pip install --no-user {opts} {packages} download = True From eb92d2e69e80282a69ebc198a77e36b9c0fb0468 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Wed, 26 Jan 2022 13:32:33 -0500 Subject: [PATCH 11/60] remove duplicate call to pytest in setup options --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 727460f0..e83913f4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,7 +41,7 @@ test = pytest [tool:pytest] collect_ignore = ["setup.py"] -addopts = --verbose tests +addopts = --verbose filterwarnings = ignore::UserWarning markers = From d9b4ab8b3a925ab1e155de26821ccc43a1362479 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 28 Jan 2022 14:01:42 -0500 Subject: [PATCH 12/60] full reimplementation with loguru --- clisops/__init__.py | 11 +---- clisops/ops/base_operation.py | 4 +- clisops/ops/subset.py | 16 ++++---- clisops/utils/common.py | 33 +++++++++++++++ clisops/utils/output_utils.py | 4 +- clisops/utils/tutorial.py | 3 +- tests/conftest.py | 15 +++++++ tests/test_logging.py | 76 ++++++++++++++++++++++++++++++----- tests/test_output_utils.py | 50 +++++++++++++---------- 9 files changed, 156 insertions(+), 56 deletions(-) diff --git a/clisops/__init__.py b/clisops/__init__.py index 2e85909f..b318ab4e 100644 --- a/clisops/__init__.py +++ b/clisops/__init__.py @@ -1,20 +1,13 @@ # -*- coding: utf-8 -*- """Top-level package for clisops.""" -import logging import os +from loguru import logger from roocs_utils.config import get_config from .__version__ import __author__, __email__, __version__ -logging.getLogger("clisops").addHandler(logging.NullHandler()) - -# logger.add( -# sys.stdout, -# format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", -# filter="clisops", -# ) -# clisops_logger = logger +logger.disable("clisops") # Workaround for roocs_utils to not re-import clisops diff --git a/clisops/ops/base_operation.py b/clisops/ops/base_operation.py index 4e9d60fc..001bf81c 100644 --- a/clisops/ops/base_operation.py +++ b/clisops/ops/base_operation.py @@ -1,15 +1,13 @@ -import logging from pathlib import Path import xarray as xr +from loguru import logger from roocs_utils.xarray_utils.xarray_utils import get_main_variable, open_xr_dataset from clisops.utils.common import expand_wildcards from clisops.utils.file_namers import get_file_namer from clisops.utils.output_utils import get_output, get_time_slices -logger = logging.getLogger("clisops") - class Operation(object): """ diff --git a/clisops/ops/subset.py b/clisops/ops/subset.py index 80078985..22eb2c9e 100644 --- a/clisops/ops/subset.py +++ b/clisops/ops/subset.py @@ -1,8 +1,8 @@ -import logging from pathlib import Path from typing import Dict, List, Optional, Tuple, Union import xarray as xr +from loguru import logger from roocs_utils.parameter import parameterise from roocs_utils.parameter.area_parameter import AreaParameter from roocs_utils.parameter.level_parameter import LevelParameter @@ -23,8 +23,6 @@ __all__ = ["Subset", "subset"] -LOGGER = logging.getLogger("clisops") - class Subset(Operation): def _resolve_params(self, **params): @@ -34,7 +32,7 @@ def _resolve_params(self, **params): level = params.get("level", None) time_comps = params.get("time_components", None) - LOGGER.debug( + logger.debug( f"Mapping parameters: time: {time}, area: {area}, " f"level: {level}, time_components: {time_comps}." ) @@ -81,7 +79,7 @@ def _calculate(self): ) # subset with space and optionally time and level - LOGGER.debug(f"subset_bbox with parameters: {self.params}") + logger.debug(f"subset_bbox with parameters: {self.params}") # bounds are always ascending, so if lon is descending rolling will not work. ds = check_lon_alignment(self.ds, self.params.get("lon_bnds")) try: @@ -115,7 +113,7 @@ def _calculate(self): # Subset over time interval if requested if any(kwargs.values()): - LOGGER.debug(f"subset_time with parameters: {kwargs}") + logger.debug(f"subset_time with parameters: {kwargs}") result = subset_time(self.ds, **kwargs) # Subset a series of time values if requested elif self.params.get("time_values"): @@ -141,18 +139,18 @@ def _calculate(self): ) self.params["first_level"], self.params["last_level"] = last, first - LOGGER.debug(f"subset_level with parameters: {kwargs}") + logger.debug(f"subset_level with parameters: {kwargs}") result = subset_level(result, **kwargs) elif self.params.get("level_values", None): kwargs = {"level_values": self.params["level_values"]} - LOGGER.debug(f"subset_level_by_values with parameters: {kwargs}") + logger.debug(f"subset_level_by_values with parameters: {kwargs}") result = subset_level_by_values(result, **kwargs) # Now apply time components if specified time_comps = self.params.get("time_components") if time_comps: - LOGGER.debug(f"subset_by_time_components with parameters: {time_comps}") + logger.debug(f"subset_by_time_components with parameters: {time_comps}") result = subset_time_by_components(result, time_components=time_comps) return result diff --git a/clisops/utils/common.py b/clisops/utils/common.py index acaad644..cd249e8b 100644 --- a/clisops/utils/common.py +++ b/clisops/utils/common.py @@ -1,9 +1,42 @@ +import sys from pathlib import Path from typing import Union +from loguru import logger + def expand_wildcards(paths: Union[str, Path]) -> list: """Expand the wildcards that may be present in Paths.""" path = Path(paths).expanduser() parts = path.parts[1:] if path.is_absolute() else path.parts return [f for f in Path(path.root).glob(str(Path("").joinpath(*parts)))] + + +def _logging_examples() -> None: + """Testing module""" + logger.trace("0") + logger.debug("1") + logger.info("2") + logger.success("2.5") + logger.warning("3") + logger.error("4") + logger.critical("5") + + +def enable_logging(): + logger.remove() + + logger.enable("clisops") + logger.add( + sys.stdout, + format="{time:YYYY-MM-DD HH:mm:ss.SSS Z UTC}" + " | {level} | {name}:{function}:{line}" + " | {message}", + level="INFO", + ) + + logger.add( + sys.stderr, + format="{time:YYYY-MM-DD HH:mm:ss.SSS Z UTC} | {level} | {name}:{function}:{line} | {message}", + level="WARNING", + ) diff --git a/clisops/utils/output_utils.py b/clisops/utils/output_utils.py index c2c218ca..bf73d833 100644 --- a/clisops/utils/output_utils.py +++ b/clisops/utils/output_utils.py @@ -1,4 +1,3 @@ -import logging import math import os import shutil @@ -11,13 +10,12 @@ import dask import pandas as pd import xarray as xr +from loguru import logger from roocs_utils.utils.common import parse_size from roocs_utils.xarray_utils import xarray_utils as xu from clisops import CONFIG, chunk_memory_limit -logger = logging.getLogger("clisops.utils") - SUPPORTED_FORMATS = { "netcdf": {"method": "to_netcdf", "extension": "nc"}, "nc": {"method": "to_netcdf", "extension": "nc"}, diff --git a/clisops/utils/tutorial.py b/clisops/utils/tutorial.py index 9a32a9f6..7f0c7823 100644 --- a/clisops/utils/tutorial.py +++ b/clisops/utils/tutorial.py @@ -1,7 +1,6 @@ """Testing and tutorial utilities module.""" # Most of this code copied and adapted from xarray, xclim, and raven import hashlib -import logging import re from pathlib import Path from typing import List, Optional, Sequence, Union @@ -10,12 +9,12 @@ from urllib.request import urlretrieve import requests +from loguru import logger from xarray import Dataset from xarray import open_dataset as _open_dataset _default_cache_dir = Path.home() / ".clisops_testing_data" -logger = logging.getLogger("clisops.utils") __all__ = ["get_file", "open_dataset", "query_folder"] diff --git a/tests/conftest.py b/tests/conftest.py index 2f616795..ad08cd79 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,10 +1,13 @@ +import logging import os import numpy as np import pandas as pd import pytest import xarray as xr +from _pytest.logging import caplog as _caplog # noqa from git import Repo +from loguru import logger from clisops.utils import get_file from tests._common import MINI_ESGF_CACHE_DIR, write_roocs_cfg @@ -14,6 +17,18 @@ ESGF_TEST_DATA_REPO_URL = "https://github.com/roocs/mini-esgf-data" +@pytest.fixture +def caplog(_caplog): # noqa + class PropogateHandler(logging.Handler): + def emit(self, record): + logging.getLogger(record.name).handle(record) + + handler_id = logger.add(PropogateHandler(), format="{message}") + yield _caplog + logger.remove(handler_id) + logger.remove() + + @pytest.fixture def tmp_netcdf_filename(tmp_path): return tmp_path.joinpath("testfile.nc") diff --git a/tests/test_logging.py b/tests/test_logging.py index 3dd6db58..5a83a1b7 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -1,13 +1,71 @@ -import logging +import sys +from loguru import logger + +from clisops.utils.common import _logging_examples, enable_logging # noqa + + +def test_logging_configuration(caplog): + # no need for logger.remove() because of caplog workaround + logger.enable("clisops") + caplog.set_level("WARNING", logger="clisops") + + _logging_examples() # noqa + + assert ("clisops.utils.common", 10, "1") not in caplog.record_tuples + assert ("clisops.utils.common", 40, "4") in caplog.record_tuples + logger.disable("clisops") + + +def test_disabled_enabled_logging(capsys): + logger.remove() + + # CLISOPS disabled by default + logger.add(sys.stderr, level="WARNING") + logger.add(sys.stdout, level="INFO") + + _logging_examples() # noqa + + captured = capsys.readouterr() + assert "WARNING" not in captured.err + assert "INFO" not in captured.out + + # re-enable CLISOPS logging + logger.enable("clisops") + logger.add(sys.stderr, level="WARNING") + logger.add(sys.stdout, level="INFO") + + _logging_examples() # noqa + + captured = capsys.readouterr() + assert "INFO" not in captured.err + assert "WARNING" in captured.err + assert "INFO" in captured.out + logger.disable("clisops") + + +def test_logging_enabler(capsys): + logger.remove() + + # CLISOPS disabled by default + logger.add(sys.stderr, level="WARNING") + logger.add(sys.stdout, level="INFO") + + _logging_examples() # noqa + + captured = capsys.readouterr() + assert "WARNING" not in captured.err + assert "INFO" not in captured.out + + enable_logging() + + _logging_examples() # noqa -def test_logging_configuration(capsys): # noqa - logger = logging.getLogger("clisops") - logger.debug("1") - logger.info("2") - logger.warning("3") - logger.error("4") - logger.critical("5") captured = capsys.readouterr() + print(captured.out) + assert "INFO" not in captured.err + assert "WARNING" in captured.err + assert "INFO" in captured.out - assert "DEBUG" not in captured.err + logger.disable("clisops") + logger.remove() # sets logging back to default diff --git a/tests/test_output_utils.py b/tests/test_output_utils.py index eff990ca..64adb1c0 100644 --- a/tests/test_output_utils.py +++ b/tests/test_output_utils.py @@ -2,11 +2,11 @@ import tempfile from glob import glob from pathlib import Path -from unittest import mock import xarray as xr +from loguru import logger -from clisops import CONFIG, logging +from clisops import CONFIG from clisops.utils.common import expand_wildcards from clisops.utils.file_namers import get_file_namer from clisops.utils.output_utils import ( @@ -17,8 +17,6 @@ ) from tests._common import CMIP5_TAS, CMIP6_TOS -LOGGER = logging.getLogger(__file__) - def _open(coll): if isinstance(coll, (str, Path)): @@ -90,9 +88,12 @@ def test_get_time_slices_multiple_slices(load_esgf_test_data): assert resp[1] == second -def test_tmp_dir_created_with_staging_dir(): +def test_tmp_dir_created_with_staging_dir(tmpdir): + staging = Path(tmpdir).joinpath("tests") + staging.mkdir(exist_ok=True) + # copy part of function that creates tmp dir to check that it is created - CONFIG["clisops:write"]["output_staging_dir"] = "tests/" + CONFIG["clisops:write"]["output_staging_dir"] = staging staging_dir = CONFIG["clisops:write"].get("output_staging_dir", "") output_path = "./output_001.nc" @@ -101,7 +102,7 @@ def test_tmp_dir_created_with_staging_dir(): tmp_dir = tempfile.TemporaryDirectory(dir=staging_dir) fname = os.path.basename(output_path) target_path = os.path.join(tmp_dir.name, fname) - LOGGER.info(f"Writing to temporary path: {target_path}") + logger.info(f"Writing to temporary path: {target_path}") else: target_path = output_path @@ -109,9 +110,6 @@ def test_tmp_dir_created_with_staging_dir(): assert len(glob("tests/tmp*")) == 1 assert "tests/tmp" in glob("tests/tmp*")[0] - # delete the temporary directory - tmp_dir.cleanup() - def test_tmp_dir_not_created_with_no_staging_dir(): # copy part of function that creates tmp dir to check that it is not created when no staging dir @@ -124,7 +122,7 @@ def test_tmp_dir_not_created_with_no_staging_dir(): tmp_dir = tempfile.TemporaryDirectory(dir=staging_dir) fname = os.path.basename(output_path) target_path = os.path.join(tmp_dir.name, fname) - LOGGER.info(f"Writing to temporary path: {target_path}") + logger.info(f"Writing to temporary path: {target_path}") else: target_path = output_path @@ -146,7 +144,7 @@ def test_no_staging_dir(caplog): def test_invalid_staging_dir(caplog): - # check stagin dir not used with invalid directory + # check staging dir not used with invalid directory CONFIG["clisops:write"]["output_staging_dir"] = "test/not/real/dir/" ds = _open(CMIP5_TAS) @@ -160,20 +158,26 @@ def test_invalid_staging_dir(caplog): os.remove("output_001.nc") -def test_staging_dir_used(caplog): - # check staging dir used when valid directory - CONFIG["clisops:write"]["output_staging_dir"] = "tests/" +def test_staging_dir_used(caplog, tmpdir): + logger.enable("clisops") + caplog.set_level("INFO", logger="clisops") + # check staging dir used when valid directory + staging = Path(tmpdir).joinpath("tests") + staging.mkdir(exist_ok=True) + CONFIG["clisops:write"]["output_staging_dir"] = str(staging) ds = _open(CMIP5_TAS) output_path = get_output( ds, output_type="nc", output_dir=".", namer=get_file_namer("simple")() ) - assert "Writing to temporary path: tests/" in caplog.text + assert f"Writing to temporary path: {staging}" in caplog.text assert output_path == "output_001.nc" - os.remove("output_001.nc") + logger.disable("clisops") + + Path("output_001.nc").unlink() def test_final_output_path_staging_dir(): @@ -198,15 +202,19 @@ def test_final_output_path_no_staging_dir(): os.remove("output_001.nc") -def test_tmp_dir_deleted(): - # check temporary directory under stagin dir gets deleted after data has bee staged - CONFIG["clisops:write"]["output_staging_dir"] = "tests/" +def test_tmp_dir_deleted(tmpdir): + # check temporary directory under staging dir gets deleted after data has bee staged + staging = Path(tmpdir).joinpath("tests") + staging.mkdir(exist_ok=True) + CONFIG["clisops:write"]["output_staging_dir"] = staging + + # CONFIG["clisops:write"]["output_staging_dir"] = "tests/" ds = _open(CMIP5_TAS) get_output(ds, output_type="nc", output_dir=".", namer=get_file_namer("simple")()) # check that no tmpdir directories exist - assert glob("tests/tmp*") == [] + assert [f for f in staging.glob("tmp*")] == [] os.remove("output_001.nc") From 7615f938071a6842ab34890dbbf37512ad8ffaec Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 28 Jan 2022 14:58:36 -0500 Subject: [PATCH 13/60] add loguru to environment.yml --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index bbe05cd9..745a6172 100644 --- a/environment.yml +++ b/environment.yml @@ -9,6 +9,7 @@ dependencies: - cftime>=1.4.1 - dask>=2.6.0 - geopandas>=0.7 + - loguru>=0.5.0 - netCDF4>=1.4 - numpy>=1.16 - pandas>=1.0.3 From aa4278dc233757c3af79a71c78f48c126304e000 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 28 Jan 2022 15:37:32 -0500 Subject: [PATCH 14/60] pin pandas below 1.4 --- environment.yml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 745a6172..06984acf 100644 --- a/environment.yml +++ b/environment.yml @@ -12,7 +12,7 @@ dependencies: - loguru>=0.5.0 - netCDF4>=1.4 - numpy>=1.16 - - pandas>=1.0.3 + - pandas>=1.0.3,<1.4 - poppler>=0.67 - pygeos>=0.9 - pyproj>=2.5 diff --git a/requirements.txt b/requirements.txt index add235ce..229e1c2c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ numpy>=1.16 xarray>=0.15 -pandas>=1.0.3 +pandas>=1.0.3,<1.4 cftime>=1.4.1 netCDF4>=1.4 shapely>=1.6 From a2c1037253f3da8ca16a6cd02a89f67f17a60ac2 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 28 Jan 2022 15:41:00 -0500 Subject: [PATCH 15/60] add some colour to tox readout --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 645dda50..4a3eed38 100644 --- a/tox.ini +++ b/tox.ini @@ -28,6 +28,7 @@ whitelist_externals = [testenv] setenv = + PYTEST_ADDOPTS = "--color=yes" PYTHONPATH = {toxinidir} GDAL_VERSION = 3.0.0 COV_CORE_SOURCE= From 31c9839420f8fc3f66326c45b1c4cf79d501dab8 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 28 Jan 2022 16:46:10 -0500 Subject: [PATCH 16/60] fix folder discovery test --- tests/test_output_utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_output_utils.py b/tests/test_output_utils.py index 64adb1c0..c750d825 100644 --- a/tests/test_output_utils.py +++ b/tests/test_output_utils.py @@ -107,8 +107,9 @@ def test_tmp_dir_created_with_staging_dir(tmpdir): target_path = output_path assert target_path != "output_001.nc" - assert len(glob("tests/tmp*")) == 1 - assert "tests/tmp" in glob("tests/tmp*")[0] + temp_test_folders = [f for f in staging.glob("tmp*")] + assert len(temp_test_folders) == 1 + assert "tests/tmp" in temp_test_folders[0].as_posix() def test_tmp_dir_not_created_with_no_staging_dir(): @@ -214,7 +215,7 @@ def test_tmp_dir_deleted(tmpdir): get_output(ds, output_type="nc", output_dir=".", namer=get_file_namer("simple")()) # check that no tmpdir directories exist - assert [f for f in staging.glob("tmp*")] == [] + assert len([f for f in staging.glob("tmp*")]) == 0 os.remove("output_001.nc") From b0595fe45e5352d27e5424430518a074e47693c5 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 29 Mar 2022 14:08:49 -0400 Subject: [PATCH 17/60] Adjustments to loguru --- clisops/__init__.py | 3 +- clisops/utils/common.py | 36 +++++++++-------- environment.yml | 2 +- requirements.txt | 2 +- requirements_dev.txt | 1 + tests/conftest.py | 15 ------- tests/test_logging.py | 87 +++++++++++++++++++---------------------- 7 files changed, 64 insertions(+), 82 deletions(-) diff --git a/clisops/__init__.py b/clisops/__init__.py index b318ab4e..7e4e47a6 100644 --- a/clisops/__init__.py +++ b/clisops/__init__.py @@ -7,7 +7,8 @@ from .__version__ import __author__, __email__, __version__ -logger.disable("clisops") +# Remove the logger that is instantiated on import +logger.remove() # Workaround for roocs_utils to not re-import clisops diff --git a/clisops/utils/common.py b/clisops/utils/common.py index cd249e8b..4aec98ec 100644 --- a/clisops/utils/common.py +++ b/clisops/utils/common.py @@ -1,7 +1,8 @@ import sys from pathlib import Path -from typing import Union +from typing import Union, List +import loguru from loguru import logger @@ -23,20 +24,21 @@ def _logging_examples() -> None: logger.critical("5") -def enable_logging(): - logger.remove() - - logger.enable("clisops") - logger.add( - sys.stdout, - format="{time:YYYY-MM-DD HH:mm:ss.SSS Z UTC}" - " | {level} | {name}:{function}:{line}" - " | {message}", - level="INFO", - ) - - logger.add( - sys.stderr, - format="{time:YYYY-MM-DD HH:mm:ss.SSS Z UTC} | {level} | {name}:{function}:{line} | {message}", - level="WARNING", +def enable_logging() -> List[int]: + config = dict( + handlers=[ + dict( + sink=sys.stdout, + format="{time:YYYY-MM-DD HH:mm:ss.SSS Z UTC}" + " | {level} | {name}:{function}:{line}" + " | {message}", + level="INFO", + ), + dict( + sink=sys.stderr, + format="{time:YYYY-MM-DD HH:mm:ss.SSS Z UTC} | {level} | {name}:{function}:{line} | {message}", + level="WARNING", + ), + ] ) + return logger.configure(**config) diff --git a/environment.yml b/environment.yml index 06984acf..3ba04f5f 100644 --- a/environment.yml +++ b/environment.yml @@ -9,7 +9,7 @@ dependencies: - cftime>=1.4.1 - dask>=2.6.0 - geopandas>=0.7 - - loguru>=0.5.0 + - loguru>=0.5.3 - netCDF4>=1.4 - numpy>=1.16 - pandas>=1.0.3,<1.4 diff --git a/requirements.txt b/requirements.txt index 229e1c2c..9b0530e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,4 +12,4 @@ requests>=2.0 roocs-utils>=0.5.0 # roocs-utils @ git+https://github.com/roocs/roocs-utils.git@master#egg=roocs-utils cf-xarray>=0.5.1 -loguru>=0.5.0 +loguru>=0.5.3 diff --git a/requirements_dev.txt b/requirements_dev.txt index f24ba9aa..cb837475 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -9,6 +9,7 @@ Sphinx sphinx-rtd-theme twine pytest +pytest-loguru pytest-runner pre-commit>=2.9.0 black>=21.12b0 diff --git a/tests/conftest.py b/tests/conftest.py index ad08cd79..00db7b85 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,3 @@ -import logging import os import numpy as np @@ -7,8 +6,6 @@ import xarray as xr from _pytest.logging import caplog as _caplog # noqa from git import Repo -from loguru import logger - from clisops.utils import get_file from tests._common import MINI_ESGF_CACHE_DIR, write_roocs_cfg @@ -17,18 +14,6 @@ ESGF_TEST_DATA_REPO_URL = "https://github.com/roocs/mini-esgf-data" -@pytest.fixture -def caplog(_caplog): # noqa - class PropogateHandler(logging.Handler): - def emit(self, record): - logging.getLogger(record.name).handle(record) - - handler_id = logger.add(PropogateHandler(), format="{message}") - yield _caplog - logger.remove(handler_id) - logger.remove() - - @pytest.fixture def tmp_netcdf_filename(tmp_path): return tmp_path.joinpath("testfile.nc") diff --git a/tests/test_logging.py b/tests/test_logging.py index 5a83a1b7..dad7c778 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -1,71 +1,64 @@ +import logging import sys +import pytest from loguru import logger from clisops.utils.common import _logging_examples, enable_logging # noqa -def test_logging_configuration(caplog): - # no need for logger.remove() because of caplog workaround - logger.enable("clisops") - caplog.set_level("WARNING", logger="clisops") +class TestLoggingFuncs: - _logging_examples() # noqa + @pytest.mark.xfail(reason="pytest-loguru does not implement logging levels for caplog yet") + def test_logging_configuration(self, caplog): + caplog.set_level(logging.WARNING, logger="clisops") - assert ("clisops.utils.common", 10, "1") not in caplog.record_tuples - assert ("clisops.utils.common", 40, "4") in caplog.record_tuples - logger.disable("clisops") + _logging_examples() # noqa + assert ("clisops.utils.common", 10, "1") not in caplog.record_tuples + assert ("clisops.utils.common", 40, "4") in caplog.record_tuples -def test_disabled_enabled_logging(capsys): - logger.remove() + def test_disabled_enabled_logging(self, capsys): + logger.disable("clisops") - # CLISOPS disabled by default - logger.add(sys.stderr, level="WARNING") - logger.add(sys.stdout, level="INFO") + # CLISOPS disabled + id1 = logger.add(sys.stderr, level="WARNING") + id2 = logger.add(sys.stdout, level="INFO") - _logging_examples() # noqa + _logging_examples() # noqa - captured = capsys.readouterr() - assert "WARNING" not in captured.err - assert "INFO" not in captured.out + captured = capsys.readouterr() + assert "WARNING" not in captured.err + assert "INFO" not in captured.out - # re-enable CLISOPS logging - logger.enable("clisops") - logger.add(sys.stderr, level="WARNING") - logger.add(sys.stdout, level="INFO") + # re-enable CLISOPS logging + logger.enable("clisops") - _logging_examples() # noqa + _logging_examples() # noqa - captured = capsys.readouterr() - assert "INFO" not in captured.err - assert "WARNING" in captured.err - assert "INFO" in captured.out - logger.disable("clisops") + captured = capsys.readouterr() + assert "INFO" not in captured.err + assert "WARNING" in captured.err + assert "INFO" in captured.out + logger.remove(id1) + logger.remove(id2) -def test_logging_enabler(capsys): - logger.remove() + def test_logging_enabler(self, capsys): + _logging_examples() # noqa - # CLISOPS disabled by default - logger.add(sys.stderr, level="WARNING") - logger.add(sys.stdout, level="INFO") + captured = capsys.readouterr() + assert "WARNING" not in captured.err + assert "INFO" not in captured.out - _logging_examples() # noqa + ids = enable_logging() - captured = capsys.readouterr() - assert "WARNING" not in captured.err - assert "INFO" not in captured.out + _logging_examples() # noqa - enable_logging() + captured = capsys.readouterr() + assert "INFO" not in captured.err + assert "WARNING" in captured.err + assert "INFO" in captured.out - _logging_examples() # noqa - - captured = capsys.readouterr() - print(captured.out) - assert "INFO" not in captured.err - assert "WARNING" in captured.err - assert "INFO" in captured.out - - logger.disable("clisops") - logger.remove() # sets logging back to default + for i in ids: + logger.remove(i) # sets logging back to default From 0e1283aa6d2e4ff1768b96c58beaec29e056442d Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 29 Mar 2022 14:11:34 -0400 Subject: [PATCH 18/60] pre-commit adjustments --- .pre-commit-config.yaml | 4 ++-- clisops/core/average.py | 1 - clisops/utils/common.py | 3 +-- tests/conftest.py | 1 + tests/test_logging.py | 7 ++++--- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9c675961..c9b74650 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ repos: - id: debug-statements language_version: python3 - repo: https://github.com/ambv/black - rev: 21.12b0 + rev: 22.3.0 hooks: - id: black language_version: python3 @@ -44,7 +44,7 @@ repos: # - id: pydocstyle # args: ["--conventions=numpy"] - repo: https://github.com/asottile/pyupgrade - rev: v2.31.0 + rev: v2.31.1 hooks: - id: pyupgrade language_version: python3 diff --git a/clisops/core/average.py b/clisops/core/average.py index c2f84faf..5f3a0095 100644 --- a/clisops/core/average.py +++ b/clisops/core/average.py @@ -3,7 +3,6 @@ from pathlib import Path from typing import Sequence, Tuple, Union - import cf_xarray import geopandas as gpd import numpy as np diff --git a/clisops/utils/common.py b/clisops/utils/common.py index 4aec98ec..cbec665b 100644 --- a/clisops/utils/common.py +++ b/clisops/utils/common.py @@ -1,8 +1,7 @@ import sys from pathlib import Path -from typing import Union, List +from typing import List, Union -import loguru from loguru import logger diff --git a/tests/conftest.py b/tests/conftest.py index 00db7b85..d8488de8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,6 +6,7 @@ import xarray as xr from _pytest.logging import caplog as _caplog # noqa from git import Repo + from clisops.utils import get_file from tests._common import MINI_ESGF_CACHE_DIR, write_roocs_cfg diff --git a/tests/test_logging.py b/tests/test_logging.py index dad7c778..faa4851e 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -1,15 +1,16 @@ import logging import sys -import pytest +import pytest from loguru import logger from clisops.utils.common import _logging_examples, enable_logging # noqa class TestLoggingFuncs: - - @pytest.mark.xfail(reason="pytest-loguru does not implement logging levels for caplog yet") + @pytest.mark.xfail( + reason="pytest-loguru does not implement logging levels for caplog yet" + ) def test_logging_configuration(self, caplog): caplog.set_level(logging.WARNING, logger="clisops") From 2b5324984ae29b44d90fffcf9dafc5a17592310e Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 29 Mar 2022 14:13:31 -0400 Subject: [PATCH 19/60] use py37 conventions --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c9b74650..7cc73fc1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,7 +20,7 @@ repos: hooks: - id: black language_version: python3 - args: ["--target-version", "py36"] + args: ["--target-version", "py37"] - repo: https://github.com/pycqa/flake8 rev: 4.0.1 hooks: From efa838cf591a406dadebb36fea5096efa5924421 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 29 Mar 2022 14:15:08 -0400 Subject: [PATCH 20/60] use py37 conventions --- clisops/core/subset.py | 4 ++-- tests/test_roll.py | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/clisops/core/subset.py b/clisops/core/subset.py index c7bbd6ff..229b8abd 100644 --- a/clisops/core/subset.py +++ b/clisops/core/subset.py @@ -648,11 +648,11 @@ def _curvilinear_grid_exterior_polygon(ds, mode="bbox"): from shapely.ops import unary_union def round_up(x, decimal=1): - f = 10 ** decimal + f = 10**decimal return math.ceil(x * f) / f def round_down(x, decimal=1): - f = 10 ** decimal + f = 10**decimal return math.floor(x * f) / f if mode == "bbox": diff --git a/tests/test_roll.py b/tests/test_roll.py index c5360d8f..45228000 100644 --- a/tests/test_roll.py +++ b/tests/test_roll.py @@ -44,8 +44,8 @@ def test_roll_lon_minus_180(load_esgf_test_data): ds, lon = setup_test() # check longitude is 0 to 360 initially - assert isclose(lon.values.min(), 0, abs_tol=10 ** 2) - assert isclose(lon.values.max(), 360, abs_tol=10 ** 2) + assert isclose(lon.values.min(), 0, abs_tol=10**2) + assert isclose(lon.values.max(), 360, abs_tol=10**2) # roll longitude by -180 ds = ds.roll(shifts={f"{lon.name}": -180}, roll_coords=True) @@ -159,8 +159,8 @@ def test_convert_lon_coords(tmpdir, load_esgf_test_data): ds.coords[lon.name] = (ds.coords[lon.name] + 180) % 360 - 180 ds = ds.sortby(ds[lon.name]) - assert isclose(ds.lon.values.min(), -180, abs_tol=10 ** 2) - assert isclose(ds.lon.values.max(), 180, abs_tol=10 ** 2) + assert isclose(ds.lon.values.min(), -180, abs_tol=10**2) + assert isclose(ds.lon.values.max(), 180, abs_tol=10**2) result = subset( ds=ds, @@ -191,8 +191,8 @@ def test_roll_convert_lon_coords(load_esgf_test_data): ds_roll.coords[lon.name] = ds_roll.coords[lon.name] - 180 - assert isclose(ds_roll.lon.values.min(), -180, abs_tol=10 ** 2) - assert isclose(ds_roll.lon.values.max(), 180, abs_tol=10 ** 2) + assert isclose(ds_roll.lon.values.min(), -180, abs_tol=10**2) + assert isclose(ds_roll.lon.values.max(), 180, abs_tol=10**2) result = subset( ds=ds_roll, @@ -253,8 +253,8 @@ def test_compare_methods(load_esgf_test_data): ds_roll.coords[lon.name] = ds_roll.coords[lon.name] - 180 - assert isclose(ds_roll.lon.values.min(), -180, abs_tol=10 ** 2) - assert isclose(ds_roll.lon.values.max(), 180, abs_tol=10 ** 2) + assert isclose(ds_roll.lon.values.min(), -180, abs_tol=10**2) + assert isclose(ds_roll.lon.values.max(), 180, abs_tol=10**2) result1 = subset( ds=ds_roll, @@ -270,8 +270,8 @@ def test_compare_methods(load_esgf_test_data): ds.coords[lon.name] = (ds.coords[lon.name] + 180) % 360 - 180 ds = ds.sortby(ds[lon.name]) - assert isclose(ds.lon.values.min(), -180, abs_tol=10 ** 2) - assert isclose(ds.lon.values.max(), 180, abs_tol=10 ** 2) + assert isclose(ds.lon.values.min(), -180, abs_tol=10**2) + assert isclose(ds.lon.values.max(), 180, abs_tol=10**2) result2 = subset( ds=ds, From 3b7f6d34449a7925b1681621ec50cc7f467b758c Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 29 Mar 2022 14:26:19 -0400 Subject: [PATCH 21/60] update pre-commit with newest versions of libraries and use py37+ conventions in black and pyupgrade --- .pre-commit-config.yaml | 32 ++++++++++---------------------- clisops/__init__.py | 1 - clisops/__version__.py | 1 - clisops/core/subset.py | 4 ++-- clisops/ops/base_operation.py | 2 +- clisops/utils/file_namers.py | 2 +- clisops/utils/tutorial.py | 2 +- docs/conf.py | 1 - setup.py | 3 +-- tests/test_file_namers.py | 2 +- tests/test_roll.py | 20 ++++++++++---------- 11 files changed, 27 insertions(+), 43 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 96ee5cbc..f824c06e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,57 +3,45 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.4.0 + rev: v4.1.0 hooks: - id: trailing-whitespace - language_version: python3 exclude: setup.cfg - id: end-of-file-fixer - language_version: python3 exclude: setup.cfg - id: check-yaml - language_version: python3 - id: debug-statements - language_version: python3 - repo: https://github.com/ambv/black - rev: 21.4b2 + rev: 22.3.0 hooks: - id: black - language_version: python3 - args: ["--target-version", "py36"] + args: ["--target-version", "py37"] - repo: https://github.com/pycqa/flake8 - rev: 3.9.1 + rev: 4.0.1 hooks: - id: flake8 - language_version: python3 args: ['--config=setup.cfg'] -#- repo: https://github.com/pre-commit/mirrors-autopep8 -# rev: v1.4.4 -# hooks: -# - id: autopep8 -# args: ['--global-config=setup.cfg','--in-place'] - repo: https://github.com/timothycrosley/isort - rev: 5.8.0 + rev: 5.10.1 hooks: - id: isort - language_version: python3 args: ['--profile', 'black'] #- repo: https://github.com/pycqa/pydocstyle -# rev: 5.0.2 +# rev: 6.1.1 # hooks: # - id: pydocstyle -# args: ["--conventions=numpy"] +# args: ["--convention=numpy"] - repo: https://github.com/asottile/pyupgrade - rev: v2.14.0 + rev: v2.31.1 hooks: - id: pyupgrade - language_version: python3 + args: ['--py37-plus'] - repo: meta hooks: - id: check-hooks-apply - id: check-useless-excludes - repo: https://github.com/kynan/nbstripout - rev: 0.4.0 + rev: 0.5.0 hooks: - id: nbstripout language_version: python3 diff --git a/clisops/__init__.py b/clisops/__init__.py index 4432bf83..8ee33e0a 100644 --- a/clisops/__init__.py +++ b/clisops/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Top-level package for clisops.""" import logging.config diff --git a/clisops/__version__.py b/clisops/__version__.py index ec28529d..4f0d25e6 100644 --- a/clisops/__version__.py +++ b/clisops/__version__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # This information is located in its own file so that it can be loaded # without importing the main package when its dependencies are not installed. # See: https://packaging.python.org/guides/single-sourcing-package-version diff --git a/clisops/core/subset.py b/clisops/core/subset.py index 0a5790f2..a111ccf5 100644 --- a/clisops/core/subset.py +++ b/clisops/core/subset.py @@ -648,11 +648,11 @@ def _curvilinear_grid_exterior_polygon(ds, mode="bbox"): from shapely.ops import unary_union def round_up(x, decimal=1): - f = 10 ** decimal + f = 10**decimal return math.ceil(x * f) / f def round_down(x, decimal=1): - f = 10 ** decimal + f = 10**decimal return math.floor(x * f) / f if mode == "bbox": diff --git a/clisops/ops/base_operation.py b/clisops/ops/base_operation.py index f5bbb0b2..501e771f 100644 --- a/clisops/ops/base_operation.py +++ b/clisops/ops/base_operation.py @@ -11,7 +11,7 @@ LOGGER = logging.getLogger(__file__) -class Operation(object): +class Operation: """ Base class for all Operations. """ diff --git a/clisops/utils/file_namers.py b/clisops/utils/file_namers.py index b2bf8c0d..db0e4ae2 100644 --- a/clisops/utils/file_namers.py +++ b/clisops/utils/file_namers.py @@ -12,7 +12,7 @@ def get_file_namer(name): return namers.get(name, StandardFileNamer) -class _BaseFileNamer(object): +class _BaseFileNamer: """File namer base class""" def __init__(self, replace=None, extra=""): diff --git a/clisops/utils/tutorial.py b/clisops/utils/tutorial.py index d37d3eb0..c5c1a391 100644 --- a/clisops/utils/tutorial.py +++ b/clisops/utils/tutorial.py @@ -36,7 +36,7 @@ def _get( ) -> Path: cache_dir = cache_dir.absolute() local_file = cache_dir / branch / fullname - md5name = fullname.with_suffix("{}.md5".format(suffix)) + md5name = fullname.with_suffix(f"{suffix}.md5") md5file = cache_dir / branch / md5name if not local_file.is_file(): diff --git a/docs/conf.py b/docs/conf.py index 72d8953a..fff69a69 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # clisops documentation build configuration file, created by # sphinx-quickstart on Fri Jun 9 13:47:02 2017. diff --git a/setup.py b/setup.py index 8522b06e..198a2927 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- """The setup script.""" import os @@ -13,7 +12,7 @@ _long_description = open(os.path.join(here, "README.rst")).read() about = dict() -with open(os.path.join(here, "clisops", "__version__.py"), "r") as f: +with open(os.path.join(here, "clisops", "__version__.py")) as f: exec(f.read(), about) requirements = [line.strip() for line in open("requirements.txt")] diff --git a/tests/test_file_namers.py b/tests/test_file_namers.py index fc2560fc..2c36700d 100644 --- a/tests/test_file_namers.py +++ b/tests/test_file_namers.py @@ -58,7 +58,7 @@ def test_SimpleFileNamer_with_chunking(load_esgf_test_data, tmpdir): def test_StandardFileNamer_no_project_match(): s = get_file_namer("standard")() - class Thing(object): + class Thing: pass mock_ds = Thing() diff --git a/tests/test_roll.py b/tests/test_roll.py index c5360d8f..45228000 100644 --- a/tests/test_roll.py +++ b/tests/test_roll.py @@ -44,8 +44,8 @@ def test_roll_lon_minus_180(load_esgf_test_data): ds, lon = setup_test() # check longitude is 0 to 360 initially - assert isclose(lon.values.min(), 0, abs_tol=10 ** 2) - assert isclose(lon.values.max(), 360, abs_tol=10 ** 2) + assert isclose(lon.values.min(), 0, abs_tol=10**2) + assert isclose(lon.values.max(), 360, abs_tol=10**2) # roll longitude by -180 ds = ds.roll(shifts={f"{lon.name}": -180}, roll_coords=True) @@ -159,8 +159,8 @@ def test_convert_lon_coords(tmpdir, load_esgf_test_data): ds.coords[lon.name] = (ds.coords[lon.name] + 180) % 360 - 180 ds = ds.sortby(ds[lon.name]) - assert isclose(ds.lon.values.min(), -180, abs_tol=10 ** 2) - assert isclose(ds.lon.values.max(), 180, abs_tol=10 ** 2) + assert isclose(ds.lon.values.min(), -180, abs_tol=10**2) + assert isclose(ds.lon.values.max(), 180, abs_tol=10**2) result = subset( ds=ds, @@ -191,8 +191,8 @@ def test_roll_convert_lon_coords(load_esgf_test_data): ds_roll.coords[lon.name] = ds_roll.coords[lon.name] - 180 - assert isclose(ds_roll.lon.values.min(), -180, abs_tol=10 ** 2) - assert isclose(ds_roll.lon.values.max(), 180, abs_tol=10 ** 2) + assert isclose(ds_roll.lon.values.min(), -180, abs_tol=10**2) + assert isclose(ds_roll.lon.values.max(), 180, abs_tol=10**2) result = subset( ds=ds_roll, @@ -253,8 +253,8 @@ def test_compare_methods(load_esgf_test_data): ds_roll.coords[lon.name] = ds_roll.coords[lon.name] - 180 - assert isclose(ds_roll.lon.values.min(), -180, abs_tol=10 ** 2) - assert isclose(ds_roll.lon.values.max(), 180, abs_tol=10 ** 2) + assert isclose(ds_roll.lon.values.min(), -180, abs_tol=10**2) + assert isclose(ds_roll.lon.values.max(), 180, abs_tol=10**2) result1 = subset( ds=ds_roll, @@ -270,8 +270,8 @@ def test_compare_methods(load_esgf_test_data): ds.coords[lon.name] = (ds.coords[lon.name] + 180) % 360 - 180 ds = ds.sortby(ds[lon.name]) - assert isclose(ds.lon.values.min(), -180, abs_tol=10 ** 2) - assert isclose(ds.lon.values.max(), 180, abs_tol=10 ** 2) + assert isclose(ds.lon.values.min(), -180, abs_tol=10**2) + assert isclose(ds.lon.values.max(), 180, abs_tol=10**2) result2 = subset( ds=ds, From 76f62d6aca257d366448671e5f9619ae35fb88b6 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 29 Mar 2022 14:52:05 -0400 Subject: [PATCH 22/60] add pytest-loguru --- environment.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 3ba04f5f..455c3ee9 100644 --- a/environment.yml +++ b/environment.yml @@ -21,5 +21,6 @@ dependencies: - shapely>=1.6 - xarray>=0.15 - xesmf>=0.6.2 -# - pip: + - pip: + - pytest-loguru # - roocs-utils @ git+https://github.com/roocs/roocs-utils.git@master#egg=roocs-utils From ea9c41b380c47e2eea40db492bb713fa7c03fb75 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 29 Mar 2022 14:55:31 -0400 Subject: [PATCH 23/60] add pre-commit CI hook --- .pre-commit-config.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f824c06e..f4b58bdf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -44,5 +44,15 @@ repos: rev: 0.5.0 hooks: - id: nbstripout - language_version: python3 files: ".ipynb" + +ci: + autofix_commit_msg: | + [pre-commit.ci] auto fixes from pre-commit.com hooks + for more information, see https://pre-commit.ci + autofix_prs: true + autoupdate_branch: '' + autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate' + autoupdate_schedule: weekly + skip: [] + submodules: false From 2677889ff56566111de0b9f28e590aadeee7d874 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 29 Mar 2022 15:05:37 -0400 Subject: [PATCH 24/60] try using mamba to reduce RAM usage --- .github/workflows/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fd9ce15b..21ef1e58 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -64,9 +64,12 @@ jobs: update-conda: true python-version: ${{ matrix.python-version }} conda-channels: conda-forge, defaults + - name: Install mamba + run: | + conda install -c conda-forge mamba - name: Conda env configuration run: | - conda env create -f environment.yml + mamba env create -f environment.yml source activate clisops pip install -e ".[dev]" - name: Test with conda From 7ae22e36a447b1fe7c36d4f440cb36b3962ea800 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 29 Mar 2022 15:12:45 -0400 Subject: [PATCH 25/60] pin pandas below v1.4 --- environment.yml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 2a8114b2..8c8cc67c 100644 --- a/environment.yml +++ b/environment.yml @@ -6,7 +6,7 @@ dependencies: - pip - numpy>=1.16 - xarray>=0.15 - - pandas>=1.0.3 + - pandas>=1.0.3,<1.4 - cftime>=1.4.1 - netCDF4>=1.4 - poppler>=0.67 diff --git a/requirements.txt b/requirements.txt index bec07c34..0b268f39 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ numpy>=1.16 xarray>=0.15 -pandas>=1.0.3 +pandas>=1.0.3,<1.4 cftime>=1.4.1 netCDF4>=1.4 shapely>=1.6 From 6c89b80f2ec7008a5ed7ff1708f5bdfec9a6f105 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 29 Mar 2022 15:27:28 -0400 Subject: [PATCH 26/60] Update HISTORY.rst --- HISTORY.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index b8ef33d9..31f6b52d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,6 +1,14 @@ Version History =============== +v0.9.0 (unreleased) +------------------- + +Other Changes +^^^^^^^^^^^^^ +* Pandas now pinned below version 1.4.0. +* Pre-commit configuration updated with code style conventions (black, pyupgrade) set to Python3.7+. + v0.8.0 (2022-01-13) ------------------- From 08aee79aa970d99c5d64ac793df596e250d69a49 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 29 Mar 2022 15:05:37 -0400 Subject: [PATCH 27/60] try using mamba to reduce RAM usage --- .github/workflows/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fd9ce15b..21ef1e58 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -64,9 +64,12 @@ jobs: update-conda: true python-version: ${{ matrix.python-version }} conda-channels: conda-forge, defaults + - name: Install mamba + run: | + conda install -c conda-forge mamba - name: Conda env configuration run: | - conda env create -f environment.yml + mamba env create -f environment.yml source activate clisops pip install -e ".[dev]" - name: Test with conda From 396cb687497285ad8cd36333e454d4f058640566 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 29 Mar 2022 15:42:30 -0400 Subject: [PATCH 28/60] add workflow step to cancel previous runs --- .github/workflows/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 21ef1e58..45580482 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,6 +13,10 @@ jobs: matrix: tox-env: [black] steps: + - name: Cancel previous runs + uses: styfle/cancel-workflow-action@0.9.1 + with: + access_token: ${{ github.token }} - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: From 1c6df3de49e9eec4534232d7a4253255623b95dd Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 1 Apr 2022 12:51:34 -0400 Subject: [PATCH 29/60] Safer testing of logs with context management, logging disabled by default, add monkeypatch to pass Warnings to loguru handlers --- clisops/__init__.py | 14 ++++- clisops/ops/average.py | 3 +- tests/_common.py | 28 +++++++++ tests/test_logging.py | 70 +++++++++++----------- tests/test_output_utils.py | 115 ++++++++++++++++++++----------------- 5 files changed, 139 insertions(+), 91 deletions(-) diff --git a/clisops/__init__.py b/clisops/__init__.py index 7e4e47a6..0747197a 100644 --- a/clisops/__init__.py +++ b/clisops/__init__.py @@ -1,13 +1,25 @@ # -*- coding: utf-8 -*- """Top-level package for clisops.""" import os +import warnings from loguru import logger from roocs_utils.config import get_config from .__version__ import __author__, __email__, __version__ -# Remove the logger that is instantiated on import + +def showwarning(message, *args, **kwargs): + """Inject warnings from `warnings.warn` into `loguru`.""" + logger.warning(message) + showwarning_(message, *args, **kwargs) + + +showwarning_ = warnings.showwarning +warnings.showwarning = showwarning + +# Disable logging for clisops and remove the logger that is instantiated on import +logger.disable("clisops") logger.remove() diff --git a/clisops/ops/average.py b/clisops/ops/average.py index 02c4ee7e..26586bc0 100644 --- a/clisops/ops/average.py +++ b/clisops/ops/average.py @@ -9,7 +9,8 @@ from clisops.core import average from clisops.ops.base_operation import Operation from clisops.utils.file_namers import get_file_namer -from clisops.utils.output_utils import get_output, get_time_slices + +# from clisops.utils.output_utils import get_output, get_time_slices __all__ = ["average_over_dims", "average_time"] diff --git a/tests/_common.py b/tests/_common.py index ca33b433..1eec135c 100644 --- a/tests/_common.py +++ b/tests/_common.py @@ -1,8 +1,10 @@ import os import tempfile from pathlib import Path +from typing import Optional import xarray as xr +from _pytest.logging import LogCaptureFixture # noqa from jinja2 import Template ROOCS_CFG = Path(tempfile.gettempdir(), "roocs.ini").as_posix() @@ -16,6 +18,32 @@ ).as_posix() +class ContextLogger: + """Helper function for safe logging management in pytests""" + + def __init__(self, caplog: Optional[LogCaptureFixture] = False): + from loguru import logger + + self.logger = logger + self.using_caplog = False + if caplog: + self.using_caplog = True + + def __enter__(self): + self.logger.enable("clisops") + return self.logger + + def __exit__(self, exc_type, exc_val, exc_tb): + """If test is supplying caplog, pytest will manage teardown.""" + + self.logger.disable("clisops") + if not self.using_caplog: + try: + self.logger.remove() + except ValueError: + pass + + def assert_vars_equal(var_id, *ds_list, extras=None): """Extract variable/DataArray `var_id` from each Dataset in the `ds_list`. Check they are all the same by comparing the arrays and common attributes. diff --git a/tests/test_logging.py b/tests/test_logging.py index faa4851e..8b57a2d7 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -1,65 +1,63 @@ -import logging import sys import pytest -from loguru import logger from clisops.utils.common import _logging_examples, enable_logging # noqa +from tests._common import ContextLogger class TestLoggingFuncs: @pytest.mark.xfail( - reason="pytest-loguru does not implement logging levels for caplog yet" + reason="pytest-loguru does not implement logging levels for caplog yet." ) def test_logging_configuration(self, caplog): - caplog.set_level(logging.WARNING, logger="clisops") + with ContextLogger(caplog): + caplog.set_level("WARNING", logger="clisops") - _logging_examples() # noqa + _logging_examples() # noqa - assert ("clisops.utils.common", 10, "1") not in caplog.record_tuples - assert ("clisops.utils.common", 40, "4") in caplog.record_tuples + assert ("clisops.utils.common", 10, "1") not in caplog.record_tuples + assert ("clisops.utils.common", 40, "4") in caplog.record_tuples def test_disabled_enabled_logging(self, capsys): - logger.disable("clisops") + with ContextLogger() as _logger: - # CLISOPS disabled - id1 = logger.add(sys.stderr, level="WARNING") - id2 = logger.add(sys.stdout, level="INFO") + _logger.disable("clisops") - _logging_examples() # noqa + # CLISOPS disabled + _logger.add(sys.stderr, level="WARNING") + _logger.add(sys.stdout, level="INFO") - captured = capsys.readouterr() - assert "WARNING" not in captured.err - assert "INFO" not in captured.out + _logging_examples() # noqa - # re-enable CLISOPS logging - logger.enable("clisops") + captured = capsys.readouterr() + assert "WARNING" not in captured.err + assert "INFO" not in captured.out - _logging_examples() # noqa + # re-enable CLISOPS logging + _logger.enable("clisops") - captured = capsys.readouterr() - assert "INFO" not in captured.err - assert "WARNING" in captured.err - assert "INFO" in captured.out + _logging_examples() # noqa - logger.remove(id1) - logger.remove(id2) + captured = capsys.readouterr() + assert "INFO" not in captured.err + assert "WARNING" in captured.err + assert "INFO" in captured.out def test_logging_enabler(self, capsys): - _logging_examples() # noqa + with ContextLogger(): - captured = capsys.readouterr() - assert "WARNING" not in captured.err - assert "INFO" not in captured.out + _logging_examples() # noqa - ids = enable_logging() + captured = capsys.readouterr() + assert "WARNING" not in captured.err + assert "INFO" not in captured.out - _logging_examples() # noqa + enable_logging() - captured = capsys.readouterr() - assert "INFO" not in captured.err - assert "WARNING" in captured.err - assert "INFO" in captured.out + _logging_examples() # noqa - for i in ids: - logger.remove(i) # sets logging back to default + captured = capsys.readouterr() + assert "INFO" not in captured.err + assert "WARNING" in captured.err + assert "INFO" in captured.out diff --git a/tests/test_output_utils.py b/tests/test_output_utils.py index c750d825..d77c1b98 100644 --- a/tests/test_output_utils.py +++ b/tests/test_output_utils.py @@ -1,10 +1,9 @@ import os +import sys import tempfile -from glob import glob from pathlib import Path import xarray as xr -from loguru import logger from clisops import CONFIG from clisops.utils.common import expand_wildcards @@ -15,7 +14,7 @@ get_output, get_time_slices, ) -from tests._common import CMIP5_TAS, CMIP6_TOS +from tests._common import CMIP5_TAS, CMIP6_TOS, ContextLogger def _open(coll): @@ -89,30 +88,36 @@ def test_get_time_slices_multiple_slices(load_esgf_test_data): def test_tmp_dir_created_with_staging_dir(tmpdir): - staging = Path(tmpdir).joinpath("tests") - staging.mkdir(exist_ok=True) + with ContextLogger() as _logger: + _logger.add(sys.stdout, level="INFO") - # copy part of function that creates tmp dir to check that it is created - CONFIG["clisops:write"]["output_staging_dir"] = staging - staging_dir = CONFIG["clisops:write"].get("output_staging_dir", "") + staging = Path(tmpdir).joinpath("tests") + staging.mkdir(exist_ok=True) - output_path = "./output_001.nc" + # copy part of function that creates tmp dir to check that it is created + CONFIG["clisops:write"]["output_staging_dir"] = staging + staging_dir = CONFIG["clisops:write"].get("output_staging_dir", "") - if os.path.isdir(staging_dir): - tmp_dir = tempfile.TemporaryDirectory(dir=staging_dir) - fname = os.path.basename(output_path) - target_path = os.path.join(tmp_dir.name, fname) - logger.info(f"Writing to temporary path: {target_path}") - else: - target_path = output_path + output_path = "./output_001.nc" - assert target_path != "output_001.nc" - temp_test_folders = [f for f in staging.glob("tmp*")] - assert len(temp_test_folders) == 1 - assert "tests/tmp" in temp_test_folders[0].as_posix() + if os.path.isdir(staging_dir): + tmp_dir = tempfile.TemporaryDirectory(dir=staging_dir) + fname = os.path.basename(output_path) + target_path = os.path.join(tmp_dir.name, fname) + _logger.info(f"Writing to temporary path: {target_path}") + else: + target_path = output_path + + assert target_path != "output_001.nc" + temp_test_folders = [f for f in staging.glob("tmp*")] + assert len(temp_test_folders) == 1 + assert "tests/tmp" in temp_test_folders[0].as_posix() def test_tmp_dir_not_created_with_no_staging_dir(): + # with ContextLogger() as _logger: + # _logger.add(sys.stdout, level="INFO") + # copy part of function that creates tmp dir to check that it is not created when no staging dir CONFIG["clisops:write"]["output_staging_dir"] = "" staging_dir = CONFIG["clisops:write"].get("output_staging_dir", "") @@ -123,7 +128,6 @@ def test_tmp_dir_not_created_with_no_staging_dir(): tmp_dir = tempfile.TemporaryDirectory(dir=staging_dir) fname = os.path.basename(output_path) target_path = os.path.join(tmp_dir.name, fname) - logger.info(f"Writing to temporary path: {target_path}") else: target_path = output_path @@ -131,54 +135,59 @@ def test_tmp_dir_not_created_with_no_staging_dir(): def test_no_staging_dir(caplog): + with ContextLogger(caplog) as _logger: + _logger.add(sys.stdout, level="INFO") + caplog.set_level("INFO", logger="clisops") - CONFIG["clisops:write"]["output_staging_dir"] = "" - ds = _open(CMIP5_TAS) - output_path = get_output( - ds, output_type="nc", output_dir=".", namer=get_file_namer("simple")() - ) + CONFIG["clisops:write"]["output_staging_dir"] = "" + ds = _open(CMIP5_TAS) + output_path = get_output( + ds, output_type="nc", output_dir=".", namer=get_file_namer("simple")() + ) - assert "Writing to temporary path: " not in caplog.text - assert output_path == "output_001.nc" + assert "Writing to temporary path: " not in caplog.text + assert output_path == "output_001.nc" - os.remove("output_001.nc") + os.remove("output_001.nc") def test_invalid_staging_dir(caplog): - # check staging dir not used with invalid directory - CONFIG["clisops:write"]["output_staging_dir"] = "test/not/real/dir/" + with ContextLogger(caplog) as _logger: + _logger.add(sys.stdout, level="INFO") + caplog.set_level("INFO", logger="clisops") - ds = _open(CMIP5_TAS) - output_path = get_output( - ds, output_type="nc", output_dir=".", namer=get_file_namer("simple")() - ) - assert "Writing to temporary path: " not in caplog.text + # check staging dir not used with invalid directory + CONFIG["clisops:write"]["output_staging_dir"] = "test/not/real/dir/" - assert output_path == "output_001.nc" + ds = _open(CMIP5_TAS) + output_path = get_output( + ds, output_type="nc", output_dir=".", namer=get_file_namer("simple")() + ) + assert "Writing to temporary path: " not in caplog.text + assert output_path == "output_001.nc" - os.remove("output_001.nc") + os.remove("output_001.nc") def test_staging_dir_used(caplog, tmpdir): - logger.enable("clisops") - caplog.set_level("INFO", logger="clisops") - - # check staging dir used when valid directory - staging = Path(tmpdir).joinpath("tests") - staging.mkdir(exist_ok=True) - CONFIG["clisops:write"]["output_staging_dir"] = str(staging) - ds = _open(CMIP5_TAS) + with ContextLogger(caplog) as _logger: + _logger.add(sys.stdout, level="INFO") + caplog.set_level("INFO", logger="clisops") - output_path = get_output( - ds, output_type="nc", output_dir=".", namer=get_file_namer("simple")() - ) + # check staging dir used when valid directory + staging = Path(tmpdir).joinpath("tests") + staging.mkdir(exist_ok=True) + CONFIG["clisops:write"]["output_staging_dir"] = str(staging) + ds = _open(CMIP5_TAS) - assert f"Writing to temporary path: {staging}" in caplog.text - assert output_path == "output_001.nc" + output_path = get_output( + ds, output_type="nc", output_dir=".", namer=get_file_namer("simple")() + ) - logger.disable("clisops") + assert f"Writing to temporary path: {staging}" in caplog.text + assert output_path == "output_001.nc" - Path("output_001.nc").unlink() + Path("output_001.nc").unlink() def test_final_output_path_staging_dir(): From 89a4faa18b3ad4e2f9357833c681913a0101cc14 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 1 Apr 2022 12:58:54 -0400 Subject: [PATCH 30/60] Update contributing and history to detail logging methods --- CONTRIBUTING.rst | 22 ++++++++++++++++++++++ HISTORY.rst | 22 ++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index cbe663db..8ca155ca 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -106,6 +106,28 @@ Ready to contribute? Here's how to set up ``clisops`` for local development. #. Submit a pull request through the GitHub website. + +Logging +------- + +``clisops`` uses the `loguru `_ library as its primary logging engine. In order to integrate this kind of logging in processes, we can use their logger: + +.. code-block:: python + from loguru import logger + logger.warning("This a warning message!") + +The mechanism for enabling log reporting in scripts/notebooks using ``loguru`` is as follows: + +.. code-block:: python + import sys + from loguru import logger + + logger.enable("clisops") + LEVEL = "INFO || DEBUG || WARNING || etc." + logger.add(sys.stdout, level=LEVEL) # for logging to stdout + # or + logger.add("my_log_file.log", level=LEVEL, enqueue=True) # for logging to a file + Pull Request Guidelines ----------------------- diff --git a/HISTORY.rst b/HISTORY.rst index b8ef33d9..4cadfa4f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,6 +1,28 @@ Version History =============== +v0.9.0 (unreleased) +------------------- + +New Features +^^^^^^^^^^^^ +* ``clisops`` now uses the `loguru `_ library as its primary logging engine. + The mechanism for enabling log reporting in scripts/notebooks using ``loguru`` is as follows: + +.. code-block:: python + import sys + from loguru import logger + + logger.activate("clisops") + LEVEL = "INFO || DEBUG || WARNING || etc." + logger.add(sys.stdout, level=LEVEL) # for logging to stdout + # or + logger.add("my_log_file.log", level=LEVEL, enqueue=True) # for logging to a file + +Other changes +^^^^^^^^^^^^^ +* ``loguru`` is now an install dependency, with ``pytest-loguru`` as a development-only dependency. + v0.8.0 (2022-01-13) ------------------- From 44fcee5e77efe6c73ac999c423dcc591c91e73e1 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 4 Apr 2022 10:33:45 -0400 Subject: [PATCH 31/60] add documentation about enable_logging function --- CONTRIBUTING.rst | 7 +++++++ clisops/__init__.py | 1 + clisops/utils/common.py | 2 ++ 3 files changed, 10 insertions(+) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 8ca155ca..45dab7da 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -128,6 +128,13 @@ The mechanism for enabling log reporting in scripts/notebooks using ``loguru`` i # or logger.add("my_log_file.log", level=LEVEL, enqueue=True) # for logging to a file +For convenience, a preset logger configuration can be enabled via `clisops.enable_logging()`. + +.. code-block:: python + from clisops import enable_logging + + enable_logging() + Pull Request Guidelines ----------------------- diff --git a/clisops/__init__.py b/clisops/__init__.py index 0747197a..a8fd3b7c 100644 --- a/clisops/__init__.py +++ b/clisops/__init__.py @@ -7,6 +7,7 @@ from roocs_utils.config import get_config from .__version__ import __author__, __email__, __version__ +from .utils.common import enable_logging def showwarning(message, *args, **kwargs): diff --git a/clisops/utils/common.py b/clisops/utils/common.py index cbec665b..a8c1c8a3 100644 --- a/clisops/utils/common.py +++ b/clisops/utils/common.py @@ -24,6 +24,8 @@ def _logging_examples() -> None: def enable_logging() -> List[int]: + logger.enable("clisops") + config = dict( handlers=[ dict( From 002557756e9cc99736b3ed33774dbc1c8268f7ed Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Apr 2022 17:07:41 +0000 Subject: [PATCH 32/60] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/pre-commit-hooks: v4.1.0 → v4.2.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.1.0...v4.2.0) - https://github.com/ambv/black → https://github.com/psf/black - https://github.com/timothycrosley/isort → https://github.com/PyCQA/isort - [github.com/asottile/pyupgrade: v2.31.1 → v2.32.0](https://github.com/asottile/pyupgrade/compare/v2.31.1...v2.32.0) --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f4b58bdf..aece2677 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v4.2.0 hooks: - id: trailing-whitespace exclude: setup.cfg @@ -11,7 +11,7 @@ repos: exclude: setup.cfg - id: check-yaml - id: debug-statements -- repo: https://github.com/ambv/black +- repo: https://github.com/psf/black rev: 22.3.0 hooks: - id: black @@ -21,7 +21,7 @@ repos: hooks: - id: flake8 args: ['--config=setup.cfg'] -- repo: https://github.com/timothycrosley/isort +- repo: https://github.com/PyCQA/isort rev: 5.10.1 hooks: - id: isort @@ -32,7 +32,7 @@ repos: # - id: pydocstyle # args: ["--convention=numpy"] - repo: https://github.com/asottile/pyupgrade - rev: v2.31.1 + rev: v2.32.0 hooks: - id: pyupgrade args: ['--py37-plus'] From 27bfa7401713baa535c6d8c5eef19aafbf495460 Mon Sep 17 00:00:00 2001 From: Martin Schupfner Date: Wed, 13 Apr 2022 11:35:34 +0200 Subject: [PATCH 33/60] Replace function utils.dataset_utils.check_lon_alignment to fix #217 (#218) * Replace function utils.dataset_utils.check_lon_alignment - replace utils.dataset_utils.check_lon_alignment by cf_convert_between_lon_frames: + converts either dataset or specified lon interval to another longitude frame if necessary + fixes issue #217 of bounds not getting adjusted by check_lon_alignment + added functionality to adjust shifted longitude frames - added utils.dataset_utils.detect_bounds and detect_coordinate - added tests for the new functions, added new test datasets - set minimum cf_xarray version to 0.7 (incl. functionality to read eg. bounds from encoding dict) - pin pandas to <1.4 because of failing TestSubsetShape tests * Updated pre-commit config and fixed linting * Added tests, added pooch as dependency. * Revert changes to average.py necessary for latest unreleased roocs_utils version. * Adjust dependencies of read the docs env. * Update setup.py Adding ipython-genutils in an attempt to fix read the docs build. * Updates to cf_convert_between_lon_frames - no longer converting projection x-coordinate in curvilinear datasets - removed sorting for converted curvilinear datasets - moved grid detection part to own function - added tests * Fixing a problem in subset/check_lons, adding test. * Adding history entry and author information Co-authored-by: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> --- AUTHORS.rst | 1 + HISTORY.rst | 1 + clisops/core/average.py | 7 +- clisops/core/subset.py | 4 +- clisops/ops/subset.py | 10 +- clisops/utils/dataset_utils.py | 372 +++++++++++++++++++++++++ environment.yml | 3 +- requirements.txt | 4 +- setup.py | 1 + tests/_common.py | 43 ++- tests/core/test_subset.py | 15 + tests/ops/test_subset.py | 30 +- tests/test_utils/test_dataset_utils.py | 269 +++++++++++++++++- 13 files changed, 727 insertions(+), 33 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 4750a7c6..bdcd7a7c 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -20,3 +20,4 @@ Contributors * Pascal Bourgault bourgault.pascal@ouranos.ca `@aulemahal `_ * David Huard huard.david@ouranos.ca `@huard `_ +* Martin Schupfner schupfner@dkrz.de `@sol1105 `_ diff --git a/HISTORY.rst b/HISTORY.rst index f43fc51a..2e8974e8 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -24,6 +24,7 @@ Other Changes * Pandas now pinned below version 1.4.0. * Pre-commit configuration updated with code style conventions (black, pyupgrade) set to Python3.7+. * ``loguru`` is now an install dependency, with ``pytest-loguru`` as a development-only dependency. +* Added function to convert the longitude axis between different longitude frames (eg. [-180, 180] and [0, 360]). v0.8.0 (2022-01-13) ------------------- diff --git a/clisops/core/average.py b/clisops/core/average.py index 5f3a0095..ee64e6ee 100644 --- a/clisops/core/average.py +++ b/clisops/core/average.py @@ -269,10 +269,15 @@ def average_time( ) # check time coordinate exists and get name + # For roocs_utils 0.5.0 t = get_coord_by_type(ds, "time", ignore_aux_coords=False) - if t is None: raise Exception("Time dimension could not be found") + # For latest roocs_utils (master) + # try: + # t = get_coord_by_type(ds, "time", ignore_aux_coords=False) + # except KeyError: + # raise Exception("Time dimension could not be found") # resample and average over time ds_t_avg = ds.resample(indexer={t.name: freqs[freq]}).mean( diff --git a/clisops/core/subset.py b/clisops/core/subset.py index 229b8abd..225b61f6 100644 --- a/clisops/core/subset.py +++ b/clisops/core/subset.py @@ -259,11 +259,11 @@ def func_checker(*args, **kwargs): f"Input longitude bounds ({kwargs[lon]}) cross the 0 degree meridian but" " dataset longitudes are all positive." ) - if np.all((ds_lon <= 0) | (np.isnan(ds_lon))) and np.any(kwargs[lon] > 0): + if np.all((ds_lon <= 0) | (np.isnan(ds_lon))) and np.any(kwargs[lon] > 180): if isinstance(kwargs[lon], float): kwargs[lon] -= 360 else: - kwargs[lon][kwargs[lon] < 0] -= 360 + kwargs[lon][kwargs[lon] <= 180] -= 360 return func(*args, **kwargs) diff --git a/clisops/ops/subset.py b/clisops/ops/subset.py index 22eb2c9e..3306f7c9 100644 --- a/clisops/ops/subset.py +++ b/clisops/ops/subset.py @@ -19,7 +19,10 @@ ) from clisops.core.subset import assign_bounds, get_lat, get_lon # noqa from clisops.ops.base_operation import Operation -from clisops.utils.dataset_utils import check_lon_alignment +from clisops.utils.common import expand_wildcards +from clisops.utils.dataset_utils import cf_convert_between_lon_frames +from clisops.utils.file_namers import get_file_namer +from clisops.utils.output_utils import get_output, get_time_slices __all__ = ["Subset", "subset"] @@ -81,7 +84,10 @@ def _calculate(self): # subset with space and optionally time and level logger.debug(f"subset_bbox with parameters: {self.params}") # bounds are always ascending, so if lon is descending rolling will not work. - ds = check_lon_alignment(self.ds, self.params.get("lon_bnds")) + ds, lb, ub = cf_convert_between_lon_frames( + self.ds, self.params.get("lon_bnds") + ) + self.params["lon_bnds"] = (lb, ub) try: kwargs = {} valid_args = [ diff --git a/clisops/utils/dataset_utils.py b/clisops/utils/dataset_utils.py index ea805606..8ef6c3c7 100644 --- a/clisops/utils/dataset_utils.py +++ b/clisops/utils/dataset_utils.py @@ -1,5 +1,11 @@ +import math +import warnings + +import cf_xarray as cfxr import cftime import numpy as np +import xarray as xr +from roocs_utils.exceptions import InvalidParameterValue from roocs_utils.utils.time_utils import str_to_AnyCalendarDateTime from roocs_utils.xarray_utils.xarray_utils import get_coord_by_type @@ -27,6 +33,230 @@ def calculate_offset(lon, first_element_value): return offset +def _crosses_0_meridian(lon_c): + """ + Determine whether grid extents over the 0-meridian. + + Assumes approximate constant width of grid cells. + + Parameters + ---------- + lon_c : TYPE + Longitude coordinate variable in the longitude frame [-180, 180]. + + Returns + ------- + bool + True for a dataset crossing the 0-meridian, False else. + """ + if not isinstance(lon_c, xr.DataArray): + raise InvalidParameterValue("Input needs to be of type xarray.DataArray.") + + # Not crossing the 0-meridian if all values are positive or negative + lon_n = lon_c.where(lon_c <= 0, 0) + lon_p = lon_c.where(lon_c >= 0, 0) + if lon_n.all() or lon_p.all(): + return False + + # Determine min/max lon values + xc_min = float(lon_c.min()) + xc_max = float(lon_c.max()) + + # Determine resolution in zonal direction + if lon_c.ndim == 1: + xc_inc = (xc_max - xc_min) / (lon_c.sizes[lon_c.dims[0]] - 1) + else: + xc_inc = (xc_max - xc_min) / (lon_c.sizes[lon_c.dims[1]] - 1) + + # Generate a histogram with bins for sections along a latitudinal circle, + # width of the bins/sections dependent on the resolution in x-direction + atol = 2.0 * xc_inc + extent_hist = np.histogram( + lon_c, + bins=np.arange(xc_min - xc_inc, xc_max + atol, atol), + ) + + # If the counts for all bins are greater than zero, the grid is considered crossing the 0-meridian + if np.all(extent_hist[0]): + return True + else: + return False + + +def _convert_interval_between_lon_frames(low, high): + """Convert a longitude interval to another longitude frame, returns Tuple of two floats.""" + diff = high - low + if low < 0 and high > 0: + raise ValueError( + "Cannot convert longitude interval if it includes the 0°- or 180°-meridian." + ) + elif low < 0: + return tuple(sorted((low + 360.0, low + 360.0 + diff))) + elif low < 180 and high > 180: + raise ValueError( + "Cannot convert longitude interval if it includes the 0°- or 180°-meridian." + ) + elif high > 180: + return tuple(sorted((high - 360.0 - diff, high - 360.0))) + else: + return float(low), float(high) + + +def cf_convert_between_lon_frames(ds_in, lon_interval): + """ + Convert ds or lon_interval (whichever deems appropriate) to the other longitude frame, if the longitude frames do not match. + + If ds and lon_interval are defined on different longitude frames ([-180, 180] and [0, 360]), + this function will convert one of the input parameters to the other longitude frame, preferably + the lon_interval. + Adjusts shifted longitude frames [0-x, 360-x] in the dataset to one of the two standard longitude + frames, dependent on the specified lon_interval. + In case of curvilinear grids featuring an additional 1D x-coordinate of the projection, + this projection x-coordinate will not get converted. + + Parameters + ---------- + ds_in : xarray.Dataset or xarray.DataArray + xarray data object with defined longitude dimension. + lon_interval : tuple or list + length-2-tuple or -list of floats or integers denoting the bounds of the longitude interval. + + Returns + ------- + Tuple(ds, lon_low, lon_high) + The xarray.Dataset and the bounds of the longitude interval, potentially adjusted in terms + of their defined longitude frame. + """ + # Collect input specs + if isinstance(ds_in, (xr.Dataset, xr.DataArray)): + lon = detect_coordinate(ds_in, "longitude") + lat = detect_coordinate(ds_in, "latitude") + lon_bnds = detect_bounds(ds_in, lon) + # lat_bnds = detect_bounds(ds_in, lat) + # Do not consider bounds in gridtype detection (yet fails due to open_mfdataset bug that adds + # time dimension to bounds - todo) + gridtype = detect_gridtype( + ds_in, lon=lon, lat=lat + ) # lat_bnds=lat_bnds, lon_bnds = lon_bnds) + ds = ds_in.copy() + else: + raise InvalidParameterValue( + "This function requires an xarray.DataArray or xarray.Dataset as input." + ) + low, high = lon_interval + lon_min, lon_max = ds.coords[lon].values.min(), ds.coords[lon].values.max() + atol = 0.5 + + # Check longitude + # For longitude frames other than [-180, 180] and [0, 360] in the dataset the following assumptions + # are being made: + # - fixpoint is the 0-meridian + # - the lon_interval is either defined in the longitude frame [-180, 180] or [0, 360] + if lon_max - lon_min > 360 + atol or lon_min < -360 - atol or lon_max > 360 + atol: + raise ValueError( + "The longitude coordinate values have to lie within the interval " + "[-360, 360] degrees and not exceed an extent of 360 degrees." + ) + + # Conversion between longitude frames if required + if (lon_min <= low or np.isclose(low, lon_min, atol=atol)) and ( + lon_max >= high or np.isclose(high, lon_max, atol=atol) + ): + return ds, low, high + + # Conversion: longitude is a singleton dimension + elif (ds[lon].ndim == 1 and ds.sizes[ds[lon].dims[0]] == 1) or ( + ds[lon].ndim > 1 and ds.sizes[ds[lon].dims[1]] == 1 + ): + if low < 0 and lon_min > 0: + ds[lon] = ds[lon].where(ds[lon] <= 180, ds[lon] - 360.0) + if lon_bnds: + ds[lon_bnds] = ds[lon_bnds].where( + ds[lon_bnds] > 180, ds[lon_bnds] - 360.0 + ) + elif low > 0 and lon_min < 0: + ds[lon] = ds[lon].where(ds[lon] >= 0, ds[lon] + 360.0) + if lon_bnds: + ds[lon_bnds] = ds[lon_bnds].where( + ds[lon_bnds] >= 0, ds[lon_bnds] + 360.0 + ) + return ds, low, high + + # Conversion: 1D or 2D longitude coordinate variable + else: + # regional [0 ... 180] + if lon_min >= 0 and lon_max <= 180: + return ds, low, high + + # shifted frame beyond -180, eg. [-300, 60] + elif lon_min < -180 - atol: + if low < 0: + ds[lon] = ds[lon].where(ds[lon] > -180, ds[lon] + 360.0) + if lon_bnds: + ds[lon_bnds] = ds[lon_bnds].where( + ds[lon_bnds] > -180, ds[lon_bnds] + 360.0 + ) + elif low >= 0: + ds[lon] = ds[lon].where(ds[lon] >= 0, ds[lon] + 360.0) + if lon_bnds: + ds[lon_bnds] = ds[lon_bnds].where( + ds[lon_bnds] >= 0, ds[lon_bnds] + 360.0 + ) + + # shifted frame beyond 0, eg. [-60, 300] + elif lon_min < -atol and lon_max > 180 + atol: + if low < 0: + ds[lon] = ds[lon].where(ds[lon] <= 180, ds[lon] - 360.0) + if lon_bnds: + ds[lon_bnds] = ds[lon_bnds].where( + ds[lon_bnds] <= 180, ds[lon_bnds] - 360.0 + ) + elif low >= 0: + ds[lon] = ds[lon].where(ds[lon] >= 0, ds[lon] + 360.0) + if lon_bnds: + ds[lon_bnds] = ds[lon_bnds].where( + ds[lon_bnds] >= 0, ds[lon_bnds] + 360.0 + ) + + # [-180 ... 180] + elif lon_min < 0: + # interval includes 180°-meridian: convert dataset to [0, 360] + if low < 180 and high > 180: + ds[lon] = ds[lon].where(ds[lon] >= 0, ds[lon] + 360.0) + if lon_bnds: + ds[lon_bnds] = ds[lon_bnds].where( + ds[lon_bnds] >= 0, ds[lon_bnds] + 360.0 + ) + # interval does not include 180°-meridian: convert interval to [-180,180] + else: + if low >= 0: + low, high = _convert_interval_between_lon_frames(low, high) + return ds, low, high + + # [0 ... 360] + else: + # interval positive, return unchanged + if low >= 0: + return ds, low, high + # interval includes 0°-meridian: convert dataset to [-180, 180] + elif high > 0: + ds[lon] = ds[lon].where(ds[lon] <= 180, ds[lon] - 360.0) + if lon_bnds: + ds[lon_bnds] = ds[lon_bnds].where( + ds[lon_bnds] <= 180, ds[lon_bnds] - 360.0 + ) + # interval negative + else: + low, high = _convert_interval_between_lon_frames(low, high) + return ds, low, high + + # 1D coordinate variable: Sort, since order might no longer be ascending / descending + if gridtype == "regular_lat_lon": + ds = ds.sortby(lon) + + return ds, low, high + + def check_lon_alignment(ds, lon_bnds): """ Check whether the longitude subset requested is within the bounds of the dataset. If not try to roll the dataset so @@ -129,3 +359,145 @@ def adjust_date_to_calendar(da, date, direction="backwards"): raise ValueError( f"Could not find an existing date near {date} in the calendar: {cal}" ) + + +def detect_coordinate(ds, coord_type): + """ + Use cf_xarray to obtain the variable name of the requested coordinate. + + Parameters + ---------- + ds: xarray.Dataset, xarray.DataArray + Dataset the coordinate variable name shall be obtained from. + coord_type : str + Coordinate type understood by cf-xarray, eg. 'lat', 'lon', ... + + Raises + ------ + AttributeError + Raised if the requested coordinate cannot be identified. + + Returns + ------- + str + Coordinate variable name. + """ + error_msg = f"A {coord_type} coordinate cannot be identified in the dataset." + + # Make use of cf-xarray accessor + try: + coord = ds.cf[coord_type] + # coord = get_coord_by_type(ds, coord_type, ignore_aux_coords=False) + except KeyError: + raise KeyError(error_msg) + + # Return the name of the coordinate variable + try: + return coord.name + except AttributeError: + raise AttributeError(error_msg) + + +def detect_bounds(ds, coordinate): + """ + Use cf_xarray to obtain the variable name of the requested coordinates bounds. + + Parameters + ---------- + ds: xarray.Dataset, xarray.DataArray + Dataset the coordinate bounds variable name shall be obtained from. + coordinate : str + Name of the coordinate variable to determine the bounds from. + + Returns + ------- + str + Returns the variable name of the requested coordinate bounds, + returns None if the variable has no bounds or they cannot be identified. + """ + try: + return ds.cf.bounds[coordinate][0] + except (KeyError, AttributeError): + warnings.warn( + "For coordinate variable '%s' no bounds can be identified." % coordinate + ) + return + + +def detect_gridtype(ds, lon, lat, lon_bnds=None, lat_bnds=None): + """ + Detect type of the grid as one of "regular_lat_lon", "curvilinear", "unstructured". + + Assumes the grid description / structure follows the CF conventions. + """ + + # 1D coordinate variables + if ds[lat].ndim == 1 and ds[lon].ndim == 1: + lat_1D = ds[lat].dims[0] + lon_1D = ds[lon].dims[0] + if not lat_bnds or not lon_bnds: + if lat_1D == lon_1D: + return "unstructured" + else: + return "regular_lat_lon" + else: + # unstructured: bounds [ncells, nvertices] + if ( + lat_1D == lon_1D + and all([ds[bnds].ndim == 2 for bnds in [lon_bnds, lat_bnds]]) + and all( + [ + ds.dims[dim] > 2 + for dim in [ + ds[lon_bnds].dims[-1], + ds[lat_bnds].dims[-1], + ] + ] + ) + ): + return "unstructured" + # rectilinear: bounds [nlat/nlon, 2] + elif ( + all([ds[bnds].ndim == 2 for bnds in [lon_bnds, lat_bnds]]) + and ds.dims[ds.cf.get_bounds_dim_name(lon)] == 2 + ): + return "regular_lat_lon" + else: + raise Exception("The grid type is not supported.") + + # 2D coordinate variables + elif ds[lat].ndim == 2 and ds[lon].ndim == 2: + # Test for curvilinear or restructure lat/lon coordinate variables + # todo: Check if regular_lat_lon despite 2D + # - requires additional function checking + # lat[:,i]==lat[:,j] for all i,j + # lon[i,:]==lon[j,:] for all i,j + # - and if that is the case to extract lat/lon and *_bnds + # lat[:]=lat[:,j], lon[:]=lon[j,:] + # lat_bnds[:, 2]=[min(lat_bnds[:,j, :]), max(lat_bnds[:,j, :])] + # lon_bnds similar + if not ds[lat].shape == ds[lon].shape: + raise InvalidParameterValue( + "The horizontal coordinate variables have differing shapes." + ) + else: + if not lat_bnds or not lon_bnds: + return "curvilinear" + else: + # Shape of curvilinear bounds either [nlat, nlon, 4] or [nlat+1, nlon+1] + if list(ds[lat].shape) + [4] == list(ds[lat_bnds].shape) and list( + ds[lon].shape + ) + [4] == list(ds[lon_bnds].shape): + return "curvilinear" + elif [si + 1 for si in ds[lat].shape] == list(ds[lat_bnds].shape) and [ + si + 1 for si in ds[lon].shape + ] == list(ds[lon_bnds].shape): + return "curvilinear" + else: + raise Exception("The grid type is not supported.") + + # >2D coordinate variables, or coordinate variables of different dimensionality + else: + raise InvalidParameterValue( + "The horizontal coordinate variables have more than 2 dimensions." + ) diff --git a/environment.yml b/environment.yml index 455c3ee9..9037a153 100644 --- a/environment.yml +++ b/environment.yml @@ -5,7 +5,7 @@ channels: dependencies: - pip - bottleneck>=1.3.1,<1.4 - - cf_xarray>=0.5.1 + - cf_xarray>=0.7.0 - cftime>=1.4.1 - dask>=2.6.0 - geopandas>=0.7 @@ -16,6 +16,7 @@ dependencies: - poppler>=0.67 - pygeos>=0.9 - pyproj>=2.5 + - pooch - requests>=2.0 - roocs-utils>=0.5.0 - shapely>=1.6 diff --git a/requirements.txt b/requirements.txt index 9b0530e7..e27565bb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,9 +7,11 @@ shapely>=1.6 geopandas>=0.7 dask[complete]>=2.6 pyproj>=2.5 +pooch +cf-xarray>=0.7.0 +#cf-xarray @ git+https://github.com/xarray-contrib/cf-xarray/@main#egg=cf-xarray bottleneck>=1.3.1 requests>=2.0 roocs-utils>=0.5.0 # roocs-utils @ git+https://github.com/roocs/roocs-utils.git@master#egg=roocs-utils -cf-xarray>=0.5.1 loguru>=0.5.3 diff --git a/setup.py b/setup.py index 198a2927..c83bd58d 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,7 @@ "nbsphinx", "pandoc", "ipython", + "ipython_genutils", "ipykernel", "jupyter_client", "matplotlib", diff --git a/tests/_common.py b/tests/_common.py index 1eec135c..33e23d9a 100644 --- a/tests/_common.py +++ b/tests/_common.py @@ -173,6 +173,12 @@ def cmip6_archive_base(): "master/test_data/badc/cmip6/data/CMIP6/ScenarioMIP/MIROC/MIROC6/ssp119/r1i1p1f1/Amon/ta/gn/files/d20190807/ta_Amon_MIROC6_ssp119_r1i1p1f1_gn_201501-202412.nc", ).as_posix() +# Dataset with julian calender +CMIP6_JULIAN = Path( + MINI_ESGF_CACHE_DIR, + "master/test_data/badc/cmip6/data/CMIP6/CMIP/CCCR-IITM/IITM-ESM/1pctCO2/r1i1p1f1/Omon/tos/gn/v20191204/tos_Omon_IITM-ESM_1pctCO2_r1i1p1f1_gn_193001-193412.nc", +).as_posix() + C3S_CORDEX_AFR_TAS = Path( MINI_ESGF_CACHE_DIR, "master/test_data/pool/data/CORDEX/data/cordex/output/AFR-22/GERICS/MPI-M-MPI-ESM-LR/historical/r1i1p1/GERICS-REMO2015/v1/day/tas/v20201015/*.nc", @@ -203,14 +209,49 @@ def cmip6_archive_base(): "c3s-cmip5/output1/BCC/bcc-csm1-1-m/historical/mon/ocean/Omon/r1i1p1/tos/v20120709/*.nc", ).as_posix() - CMIP6_TOS = Path( MINI_ESGF_CACHE_DIR, "master/test_data/badc/cmip6/data/CMIP6/CMIP/MPI-M/MPI-ESM1-2-LR/historical/r1i1p1f1/Omon/tos/gn/v20190710/tos_Omon_MPI-ESM1-2-LR_historical_r1i1p1f1_gn_185001-186912.nc", ).as_posix() +CMIP6_TAS_ONE_TIME_STEP = Path( + MINI_ESGF_CACHE_DIR, + "master/test_data/badc/cmip6/data/CMIP6/CMIP/CAS/FGOALS-g3/historical/r1i1p1f1/Amon/tas/gn/v20190818/tas_Amon_FGOALS-g3_historical_r1i1p1f1_gn_185001.nc", +).as_posix() CMIP6_TOS_ONE_TIME_STEP = Path( MINI_ESGF_CACHE_DIR, "master/test_data/badc/cmip6/data/CMIP6/CMIP/MPI-M/MPI-ESM1-2-HR/historical/r1i1p1f1/Omon/tos/gn/v20190710/tos_Omon_MPI-ESM1-2-HR_historical_r1i1p1f1_gn_185001.nc", ).as_posix() + +# CMIP6 dataset with weird range in its longitude coordinate (-300, 60) +CMIP6_GFDL_EXTENT = Path( + MINI_ESGF_CACHE_DIR, + "master/test_data/badc/cmip6/data/CMIP6/CMIP/NOAA-GFDL/GFDL-CM4/historical/r1i1p1f1/Omon/sos/gn/v20180701/sos_Omon_GFDL-CM4_historical_r1i1p1f1_gn_185001.nc", +).as_posix() + +# CMIP6 2nd dataset with weird range in its longitude coordinate (-280, 80) +CMIP6_IITM_EXTENT = Path( + MINI_ESGF_CACHE_DIR, + "master/test_data/badc/cmip6/data/CMIP6/CMIP/CCCR-IITM/IITM-ESM/1pctCO2/r1i1p1f1/Omon/tos/gn/v20191204/tos_Omon_IITM-ESM_1pctCO2_r1i1p1f1_gn_193001.nc", +).as_posix() + +CMIP6_OCE_HALO_CNRM = Path( + MINI_ESGF_CACHE_DIR, + "master/test_data/badc/cmip6/data/CMIP6/CMIP/CNRM-CERFACS/CNRM-CM6-1-HR/historical/r1i1p1f2/Omon/tos/gn/v20191021/tos_Omon_CNRM-CM6-1-HR_historical_r1i1p1f2_gn_185001.nc", +).as_posix() + +CMIP6_UNSTR_FESOM_LR = Path( + MINI_ESGF_CACHE_DIR, + "master/test_data/badc/cmip6/data/CMIP6/CMIP/AWI/AWI-ESM-1-1-LR/historical/r1i1p1f1/Omon/tos/gn/v20200212/tos_Omon_AWI-ESM-1-1-LR_historical_r1i1p1f1_gn_185001.nc", +).as_posix() + +CMIP6_UNSTR_ICON_A = Path( + MINI_ESGF_CACHE_DIR, + "master/test_data/badc/cmip6/data/CMIP6/CMIP/MPI-M/ICON-ESM-LR/historical/r1i1p1f1/Amon/tas/gn/v20210215/tas_Amon_ICON-ESM-LR_historical_r1i1p1f1_gn_185001.nc", +).as_posix() + +CORDEX_TAS_ONE_TIMESTEP = Path( + MINI_ESGF_CACHE_DIR, + "master/test_data/pool/data/CORDEX/data/cordex/output/EUR-22/GERICS/MPI-M-MPI-ESM-LR/rcp85/r1i1p1/GERICS-REMO2015/v1/mon/tas/v20191029/tas_EUR-22_MPI-M-MPI-ESM-LR_rcp85_r1i1p1_GERICS-REMO2015_v1_mon_202101.nc", +).as_posix() diff --git a/tests/core/test_subset.py b/tests/core/test_subset.py index 44d08542..9947aff5 100644 --- a/tests/core/test_subset.py +++ b/tests/core/test_subset.py @@ -618,6 +618,7 @@ def test_time(self): np.testing.assert_array_equal(out.time.max().dt.day, 15) def test_raise(self): + # 1st case da = xr.open_dataset(self.nc_poslons).tas with pytest.raises(ValueError): subset.subset_bbox( @@ -628,10 +629,24 @@ def test_raise(self): end_date="2055", ) + # 2nd case da = xr.open_dataset(self.nc_2dlonlat).tasmax.drop_vars(names=["lon", "lat"]) with pytest.raises(Exception): subset.subset_bbox(da, lon_bnds=self.lon, lat_bnds=self.lat) + # 3rd case + ds = xr.Dataset( + data_vars={"var": (("lat", "lon"), np.ones((5, 10)))}, + coords={ + "lat": ("lat", np.zeros(5)), + "lon": ("lon", np.arange(-10, 0, 1)), + }, + ) + ds["lat"].attrs["standard_name"] = "latitude" + ds["lon"].attrs["standard_name"] = "longitude" + with pytest.raises(ValueError): + subset.subset_bbox(ds, lon_bnds=(-0.1, 1.0)) + def test_warnings(self): da = xr.open_dataset(self.nc_poslons).tas da = da.assign_coords(lon=(da.lon - 360)) diff --git a/tests/ops/test_subset.py b/tests/ops/test_subset.py index ed746bf0..fb9c181e 100644 --- a/tests/ops/test_subset.py +++ b/tests/ops/test_subset.py @@ -1211,27 +1211,19 @@ def test_reverse_lon_curvilinear(self, load_esgf_test_data): np.testing.assert_array_equal(result[0].tos, result_rev[0].tos) def test_reverse_lon_cross_meridian_curvilinear(self, load_esgf_test_data): - # can't roll because ds has a curvilinear grid - with pytest.raises(Exception) as exc: - subset( - ds=CMIP6_TOS_ONE_TIME_STEP, - area=(-70, -45, 240, 45), - output_type="xarray", - ) - - # can't roll because ds has a curvilinear grid - with pytest.raises(Exception) as exc_rev: - subset( - ds=CMIP6_TOS_ONE_TIME_STEP, - area=(240, -45, -70, 45), - output_type="xarray", - ) + result = subset( + ds=CMIP6_TOS_ONE_TIME_STEP, + area=(-70, -45, 120, 45), + output_type="xarray", + ) - assert ( - str(exc.value) - == "The requested longitude subset (-70.0, 240.0) is not within the longitude bounds of this dataset and the data could not be converted to this longitude frame successfully. Please re-run your request with longitudes within the bounds of the dataset: (0.01, 360.00)" + result_rev = subset( + ds=CMIP6_TOS_ONE_TIME_STEP, + area=(120, -45, -70, 45), + output_type="xarray", ) - assert str(exc.value) == str(exc_rev.value) + + np.testing.assert_array_equal(result[0].tos, result_rev[0].tos) def test_reverse_lat_and_lon_curvilinear(self, load_esgf_test_data): result = subset( diff --git a/tests/test_utils/test_dataset_utils.py b/tests/test_utils/test_dataset_utils.py index 47ed1ed2..a9ed9e2c 100644 --- a/tests/test_utils/test_dataset_utils.py +++ b/tests/test_utils/test_dataset_utils.py @@ -1,16 +1,29 @@ +import cf_xarray as cfxr +import numpy as np import pytest import xarray as xr -from clisops.utils.dataset_utils import adjust_date_to_calendar +import clisops.utils.dataset_utils as clidu -from .._common import CMIP6_SICONC +from .._common import ( + C3S_CORDEX_AFR_TAS, + C3S_CORDEX_ANT_SFC_WIND, + CMIP6_GFDL_EXTENT, + CMIP6_IITM_EXTENT, + CMIP6_OCE_HALO_CNRM, + CMIP6_SICONC, + CMIP6_TAS_ONE_TIME_STEP, + CMIP6_TOS_ONE_TIME_STEP, + CMIP6_UNSTR_ICON_A, + CORDEX_TAS_ONE_TIMESTEP, +) def test_add_day(): da = xr.open_dataset(CMIP6_SICONC, use_cftime=True) date = "2012-02-29T00:00:00" - new_date = adjust_date_to_calendar(da, date, "forwards") + new_date = clidu.adjust_date_to_calendar(da, date, "forwards") assert new_date == "2012-03-01T00:00:00" @@ -19,7 +32,7 @@ def test_sub_day(): da = xr.open_dataset(CMIP6_SICONC, use_cftime=True) date = "2012-02-30T00:00:00" - new_date = adjust_date_to_calendar(da, date, "backwards") + new_date = clidu.adjust_date_to_calendar(da, date, "backwards") assert new_date == "2012-02-28T00:00:00" @@ -29,7 +42,7 @@ def test_invalid_day(): date = "2012-02-29T00:00:00" with pytest.raises(Exception) as exc: - adjust_date_to_calendar(da, date, "odd") + clidu.adjust_date_to_calendar(da, date, "odd") assert ( str(exc.value) == "Invalid value for direction: odd. This should be either 'backwards' to indicate subtracting a day or 'forwards' for adding a day." @@ -41,7 +54,251 @@ def test_date_out_of_expected_range(): date = "2012-00-01T00:00:00" with pytest.raises(Exception) as exc: - adjust_date_to_calendar(da, date, "forwards") + clidu.adjust_date_to_calendar(da, date, "forwards") assert ( str(exc.value) == "Invalid input 0 for month. Expected value between 1 and 12." ) + + +def test_detect_coordinate_and_bounds(): + "Test detect_bounds and detect_coordinate functions." + ds_a = xr.open_mfdataset(C3S_CORDEX_AFR_TAS, use_cftime=True, combine="by_coords") + ds_b = xr.open_mfdataset( + C3S_CORDEX_ANT_SFC_WIND, use_cftime=True, combine="by_coords" + ) + ds_c = xr.open_dataset(CMIP6_UNSTR_ICON_A) + ds_d = xr.open_dataset(CMIP6_OCE_HALO_CNRM) + + # check lat, lon are found + lat_a = clidu.detect_coordinate(ds_a, "latitude") + lon_a = clidu.detect_coordinate(ds_a, "longitude") + lat_b = clidu.detect_coordinate(ds_b, "latitude") + lon_b = clidu.detect_coordinate(ds_b, "longitude") + lat_c = clidu.detect_coordinate(ds_c, "latitude") + lon_c = clidu.detect_coordinate(ds_c, "longitude") + lat_d = clidu.detect_coordinate(ds_d, "latitude") + lon_d = clidu.detect_coordinate(ds_d, "longitude") + + # assert the correct variables have been found + assert lat_a == "lat" + assert lon_a == "lon" + assert lat_b == "lat" + assert lon_b == "lon" + assert lat_c == "latitude" + assert lon_c == "longitude" + assert lat_d == "lat" + assert lon_d == "lon" + + # assert detected bounds + assert clidu.detect_bounds(ds_a, lat_a) == "lat_vertices" + assert clidu.detect_bounds(ds_a, lon_a) == "lon_vertices" + assert clidu.detect_bounds(ds_b, lat_b) is None + assert clidu.detect_bounds(ds_b, lon_b) is None + assert clidu.detect_bounds(ds_c, lat_c) == "latitude_bnds" + assert clidu.detect_bounds(ds_c, lon_c) == "longitude_bnds" + assert clidu.detect_bounds(ds_d, lat_d) == "lat_bnds" + assert clidu.detect_bounds(ds_d, lon_d) == "lon_bnds" + + # test that latitude and longitude are still found when they are data variables + # reset coords sets lat and lon as data variables + ds_a = ds_a.reset_coords([lat_a, lon_a]) + ds_b = ds_b.reset_coords([lat_b, lon_b]) + ds_c = ds_c.reset_coords([lat_c, lon_c]) + ds_d = ds_d.reset_coords([lat_d, lon_d]) + assert lat_a == clidu.detect_coordinate(ds_a, "latitude") + assert lon_a == clidu.detect_coordinate(ds_a, "longitude") + assert lat_b == clidu.detect_coordinate(ds_b, "latitude") + assert lon_b == clidu.detect_coordinate(ds_b, "longitude") + assert lat_c == clidu.detect_coordinate(ds_c, "latitude") + assert lon_c == clidu.detect_coordinate(ds_c, "longitude") + assert lat_d == clidu.detect_coordinate(ds_d, "latitude") + assert lon_d == clidu.detect_coordinate(ds_d, "longitude") + + +def test_detect_gridtype(): + "Test the function detect_gridtype" + ds_a = xr.open_dataset(CMIP6_UNSTR_ICON_A, use_cftime=True) + ds_b = xr.open_dataset(CMIP6_TOS_ONE_TIME_STEP, use_cftime=True) + ds_c = xr.open_dataset(CMIP6_TAS_ONE_TIME_STEP, use_cftime=True) + assert ( + clidu.detect_gridtype( + ds_a, + lat="latitude", + lon="longitude", + lat_bnds="latitude_bnds", + lon_bnds="longitude_bnds", + ) + == "unstructured" + ) + assert ( + clidu.detect_gridtype( + ds_b, + lat="latitude", + lon="longitude", + lat_bnds="vertices_latitude", + lon_bnds="vertices_longitude", + ) + == "curvilinear" + ) + assert ( + clidu.detect_gridtype( + ds_c, lat="lat", lon="lon", lat_bnds="lat_bnds", lon_bnds="lon_bnds" + ) + == "regular_lat_lon" + ) + + +def test_crosses_0_meridian(): + "Test the _crosses_0_meridian function" + # Case 1 - longitude crossing 180° meridian + lon = np.arange(160.0, 200.0, 1.0) + + # convert to [-180, 180], min and max now suggest 0-meridian crossing + lon = np.where(lon > 180, lon - 360, lon) + + da = xr.DataArray(dims=["x"], coords={"x": lon}) + assert not clidu._crosses_0_meridian(da["x"]) + + # Case 2 - regional dataset ranging [315 .. 66] but for whatever reason not defined on + # [-180, 180] longitude frame + ds = xr.open_dataset(CORDEX_TAS_ONE_TIMESTEP) + assert np.isclose(ds["lon"].min(), 0, atol=0.1) + assert np.isclose(ds["lon"].max(), 360, atol=0.1) + + # Convert to -180, 180 frame and confirm crossing 0-meridian + ds, ll, lu = clidu.cf_convert_between_lon_frames(ds, (-180, 180)) + assert np.isclose(ds["lon"].min(), -45.4, atol=0.1) + assert np.isclose(ds["lon"].max(), 66.1, atol=0.1) + assert clidu._crosses_0_meridian(ds["lon"]) + + +def test_convert_interval_between_lon_frames(): + "Test the helper function _convert_interval_between_lon_frames" + # Convert from 0,360 to -180,180 longitude frame and vice versa + assert clidu._convert_interval_between_lon_frames(20, 60) == (20, 60) + assert clidu._convert_interval_between_lon_frames(190, 200) == (-170, -160) + assert clidu._convert_interval_between_lon_frames(-20, -90) == (270, 340) + + # Exception when crossing 0°- or 180°-meridian + with pytest.raises( + Exception, + match="Cannot convert longitude interval if it includes the 0°- or 180°-meridian.", + ): + clidu._convert_interval_between_lon_frames(170, 300) + with pytest.raises( + Exception, + match="Cannot convert longitude interval if it includes the 0°- or 180°-meridian.", + ): + clidu._convert_interval_between_lon_frames(-30, 10) + + +def test_convert_lon_frame_bounds(): + "Test the function cf_convert_between_lon_frames" + # Load tutorial dataset defined on [200,330] + ds = xr.tutorial.open_dataset("air_temperature") + assert ds["lon"].min() == 200.0 + assert ds["lon"].max() == 330.0 + + # Create bounds + dsb = ds.cf.add_bounds("lon") + + # Convert to other lon frame + conv, ll, lu = clidu.cf_convert_between_lon_frames(dsb, (-180, 180)) + + assert conv["lon"].values[0] == -160.0 + assert conv["lon"].values[-1] == -30.0 + + # Check bounds are containing the respective cell centers + assert np.all(conv["lon"].values[:] > conv["lon_bounds"].values[0, :]) + assert np.all(conv["lon"].values[:] < conv["lon_bounds"].values[1, :]) + + # Convert only lon_interval + conv, ll, lu = clidu.cf_convert_between_lon_frames(dsb, (-180, -10)) + + assert conv["lon"].min() == 200.0 + assert conv["lon"].max() == 330.0 + assert ll == 180.0 + assert lu == 350.0 + + +def test_convert_lon_frame_shifted_bounds(): + ds = xr.open_dataset(CMIP6_GFDL_EXTENT, use_cftime=True) + + # confirm shifted frame + assert np.isclose(ds["lon"].min(), -300.0, atol=0.5) + assert np.isclose(ds["lon"].max(), 60.0, atol=0.5) + + # convert to [-180, 180] + ds_a, ll, lu = clidu.cf_convert_between_lon_frames(ds, (-180, 180)) + assert (ll, lu) == (-180, 180) + assert np.isclose(ds_a["lon"].min(), -180.0, atol=0.5) + assert np.isclose(ds_a["lon"].max(), 180.0, atol=0.5) + assert np.isclose(ds_a["lon_bnds"].min(), -180.0, atol=0.5) + assert np.isclose(ds_a["lon_bnds"].max(), 180.0, atol=0.5) + + # convert to [0, 360] + ds_b, ll, lu = clidu.cf_convert_between_lon_frames(ds, (0, 360)) + assert (ll, lu) == (0, 360) + assert np.isclose(ds_b["lon"].min(), 0.0, atol=0.5) + assert np.isclose(ds_b["lon"].max(), 360.0, atol=0.5) + assert np.isclose(ds_b["lon_bnds"].min(), 0.0, atol=0.5) + assert np.isclose(ds_b["lon_bnds"].max(), 360.0, atol=0.5) + + # convert intermediate result to [0, 360] + ds_c, ll, lu = clidu.cf_convert_between_lon_frames(ds_a, (0, 360)) + assert (ll, lu) == (0, 360) + assert np.isclose(ds_c["lon"].min(), 0.0, atol=0.5) + assert np.isclose(ds_c["lon"].max(), 360.0, atol=0.5) + assert np.isclose(ds_c["lon_bnds"].min(), 0.0, atol=0.5) + assert np.isclose(ds_c["lon_bnds"].max(), 360.0, atol=0.5) + + # convert intermediate result to [-180, 180] + ds_d, ll, lu = clidu.cf_convert_between_lon_frames(ds_a, (-180, 180)) + assert (ll, lu) == (-180, 180) + assert np.isclose(ds_d["lon"].min(), -180.0, atol=0.5) + assert np.isclose(ds_d["lon"].max(), 180.0, atol=0.5) + assert np.isclose(ds_d["lon_bnds"].min(), -180.0, atol=0.5) + assert np.isclose(ds_d["lon_bnds"].max(), 180.0, atol=0.5) + + # assert projection coordinate sorted + assert np.all(ds_d["x"].values[1:] - ds_d["x"].values[:-1] > 0.0) + assert np.all(ds_c["x"].values[1:] - ds_c["x"].values[:-1] > 0.0) + + +def test_convert_lon_frame_shifted_no_bounds(): + ds = xr.open_dataset(CMIP6_IITM_EXTENT, use_cftime=True) + + # confirm shifted frame + assert np.isclose(ds["longitude"].min(), -280.0, atol=1.0) + assert np.isclose(ds["longitude"].max(), 80.0, atol=1.0) + + # convert to [-180, 180] + ds_a, ll, lu = clidu.cf_convert_between_lon_frames(ds, (-180, 180)) + assert (ll, lu) == (-180, 180) + assert np.isclose(ds_a["longitude"].min(), -180.0, atol=1.0) + assert np.isclose(ds_a["longitude"].max(), 180.0, atol=1.0) + + # convert to [0, 360] + ds_b, ll, lu = clidu.cf_convert_between_lon_frames(ds, (0, 360)) + assert (ll, lu) == (0, 360) + assert np.isclose(ds_b["longitude"].min(), 0.0, atol=1.0) + assert np.isclose(ds_b["longitude"].max(), 360.0, atol=1.0) + + # convert intermediate result to [0, 360] + ds_c, ll, lu = clidu.cf_convert_between_lon_frames(ds_a, (0, 360)) + assert (ll, lu) == (0, 360) + assert np.isclose(ds_c["longitude"].min(), 0.0, atol=1.0) + assert np.isclose(ds_c["longitude"].max(), 360.0, atol=1.0) + + # convert intermediate result to [-180, 180] + ds_d, ll, lu = clidu.cf_convert_between_lon_frames(ds_a, (-180, 180)) + assert (ll, lu) == (-180, 180) + assert np.isclose(ds_d["longitude"].min(), -180.0, atol=1.0) + assert np.isclose(ds_d["longitude"].max(), 180.0, atol=1.0) + + # assert projection coordinate sorted + assert np.all(ds_d["x"].values[1:] - ds_d["x"].values[:-1] > 0.0) + assert np.all(ds_c["x"].values[1:] - ds_c["x"].values[:-1] > 0.0) + + +# todo: add a few more tests of cf_convert_lon_frame using xe.util functions to create regional and global datasets From 5346657b3c8abff2a78fba1b7b9936a792efc7ed Mon Sep 17 00:00:00 2001 From: MacPingu Date: Wed, 13 Apr 2022 21:08:10 +0200 Subject: [PATCH 34/60] prepare release 0.9.0 (#222) * Update HISTORY.rst Co-authored-by: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> --- HISTORY.rst | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2e8974e8..838525d1 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,12 +1,15 @@ Version History =============== -v0.9.0 (unreleased) +v0.9.0 (2022-04-13) ------------------- New Features ^^^^^^^^^^^^ -* ``clisops`` now uses the `loguru `_ library as its primary logging engine. +* ``clisops.ops.average.average_time`` and ``clisops.core.average.average_time`` added (#211). Allowing averaging over time frequencies of day, month and year. +* New function ``create_time_bounds`` in ``clisops.utils.time_utils``, to generate time bounds for temporally averaged datasets. + +* ``clisops`` now uses the `loguru `_ library as its primary logging engine (#216). The mechanism for enabling log reporting in scripts/notebooks using ``loguru`` is as follows: .. code-block:: python @@ -22,9 +25,9 @@ New Features Other Changes ^^^^^^^^^^^^^ * Pandas now pinned below version 1.4.0. -* Pre-commit configuration updated with code style conventions (black, pyupgrade) set to Python3.7+. +* Pre-commit configuration updated with code style conventions (black, pyupgrade) set to Python3.7+ (#219). * ``loguru`` is now an install dependency, with ``pytest-loguru`` as a development-only dependency. -* Added function to convert the longitude axis between different longitude frames (eg. [-180, 180] and [0, 360]). +* Added function to convert the longitude axis between different longitude frames (eg. [-180, 180] and [0, 360]) (#217, #218). v0.8.0 (2022-01-13) ------------------- From 3af7522ac4f25d493bae1a088f969c0d05003a1d Mon Sep 17 00:00:00 2001 From: Carsten Ehbrecht Date: Wed, 13 Apr 2022 21:10:17 +0200 Subject: [PATCH 35/60] =?UTF-8?q?Bump=20version:=200.8.0=20=E2=86=92=200.9?= =?UTF-8?q?.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- clisops/__version__.py | 2 +- docs/conf.py | 2 +- setup.cfg | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/clisops/__version__.py b/clisops/__version__.py index 4f0d25e6..7a235f06 100644 --- a/clisops/__version__.py +++ b/clisops/__version__.py @@ -4,4 +4,4 @@ __author__ = "Elle Smith" __email__ = "eleanor.smith@stfc.ac.uk" -__version__ = "0.8.0" +__version__ = "0.9.0" diff --git a/docs/conf.py b/docs/conf.py index fff69a69..e561e20d 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -74,7 +74,7 @@ # the built documents. # # The short X.Y version. -version = "0.8.0" +version = "0.9.0" # The full version, including alpha/beta/rc tags. release = version diff --git a/setup.cfg b/setup.cfg index e83913f4..a0763b14 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.8.0 +current_version = 0.9.0 commit = True tag = True From 9c9a2b39c1385b7a5d4c841df2e2fde3b89df8a2 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 19 Apr 2022 13:05:32 -0400 Subject: [PATCH 36/60] Update environment.yml --- environment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/environment.yml b/environment.yml index 9037a153..1a8bf945 100644 --- a/environment.yml +++ b/environment.yml @@ -23,5 +23,4 @@ dependencies: - xarray>=0.15 - xesmf>=0.6.2 - pip: - - pytest-loguru # - roocs-utils @ git+https://github.com/roocs/roocs-utils.git@master#egg=roocs-utils From 7d8d27dee4b47ac7df33497a097e224768cb0dc3 Mon Sep 17 00:00:00 2001 From: MacPingu Date: Tue, 3 May 2022 20:10:53 +0200 Subject: [PATCH 37/60] using roocs-utils 0.6.2 (#226) * using roocs-utils 0.6.2 --- environment.yml | 4 +--- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/environment.yml b/environment.yml index 1a8bf945..590fdc74 100644 --- a/environment.yml +++ b/environment.yml @@ -18,9 +18,7 @@ dependencies: - pyproj>=2.5 - pooch - requests>=2.0 - - roocs-utils>=0.5.0 + - roocs-utils>=0.6.2,<0.7 - shapely>=1.6 - xarray>=0.15 - xesmf>=0.6.2 - - pip: -# - roocs-utils @ git+https://github.com/roocs/roocs-utils.git@master#egg=roocs-utils diff --git a/requirements.txt b/requirements.txt index e27565bb..e554b0c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,6 @@ cf-xarray>=0.7.0 #cf-xarray @ git+https://github.com/xarray-contrib/cf-xarray/@main#egg=cf-xarray bottleneck>=1.3.1 requests>=2.0 -roocs-utils>=0.5.0 +roocs-utils>=0.6.2,<0.7 # roocs-utils @ git+https://github.com/roocs/roocs-utils.git@master#egg=roocs-utils loguru>=0.5.3 From 260edfdd5b49582f5c08ab268e8828adf2edc3a1 Mon Sep 17 00:00:00 2001 From: MacPingu Date: Tue, 3 May 2022 20:34:59 +0200 Subject: [PATCH 38/60] Fix 224 inconsistent bounds (#225) * added fix for bounds * update test --- clisops/ops/base_operation.py | 31 +++++++++++++++++++++++++++++-- tests/ops/test_subset.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/clisops/ops/base_operation.py b/clisops/ops/base_operation.py index 975bc030..b2230328 100644 --- a/clisops/ops/base_operation.py +++ b/clisops/ops/base_operation.py @@ -71,13 +71,15 @@ def _remove_redundant_fill_values(self, ds): """ Get coordinate variables and remove fill values added by xarray (CF conventions say that coordinate variables cannot have missing values). Get bounds variables and remove fill values added by xarray. + + See issue: https://github.com/roocs/clisops/issues/224 """ if isinstance(ds, xr.Dataset): main_var = get_main_variable(ds) for coord_id in ds[main_var].coords: # remove fill value from coordinate variables - if ds.coords[coord_id].dims == (coord_id,): - ds[coord_id].encoding["_FillValue"] = None + # if ds.coords[coord_id].dims == (coord_id,): + ds[coord_id].encoding["_FillValue"] = None # remove fill value from bounds variables if they exist try: bnd = ds.cf.get_bounds(coord_id).name @@ -86,6 +88,29 @@ def _remove_redundant_fill_values(self, ds): continue return ds + def _remove_redundant_coordinates_from_bounds(self, ds): + """ + This method removes redundant coordinates from bounds, example: + + double time_bnds(time, bnds) ; + time_bnds:coordinates = "height" ; + + Programs like cdo will complain about this: + + Warning (cdf_set_var): Inconsistent variable definition for time_bnds! + + See issue: https://github.com/roocs/clisops/issues/224 + """ + if isinstance(ds, xr.Dataset): + main_var = get_main_variable(ds) + for coord_id in ds[main_var].coords: + try: + bnd = ds.cf.get_bounds(coord_id).name + ds[bnd].encoding["coordinates"] = None + except KeyError: + continue + return ds + def process(self): """ Main processing method used by all sub-classes. @@ -108,6 +133,8 @@ def process(self): # remove fill values from lat/lon/time if required processed_ds = self._remove_redundant_fill_values(processed_ds) + # remove redundant coordinates from bounds + processed_ds = self._remove_redundant_coordinates_from_bounds(processed_ds) # Work out how many outputs should be created based on the size # of the array. Manage this as a list of time slices. diff --git a/tests/ops/test_subset.py b/tests/ops/test_subset.py index fb9c181e..3eb042b2 100644 --- a/tests/ops/test_subset.py +++ b/tests/ops/test_subset.py @@ -1604,8 +1604,10 @@ def test_subset_nc_no_fill_value(cmip5_tas_file, tmpdir): ds.to_netcdf(f"{tmpdir}/test_fill_values.nc") ds = _load_ds(f"{tmpdir}/test_fill_values.nc") + # assert np.isnan(float(ds.time.encoding.get("_FillValue"))) assert np.isnan(float(ds.lat.encoding.get("_FillValue"))) assert np.isnan(float(ds.lon.encoding.get("_FillValue"))) + assert np.isnan(float(ds.height.encoding.get("_FillValue"))) assert np.isnan(float(ds.lat_bnds.encoding.get("_FillValue"))) assert np.isnan(float(ds.lon_bnds.encoding.get("_FillValue"))) @@ -1613,9 +1615,36 @@ def test_subset_nc_no_fill_value(cmip5_tas_file, tmpdir): # check that there is no fill value in encoding for coordinate variables and bounds res = _load_ds(result) + assert "_FillValue" not in res.time.encoding assert "_FillValue" not in res.lat.encoding assert "_FillValue" not in res.lon.encoding + assert "_FillValue" not in res.height.encoding assert "_FillValue" not in res.lat_bnds.encoding assert "_FillValue" not in res.lon_bnds.encoding assert "_FillValue" not in res.time_bnds.encoding + + +def test_subset_nc_consistent_bounds(cmip5_tas_file, tmpdir): + """Tests clisops subset function with a time subset.""" + result = subset( + ds=CMIP5_TAS, + time=time_interval("2005-01-01T00:00:00", "2020-12-30T00:00:00"), + output_dir=tmpdir, + output_type="nc", + file_namer="simple", + ) + res = _load_ds(result) + # check fill value in bounds + assert "_FillValue" not in res.lat_bnds.encoding + assert "_FillValue" not in res.lon_bnds.encoding + assert "_FillValue" not in res.time_bnds.encoding + # check fill value in coordinates + assert "_FillValue" not in res.time.encoding + assert "_FillValue" not in res.lat.encoding + assert "_FillValue" not in res.lon.encoding + assert "_FillValue" not in res.height.encoding + # check coordinates in bounds + assert "coordinates" not in res.lat_bnds.encoding + assert "coordinates" not in res.lon_bnds.encoding + assert "coordinates" not in res.time_bnds.encoding From d0a03c73855e96ac3a3401de38242a58a69c2b43 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 9 May 2022 16:58:19 +0000 Subject: [PATCH 39/60] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v2.32.0 → v2.32.1](https://github.com/asottile/pyupgrade/compare/v2.32.0...v2.32.1) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aece2677..afb6d0ea 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,7 +32,7 @@ repos: # - id: pydocstyle # args: ["--convention=numpy"] - repo: https://github.com/asottile/pyupgrade - rev: v2.32.0 + rev: v2.32.1 hooks: - id: pyupgrade args: ['--py37-plus'] From 5c43f623c01e1b03b2f81b8176a0ace22e57f651 Mon Sep 17 00:00:00 2001 From: Carsten Ehbrecht Date: Wed, 11 May 2022 22:08:56 +0200 Subject: [PATCH 40/60] added another metadata test for cmip6 --- tests/_common.py | 5 +++++ tests/ops/test_subset.py | 30 ++++++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/tests/_common.py b/tests/_common.py index 33e23d9a..33fd283b 100644 --- a/tests/_common.py +++ b/tests/_common.py @@ -173,6 +173,11 @@ def cmip6_archive_base(): "master/test_data/badc/cmip6/data/CMIP6/ScenarioMIP/MIROC/MIROC6/ssp119/r1i1p1f1/Amon/ta/gn/files/d20190807/ta_Amon_MIROC6_ssp119_r1i1p1f1_gn_201501-202412.nc", ).as_posix() +CMIP6_TASMIN = Path( + MINI_ESGF_CACHE_DIR, + "master/test_data/badc/cmip6/data/CMIP6/CMIP/MPI-M/MPI-ESM1-2-HR/historical/r1i1p1f1/Amon/tasmin/gn/v20190710/tasmin_Amon_MPI-ESM1-2-HR_historical_r1i1p1f1_gn_201001-201412.nc", +).as_posix() + # Dataset with julian calender CMIP6_JULIAN = Path( MINI_ESGF_CACHE_DIR, diff --git a/tests/ops/test_subset.py b/tests/ops/test_subset.py index 3eb042b2..8b794153 100644 --- a/tests/ops/test_subset.py +++ b/tests/ops/test_subset.py @@ -30,6 +30,7 @@ CMIP6_SICONC, CMIP6_SICONC_DAY, CMIP6_TA, + CMIP6_TASMIN, CMIP6_TOS, CMIP6_TOS_ONE_TIME_STEP, _check_output_nc, @@ -1625,8 +1626,8 @@ def test_subset_nc_no_fill_value(cmip5_tas_file, tmpdir): assert "_FillValue" not in res.time_bnds.encoding -def test_subset_nc_consistent_bounds(cmip5_tas_file, tmpdir): - """Tests clisops subset function with a time subset.""" +def test_subset_cmip5_nc_consistent_bounds(cmip5_tas_file, tmpdir): + """Tests clisops subset function with a time subset and check the metadata""" result = subset( ds=CMIP5_TAS, time=time_interval("2005-01-01T00:00:00", "2020-12-30T00:00:00"), @@ -1648,3 +1649,28 @@ def test_subset_nc_consistent_bounds(cmip5_tas_file, tmpdir): assert "coordinates" not in res.lat_bnds.encoding assert "coordinates" not in res.lon_bnds.encoding assert "coordinates" not in res.time_bnds.encoding + + +def test_subset_cmip6_nc_consistent_bounds(cmip5_tas_file, tmpdir): + """Tests clisops subset function with a time subset and check the metadata""" + result = subset( + ds=CMIP6_TASMIN, + time=time_interval("2010-01-01T00:00:00", "2010-12-31T00:00:00"), + output_dir=tmpdir, + output_type="nc", + file_namer="simple", + ) + res = _load_ds(result) + # check fill value in bounds + assert "_FillValue" not in res.lat_bnds.encoding + assert "_FillValue" not in res.lon_bnds.encoding + assert "_FillValue" not in res.time_bnds.encoding + # check fill value in coordinates + assert "_FillValue" not in res.time.encoding + assert "_FillValue" not in res.lat.encoding + assert "_FillValue" not in res.lon.encoding + assert "_FillValue" not in res.height.encoding + # check coordinates in bounds + assert "coordinates" not in res.lat_bnds.encoding + assert "coordinates" not in res.lon_bnds.encoding + assert "coordinates" not in res.time_bnds.encoding From 1d61c6eeb5af554571abd023494846a71d95efb6 Mon Sep 17 00:00:00 2001 From: Carsten Ehbrecht Date: Wed, 11 May 2022 22:40:17 +0200 Subject: [PATCH 41/60] update changes --- HISTORY.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 838525d1..343dcc33 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,6 +1,19 @@ Version History =============== +v0.9.1 (2022-05-12) +------------------- + +Bug fixes +^^^^^^^^^ +* Fix inconsistent bounds in metadata after subset operation (#224). + +Other Changes +^^^^^^^^^^^^^ +* Use ``roocs-utils`` 0.6.2 to avoid test failure (#226). +* Removed unneeded testing dep from environment.yml (#223). +* Merged pre-commit autoupdate (#227). + v0.9.0 (2022-04-13) ------------------- From 55f61e8605124f06c4d3772d0168cf93fe5353d0 Mon Sep 17 00:00:00 2001 From: Carsten Ehbrecht Date: Wed, 11 May 2022 23:40:57 +0200 Subject: [PATCH 42/60] =?UTF-8?q?Bump=20version:=200.9.0=20=E2=86=92=200.9?= =?UTF-8?q?.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- clisops/__version__.py | 2 +- docs/conf.py | 2 +- setup.cfg | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/clisops/__version__.py b/clisops/__version__.py index 7a235f06..9efa15fb 100644 --- a/clisops/__version__.py +++ b/clisops/__version__.py @@ -4,4 +4,4 @@ __author__ = "Elle Smith" __email__ = "eleanor.smith@stfc.ac.uk" -__version__ = "0.9.0" +__version__ = "0.9.1" diff --git a/docs/conf.py b/docs/conf.py index e561e20d..24f257e8 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -74,7 +74,7 @@ # the built documents. # # The short X.Y version. -version = "0.9.0" +version = "0.9.1" # The full version, including alpha/beta/rc tags. release = version diff --git a/setup.cfg b/setup.cfg index a0763b14..2664911a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.9.0 +current_version = 0.9.1 commit = True tag = True From 8511f4e7a896c970e6ee033cb6db621ec501fa10 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 27 May 2022 14:59:33 -0400 Subject: [PATCH 43/60] Fix broken notebooks alias --- docs/notebooks | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/notebooks b/docs/notebooks index f411073e..8f9a5b2e 120000 --- a/docs/notebooks +++ b/docs/notebooks @@ -1 +1 @@ -../notebooks +../notebooks \ No newline at end of file From eabe71c42f83de39365f4f12d0d013cf20a139a8 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 27 May 2022 15:54:20 -0400 Subject: [PATCH 44/60] Fix conf.py error that prevented notebooks from rendering in RtD --- docs/conf.py | 4 +++- notebooks/average_over_dims.ipynb | 40 +++++++++++++++---------------- notebooks/core_subset.ipynb | 4 ++-- notebooks/subset.ipynb | 21 ++++++++-------- 4 files changed, 36 insertions(+), 33 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 24f257e8..d8ccc1c8 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -49,6 +49,7 @@ napoleon_use_param = False napoleon_use_ivar = True +nbsphinx_allow_errors = True nbsphinx_execute = "auto" nbsphinx_timeout = 300 @@ -59,7 +60,8 @@ # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = [".rst", ".ipynb"] +source_suffix = [".rst"] +# note: do not add .ipynb when nbspinx is enabled, otherwise you get the "missing title" error # The master toctree document. master_doc = "index" diff --git a/notebooks/average_over_dims.ipynb b/notebooks/average_over_dims.ipynb index 37221623..3e052143 100644 --- a/notebooks/average_over_dims.ipynb +++ b/notebooks/average_over_dims.ipynb @@ -1,5 +1,16 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Averaging over dimensions of the dataset\n", + "\n", + "The average over dimensions operation makes use of `clisops.core.average` to process the datasets and to set the output type and the output file names.\n", + "\n", + "It is possible to average over none or any number of time, longitude, latitude or level dimensions in the dataset." + ] + }, { "cell_type": "code", "execution_count": null, @@ -26,18 +37,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Averaging over dimensions of the dataset\n", - "\n", - "The average over dimensions operation makes use of `clisops.core.average` to process the datasets and to set the output type and the output file names.\n", - "\n", - "It is possible to average over none or any number of time, longitude, latitude or level dimensions in the dataset." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Parameters\n", + "## Parameters\n", "\n", "Parameters taken by the `average_over_dims` are below:\n", "\n", @@ -83,7 +83,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Average over one dimension" + "## Average over one dimension" ] }, { @@ -108,7 +108,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Average over two dimensions\n", + "## Average over two dimensions\n", "\n", "Averaging over two dimensions is just as simple as averaging over one. The dimensions to be averaged over should be passed in as a sequence." ] @@ -135,7 +135,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Allowed dimensions\n", + "## Allowed dimensions\n", "\n", "It is only possible to average over longtiude, latitude, level and time. If a different dimension is provided to average over an error will be raised." ] @@ -161,7 +161,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Dimensions not found" + "## Dimensions not found" ] }, { @@ -223,7 +223,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# No dimensions supplied" + "## No dimensions supplied" ] }, { @@ -253,7 +253,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# An example of averaging over level" + "## An example of averaging over level" ] }, { @@ -287,7 +287,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -301,7 +301,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.6" + "version": "3.9.12" } }, "nbformat": 4, diff --git a/notebooks/core_subset.ipynb b/notebooks/core_subset.ipynb index 1aa4e65d..bfa79d53 100644 --- a/notebooks/core_subset.ipynb +++ b/notebooks/core_subset.ipynb @@ -1682,7 +1682,7 @@ "metadata": { "keep_output": true, "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -1696,7 +1696,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.6" + "version": "3.9.12" } }, "nbformat": 4, diff --git a/notebooks/subset.ipynb b/notebooks/subset.ipynb index 56db9c40..7c1c9a40 100644 --- a/notebooks/subset.ipynb +++ b/notebooks/subset.ipynb @@ -1,5 +1,14 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Subsetting\n", + "\n", + "The subset operation makes use of `clisops.core.subset` to process the datasets and to set the output type and the output file names." + ] + }, { "cell_type": "code", "execution_count": null, @@ -32,15 +41,6 @@ " os.remove(\"./output_001.nc\")" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Subsetting\n", - "\n", - "The subset operation makes use of `clisops.core.subset` to process the datasets and to set the output type and the output file names." - ] - }, { "cell_type": "code", "execution_count": null, @@ -330,6 +330,7 @@ } ], "metadata": { + "celltoolbar": "Edit Metadata", "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", @@ -345,7 +346,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.10" + "version": "3.9.12" } }, "nbformat": 4, From 65eeb3cf452409336dc2dabd6bea1bf4c2642663 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 27 May 2022 16:31:29 -0400 Subject: [PATCH 45/60] rename some sections that were conflicting in notebooks, adjust some function docstrings that were malformed, fix sections heading underlines that were too short, fix code-blocks that were malformed, use autosectionlabel options to prevent errors in HISTORY.rst --- HISTORY.rst | 23 +++++------ README.rst | 8 +--- clisops/ops/subset.py | 24 +++-------- clisops/utils/dataset_utils.py | 74 ++++++++++++++++++---------------- docs/api.rst | 4 +- docs/conf.py | 3 ++ notebooks/subset.ipynb | 3 +- 7 files changed, 61 insertions(+), 78 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 343dcc33..c2c6dc07 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -26,6 +26,7 @@ New Features The mechanism for enabling log reporting in scripts/notebooks using ``loguru`` is as follows: .. code-block:: python + import sys from loguru import logger @@ -64,7 +65,6 @@ Other Changes * Minimum pygeos version set to 0.9. * Replace ``cascaded_union`` by ``unary_union`` to anticipate a `shapely` deprecation. - v0.7.0 (2021-10-26) ------------------- @@ -86,7 +86,6 @@ Other Changes ^^^^^^^^^^^^^ * Python 3.6 no longer tested in GitHub actions. - v0.6.5 (2021-06-10) ------------------- @@ -130,7 +129,6 @@ Other Changes ^^^^^^^^^^^^^ * Error message improved to include longitude bounds of the dataset when the bounds requested in ``ops.subset.subset`` are not within range and rolling could not be completed. - v0.6.2 (2021-03-22) ------------------- @@ -146,17 +144,18 @@ New Features v0.6.1 (2021-02-23) ------------------- + Bug Fixes ^^^^^^^^^ * Add ``cf-xarray`` as dependency. This is a dependency of ``roocs-utils``>=0.2.1 so is not a breaking change. * Remove ``python-dateutil``, ``fiona`` and ``geojson`` as dependencies, no longer needed. - v0.6.0 (2021-02-22) ------------------- + Breaking Changes ^^^^^^^^^^^^^^^^ -* New dev dependency: ``GitPython``==3.1.12 +* New dev dependency: ``GitPython``\ ==3.1.12 * ``roocs-utils``>=0.2.1 required. New Features @@ -181,7 +180,6 @@ Other Changes * Added functionality to ``core.subset.create_mask`` so it can accept ``GeoDataFrames`` with non-integer indexes. * ``clisops.utils.file_namers`` adjusted to allow values to be overwritten and extras to be added to the end before the file extension. - v0.5.1 (2021-01-11) ------------------- @@ -197,7 +195,7 @@ Other Changes v0.5.0 (2020-12-17) ------------------- +------------------- Breaking Changes ^^^^^^^^^^^^^^^^ @@ -214,7 +212,7 @@ Other Changes v0.4.0 (2020-11-10) ------------------ +------------------- Adding new features, updating doc strings and documentation and inclusion of static type support. @@ -250,7 +248,6 @@ Bug Fixes * Nudging time values to nearest available in dataset to fix a bug where subsetting failed when the exact date did not exist in the dataset. - Other Changes ^^^^^^^^^^^^^ @@ -261,7 +258,6 @@ Other Changes * md files changed to rst. * tests now use ``mini-esgf-data`` by default. - v0.3.1 (2020-08-04) ------------------- @@ -269,9 +265,8 @@ Other Changes ^^^^^^^^^^^^^ * Add missing ``rtree`` dependency to ensure correct spatial indexing. - v0.3.0 (2020-07-23) ------------------- +------------------- Other Changes ^^^^^^^^^^^^^ @@ -287,7 +282,7 @@ Other Changes v0.2.0 (2020-06-19) ------------------- +------------------- New Features ^^^^^^^^^^^^^ @@ -303,6 +298,6 @@ Other Changes v0.1.0 (2020-04-22) ------------------- +------------------- * First release. diff --git a/README.rst b/README.rst index 3c102c90..5112eebc 100644 --- a/README.rst +++ b/README.rst @@ -1,8 +1,6 @@ - clisops - climate simulation operations ======================================= - .. image:: https://img.shields.io/pypi/v/clisops.svg :target: https://pypi.python.org/pypi/clisops :alt: Pypi @@ -15,7 +13,6 @@ clisops - climate simulation operations :target: https://clisops.readthedocs.io/en/latest/?badge=latest :alt: Documentation - The ``clisops`` package (pronounced "clie-sops") provides a python library for running *data-reduction* operations on `Xarray `_ data sets or files that can be interpreted by Xarray. These basic operations (subsetting, averaging and @@ -30,7 +27,6 @@ the results. ``clisops`` can be used stand-alone to read individual, or groups of, NetCDF files directly. - * Free software: BSD * Documentation: https://clisops.readthedocs.io. @@ -45,15 +41,13 @@ The package provides the following operations: * regrid Credits -======= +------- This package was created with ``Cookiecutter`` and the ``audreyr/cookiecutter-pypackage`` project template. - * Cookiecutter: https://github.com/audreyr/cookiecutter * cookiecutter-pypackage: https://github.com/audreyr/cookiecutter-pypackage - .. image:: https://img.shields.io/badge/code%20style-black-000000.svg :target: https://github.com/python/black :alt: Python Black diff --git a/clisops/ops/subset.py b/clisops/ops/subset.py index 3306f7c9..a92fccce 100644 --- a/clisops/ops/subset.py +++ b/clisops/ops/subset.py @@ -189,26 +189,14 @@ def subset( split_method="time:auto", file_namer="standard", ) -> List[Union[xr.Dataset, str]]: - """ + """Subset operation. Parameters ---------- ds: Union[xr.Dataset, str] time: Optional[Union[str, Tuple[str, str], TimeParameter]] = None, - area: Optional[ - Union[ - str, - Tuple[ - Union[int, float, str], - Union[int, float, str], - Union[int, float, str], - Union[int, float, str], - ], - AreaParameter - ] - ] = None, - level: Optional[Union[str, Tuple[Union[int, float, str], Union[int, float, str]], - LevelParameter] = None, + area: str or AreaParameter or Tuple[Union[int, float, str], Union[int, float, str], Union[int, float, str], Union[int, float, str]], optional + level: Optional[Union[str, Tuple[Union[int, float, str], Union[int, float, str]], LevelParameter] = None, time_components: Optional[Union[str, Dict, TimeComponentsParameter]] = None, output_dir: Optional[Union[str, Path]] = None output_type: {"netcdf", "nc", "zarr", "xarray"} @@ -236,9 +224,9 @@ def subset( Note ---- - If you request a selection range (such as level, latitude or longitude) that specifies the lower - and upper bounds in the opposite direction to the actual coordinate values then clisops.ops.subset - will detect this issue and reverse your selection before returning the data subset. + If you request a selection range (such as level, latitude or longitude) that specifies the lower + and upper bounds in the opposite direction to the actual coordinate values then clisops.ops.subset + will detect this issue and reverse your selection before returning the data subset. """ op = Subset(**locals()) return op.process() diff --git a/clisops/utils/dataset_utils.py b/clisops/utils/dataset_utils.py index 8ef6c3c7..baf27659 100644 --- a/clisops/utils/dataset_utils.py +++ b/clisops/utils/dataset_utils.py @@ -1,5 +1,6 @@ import math import warnings +from typing import Optional import cf_xarray as cfxr import cftime @@ -11,12 +12,14 @@ def calculate_offset(lon, first_element_value): - """ - Calculate the number of elements to roll the dataset by in order to have - longitude from within requested bounds. + """Calculate the number of elements to roll the dataset by in order to have longitude from within requested bounds. - :param lon: longitude coordinate of xarray dataset. - :param first_element_value: the value of the first element of the longitude array to roll to. + Parameters + ---------- + lon + Longitude coordinate of xarray dataset. + first_element_value + The value of the first element of the longitude array to roll to. """ # get resolution of data res = lon.values[1] - lon.values[0] @@ -33,15 +36,14 @@ def calculate_offset(lon, first_element_value): return offset -def _crosses_0_meridian(lon_c): - """ - Determine whether grid extents over the 0-meridian. +def _crosses_0_meridian(lon_c: xr.DataArray): + """Determine whether grid extents over the 0-meridian. Assumes approximate constant width of grid cells. Parameters ---------- - lon_c : TYPE + lon_c: xr.DataArray Longitude coordinate variable in the longitude frame [-180, 180]. Returns @@ -103,8 +105,7 @@ def _convert_interval_between_lon_frames(low, high): def cf_convert_between_lon_frames(ds_in, lon_interval): - """ - Convert ds or lon_interval (whichever deems appropriate) to the other longitude frame, if the longitude frames do not match. + """Convert ds or lon_interval (whichever deems appropriate) to the other longitude frame, if the longitude frames do not match. If ds and lon_interval are defined on different longitude frames ([-180, 180] and [0, 360]), this function will convert one of the input parameters to the other longitude frame, preferably @@ -116,9 +117,9 @@ def cf_convert_between_lon_frames(ds_in, lon_interval): Parameters ---------- - ds_in : xarray.Dataset or xarray.DataArray + ds_in: xarray.Dataset or xarray.DataArray xarray data object with defined longitude dimension. - lon_interval : tuple or list + lon_interval: tuple or list length-2-tuple or -list of floats or integers denoting the bounds of the longitude interval. Returns @@ -313,20 +314,27 @@ def check_lon_alignment(ds, lon_bnds): def adjust_date_to_calendar(da, date, direction="backwards"): - """ - Check that the date specified exists in the calendar type of the dataset. If not, - change the date a day at a time (up to a maximum of 5 times) to find a date that does exist. + """Check that the date specified exists in the calendar type of the dataset. + If not present, changes the date a day at a time (up to a maximum of 5 times) to find a date that does exist. The direction to change the date by is indicated by 'direction'. - :param da: xarray.Dataset or xarray.DataArray - :param date: The date to check, as a string. - :param direction: The direction to move in days to find a date that does exist. - 'backwards' means the search will go backwards in time until an existing date is found. - 'forwards' means the search will go forwards in time. - The default is 'backwards'. + Parameters + ---------- + da: xarray.Dataset or xarray.DataArray + The data to examine. + date: str + The date to check. + direction: str + The direction to move in days to find a date that does exist. + 'backwards' means the search will go backwards in time until an existing date is found. + 'forwards' means the search will go forwards in time. + The default is 'backwards'. - :return: (str) The next possible existing date in the calendar of the dataset. + Returns + ------- + str + The next possible existing date in the calendar of the dataset. """ # turn date into AnyCalendarDateTime object d = str_to_AnyCalendarDateTime(date) @@ -362,14 +370,13 @@ def adjust_date_to_calendar(da, date, direction="backwards"): def detect_coordinate(ds, coord_type): - """ - Use cf_xarray to obtain the variable name of the requested coordinate. + """Use cf_xarray to obtain the variable name of the requested coordinate. Parameters ---------- ds: xarray.Dataset, xarray.DataArray Dataset the coordinate variable name shall be obtained from. - coord_type : str + coord_type: str Coordinate type understood by cf-xarray, eg. 'lat', 'lon', ... Raises @@ -398,9 +405,8 @@ def detect_coordinate(ds, coord_type): raise AttributeError(error_msg) -def detect_bounds(ds, coordinate): - """ - Use cf_xarray to obtain the variable name of the requested coordinates bounds. +def detect_bounds(ds, coordinate) -> Optional[str]: + """Use cf_xarray to obtain the variable name of the requested coordinates bounds. Parameters ---------- @@ -411,9 +417,9 @@ def detect_bounds(ds, coordinate): Returns ------- - str - Returns the variable name of the requested coordinate bounds, - returns None if the variable has no bounds or they cannot be identified. + str or None + Returns the variable name of the requested coordinate bounds. + Returns None if the variable has no bounds or if they cannot be identified. """ try: return ds.cf.bounds[coordinate][0] @@ -425,12 +431,10 @@ def detect_bounds(ds, coordinate): def detect_gridtype(ds, lon, lat, lon_bnds=None, lat_bnds=None): - """ - Detect type of the grid as one of "regular_lat_lon", "curvilinear", "unstructured". + """Detect type of the grid as one of "regular_lat_lon", "curvilinear", "unstructured". Assumes the grid description / structure follows the CF conventions. """ - # 1D coordinate variables if ds[lat].ndim == 1 and ds[lon].ndim == 1: lat_1D = ds[lat].dims[0] diff --git a/docs/api.rst b/docs/api.rst index 8f38233f..435cc11b 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -11,7 +11,7 @@ Core subset functionality :show-inheritance: Core average functionality -========================= +========================== .. automodule:: clisops.core.average :members: @@ -70,7 +70,7 @@ File Namers Dataset Utilities -================ +================= .. automodule:: clisops.utils.dataset_utils :noindex: diff --git a/docs/conf.py b/docs/conf.py index d8ccc1c8..f79c6f03 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -44,6 +44,9 @@ "IPython.sphinxext.ipython_console_highlighting", ] +autosectionlabel_prefix_document = True +autosectionlabel_maxdepth = 2 + napoleon_numpy_docstring = True napoleon_use_rtype = False napoleon_use_param = False diff --git a/notebooks/subset.ipynb b/notebooks/subset.ipynb index 7c1c9a40..86523c75 100644 --- a/notebooks/subset.ipynb +++ b/notebooks/subset.ipynb @@ -57,8 +57,7 @@ "source": [ "The `subset` process takes several parameters:\n", "\n", - "Parameters\n", - "----------\n", + "## Subsetting Parameters\n", "\n", " ds: Union[xr.Dataset, str, Path]\n", " time: Optional[Union[str, TimeParameter]]\n", From b3b528c5f4e3c2d08f1c4f46f086e4a00da15963 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Fri, 27 May 2022 16:42:26 -0400 Subject: [PATCH 46/60] Set ReadTheDocs to fail on warning --- .readthedocs.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index cefa3f5f..3bc609f3 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -8,6 +8,7 @@ version: 2 # Build documentation in the docs/ directory with Sphinx sphinx: configuration: docs/conf.py + fail_on_warning: true # Build documentation with MkDocs #mkdocs: @@ -17,17 +18,14 @@ sphinx: formats: all # Optionally set the version of Python and requirements required to build your docs -#python: -# version: 3.7 python: - version: 3.7 + version: "3.7" install: - method: pip path: . extra_requirements: - docs - # conda: # environment: docs/environment.yml From 0bf8e94a00b0c88817ae3dcf6ea9914c388c5d02 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 13 Jun 2022 17:09:52 +0000 Subject: [PATCH 47/60] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/pre-commit-hooks: v4.2.0 → v4.3.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.2.0...v4.3.0) - [github.com/asottile/pyupgrade: v2.32.1 → v2.34.0](https://github.com/asottile/pyupgrade/compare/v2.32.1...v2.34.0) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index afb6d0ea..f3e917cc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.2.0 + rev: v4.3.0 hooks: - id: trailing-whitespace exclude: setup.cfg @@ -32,7 +32,7 @@ repos: # - id: pydocstyle # args: ["--convention=numpy"] - repo: https://github.com/asottile/pyupgrade - rev: v2.32.1 + rev: v2.34.0 hooks: - id: pyupgrade args: ['--py37-plus'] From e7b708bcdb6eb44de5da1bcea78a13f88104784e Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 13 Jun 2022 15:37:05 -0400 Subject: [PATCH 48/60] Retire Python3.7, add support for Python3.10 --- .github/CONTRIBUTING.rst | 2 +- .github/workflows/main.yml | 18 +++++++++--------- .pre-commit-config.yaml | 4 ++-- .readthedocs.yml | 2 +- CONTRIBUTING.rst | 4 ++-- environment.yml | 1 + setup.py | 4 ++-- tox.ini | 4 ++-- 8 files changed, 20 insertions(+), 19 deletions(-) diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst index dd725fdf..29e8495a 100644 --- a/.github/CONTRIBUTING.rst +++ b/.github/CONTRIBUTING.rst @@ -139,7 +139,7 @@ Before you submit a pull request, please follow these guidelines: * `numpydoc`_ * `reStructuredText (ReST)`_ -5. The pull request should work for Python 3.7 as well as raise test coverage. +5. The pull request should work for Python 3.8+ as well as raise test coverage. Pull requests are also checked for documentation build status and for `PEP8`_ compliance. The build statuses and build errors for pull requests can be found at: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 45580482..948cb064 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,10 +17,10 @@ jobs: uses: styfle/cancel-workflow-action@0.9.1 with: access_token: ${{ github.token }} - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: "3.8" - name: Install tox run: pip install tox - name: Run linting suite @@ -32,19 +32,19 @@ jobs: strategy: matrix: include: - - python-version: 3.7 + - python-version: "3.8" tox-env: py37 allowed_to_fail: false - - python-version: 3.8 + - python-version: "3.9" tox-env: py38 allowed_to_fail: false - - python-version: 3.9 + - python-version: "3.10" tox-env: py39 allowed_to_fail: false steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install tox @@ -61,7 +61,7 @@ jobs: matrix: python-version: [3.8] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Setup conda with Python ${{ matrix.python-version }} uses: s-weigand/setup-conda@v1 with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index afb6d0ea..fafb2f81 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: rev: 22.3.0 hooks: - id: black - args: ["--target-version", "py37"] + args: ["--target-version", "py38"] - repo: https://github.com/pycqa/flake8 rev: 4.0.1 hooks: @@ -35,7 +35,7 @@ repos: rev: v2.32.1 hooks: - id: pyupgrade - args: ['--py37-plus'] + args: ['--py38-plus'] - repo: meta hooks: - id: check-hooks-apply diff --git a/.readthedocs.yml b/.readthedocs.yml index 3bc609f3..4f3978ab 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -19,7 +19,7 @@ formats: all # Optionally set the version of Python and requirements required to build your docs python: - version: "3.7" + version: "3.8" install: - method: pip path: . diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 45dab7da..598d5e55 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -85,7 +85,7 @@ Ready to contribute? Here's how to set up ``clisops`` for local development. tests, including testing other Python versions with tox: $ flake8 clisops tests - $ black --target-version py36 clisops tests + $ black --target-version py38 clisops tests $ python setup.py test # (or pytest) $ tox @@ -145,6 +145,6 @@ Before you submit a pull request, check that it meets these guidelines: #. If the pull request adds functionality, the docs should be updated. Put your new functionality into a function with a docstring, and add the feature to the list in README.md. -#. The pull request should work for Python 3.7, 3.8, and 3.9. Check +#. The pull request should work for Python 3.8, 3.9, and 3.10. Check https://github.com/roocs/clisops/actions and make sure that the tests pass for all supported Python versions. diff --git a/environment.yml b/environment.yml index 590fdc74..83497ffb 100644 --- a/environment.yml +++ b/environment.yml @@ -3,6 +3,7 @@ channels: - conda-forge - defaults dependencies: + - python >=3.8 - pip - bottleneck>=1.3.1,<1.4 - cf_xarray>=0.7.0 diff --git a/setup.py b/setup.py index c83bd58d..e472430e 100644 --- a/setup.py +++ b/setup.py @@ -58,9 +58,9 @@ "Operating System :: POSIX :: Linux", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Topic :: Security", "Topic :: Internet", "Topic :: Scientific/Engineering", @@ -70,7 +70,7 @@ ], description="clisops - climate simulation operations.", license=__license__, - python_requires=">=3.7.0", + python_requires=">=3.8.0", install_requires=requirements, long_description=_long_description, long_description_content_type="text/x-rst", diff --git a/tox.ini b/tox.ini index 4a3eed38..d0c88553 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{37,38,39}, black, docs +envlist = py{38,39,310}, black, docs requires = pip >= 21.0 opts = -v @@ -16,7 +16,7 @@ deps = black commands = flake8 clisops tests - black --check --target-version py37 clisops tests --exclude tests/mini-esgf-data + black --check --target-version py38 clisops tests --exclude tests/mini-esgf-data [testenv:docs] extras = docs From 4da414a064e4c4b0bfb1a8220658ee4def96f9d4 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 13 Jun 2022 15:40:00 -0400 Subject: [PATCH 49/60] Python3.7 needed for RtD --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 4f3978ab..3bc609f3 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -19,7 +19,7 @@ formats: all # Optionally set the version of Python and requirements required to build your docs python: - version: "3.8" + version: "3.7" install: - method: pip path: . From 5cb336869309795da796d5aecb071e619ee1632b Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 13 Jun 2022 15:45:19 -0400 Subject: [PATCH 50/60] adjust tox builds --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 948cb064..7b722b41 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -33,13 +33,13 @@ jobs: matrix: include: - python-version: "3.8" - tox-env: py37 + tox-env: py38 allowed_to_fail: false - python-version: "3.9" - tox-env: py38 + tox-env: py39 allowed_to_fail: false - python-version: "3.10" - tox-env: py39 + tox-env: py310 allowed_to_fail: false steps: - uses: actions/checkout@v3 From 32a779db770fc7b5e71d2ac0a2b72219d22dc6ec Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 13 Jun 2022 15:53:53 -0400 Subject: [PATCH 51/60] new sphinx requires explicitly set language --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index f79c6f03..7132dd73 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -88,7 +88,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. From 2fda6b8d5cbc8935149df633affd4e5515770e5d Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 13 Jun 2022 15:57:27 -0400 Subject: [PATCH 52/60] update HISTORY.rst --- HISTORY.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index c2c6dc07..219fbad5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,6 +1,13 @@ Version History =============== +v0.9.2 (unreleased) +------------------- + +Breaking Changes +^^^^^^^^^^^^^^^^ +* Support has been dropped for Python3.7 and extended to Python3.10. Python3.7 is no longer tested in GitHub actions. + v0.9.1 (2022-05-12) ------------------- From 5dae75e2129b665c56f10b4eece265175d967551 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 13 Jun 2022 17:28:26 -0400 Subject: [PATCH 53/60] use newer readthedocs build image --- .readthedocs.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 3bc609f3..3f1da880 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -10,24 +10,18 @@ sphinx: configuration: docs/conf.py fail_on_warning: true -# Build documentation with MkDocs -#mkdocs: -# configuration: mkdocs.yml - # Optionally build your docs in additional formats such as PDF and ePub formats: all # Optionally set the version of Python and requirements required to build your docs python: - version: "3.7" install: - method: pip path: . extra_requirements: - docs -# conda: -# environment: docs/environment.yml - build: - image: stable + os: ubuntu-20.04 + tools: + python: "3.8" From 918e3c45cd3a806e414896b097c6459853fbf972 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 14 Jun 2022 10:46:51 -0400 Subject: [PATCH 54/60] do not build epud --- .readthedocs.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 3f1da880..a05caf46 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -11,7 +11,8 @@ sphinx: fail_on_warning: true # Optionally build your docs in additional formats such as PDF and ePub -formats: all +formats: + - pdf # Optionally set the version of Python and requirements required to build your docs python: From 041ab84cf568dc1cc2dca284b52c726c63e84ae2 Mon Sep 17 00:00:00 2001 From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 14 Jun 2022 11:08:51 -0400 Subject: [PATCH 55/60] Modify xarray module path in documentation for nicer type hinting --- docs/conf.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 7132dd73..1845c872 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -22,9 +22,12 @@ import os import sys -sys.path.insert(0, os.path.abspath("..")) +import xarray + +xarray.DataArray.__module__ = "xarray" +xarray.Dataset.__module__ = "xarray" -import sphinx_rtd_theme +sys.path.insert(0, os.path.abspath("..")) # -- General configuration --------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. From 7b83753b990ec44c401d2ec72840ecc62205c2d7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 4 Jul 2022 10:10:10 +0000 Subject: [PATCH 56/60] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- clisops/core/regrid.py | 21 ++++++++++----------- clisops/ops/regrid.py | 2 +- tests/ops/test_regrid.py | 1 - 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/clisops/core/regrid.py b/clisops/core/regrid.py index 56e78d32..e99b7966 100644 --- a/clisops/core/regrid.py +++ b/clisops/core/regrid.py @@ -1,16 +1,15 @@ """Regrid module.""" +import functools import warnings from pathlib import Path from typing import Tuple, Union -import functools -from pkg_resources import parse_version import cf_xarray as cfxr import numpy as np import roocs_grids import scipy import xarray as xr - +from pkg_resources import parse_version XESMF_MINIMUM_VERSION = "0.6.0" @@ -288,21 +287,21 @@ def __str__(self): def __repr__(self): info = ( - "clisops {}\n".format(self.__str__()) + f"clisops {self.__str__()}\n" + ( - "Lat x Lon: {} x {}\n".format(self.nlat, self.nlon) + f"Lat x Lon: {self.nlat} x {self.nlon}\n" if self.type != "irregular" else "" ) - + "Gridcells: {}\n".format(self.ncells) - + "Format: {}\n".format(self.format) - + "Type: {}\n".format(self.type) - + "Extent: {}\n".format(self.extent) - + "Source: {}\n".format(self.source) + + f"Gridcells: {self.ncells}\n" + + f"Format: {self.format}\n" + + f"Type: {self.type}\n" + + f"Extent: {self.extent}\n" + + f"Source: {self.source}\n" + "Bounds? {}\n".format( self.lat_bnds is not None and self.lon_bnds is not None ) - + "Permanent Mask: {}".format(self.mask) + + f"Permanent Mask: {self.mask}" ) return info diff --git a/clisops/ops/regrid.py b/clisops/ops/regrid.py index 385bcce3..cfd55a74 100644 --- a/clisops/ops/regrid.py +++ b/clisops/ops/regrid.py @@ -2,8 +2,8 @@ from typing import List, Optional, Tuple, Union import xarray as xr - from loguru import logger + from clisops.core import Grid, Weights from clisops.core import regrid as core_regrid from clisops.ops.base_operation import Operation diff --git a/tests/ops/test_regrid.py b/tests/ops/test_regrid.py index b107ca38..d9409264 100644 --- a/tests/ops/test_regrid.py +++ b/tests/ops/test_regrid.py @@ -13,7 +13,6 @@ from .._common import CMIP5_MRSOS_ONE_TIME_STEP - XESMF_IMPORT_MSG = "xESMF >= 0.6.0 is needed for regridding functionalities." From 2aad0e41033796ea4d7a96707f7c05fc00b5066d Mon Sep 17 00:00:00 2001 From: Martin Schupfner Date: Mon, 4 Jul 2022 15:45:55 +0200 Subject: [PATCH 57/60] Edit import order core/__init__.py --- clisops/core/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clisops/core/__init__.py b/clisops/core/__init__.py index 75479579..bccb53d7 100644 --- a/clisops/core/__init__.py +++ b/clisops/core/__init__.py @@ -1,4 +1,3 @@ -from .regrid import Grid, Weights, regrid from .subset import ( create_mask, subset_bbox, @@ -10,3 +9,4 @@ subset_time_by_components, subset_time_by_values, ) +from .regrid import Grid, Weights, regrid From b57083e625068a0c0c94ac7b5d943bcaddf0ee48 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 4 Jul 2022 13:46:38 +0000 Subject: [PATCH 58/60] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- clisops/core/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clisops/core/__init__.py b/clisops/core/__init__.py index bccb53d7..75479579 100644 --- a/clisops/core/__init__.py +++ b/clisops/core/__init__.py @@ -1,3 +1,4 @@ +from .regrid import Grid, Weights, regrid from .subset import ( create_mask, subset_bbox, @@ -9,4 +10,3 @@ subset_time_by_components, subset_time_by_values, ) -from .regrid import Grid, Weights, regrid From 6058eb92ce8d57d79f51878d86fa1a6d798e664b Mon Sep 17 00:00:00 2001 From: sol1105 Date: Tue, 5 Jul 2022 18:28:13 +0200 Subject: [PATCH 59/60] Updating environment.yml - adding numpy requirements of numba, exempting clisops/core/__init__.py from pre-commit import reordering --- .pre-commit-config.yaml | 1 + clisops/core/__init__.py | 3 ++- environment.yml | 4 ++-- requirements.txt | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index def4f623..9b375770 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,6 +26,7 @@ repos: hooks: - id: isort args: ['--profile', 'black'] + exclude: clisops/core/__init__.py #- repo: https://github.com/pycqa/pydocstyle # rev: 6.1.1 # hooks: diff --git a/clisops/core/__init__.py b/clisops/core/__init__.py index 75479579..6800c6cc 100644 --- a/clisops/core/__init__.py +++ b/clisops/core/__init__.py @@ -1,4 +1,3 @@ -from .regrid import Grid, Weights, regrid from .subset import ( create_mask, subset_bbox, @@ -10,3 +9,5 @@ subset_time_by_components, subset_time_by_values, ) + +from .regrid import Grid, Weights, regrid diff --git a/environment.yml b/environment.yml index 83497ffb..8ef4b51e 100644 --- a/environment.yml +++ b/environment.yml @@ -3,7 +3,7 @@ channels: - conda-forge - defaults dependencies: - - python >=3.8 + - python>=3.8 - pip - bottleneck>=1.3.1,<1.4 - cf_xarray>=0.7.0 @@ -12,7 +12,7 @@ dependencies: - geopandas>=0.7 - loguru>=0.5.3 - netCDF4>=1.4 - - numpy>=1.16 + - numpy>=1.16<1.22 - pandas>=1.0.3,<1.4 - poppler>=0.67 - pygeos>=0.9 diff --git a/requirements.txt b/requirements.txt index ba08e41b..24f2f113 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -numpy>=1.16 +numpy>=1.16<1.22 xarray>=0.15 pandas>=1.0.3,<1.4 cftime>=1.4.1 From 6798264dc3eac5c39378983a84a463b762ce3703 Mon Sep 17 00:00:00 2001 From: sol1105 Date: Tue, 5 Jul 2022 18:32:07 +0200 Subject: [PATCH 60/60] Fix typo in environment.yml --- environment.yml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 8ef4b51e..59d6fddd 100644 --- a/environment.yml +++ b/environment.yml @@ -12,7 +12,7 @@ dependencies: - geopandas>=0.7 - loguru>=0.5.3 - netCDF4>=1.4 - - numpy>=1.16<1.22 + - numpy>=1.16,<1.22 - pandas>=1.0.3,<1.4 - poppler>=0.67 - pygeos>=0.9 diff --git a/requirements.txt b/requirements.txt index 24f2f113..f853ca10 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -numpy>=1.16<1.22 +numpy>=1.16,<1.22 xarray>=0.15 pandas>=1.0.3,<1.4 cftime>=1.4.1