Skip to content

Commit

Permalink
Fixing a bug with the attribute time conversions in xarray_to_cdf (#256)
Browse files Browse the repository at this point in the history 
* Fixing a bug with the attribute time conversions in xarray_to_cdf

* Fixing unit tests from previous changes

* Fixing issues that occurred with Windows ints in the unit tests
  • Loading branch information
@bryan-harter
bryan-harter authored May 8, 2024
1 parent e71ef36 commit d94252c
Show file tree
Hide file tree
Showing 5 changed files with 79 additions and 73 deletions.
4 changes: 2 additions & 2 deletions cdflib/epochs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,11 +181,11 @@ def to_datetime(cls, cdf_time: epoch_types) -> npt.NDArray[np.datetime64]:
"""
Converts CDF epoch argument to numpy.datetime64. This method
converts a scalar, or array-like. Precision is only kept to the
nearest microsecond.
nearest nanosecond.
"""
times = cls.breakdown(cdf_time)
times = np.atleast_2d(times)
return cls._compose_date(*times.T[:9]).astype("datetime64[us]")
return cls._compose_date(*times.T[:9]).astype("datetime64[ns]")

@staticmethod
def unixtime(cdf_time: npt.ArrayLike) -> Union[float, npt.NDArray]:
Expand Down
2 changes: 1 addition & 1 deletion cdflib/xarray/cdf_to_xarray.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -680,7 +680,7 @@ def _verify_dimension_sizes(created_data_vars: Dict[str, xr.Variable], created_c
)


def cdf_to_xarray(filename: str, to_datetime: bool = False, to_unixtime: bool = False, fillval_to_nan: bool = False) -> xr.Dataset:
def cdf_to_xarray(filename: str, to_datetime: bool = True, to_unixtime: bool = False, fillval_to_nan: bool = False) -> xr.Dataset:
"""
This function converts CDF files into XArray Dataset Objects.
Expand Down
65 changes: 37 additions & 28 deletions cdflib/xarray/xarray_to_cdf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -683,8 +683,14 @@ def _unixtime_to_cdf_time(unixtime_data: xr.DataArray, cdf_epoch: bool = False,
return cdfepoch.timestamp_to_tt2000(unixtime_data)


def _datetime_to_cdf_time(datetime_array: xr.DataArray, cdf_epoch: bool = False, cdf_epoch16: bool = False) -> npt.NDArray:
datetime_data = datetime_array.data
def _datetime_to_cdf_time(
datetime_array: xr.DataArray, cdf_epoch: bool = False, cdf_epoch16: bool = False, attribute_name: str = ""
) -> object:
if attribute_name:
datetime_data = datetime_array.attrs[attribute_name]
else:
datetime_data = datetime_array.data
datetime64_data = np.array(datetime_data, dtype="datetime64[ns]")
cdf_epoch = False
cdf_epoch16 = False
if "CDF_DATA_TYPE" in datetime_array.attrs:
Expand All @@ -693,18 +699,25 @@ def _datetime_to_cdf_time(datetime_array: xr.DataArray, cdf_epoch: bool = False,
elif datetime_array.attrs["CDF_DATA_TYPE"] == "CDF_EPOCH16":
cdf_epoch16 = True

cdf_time_data = np.array([])
for dd in datetime_data:
if cdf_epoch16 or cdf_epoch:
dtype_for_time_data = np.float64
else:
dtype_for_time_data = np.int64 # type: ignore

cdf_time_data = np.array([], dtype=dtype_for_time_data)
for dd in datetime64_data:
year = dd.astype("datetime64[Y]").astype(int) + 1970
month = dd.astype("datetime64[M]").astype(int) % 12 + 1
dd_to_convert = [
dd.year,
dd.month,
dd.day,
dd.hour,
dd.minute,
dd.second,
int(dd.microsecond / 1000),
int(dd.microsecond % 1000),
0,
dd.astype("datetime64[Y]").astype("int64") + 1970,
dd.astype("datetime64[M]").astype("int64") % 12 + 1,
((dd - np.datetime64(f"{year}-{month:02d}", "M")) / 86400000000000).astype("int64") + 1,
dd.astype("datetime64[h]").astype("int64") % 24,
dd.astype("datetime64[m]").astype("int64") % 60,
dd.astype("datetime64[s]").astype("int64") % 60,
dd.astype("datetime64[ms]").astype("int64") % 1000,
dd.astype("datetime64[us]").astype("int64") % 1000,
dd.astype("datetime64[ns]").astype("int64") % 1000,
0,
]
if cdf_epoch16:
Expand All @@ -713,7 +726,7 @@ def _datetime_to_cdf_time(datetime_array: xr.DataArray, cdf_epoch: bool = False,
converted_data = cdfepoch.compute(dd_to_convert[0:7])
else:
converted_data = cdfepoch.compute(dd_to_convert[0:9])
np.append(cdf_time_data, converted_data)
cdf_time_data = np.append(cdf_time_data, converted_data)
return cdf_time_data


Expand Down Expand Up @@ -984,11 +997,10 @@ def xarray_to_cdf(
elif d[var].attrs["CDF_DATA_TYPE"] == "CDF_EPOCH16":
cdf_epoch16 = True

if _is_datetime_array(d[var].data) and datetime_to_cdftt2000:
var_data = _datetime_to_cdf_time(d[var].data, cdf_epoch=cdf_epoch, cdf_epoch16=cdf_epoch16)
elif _is_datetime64_array(d[var].data) and datetime64_to_cdftt2000:
unixtime_from_datetime64 = d[var].data.astype("datetime64[ns]").astype("int64") / 1000000000
var_data = _unixtime_to_cdf_time(unixtime_from_datetime64, cdf_epoch=cdf_epoch, cdf_epoch16=cdf_epoch16)
if (_is_datetime_array(d[var].data) and datetime_to_cdftt2000) or (
_is_datetime64_array(d[var].data) and datetime64_to_cdftt2000
):
var_data = _datetime_to_cdf_time(d[var], cdf_epoch=cdf_epoch, cdf_epoch16=cdf_epoch16)
elif unix_time_to_cdf_time:
if _is_istp_epoch_variable(var) or (
DATATYPES_TO_STRINGS[cdf_data_type] in ("CDF_EPOCH", "CDF_EPOCH16", "CDF_TIME_TT2000")
Expand All @@ -998,23 +1010,20 @@ def xarray_to_cdf(
# Grab the attributes from xarray, and attempt to convert VALIDMIN and VALIDMAX to the same data type as the variable
var_att_dict = {}
for att in d[var].attrs:
if _is_datetime_array(d[var].attrs[att]):
att_data = _datetime_to_cdf_time(d[var].attrs[att], cdf_epoch=cdf_epoch, cdf_epoch16=cdf_epoch16)
var_att_dict[att] = [att_data, DATATYPES_TO_STRINGS[cdf_data_type]]
elif _is_datetime64_array(d[var].attrs[att]):
unixtime_from_datetime64 = d[var].attrs[att].astype("datetime64[ns]").astype("int64") / 1000000000
att_data = _unixtime_to_cdf_time(unixtime_from_datetime64, cdf_epoch=cdf_epoch, cdf_epoch16=cdf_epoch16)
var_att_dict[att] = d[var].attrs[att]
if (_is_datetime_array(d[var].attrs[att]) and datetime_to_cdftt2000) or (
_is_datetime64_array(d[var].attrs[att]) and datetime64_to_cdftt2000
):
att_data = _datetime_to_cdf_time(d[var], cdf_epoch=cdf_epoch, cdf_epoch16=cdf_epoch16, attribute_name=att)
var_att_dict[att] = [att_data, DATATYPES_TO_STRINGS[cdf_data_type]]
elif unix_time_to_cdf_time:
if "TIME_ATTRS" in d[var].attrs:
if att in d[var].attrs["TIME_ATTRS"]:
if DATATYPES_TO_STRINGS[cdf_data_type] in ("CDF_EPOCH", "CDF_EPOCH16", "CDF_TIME_TT2000"):
att_data = _unixtime_to_cdf_time(d[var].attrs[att], cdf_epoch=cdf_epoch, cdf_epoch16=cdf_epoch16)
var_att_dict[att] = [att_data, DATATYPES_TO_STRINGS[cdf_data_type]]
if (att == "VALIDMIN" or att == "VALIDMAX" or att == "FILLVAL") and istp:
elif (att == "VALIDMIN" or att == "VALIDMAX" or att == "FILLVAL") and istp:
var_att_dict[att] = [d[var].attrs[att], DATATYPES_TO_STRINGS[cdf_data_type]]
else:
var_att_dict[att] = d[var].attrs[att]

var_spec = {
"Variable": var,
Expand Down
4 changes: 2 additions & 2 deletions tests/test_epochs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ def test_parse_cdfepoch16():
parsed = cdfepoch.parse(x)
assert parsed == input_time

assert cdfepoch().to_datetime(parsed) == datetime(1694, 5, 1, 7, 42, 23, 543218)
assert cdfepoch().to_datetime(parsed).astype("datetime64[us]").item() == datetime(1694, 5, 1, 7, 42, 23, 543218)


def test_parse_cdftt2000():
Expand All @@ -227,7 +227,7 @@ def test_parse_cdftt2000():
parsed = cdfepoch.parse(x)
assert parsed == input_time

assert cdfepoch().to_datetime(parsed) == datetime(2004, 3, 1, 12, 24, 22, 351793)
assert cdfepoch().to_datetime(parsed).astype("datetime64[us]").item() == datetime(2004, 3, 1, 12, 24, 22, 351793)


def test_findepochrange_cdfepoch():
Expand Down
Loading

0 comments on commit d94252c

Please sign in to comment.