Merge branch 'main' into numpy2-and-tox

.github/workflows/ci.yml

@@ -8,10 +8,30 @@ on:
   # Allow manual runs through the web UI
   workflow_dispatch:
 
+<<<<<<< numpy2-and-tox
 # Only allow one run per git ref at a time
 concurrency:
   group: '${{ github.workflow }}-${{ github.ref }}'
   cancel-in-progress: true
+=======
+jobs:
+  tests:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ['3.9', '3.10', '3.11']
+    steps:
+    - uses: actions/checkout@v3
+
+    - uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install package and test requirements
+      run: |
+        pip install -e .[tests]
+        pip check
+>>>>>>> main
 
 jobs:
   core:

cdflib/cdfwrite.py

@@ -978,12 +978,18 @@ def _write_var_data_nonsparse(
             epoch16 = []
             if hasattr(indata, "__len__") and not isinstance(indata, str):
                 adata = indata[0]
-                if isinstance(adata, complex):
-                    recs = len(indata)
-                    for x in range(0, recs):
-                        epoch16.append(indata[x].real)
-                        epoch16.append(indata[x].imag)
-                    indata = np.array(epoch16)
+                if not isinstance(adata, complex):
+                    try:
+                        indata = np.asarray(indata).astype(np.complex128)
+                    except:
+                        raise ValueError(
+                            f"Data for variable {var} must be castable to a 128-bit complex number when data type is CDF_EPOCH16."
+                        )
+                recs = len(indata)
+                for x in range(0, recs):
+                    epoch16.append(indata[x].real)
+                    epoch16.append(indata[x].imag)
+                indata = np.array(epoch16)
             else:
                 if isinstance(indata, complex):
                     epoch16.append(indata.real)
@@ -2330,11 +2336,11 @@ def _convert_data(self, data_type: int, num_elems: int, num_values: int, indata:
             return recs, struct.pack(form2, *datau)
         elif isinstance(indata, np.ndarray):
             if data_type == self.CDF_CHAR or data_type == self.CDF_UCHAR:
-                size = indata.size
+                size = len(np.atleast_1d(indata))
                 odata = ""
                 if size >= 1:
                     for x in range(0, size):
-                        if hasattr(indata, "__len__"):
+                        if indata.ndim > 0:
                             adata = indata[x]
                         else:
                             adata = indata
@@ -2345,7 +2351,7 @@ def _convert_data(self, data_type: int, num_elems: int, num_values: int, indata:
                         elif isinstance(adata, np.ndarray):
                             size2 = adata.size
                             for y in range(0, size2):
-                                if hasattr(adata, "__len__"):
+                                if adata.ndim > 0:
                                     bdata = adata[y]
                                 else:
                                     bdata = adata
@@ -2549,7 +2555,7 @@ def _checklistofNums(obj: Any) -> bool:
         or if any pre-processing needs to occur. Numbers and datetime64 objects can be immediately converted.
         """
         if hasattr(obj, "__len__"):
-            return bool(all(obj)) and all((isinstance(elem, numbers.Number) or isinstance(elem, np.datetime64)) for elem in obj)
+            return all((isinstance(elem, numbers.Number) or isinstance(elem, np.datetime64)) for elem in obj)
         else:
             return isinstance(obj, numbers.Number) or isinstance(obj, np.datetime64)
 

cdflib/epochs.py

@@ -150,10 +150,11 @@ def breakdown(epochs: epoch_types) -> np.ndarray:
         elif epochs.dtype.type == np.complex128:
             return CDFepoch.breakdown_epoch16(epochs)
         else:
-            raise TypeError("Not sure how to handle type {}".format(type(epochs)))
+            raise TypeError(f"Not sure how to handle type {epochs.dtype}")
 
     @staticmethod
     def _compose_date(
+        nat_positions: npt.NDArray,
         years: npt.NDArray,
         months: npt.NDArray,
         days: npt.NDArray,
@@ -174,18 +175,29 @@ def _compose_date(
         vals = (v for v in (years, months, days, hours, minutes, seconds, milliseconds, microseconds, nanoseconds) if v is not None)
 
         arrays: List[npt.NDArray[np.datetime64]] = [np.array(v, dtype=t) for t, v in zip(types, vals)]
-        return np.array(sum(arrays))
+        total_datetime = np.array(sum(arrays))
+        total_datetime = np.where(nat_positions, np.datetime64("NaT"), total_datetime)
+        return total_datetime
 
     @classmethod
     def to_datetime(cls, cdf_time: epoch_types) -> npt.NDArray[np.datetime64]:
         """
-        Converts CDF epoch argument to numpy.datetime64. This method
-        converts a scalar, or array-like. Precision is only kept to the
-        nearest microsecond.
+        Converts CDF epoch argument to numpy.datetime64.
+
+        Parameters:
+            cdf_time: NumPy scalar/arrays to convert. np.int64 will be converted to cdf_tt2000, np.complex128 will be converted to cdf_epoch16, and floats will be converted to cdf_epoch.
+
+        Notes:
+            Because of datetime64 limitations, CDF_EPOCH16 precision is only kept to the nearest nanosecond.
+
         """
         times = cls.breakdown(cdf_time)
         times = np.atleast_2d(times)
-        return cls._compose_date(*times.T[:9]).astype("datetime64[us]")
+
+        fillval_locations = np.all((times[:, 0:7] == [9999, 12, 31, 23, 59, 59, 999]), axis=1)
+        padval_locations = np.all((times[:, 0:7] == [0, 1, 1, 0, 0, 0, 0]), axis=1)
+        nan_locations = np.logical_or(fillval_locations, padval_locations)
+        return cls._compose_date(nan_locations, *times.T[:9]).astype("datetime64[ns]")
 
     @staticmethod
     def unixtime(cdf_time: npt.ArrayLike) -> Union[float, npt.NDArray]:
@@ -216,7 +228,96 @@ def unixtime(cdf_time: npt.ArrayLike) -> Union[float, npt.NDArray]:
         return _squeeze_or_scalar_real(unixtime)
 
     @staticmethod
-    def compute(datetimes: npt.ArrayLike) -> Union[float, complex, npt.NDArray]:
+    def timestamp_to_cdfepoch(unixtime_data: npt.ArrayLike) -> np.ndarray:
+        """
+        Converts a unix timestamp to CDF_EPOCH, the number of milliseconds since the year 0.
+        """
+        # Make sure the object is iterable.  Sometimes numpy arrays claim to be iterable when they aren't.
+        times = np.atleast_1d(unixtime_data)
+
+        cdf_time_data = []
+        for ud in times:
+            if not np.isnan(ud):
+                dt = np.datetime64(int(ud * 1000), "ms")
+                dt_to_convert = [
+                    dt.item().year,
+                    dt.item().month,
+                    dt.item().day,
+                    dt.item().hour,
+                    dt.item().minute,
+                    dt.item().second,
+                    int(dt.item().microsecond / 1000),
+                ]
+                converted_data = CDFepoch.compute(dt_to_convert)
+            else:
+                converted_data = np.nan
+            cdf_time_data.append(converted_data)
+
+        return np.array(cdf_time_data)
+
+    @staticmethod
+    def timestamp_to_cdfepoch16(unixtime_data: npt.ArrayLike) -> np.ndarray:
+        """
+        Converts a unix timestamp to CDF_EPOCH16
+        """
+        # Make sure the object is iterable.  Sometimes numpy arrays claim to be iterable when they aren't.
+        times = np.atleast_1d(unixtime_data)
+
+        cdf_time_data = []
+        for ud in times:
+            if not np.isnan(ud):
+                dt = np.datetime64(int(ud * 1000000), "us")
+                dt_to_convert = [
+                    dt.item().year,
+                    dt.item().month,
+                    dt.item().day,
+                    dt.item().hour,
+                    dt.item().minute,
+                    dt.item().second,
+                    int(dt.item().microsecond / 1000),
+                    int(dt.item().microsecond % 1000),
+                    0,
+                    0,
+                ]
+                converted_data = CDFepoch.compute(dt_to_convert)
+            else:
+                converted_data = np.nan
+            cdf_time_data.append(converted_data)
+
+        return np.array(cdf_time_data)
+
+    @staticmethod
+    def timestamp_to_tt2000(unixtime_data: npt.ArrayLike) -> np.ndarray:
+        """
+        Converts a unix timestamp to CDF_TIME_TT2000
+        """
+        # Make sure the object is iterable.  Sometimes numpy arrays claim to be iterable when they aren't.
+        times = np.atleast_1d(unixtime_data)
+
+        cdf_time_data = []
+        for ud in times:
+            if not np.isnan(ud):
+                dt = np.datetime64(int(ud * 1000000), "us")
+                dt_to_convert = [
+                    dt.item().year,
+                    dt.item().month,
+                    dt.item().day,
+                    dt.item().hour,
+                    dt.item().minute,
+                    dt.item().second,
+                    int(dt.item().microsecond / 1000),
+                    int(dt.item().microsecond % 1000),
+                    0,
+                ]
+                converted_data = CDFepoch.compute(dt_to_convert)
+            else:
+                converted_data = np.nan
+            cdf_time_data.append(converted_data)
+
+        return np.array(cdf_time_data)
+
+    @staticmethod
+    def compute(datetimes: npt.ArrayLike) -> Union[int, float, complex, npt.NDArray]:
         """
         Computes the provided date/time components into CDF epoch value(s).
 
@@ -366,18 +467,26 @@ def breakdown_tt2000(tt2000: cdf_tt2000_type) -> np.ndarray:
         """
         Breaks down the epoch(s) into UTC components.
 
-        For CDF_EPOCH:
-                they are 7 date/time components: year, month, day,
-                hour, minute, second, and millisecond
-        For CDF_EPOCH16:
-                they are 10 date/time components: year, month, day,
-                hour, minute, second, and millisecond, microsecond,
-                nanosecond, and picosecond.
-        For TT2000:
-                they are 9 date/time components: year, month, day,
-                hour, minute, second, millisecond, microsecond,
-                nanosecond.
+        Calculate date and time from cdf_time_tt2000 integers
+
+        Parameters
+        ----------
+        epochs : array-like
+            Single, list, tuple, or np.array of tt2000 values
+
+        Returns
+        -------
+        components : ndarray
+            List or array of date and time values.  The last axis contains
+            (in order): year, month, day, hour, minute, second, millisecond,
+            microsecond, and nanosecond
+
+        Notes
+        -----
+        If a bad epoch is supplied, a fill date of 9999-12-31 23:59:59 and 999 ms, 999 us, and
+        999 ns is returned.
         """
+
         new_tt2000 = np.atleast_1d(tt2000).astype(np.longlong)
         count = len(new_tt2000)
         toutcs = np.zeros((9, count), dtype=int)
@@ -487,7 +596,14 @@ def breakdown_tt2000(tt2000: cdf_tt2000_type) -> np.ndarray:
         toutcs[7, :] = ma1
         toutcs[8, :] = na1
 
-        return np.squeeze(toutcs.T)
+        # Check standard fill and pad values
+        cdf_epoch_time_tt2000 = np.atleast_2d(toutcs.T)
+        fillval_locations = np.all(cdf_epoch_time_tt2000 == [1707, 9, 22, 12, 12, 10, 961, 224, 192], axis=1)
+        cdf_epoch_time_tt2000[fillval_locations] = [9999, 12, 31, 23, 59, 59, 999, 999, 999]
+        padval_locations = np.all(cdf_epoch_time_tt2000 == [1707, 9, 22, 12, 12, 10, 961, 224, 193], axis=1)
+        cdf_epoch_time_tt2000[padval_locations] = [0, 1, 1, 0, 0, 0, 0, 0, 0]
+
+        return np.squeeze(cdf_epoch_time_tt2000)
 
     @staticmethod
     def compute_tt2000(datetimes: npt.ArrayLike) -> Union[int, npt.NDArray[np.int64]]:
@@ -1098,7 +1214,10 @@ def breakdown_epoch16(epochs: cdf_epoch16_type) -> npt.NDArray:
         components = np.full(shape=cshape, fill_value=[9999, 12, 31, 23, 59, 59, 999, 999, 999, 999])
         for i, epoch16 in enumerate(new_epochs):
             # Ignore fill values
-            if (epoch16.real != -1.0e31) or (epoch16.imag != -1.0e31):
+            if (epoch16.real != -1.0e31) or (epoch16.imag != -1.0e31) or np.isnan(epoch16):
+                if (epoch16.imag == -1.0e30) or (epoch16.imag == -1.0e30):
+                    components[i] = [0, 1, 1, 0, 0, 0, 0, 0, 0, 0]
+                    continue
                 esec = -epoch16.real if epoch16.real < 0.0 else epoch16.real
                 efra = -epoch16.imag if epoch16.imag < 0.0 else epoch16.imag
 
@@ -1429,8 +1548,8 @@ def breakdown_epoch(epochs: cdf_epoch_type) -> np.ndarray:
         cshape.append(7)
         components = np.full(shape=cshape, fill_value=[9999, 12, 31, 23, 59, 59, 999])
         for i, epoch in enumerate(new_epochs):
-            # Ignore fill values
-            if epoch != -1.0e31:
+            # Ignore fill values and NaNs
+            if (epoch != -1.0e31) and not np.isnan(epoch):
                 esec = -epoch / 1000.0 if epoch < 0.0 else epoch / 1000.0
                 date_time = CDFepoch._calc_from_julian(esec, 0.0)
 
@@ -1441,6 +1560,8 @@ def breakdown_epoch(epochs: cdf_epoch_type) -> np.ndarray:
                     components = date_time[..., :7]
                 else:
                     components[i] = date_time[..., :7]
+            elif epoch == 0:
+                components[i] = [0, 1, 1, 0, 0, 0, 0]
 
         return np.squeeze(components)
 

cdflib/utils.py

@@ -8,22 +8,22 @@
 def _squeeze_or_scalar(arr: npt.ArrayLike) -> Union[npt.NDArray, Number]:
     arr = np.squeeze(arr)
     if arr.ndim == 0:
-        return arr.item()
+        return arr[()]
     else:
         return arr
 
 
 def _squeeze_or_scalar_real(arr: npt.ArrayLike) -> Union[npt.NDArray, float]:
     arr = np.squeeze(arr)
     if arr.ndim == 0:
-        return arr.item()
+        return arr[()]
     else:
         return arr
 
 
 def _squeeze_or_scalar_complex(arr: npt.ArrayLike) -> Union[npt.NDArray, complex]:
     arr = np.squeeze(arr)
     if arr.ndim == 0:
-        return arr.item()
+        return arr[()]
     else:
         return arr