WIP: PR for errors

[maxinelasp]

error in tests/glows/test_glows_l1a_cdf test_glows_cdf_generation

Error output:

self = <cdflib.cdfwrite.CDF object at 0x15e4a5030>, data_type = 51
num_elems = 1, num_values = 1, indata = -9223372036854775808

    def _convert_data(self, data_type: int, num_elems: int, num_values: int, indata: Any) -> Tuple[int, bytes]:
        """
        Converts "indata" into a byte stream
    
        Parameters
        ----------
        data_type : int
            The CDF file data type
    
        num_elems : int
            The number of elements in the data
    
        num_values : int
            The number of values in each record
    
        indata : (varies)
            The data to be converted
    
        Returns
        -------
        recs : int
            The number of records generated by converting indata
        odata : byte stream
            The stream of bytes to write to the CDF file
        """
    
        recSize = self._datatype_size(data_type, num_elems) * num_values
        # List or Tuple data
        if isinstance(indata, list) or isinstance(indata, tuple):
            if (num_values != 1) and (len(indata) != num_values):
                raise Exception("Use numpy for inputting multidimensional arrays")
            size = len(indata)
            if data_type == self.CDF_CHAR or data_type == self.CDF_UCHAR:
                odata = ""
                for x in range(0, size):
                    adata = indata[x]
                    if isinstance(adata, list) or isinstance(adata, tuple):
                        size2 = len(adata)
                        for y in range(0, size2):
                            odata += adata[y].ljust(num_elems, "\x00")
                    else:
                        size2 = 1
                        odata += adata.ljust(num_elems, "\x00")
                recs = int((size * size2) / num_values)
                return recs, odata.encode()
            else:
                try:
                    return self._numpy_to_bytes(data_type, num_values, num_elems, np.array(indata))
                except:
                    tofrom = self._convert_option()
                    dt_string = self._convert_type(data_type)
                    recs = int(size / num_values)
                    if data_type == self.CDF_EPOCH16 and isinstance(indata[0], complex):
                        complex_data = []
                        for x in range(0, recs):
                            complex_data.append(indata[x].real)
                            complex_data.append(indata[x].imag)
                        size = 2 * size
                        indata = complex_data
                    if data_type == self.CDF_EPOCH16 and not isinstance(indata[0], complex):
                        recs = int(recs / 2)
                    form = tofrom + str(size) + dt_string
                    return recs, struct.pack(form, *indata)
        elif isinstance(indata, bytes):
            tofrom = self._convert_option()
            recs = int(len(indata) / recSize)
            dt_string = self._convert_type(data_type)
            size = recs * num_values * num_elems
            if data_type == self.CDF_EPOCH16:
                size = size * 2
            form = str(size) + dt_string
            form2 = tofrom + form
            datau = struct.unpack(form, indata)
            return recs, struct.pack(form2, *datau)
        elif isinstance(indata, np.ndarray):
            if data_type == self.CDF_CHAR or data_type == self.CDF_UCHAR:
                size = indata.size
                odata = ""
                if size >= 1:
                    for x in range(0, size):
                        if hasattr(indata, "__len__"):
                            adata = indata[x]
                        else:
                            adata = indata
                        if isinstance(adata, list) or isinstance(adata, tuple):
                            size2 = len(adata)
                            for y in range(0, size2):
                                odata += str(adata[y]).ljust(num_elems, "\x00")
                        elif isinstance(adata, np.ndarray):
                            size2 = adata.size
                            for y in range(0, size2):
                                if hasattr(adata, "__len__"):
                                    bdata = adata[y]
                                else:
                                    bdata = adata
                                odata += str(bdata).ljust(num_elems, "\x00")
                        else:
                            size2 = 1
                            odata += str(adata).ljust(num_elems, "\x00")
                else:
                    adata = ""
                    size2 = 1
                    odata += str(adata).ljust(num_elems, "\x00")
                recs = int((size * size2) / num_values)
                return recs, odata.encode()
            else:
                return self._numpy_to_bytes(data_type, num_values, num_elems, indata)
        elif isinstance(indata, str):
            return 1, indata.ljust(num_elems, "\x00").encode()
        else:
            try:
                # Try converting the data to numpy
>               return self._numpy_to_bytes(data_type, num_values, num_elems, np.array(indata))

/Users/maha7656/Library/Caches/pypoetry/virtualenvs/imap-processing-pC-Ppmnw-py3.10/lib/python3.10/site-packages/cdflib/cdfwrite.py:2367: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <cdflib.cdfwrite.CDF object at 0x15e4a5030>, data_type = 51
num_values = 1, num_elems = 1, indata = array(-9223372036854775808)

    def _numpy_to_bytes(self, data_type: int, num_values: int, num_elems: int, indata: npt.NDArray) -> Tuple[int, bytes]:
        """
        Converts a numpy array of numbers into a byte stream
    
        Parameters
        ----------
        data_type : int
            The CDF file data type
    
        num_values : int
            The number of values in each record
    
        num_elems: int
            The number of elements in each value
    
        indata : (varies)
            The data to be converted
    
        Returns
        -------
        recs : int
            The number of records generated by converting indata
        odata : byte stream
            The stream of bytes to write to the CDF file
        """
        tofrom = self._convert_option()
        npdata = self._convert_nptype(data_type, indata)
        if indata.size == 0:  # Check if the data being read in is zero size
            recs = 0
        elif indata.size == num_values * num_elems:  # Check if only one record is being read in
            recs = 1
        else:
            recs = len(indata)
        dt_string = self._convert_type(data_type)
        if data_type == self.CDF_EPOCH16:
            num_elems = 2 * num_elems
        form = str(recs * num_values * num_elems) + dt_string
        form2 = tofrom + str(recs * num_values * num_elems) + dt_string
>       datau = struct.unpack(form, npdata)
E       struct.error: unpack requires a buffer of 1 bytes

/Users/maha7656/Library/Caches/pypoetry/virtualenvs/imap-processing-pC-Ppmnw-py3.10/lib/python3.10/site-packages/cdflib/cdfwrite.py:2252: error

During handling of the above exception, another exception occurred:

    def test_glows_cdf_generation():
        current_directory = Path(__file__).parent
        packet_path = current_directory / "glows_test_packet_20110921_v01.pkts"
>       output_files = glows_l1a(packet_path, "v001")

test_glows_l1a_cdf.py:34: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../../glows/l1/glows_l1a.py:35: in glows_l1a
    generated_files.append(write_cdf(dataset))
../../cdf/utils.py:101: in write_cdf
    xarray_to_cdf(
/Users/maha7656/Library/Caches/pypoetry/virtualenvs/imap-processing-pC-Ppmnw-py3.10/lib/python3.10/site-packages/cdflib/xarray/xarray_to_cdf.py:836: in xarray_to_cdf
    x.write_var(var_spec, var_attrs=var_att_dict, var_data=var_data)
/Users/maha7656/Library/Caches/pypoetry/virtualenvs/imap-processing-pC-Ppmnw-py3.10/lib/python3.10/site-packages/cdflib/cdfwrite.py:32: in ensure_open
    return func(self, *args, **kwargs)
/Users/maha7656/Library/Caches/pypoetry/virtualenvs/imap-processing-pC-Ppmnw-py3.10/lib/python3.10/site-packages/cdflib/cdfwrite.py:790: in write_var
    self._write_var_attrs(f, varNum, var_attrs, zVar)
/Users/maha7656/Library/Caches/pypoetry/virtualenvs/imap-processing-pC-Ppmnw-py3.10/lib/python3.10/site-packages/cdflib/cdfwrite.py:921: in _write_var_attrs
    offset = self._write_aedr(f, False, attrNum, varNum, data, dataType, numElems, zVar)
/Users/maha7656/Library/Caches/pypoetry/virtualenvs/imap-processing-pC-Ppmnw-py3.10/lib/python3.10/site-packages/cdflib/cdfwrite.py:1738: in _write_aedr
    recs, cdata = self._convert_data(dataType, numElems, 1, value)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <cdflib.cdfwrite.CDF object at 0x15e4a5030>, data_type = 51
num_elems = 1, num_values = 1, indata = -9223372036854775808

    def _convert_data(self, data_type: int, num_elems: int, num_values: int, indata: Any) -> Tuple[int, bytes]:
        """
        Converts "indata" into a byte stream
    
        Parameters
        ----------
        data_type : int
            The CDF file data type
    
        num_elems : int
            The number of elements in the data
    
        num_values : int
            The number of values in each record
    
        indata : (varies)
            The data to be converted
    
        Returns
        -------
        recs : int
            The number of records generated by converting indata
        odata : byte stream
            The stream of bytes to write to the CDF file
        """
    
        recSize = self._datatype_size(data_type, num_elems) * num_values
        # List or Tuple data
        if isinstance(indata, list) or isinstance(indata, tuple):
            if (num_values != 1) and (len(indata) != num_values):
                raise Exception("Use numpy for inputting multidimensional arrays")
            size = len(indata)
            if data_type == self.CDF_CHAR or data_type == self.CDF_UCHAR:
                odata = ""
                for x in range(0, size):
                    adata = indata[x]
                    if isinstance(adata, list) or isinstance(adata, tuple):
                        size2 = len(adata)
                        for y in range(0, size2):
                            odata += adata[y].ljust(num_elems, "\x00")
                    else:
                        size2 = 1
                        odata += adata.ljust(num_elems, "\x00")
                recs = int((size * size2) / num_values)
                return recs, odata.encode()
            else:
                try:
                    return self._numpy_to_bytes(data_type, num_values, num_elems, np.array(indata))
                except:
                    tofrom = self._convert_option()
                    dt_string = self._convert_type(data_type)
                    recs = int(size / num_values)
                    if data_type == self.CDF_EPOCH16 and isinstance(indata[0], complex):
                        complex_data = []
                        for x in range(0, recs):
                            complex_data.append(indata[x].real)
                            complex_data.append(indata[x].imag)
                        size = 2 * size
                        indata = complex_data
                    if data_type == self.CDF_EPOCH16 and not isinstance(indata[0], complex):
                        recs = int(recs / 2)
                    form = tofrom + str(size) + dt_string
                    return recs, struct.pack(form, *indata)
        elif isinstance(indata, bytes):
            tofrom = self._convert_option()
            recs = int(len(indata) / recSize)
            dt_string = self._convert_type(data_type)
            size = recs * num_values * num_elems
            if data_type == self.CDF_EPOCH16:
                size = size * 2
            form = str(size) + dt_string
            form2 = tofrom + form
            datau = struct.unpack(form, indata)
            return recs, struct.pack(form2, *datau)
        elif isinstance(indata, np.ndarray):
            if data_type == self.CDF_CHAR or data_type == self.CDF_UCHAR:
                size = indata.size
                odata = ""
                if size >= 1:
                    for x in range(0, size):
                        if hasattr(indata, "__len__"):
                            adata = indata[x]
                        else:
                            adata = indata
                        if isinstance(adata, list) or isinstance(adata, tuple):
                            size2 = len(adata)
                            for y in range(0, size2):
                                odata += str(adata[y]).ljust(num_elems, "\x00")
                        elif isinstance(adata, np.ndarray):
                            size2 = adata.size
                            for y in range(0, size2):
                                if hasattr(adata, "__len__"):
                                    bdata = adata[y]
                                else:
                                    bdata = adata
                                odata += str(bdata).ljust(num_elems, "\x00")
                        else:
                            size2 = 1
                            odata += str(adata).ljust(num_elems, "\x00")
                else:
                    adata = ""
                    size2 = 1
                    odata += str(adata).ljust(num_elems, "\x00")
                recs = int((size * size2) / num_values)
                return recs, odata.encode()
            else:
                return self._numpy_to_bytes(data_type, num_values, num_elems, indata)
        elif isinstance(indata, str):
            return 1, indata.ljust(num_elems, "\x00").encode()
        else:
            try:
                # Try converting the data to numpy
                return self._numpy_to_bytes(data_type, num_values, num_elems, np.array(indata))
            except:
                tofrom = self._convert_option()
                dt_string = self._convert_type(data_type)
                if data_type == self.CDF_EPOCH16:
                    num_elems = 2 * num_elems
                try:
                    recs = int(len(indata) / recSize)
                except Exception:
                    recs = 1
                if data_type == self.CDF_EPOCH16:
                    complex_data = []
                    if recs > 1:
                        for x in range(0, recs):
                            complex_data.append(indata[x].real)
                            complex_data.append(indata[x].imag)
                    else:
                        complex_data.append(indata.real)
                        complex_data.append(indata.imag)
                    indata = complex_data
                form = tofrom + str(recs * num_values * num_elems) + dt_string
                if recs * num_values * num_elems > 1:
                    return recs, struct.pack(form, *indata)
                else:
>                   return recs, struct.pack(form, indata)
E                   struct.error: argument for 's' must be a bytes object


/Users/maha7656/Library/Caches/pypoetry/virtualenvs/imap-processing-pC-Ppmnw-py3.10/lib/python3.10/site-packages/cdflib/cdfwrite.py:2391: error