MNT: space_packet_parser update

imap_processing/ccsds/ccsds_data.py

@@ -42,10 +42,7 @@ class CcsdsData:
     def __init__(self, packet_header: dict):
         attributes = [field.name for field in fields(self)]
 
-        for key, item in packet_header.items():
-            value = (
-                item.derived_value if item.derived_value is not None else item.raw_value
-            )
+        for key, value in packet_header.items():
             if key in attributes:
                 setattr(self, key, value)
             else:

imap_processing/cdf/config/imap_ultra_l1a_variable_attrs.yaml

@@ -30,9 +30,11 @@ string_base_attrs:
   CATDESC: string metadata
   FIELDNAM: string_metadata
   FORMAT: A80
-  VAR_TYPE: metadata
+  LABLAXIS: "none"
+  VAR_TYPE: support_data
   DISPLAY_TYPE: no_plot
   DEPEND_0: epoch
+  UNITS: " "
 
 packet_data_attrs:
   CATDESC: packet data

imap_processing/codice/codice_l1a.py

@@ -28,6 +28,7 @@
 from imap_processing.codice.codice_l0 import decom_packets
 from imap_processing.codice.decompress import decompress
 from imap_processing.codice.utils import CODICEAPID
+from imap_processing.utils import convert_to_binary_string
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -541,6 +542,7 @@ def process_codice_l1a(file_path: Path, data_version: str) -> xr.Dataset:
         elif apid in constants.APIDS_FOR_SCIENCE_PROCESSING:
             # Extract the data
             science_values = packet_dataset.data.data[0]
+            science_values = convert_to_binary_string(science_values)
 
             # Get the four "main" parameters for processing
             table_id, plan_id, plan_step, view_id = get_params(packet_dataset)

imap_processing/codice/utils.py

@@ -7,8 +7,6 @@
 
 from enum import IntEnum
 
-import space_packet_parser
-
 
 class CODICEAPID(IntEnum):
     """Create ENUM for CoDICE APIDs."""
@@ -57,39 +55,3 @@ class CoDICECompression(IntEnum):
     LOSSLESS = 3
     LOSSY_A_LOSSLESS = 4
     LOSSY_B_LOSSLESS = 5
-
-
-def add_metadata_to_array(packet: space_packet_parser, metadata_arrays: dict) -> dict:
-    """
-    Add metadata to the metadata_arrays.
-
-    Parameters
-    ----------
-    packet : space_packet_parser.parser.Packet
-        CODICE data packet.
-    metadata_arrays : dict
-        Metadata arrays.
-
-    Returns
-    -------
-    metadata_arrays : dict
-        Updated metadata arrays with values.
-    """
-    ignore_list = [
-        "SPARE_1",
-        "SPARE_2",
-        "SPARE_3",
-        "SPARE_4",
-        "SPARE_5",
-        "SPARE_6",
-        "CHECKSUM",
-    ]
-
-    for key, value in packet.header.items():
-        metadata_arrays.setdefault(key, []).append(value.raw_value)
-
-    for key, value in packet.data.items():
-        if key not in ignore_list:
-            metadata_arrays.setdefault(key, []).append(value.raw_value)
-
-    return metadata_arrays

imap_processing/decom.py

@@ -8,7 +8,7 @@
 from pathlib import Path
 from typing import Union
 
-from space_packet_parser import parser, xtcedef
+from space_packet_parser import definitions
 
 
 def decom_packets(
@@ -32,9 +32,8 @@ def decom_packets(
     list
         List of all the unpacked data.
     """
-    packet_definition = xtcedef.XtcePacketDefinition(xtce_packet_definition)
-    packet_parser = parser.PacketParser(packet_definition)
+    packet_definition = definitions.XtcePacketDefinition(xtce_packet_definition)
 
     with open(packet_file, "rb") as binary_data:
-        packet_generator = packet_parser.generator(binary_data)
+        packet_generator = packet_definition.packet_generator(binary_data)
         return list(packet_generator)

imap_processing/glows/l0/decom_glows.py

@@ -3,7 +3,7 @@
 from enum import Enum
 from pathlib import Path
 
-from space_packet_parser import parser, xtcedef
+from space_packet_parser import definitions
 
 from imap_processing import imap_module_directory
 from imap_processing.ccsds.ccsds_data import CcsdsData
@@ -49,39 +49,28 @@ def decom_packets(
         f"{imap_module_directory}/glows/packet_definitions/GLX_COMBINED.xml"
     )
 
-    packet_definition = xtcedef.XtcePacketDefinition(xtce_document)
-    glows_parser = parser.PacketParser(packet_definition)
+    packet_definition = definitions.XtcePacketDefinition(xtce_document)
 
     histdata = []
     dedata = []
 
     filename = packet_file_path.name
 
     with open(packet_file_path, "rb") as binary_data:
-        glows_packets = glows_parser.generator(binary_data)
+        glows_packets = packet_definition.packet_generator(binary_data)
 
         for packet in glows_packets:
-            apid = packet.header["PKT_APID"].derived_value
+            apid = packet["PKT_APID"]
             # Do something with the packet data
             if apid == GlowsParams.HIST_APID.value:
-                values = [
-                    item.derived_value
-                    if item.derived_value is not None
-                    else item.raw_value
-                    for item in packet.data.values()
-                ]
+                values = [item.raw_value for item in packet.user_data.values()]
                 hist_l0 = HistogramL0(
                     __version__, filename, CcsdsData(packet.header), *values
                 )
                 histdata.append(hist_l0)
 
             if apid == GlowsParams.DE_APID.value:
-                values = [
-                    item.derived_value
-                    if item.derived_value is not None
-                    else item.raw_value
-                    for item in packet.data.values()
-                ]
+                values = [item.raw_value for item in packet.user_data.values()]
 
                 de_l0 = DirectEventL0(
                     __version__, filename, CcsdsData(packet.header), *values

imap_processing/glows/l0/glows_l0_data.py

@@ -114,17 +114,7 @@ class HistogramL0(GlowsL0):
     ELAVG: int
     ELVAR: int
     EVENTS: int
-    HISTOGRAM_DATA: bytearray
-
-    def __post_init__(self) -> None:
-        """Convert HISTOGRAM_DATA attribute from string to bytearray if needed."""
-        if isinstance(self.HISTOGRAM_DATA, str):
-            # Convert string output from space_packet_parser to bytearray
-            self.HISTOGRAM_DATA = bytearray(
-                int(self.HISTOGRAM_DATA, 2).to_bytes(
-                    len(self.HISTOGRAM_DATA) // 8, "big"
-                )
-            )
+    HISTOGRAM_DATA: bytes
 
 
 @dataclass

imap_processing/hi/l1a/histogram.py

@@ -4,6 +4,7 @@
 import xarray as xr
 
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.utils import convert_to_binary_string
 
 # define the names of the 24 counter arrays
 # contained in the histogram packet
@@ -59,10 +60,11 @@ def create_dataset(input_ds: xr.Dataset) -> xr.Dataset:
     # TODO: Look into avoiding the for-loops below
     #       It seems like we could try to reshape the arrays and do some numpy
     #       broadcasting rather than for-loops directly here
-    for i_epoch, counters_binary_data in enumerate(input_ds["counters"].data):
+    for i_epoch, counters_bytes_data in enumerate(input_ds["counters"].data):
+        binary_str_val = convert_to_binary_string(counters_bytes_data)
         # unpack 24 arrays of 90 12-bit unsigned integers
         counter_ints = [
-            int(counters_binary_data[i * 12 : (i + 1) * 12], 2) for i in range(90 * 24)
+            int(binary_str_val[i * 12 : (i + 1) * 12], 2) for i in range(90 * 24)
         ]
         # populate the dataset with the unpacked integers
         for i_counter, counter in enumerate(

imap_processing/hi/l1a/science_direct_event.py

@@ -5,6 +5,7 @@
 
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.spice.time import met_to_j2000ns
+from imap_processing.utils import convert_to_binary_string
 
 # TODO: read LOOKED_UP_DURATION_OF_TICK from
 # instrument status summary later. This value
@@ -328,8 +329,9 @@ def science_direct_event(packets_data: xr.Dataset) -> xr.Dataset:
     # end of the list. This way, I don't need to flatten
     # the list later.
     for i, data in enumerate(packets_data["de_tof"].data):
+        binary_str_val = convert_to_binary_string(data)
         # break binary stream data into unit of 48-bits
-        event_48bits_list = break_into_bits_size(data)
+        event_48bits_list = break_into_bits_size(binary_str_val)
         # parse 48-bits into meaningful data such as metaevent or direct event
         de_data_list.extend([parse_direct_event(event) for event in event_48bits_list])
         # add packet time to packet_met_time

imap_processing/hit/l0/data_classes/housekeeping.py

@@ -213,7 +213,7 @@ class Housekeeping(HITBase):
 
     def __init__(
         self,
-        packet: space_packet_parser.parser.Packet,
+        packet: space_packet_parser.packets.CCSDSPacket,
         software_version: str,
         packet_file_name: str,
     ):

imap_processing/hit/l0/decom_hit.py

@@ -5,6 +5,8 @@
 import numpy as np
 import xarray as xr
 
+from imap_processing.utils import convert_to_binary_string
+
 # TODO: Consider moving global values into a config file
 
 # Structure to hold binary details for a
@@ -357,7 +359,9 @@ def assemble_science_frames(sci_dataset: xr.Dataset) -> xr.Dataset:
     # Convert sequence flags and counters to NumPy arrays for vectorized operations
     seq_flgs = sci_dataset.seq_flgs.values
     seq_ctrs = sci_dataset.src_seq_ctr.values
-    science_data = sci_dataset.science_data.values
+    science_data = [
+        convert_to_binary_string(data) for data in sci_dataset.science_data.values
+    ]
     epoch_data = sci_dataset.epoch.values
 
     # Number of packets in the file

imap_processing/hit/l0/utils/hit_base.py

@@ -31,19 +31,19 @@ class HITBase:
     packet_file_name: str
     ccsds_header: CcsdsData
 
-    def parse_data(self, packet: space_packet_parser.parser.Packet) -> None:
+    def parse_data(self, packet: space_packet_parser.packets.CCSDSPacket) -> None:
         """
         Parse Lo L0 packet data.
 
         Parameters
         ----------
-        packet : space_packet_parser.parser.Packet
+        packet : space_packet_parser.packets.CCSDSPacket
             A single Lo L0 packet from space packet parser.
         """
         attributes = [field.name for field in fields(self)]
 
         # For each item in packet, assign it to the matching attribute in the class.
-        for key, item in packet.data.items():
+        for key, item in packet.user_data.items():
             value = (
                 item.derived_value if item.derived_value is not None else item.raw_value
             )