Process IMAP-Hi Housekeeping data and create CDF file (#371)

- added housekeeping process using new space_packet_parser.
- create CDF
- updated direct event code to have strong datatype in function definition

imap_processing/hi/hi_cdf_attrs.py

@@ -137,8 +137,25 @@
 # descriptor: instrument name - This is in global attributes
 # data_type: <data_level>_<descriptor> - this is here in DataLevelAttrs
 hi_de_l1a_attrs = GlobalDataLevelAttrs(
-    data_type="L1A>l1a_de",
+    data_type="L1A-de>Level-1A Direct Event",
     logical_source="imap_hi_l1a_de",
     logical_source_desc=("IMAP-HI Instrument Level-1A Direct Event Data."),
     instrument_base=hi_base,
 )
+
+hi_hk_l1a_attrs = GlobalDataLevelAttrs(
+    data_type="L1A-hk>Level-1A Housekeeping",
+    logical_source="imap_hi_l1a_hk",
+    logical_source_desc=("IMAP-HI Instrument Level-1A Housekeeping Data."),
+    instrument_base=hi_base,
+)
+
+hi_hk_l1a_metadata_attrs = ScienceAttrs(
+    validmin=0,
+    validmax=GlobalConstants.INT_MAXVAL,
+    depend_0="epoch",
+    format="I12",
+    units="int",
+    var_type="support_data",
+    variable_purpose="PRIMARY",
+)

imap_processing/hi/l1a/hi_l1a.py

@@ -2,10 +2,13 @@
 import logging
 
 from imap_processing.hi.l0 import decom_hi
+from imap_processing.hi.l1a.housekeeping import create_dataset
 from imap_processing.hi.l1a.science_direct_event import science_direct_event
 from imap_processing.hi.utils import HIAPID
 from imap_processing.utils import group_by_apid
 
+logger = logging.getLogger(__name__)
+
 
 def hi_l1a(packet_file_path: str):
     """Process IMAP raw data to l1a.
@@ -32,14 +35,17 @@ def hi_l1a(packet_file_path: str):
             # TODO: Add processing for science count data
             continue
         elif apid == HIAPID.H45_SCI_DE:
-            logging.debug(
+            logger.info(
                 "Processing direct event data for [%s] packets", HIAPID.H45_SCI_CNT.name
             )
 
             data = science_direct_event(grouped_data[apid])
             processed_data.append(data)
         elif apid == HIAPID.H45_APP_NHK:
-            # TODO: Add processing for housekeeping data
-            continue
+            logger.info(
+                "Processing housekeeping data for [%s] packets", HIAPID.H45_APP_NHK.name
+            )
+            data = create_dataset(grouped_data[apid])
+            processed_data.append(data)
 
     return processed_data

imap_processing/hi/l1a/housekeeping.py

@@ -0,0 +1,80 @@
+"""Unpack IMAP-Hi housekeeping data."""
+import collections
+import dataclasses
+
+import xarray as xr
+from space_packet_parser.parser import Packet
+
+from imap_processing.cdf.global_attrs import ConstantCoordinates
+from imap_processing.cdf.utils import calc_start_time
+from imap_processing.hi.hi_cdf_attrs import (
+    hi_hk_l1a_attrs,
+    hi_hk_l1a_metadata_attrs,
+)
+
+
+def create_dataset(packets: list[Packet]) -> xr.Dataset:
+    """Create dataset for each metadata field.
+
+    Parameters
+    ----------
+    packets : list[Packet]
+        packet list
+
+    Returns
+    -------
+    xr.dataset
+        dataset with all metadata field data in xr.DataArray
+    """
+    metadata_arrays = collections.defaultdict(list)
+    description_dict = {}
+
+    for data_packet in packets:
+        # Add metadata to array
+        for key, value in (data_packet.header | data_packet.data).items():
+            # convert key to lower case to match SPDF requirement
+            data_key = key.lower()
+            metadata_arrays[data_key].append(value.raw_value)
+            # description should be same for all packets
+            description_dict[data_key] = (
+                value.long_description or value.short_description
+            )
+
+    epoch_converted_time = [
+        calc_start_time(sc_time) for sc_time in metadata_arrays["ccsds_met"]
+    ]
+
+    epoch_time = xr.DataArray(
+        epoch_converted_time,
+        name="epoch",
+        dims=["epoch"],
+        attrs=ConstantCoordinates.EPOCH,
+    )
+
+    dataset = xr.Dataset(
+        coords={"epoch": epoch_time},
+        attrs=hi_hk_l1a_attrs.output(),
+    )
+
+    # create xarray dataset for each metadata field
+    for key, value in metadata_arrays.items():
+        if key == "instr_specific":
+            # TODO: find out why this key has data
+            # type as byte instead of uint or int like
+            # other keys. Ask Vivek.
+            continue
+        # replace description and fieldname
+        data_attrs = dataclasses.replace(
+            hi_hk_l1a_metadata_attrs,
+            catdesc=description_dict[key],
+            fieldname=key,
+            label_axis=key,
+            depend_0="epoch",
+        )
+        dataset[key] = xr.DataArray(
+            value,
+            dims=["epoch"],
+            attrs=data_attrs.output(),
+        )
+
+    return dataset

imap_processing/hi/l1a/science_direct_event.py

@@ -2,6 +2,7 @@
 
 import numpy as np
 import xarray as xr
+from space_packet_parser.parser import Packet
 
 from imap_processing import launch_time
 from imap_processing.cdf.global_attrs import ConstantCoordinates
@@ -19,7 +20,7 @@
 MICROSECOND_TO_NS = 1e3
 
 
-def get_direct_event_time(time_in_ns):
+def get_direct_event_time(time_in_ns: int) -> np.datetime64:
     """Create MET(Mission Elapsed Time) time using input times.
 
     Parameters
@@ -37,7 +38,7 @@ def get_direct_event_time(time_in_ns):
     return met_datetime
 
 
-def parse_direct_event(event_data: str):
+def parse_direct_event(event_data: str) -> dict:
     """Parse event data.
 
     IMAP-Hi direct event data information is stored in
@@ -144,7 +145,7 @@ def parse_direct_event(event_data: str):
     }
 
 
-def break_into_bits_size(binary_data: str):
+def break_into_bits_size(binary_data: str) -> list:
     """Break binary stream data into 48-bits.
 
     Parameters
@@ -166,7 +167,7 @@ def break_into_bits_size(binary_data: str):
     ]
 
 
-def create_dataset(de_data_list: list, packet_met_time: list):
+def create_dataset(de_data_list: list, packet_met_time: list) -> xr.Dataset:
     """Create xarray dataset.
 
     Parameters
@@ -301,7 +302,7 @@ def create_dataset(de_data_list: list, packet_met_time: list):
     return dataset
 
 
-def science_direct_event(packets_data: list):
+def science_direct_event(packets_data: list[Packet]) -> xr.Dataset:
     """Unpack IMAP-Hi direct event data.
 
     Processing step:
@@ -311,7 +312,7 @@ def science_direct_event(packets_data: list):
 
     Parameters
     ----------
-    packets_data : list
+    packets_data : list[Packet]
         List of packets data
 
     Returns

imap_processing/tests/hi/test_l1a.py

@@ -3,8 +3,10 @@
 import numpy as np
 import pytest
 
+from imap_processing import imap_module_directory
 from imap_processing.cdf.utils import write_cdf
 from imap_processing.hi.l1a.hi_l1a import hi_l1a
+from imap_processing.hi.utils import HIAPID
 from imap_processing.tests.conftest import ccsds_header_data, check_sum
 
 
@@ -126,3 +128,24 @@ def test_sci_de_decom(create_de_data, tmp_path):
     cdf_filepath = write_cdf(processed_data[0])
 
     assert cdf_filepath.name == cdf_filename
+
+
+def test_app_nhk_decom():
+    """Test housekeeping data"""
+
+    # Unpack housekeeping data
+    test_path = imap_module_directory / "tests/hi/l0_test_data"
+    bin_data_path = test_path / "20231030_H45_APP_NHK.bin"
+    processed_data = hi_l1a(packet_file_path=bin_data_path)
+
+    assert np.unique(processed_data[0]["pkt_apid"].values) == HIAPID.H45_APP_NHK.value
+    assert processed_data[0].attrs["Logical_source"] == "imap_hi_l1a_hk"
+    assert processed_data[0].attrs["Data_version"] == "001"
+    # TODO: compare with validation data once we have it
+
+    # Write CDF
+    cem_raw_cdf_filepath = write_cdf(processed_data[0])
+
+    # TODO: ask Vivek about this date mismatch between the file name
+    # and the data. May get resolved when we have good sample data.
+    assert cem_raw_cdf_filepath.name == "imap_hi_l1a_hk_20310824_v001.cdf"

pyproject.toml

@@ -33,7 +33,7 @@ bitstring = ">=4.0.1"
 cdflib = "==1.2.6"
 imap-data-access = ">=0.5.0"
 python = ">=3.9,<4"
-space_packet_parser = ">=4.1.0"
+space_packet_parser = ">=4.2.0rc3"
 spiceypy = ">=6.0.0"
 xarray = '>=2023.0.0'