From 7e1f0fdf34935f815084c9158ae3923faf7ca20d Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Thu, 29 Aug 2024 16:41:04 -0600 Subject: [PATCH] changes to decom --- imap_processing/ialirt/l0/decom_ialirt.py | 87 +++---------------- .../test_data/l0/hit_ialirt_sample_header.csv | 1 - .../tests/ialirt/unit/test_decom_ialirt.py | 57 ++---------- .../tests/ialirt/unit/test_process_hit.py | 4 +- 4 files changed, 17 insertions(+), 132 deletions(-) delete mode 100644 imap_processing/tests/ialirt/test_data/l0/hit_ialirt_sample_header.csv diff --git a/imap_processing/ialirt/l0/decom_ialirt.py b/imap_processing/ialirt/l0/decom_ialirt.py index fc6b2ae99..d01dbbb24 100644 --- a/imap_processing/ialirt/l0/decom_ialirt.py +++ b/imap_processing/ialirt/l0/decom_ialirt.py @@ -1,19 +1,16 @@ -"""Decommutates i-alert packets and creates L1 data products.""" +"""Decommutates i-alirt packets and creates L1 data products.""" -import collections import logging -from typing import Optional +from typing import cast import xarray as xr -from imap_processing.decom import decom_packets +from imap_processing.utils import packet_file_to_datasets logger = logging.getLogger(__name__) -def generate_xarray( - packet_file: str, xtce: str, time_keys: Optional[dict] = None -) -> xr.Dataset: +def generate_xarray(packet_file: str, xtce: str) -> dict[int, xr.Dataset]: """ Generate xarray from unpacked data. @@ -23,77 +20,15 @@ def generate_xarray( Path to the CCSDS data packet file. xtce : str Path to the XTCE packet definition file. - time_keys : dict - Keys used for creating xarray dataset. Returns ------- - dataset : xarray.Dataset - A dataset containing the decoded data fields with 'time' as the coordinating - dimension. - - Examples - -------- - # This is an example of what the xarray dataset might look like - # after being processed by this function. - - - Dimensions: (SC_SCLK_SEC: 5) - Coordinates: - * SC_SCLK_SEC (SC_SCLK_SEC) int64 322168 322169 322170 322171 322172 - Data variables: - SC_MAG_STATUS (SC_SCLK_SEC) int64 0 1 0 1 0 - SC_HIT_STATUS (SC_SCLK_SEC) int64 1 0 1 0 1 - - This example shows a dataset with 'SC_SCLK_SEC' as the coordinate - and two data variables 'SC_MAG_STATUS' and 'SC_HIT_STATUS'. + alirt_dict : dict + A dictionary of the dataset containing the decoded data fields. """ - packets = decom_packets(packet_file, xtce) - - logger.info(f"Decommutated {len(packets)} packets from {packet_file}.") - - if time_keys is None: - time_keys = { - "SC": "SC_SCLK_SEC", - "HIT": "HIT_MET", - "MAG": "MAG_ACQ", - "COD_LO": "COD_LO_ACQ", - "COD_HI": "COD_HI_ACQ", - "SWE": "SWE_ACQ_SEC", - "SWAPI": "SWAPI_ACQ", - } - - instruments = list(time_keys.keys()) - - # Initialize storage dictionary using defaultdict - data_storage: dict = {inst: collections.defaultdict(list) for inst in instruments} - - for packet in packets: - for key, value in packet.data.items(): - key_matched = False - for inst in instruments: - if key.startswith(inst): - # Directly append to the list - data_storage[inst][key].append(value.derived_value) - key_matched = True - break - - if not key_matched: - # If after checking all instruments, none match, raise an error. - raise ValueError(f"Unexpected key '{key}' found in packet data.") - - logger.info("Generating datasets for each instrument.") - - # Generate xarray dataset for each instrument and spacecraft - datasets = {} - for inst in instruments: - dataset_dict = { - key: (time_keys[inst], data_storage[inst][key]) - for key in data_storage[inst] - if key != time_keys[inst] - } - datasets[inst] = xr.Dataset( - dataset_dict, coords={time_keys[inst]: data_storage[inst][time_keys[inst]]} - ) + alirt_dict = cast( + dict[int, xr.Dataset], + packet_file_to_datasets(packet_file, xtce, use_derived_value=False), + ) - return datasets + return alirt_dict diff --git a/imap_processing/tests/ialirt/test_data/l0/hit_ialirt_sample_header.csv b/imap_processing/tests/ialirt/test_data/l0/hit_ialirt_sample_header.csv deleted file mode 100644 index e7a6a292d..000000000 --- a/imap_processing/tests/ialirt/test_data/l0/hit_ialirt_sample_header.csv +++ /dev/null @@ -1 +0,0 @@ -CCSDS_VERSION,CCSDS_TYPE,CCSDS_SEC_HDR_FLAG,CCSDS_APPID,CCSDS_GRP_FLAG,CCSDS_SEQ_CNT,CCSDS_LENGTH,SC_TICK,MET,STATUS,SUBCOM,FAST_RATE_1,L1A_TRIG_00,IA_EVNT_TRIG_00,A_EVNT_TRIG_00,L3A_TRIG_00,L1A_TRIG_01,IA_EVNT_TRIG_01,A_EVNT_TRIG_01,L3A_TRIG_01,L1A_TRIG_02,IA_EVNT_TRIG_02,A_EVNT_TRIG_02,L3A_TRIG_02,L1A_TRIG_03,IA_EVNT_TRIG_03,A_EVNT_TRIG_03,L3A_TRIG_03,L1A_TRIG_04,IA_EVNT_TRIG_04,A_EVNT_TRIG_04,L3A_TRIG_04,L1A_TRIG_05,IA_EVNT_TRIG_05,A_EVNT_TRIG_05,L3A_TRIG_05,L1A_TRIG_06,IA_EVNT_TRIG_06,A_EVNT_TRIG_06,L3A_TRIG_06,L1A_TRIG_07,IA_EVNT_TRIG_07,A_EVNT_TRIG_07,L3A_TRIG_07,L1A_TRIG_08,IA_EVNT_TRIG_08,A_EVNT_TRIG_08,L3A_TRIG_08,L1A_TRIG_09,IA_EVNT_TRIG_09,A_EVNT_TRIG_09,L3A_TRIG_09,L1A_TRIG_10,IA_EVNT_TRIG_10,A_EVNT_TRIG_10,L3A_TRIG_10,L1A_TRIG_11,IA_EVNT_TRIG_11,A_EVNT_TRIG_11,L3A_TRIG_11,L1A_TRIG_12,IA_EVNT_TRIG_12,A_EVNT_TRIG_12,L3A_TRIG_12,L1A_TRIG_13,IA_EVNT_TRIG_13,A_EVNT_TRIG_13,L3A_TRIG_13,L1A_TRIG_14,IA_EVNT_TRIG_14,A_EVNT_TRIG_14,L3A_TRIG_14,L1A_TRIG,IA_EVNT_TRIG,A_EVNT_TRIG,L3A_TRIG,FAST_RATE_2,L1B_TRIG_00,IB_EVNT_TRIG_00,B_EVNT_TRIG_00,L3B_TRIG_00,L1B_TRIG_01,IB_EVNT_TRIG_01,B_EVNT_TRIG_01,L3B_TRIG_01,L1B_TRIG_02,IB_EVNT_TRIG_02,B_EVNT_TRIG_02,L3B_TRIG_02,L1B_TRIG_03,IB_EVNT_TRIG_03,B_EVNT_TRIG_03,L3B_TRIG_03,L1B_TRIG_04,IB_EVNT_TRIG_04,B_EVNT_TRIG_04,L3B_TRIG_04,L1B_TRIG_05,IB_EVNT_TRIG_05,B_EVNT_TRIG_05,L3B_TRIG_05,L1B_TRIG_06,IB_EVNT_TRIG_06,B_EVNT_TRIG_06,L3B_TRIG_06,L1B_TRIG_07,IB_EVNT_TRIG_07,B_EVNT_TRIG_07,L3B_TRIG_07,L1B_TRIG_08,IB_EVNT_TRIG_08,B_EVNT_TRIG_08,L3B_TRIG_08,L1B_TRIG_09,IB_EVNT_TRIG_09,B_EVNT_TRIG_09,L3B_TRIG_09,L1B_TRIG_10,IB_EVNT_TRIG_10,B_EVNT_TRIG_10,L3B_TRIG_10,L1B_TRIG_11,IB_EVNT_TRIG_11,B_EVNT_TRIG_11,L3B_TRIG_11,L1B_TRIG_12,IB_EVNT_TRIG_12,B_EVNT_TRIG_12,L3B_TRIG_12,L1B_TRIG_13,IB_EVNT_TRIG_13,B_EVNT_TRIG_13,L3B_TRIG_13,L1B_TRIG_14,IB_EVNT_TRIG_14,B_EVNT_TRIG_14,L3B_TRIG_14,L1B_TRIG,IB_EVNT_TRIG,B_EVNT_TRIG,L3B_TRIG,SLOW_RATE,L1A,L2A,L3A,L1A0AHG,L1B0AHG,L1C0AHG,L4IAHG,L4OAHG,SLOW_RATE_08,SLOW_RATE_09,SLOW_RATE_10,L1A0BHG,L1B0BHG,L1C0BHG,L4IBHG,L4OBHG,IALRT_RATE_1,IALRT_RATE_2,IALRT_RATE_3,IALRT_RATE_4,IALRT_RATE_5,IALRT_RATE_6,IALRT_RATE_7,IALRT_RATE_8,IALRT_RATE_9,IALRT_RATE_10,IALRT_RATE_11,IALRT_RATE_12,IALRT_RATE_13,IALRT_RATE_14,IALRT_RATE_15,IALRT_RATE_16,IALRT_RATE_17,IALRT_RATE_18,IALRT_RATE_19,IALRT_RATE_20,TRIG_IA_EVNT,TRIG_IB_EVNT,NASIDE_IALRT,NBSIDE_IALRT,ERATE_1,ERATE_2,ERATE_3,ERATE_4,ERATE_5,L12A,L123A,PENA,L12B,L123B,PENB,SLOW_RATE_51,SLOW_RATE_52,SLOW_RATE_53,SLOW_RATE_54,H_06_08,H_12_15,H_15_70,HE4_06_08,HE4_15_70,EVENT_DATA_00,EVENT_DATA_01,EVENT_DATA_02,EVENT_DATA_03,EVENT_DATA_04,EVENT_DATA_05,EVENT_DATA_06,EVENT_DATA_07,EVENT_DATA_08,EVENT_DATA_09,EVENT_DATA_10,CHECKSUM diff --git a/imap_processing/tests/ialirt/unit/test_decom_ialirt.py b/imap_processing/tests/ialirt/unit/test_decom_ialirt.py index 50897f472..8add52c81 100644 --- a/imap_processing/tests/ialirt/unit/test_decom_ialirt.py +++ b/imap_processing/tests/ialirt/unit/test_decom_ialirt.py @@ -85,57 +85,10 @@ def test_enumerated(decom_packets_data): def test_generate_xarray(binary_packet_path, xtce_ialirt_path, decom_packets_data): - """This function checks that all instrument parameters are accounted for.""" + """This function checks that all instrument parameters are correct length.""" - instrument_lengths = {} - instruments = ["SC", "HIT", "MAG", "COD_LO", "COD_HI", "SWE", "SWAPI"] + apid = 478 + xarray_data = generate_xarray(binary_packet_path, xtce_ialirt_path)[apid] - xarray_data = generate_xarray(binary_packet_path, xtce_ialirt_path) - - # Assert that all parameters are the same - # between packet data and generated xarray - for instrument in instruments: - instrument_list = list(xarray_data[instrument].coords) - instrument_list.extend(list(xarray_data[instrument].data_vars)) - - # Create a dictionary of the number of parameters for each instrument - instrument_lengths[instrument] = len(instrument_list) - - packet_data_keys_set = set(decom_packets_data[0].data.keys()) - instrument_list_set = set(instrument_list) - - # Assert that the instrument parameters from the xarray are - # a subset of the packet data - assert instrument_list_set.issubset(packet_data_keys_set) is True - - # Assert that the length of parameters is the same - # between packet data and generated xarray - assert sum(instrument_lengths.values()) == len( - list(decom_packets_data[0].data.keys()) - ) - - # Check that all the dimensions are the correct length - # example:len(xarray_data['HIT'].coords['HIT_SC_TICK'].values) == 32 - for instrument in instruments: - for dimension_name in xarray_data[instrument].dims: - assert len(xarray_data[instrument].coords[dimension_name].values) == 32 - - -def test_unexpected_key(binary_packet_path, xtce_ialirt_path, decom_packets_data): - """ - This function tests that a ValueError is raised when an unexpected - key is encountered. - """ - - time_keys = { - "UNEXPECTED": "SC_SCLK_SEC", - "HIT": "HIT_SC_TICK", - "MAG": "MAG_ACQ", - "COD_LO": "COD_LO_ACQ", - "COD_HI": "COD_HI_ACQ", - "SWE": "SWE_ACQ_SEC", - "SWAPI": "SWAPI_ACQ", - } - - with pytest.raises(ValueError, match=r"Unexpected key '.*' found in packet data."): - generate_xarray(binary_packet_path, xtce_ialirt_path, time_keys=time_keys) + for key in xarray_data.keys(): + assert len(xarray_data[key]) == 32 diff --git a/imap_processing/tests/ialirt/unit/test_process_hit.py b/imap_processing/tests/ialirt/unit/test_process_hit.py index 77f7a58c0..d61e8ac57 100644 --- a/imap_processing/tests/ialirt/unit/test_process_hit.py +++ b/imap_processing/tests/ialirt/unit/test_process_hit.py @@ -34,9 +34,7 @@ def xarray_data(binary_packet_path, xtce_hit_path): """Create xarray data""" apid = 1253 - xarray_data = generate_xarray( - binary_packet_path, xtce_hit_path, time_keys={"HIT": "HIT_SC_TICK"} - )[apid] + xarray_data = generate_xarray(binary_packet_path, xtce_hit_path)[apid] return xarray_data