From 427cbf195eaf54821805e6df287d3f6d9d853d18 Mon Sep 17 00:00:00 2001 From: Greg Lucas Date: Fri, 30 Aug 2024 07:43:02 -0600 Subject: [PATCH] MNT: Add useful error message when packet to datasets fails (#763) This occurs when there is a nested packet file definition with different variables for packets with the same apid. For example, with data spread across multiple packets and that definition being written into the XTCE logic. --- imap_processing/tests/test_utils.py | 10 ++++++++++ imap_processing/utils.py | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/imap_processing/tests/test_utils.py b/imap_processing/tests/test_utils.py index 787289a6d..d33051777 100644 --- a/imap_processing/tests/test_utils.py +++ b/imap_processing/tests/test_utils.py @@ -97,6 +97,16 @@ def test_packet_file_to_datasets(use_derived_value, expected_mode): np.testing.assert_array_equal(data["mode"], [expected_mode] * len(data["mode"])) +def test_packet_file_to_datasets_flat_definition(): + test_file = "tests/idex/imap_idex_l0_raw_20230725_v001.pkts" + packet_files = imap_module_directory / test_file + packet_definition = ( + imap_module_directory / "idex/packet_definitions/idex_packet_definition.xml" + ) + with pytest.raises(ValueError, match="Packet fields do not match"): + utils.packet_file_to_datasets(packet_files, packet_definition) + + def test__create_minimum_dtype_array(): """Test expected return types for minimum data types.""" result = utils._create_minimum_dtype_array([1, 2, 3], "uint8") diff --git a/imap_processing/utils.py b/imap_processing/utils.py index 680cc3f2d..c2949ca62 100644 --- a/imap_processing/utils.py +++ b/imap_processing/utils.py @@ -348,6 +348,8 @@ def packet_file_to_datasets( data_dict: dict[int, dict] = dict() # Also keep track of the datatype mapping for each field datatype_mapping: dict[int, dict] = dict() + # Keep track of which variables (keys) are in the dataset + variable_mapping: dict[int, set] = dict() # Set up the parser from the input packet definition packet_definition = xtcedef.XtcePacketDefinition(xtce_packet_definition) @@ -361,6 +363,14 @@ def packet_file_to_datasets( # This is the first packet for this APID data_dict[apid] = collections.defaultdict(list) datatype_mapping[apid] = dict() + variable_mapping[apid] = packet.data.keys() + if variable_mapping[apid] != packet.data.keys(): + raise ValueError( + f"Packet fields do not match for APID {apid}. This could be " + f"due to a conditional packet definition in the XTCE, while this " + f"function currently only supports flat packet definitions." + f"\nExpected: {variable_mapping[apid]},\ngot: {packet.data.keys()}" + ) # TODO: Do we want to give an option to remove the header content? packet_content = packet.data | packet.header