From 8f80837ca31a5717b14dae06b2d6d708c5dd0f96 Mon Sep 17 00:00:00 2001 From: Greg Lucas Date: Fri, 30 Aug 2024 14:31:23 -0600 Subject: [PATCH] MNT/TST: Update SWAPI packet definition and tests This removes some unnecessary utils functions and moves over to packet_file_to_datasets() --- imap_processing/swapi/l1/swapi_l1.py | 4 +- .../swapi_packet_definition.xml | 1981 +++++++++++++---- .../tests/swapi/test_swapi_decom.py | 153 +- imap_processing/tests/swapi/test_swapi_l1.py | 8 +- 4 files changed, 1606 insertions(+), 540 deletions(-) diff --git a/imap_processing/swapi/l1/swapi_l1.py b/imap_processing/swapi/l1/swapi_l1.py index d82796c22..039fd3e6c 100644 --- a/imap_processing/swapi/l1/swapi_l1.py +++ b/imap_processing/swapi/l1/swapi_l1.py @@ -35,7 +35,7 @@ def filter_good_data(full_sweep_sci: xr.Dataset) -> npt.NDArray: """ # PLAN_ID for current sweep should all be one value and # SWEEP_TABLE should all be one value. - plan_id = full_sweep_sci["plan_id_science"].data.reshape(-1, 12) + plan_id = full_sweep_sci["plan_id"].data.reshape(-1, 12) sweep_table = full_sweep_sci["sweep_table"].data.reshape(-1, 12) mode = full_sweep_sci["mode"].data.reshape(-1, 12) @@ -493,7 +493,7 @@ def process_swapi_science(sci_dataset: xr.Dataset, data_version: str) -> xr.Data cdf_manager.add_global_attribute("Data_version", data_version) l1_global_attrs = cdf_manager.get_global_attributes("imap_swapi_l1_sci") l1_global_attrs["Sweep_table"] = f"{sci_dataset['sweep_table'].data[0]}" - l1_global_attrs["Plan_id"] = f"{sci_dataset['plan_id_science'].data[0]}" + l1_global_attrs["Plan_id"] = f"{sci_dataset['plan_id'].data[0]}" l1_global_attrs["Apid"] = f"{sci_dataset['pkt_apid'].data[0]}" dataset = xr.Dataset( diff --git a/imap_processing/swapi/packet_definitions/swapi_packet_definition.xml b/imap_processing/swapi/packet_definitions/swapi_packet_definition.xml index e349e2c83..e570b9b6a 100644 --- a/imap_processing/swapi/packet_definitions/swapi_packet_definition.xml +++ b/imap_processing/swapi/packet_definitions/swapi_packet_definition.xml @@ -1,525 +1,1627 @@ - - + + - + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + - - + + + + + + + + - - + + + + + + + + - - + + + + + + + + - - + + + + + + + + - - + + + + + + + + - - + + + + + + + + + + + - + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + CCSDS Packet Version Number (always 0) - + CCSDS Packet Type Indicator (0=telemetry) - + CCSDS Packet Secondary Header Flag (always 1) - + CCSDS Packet Application Process ID - + CCSDS Packet Grouping Flags (3=not part of group) - + CCSDS Packet Sequence Count (increments with each new packet) - + CCSDS Packet Length (number of bytes after Packet length minus 1) - + CCSDS Packet Time (SCLK) at which C&DH software created packet) - - - LVENG, LVSCI, HVENG or HVSCI - - - Spare - - - - - Sequence number of set of steps in energy sweep - - - Sweep ID - - - Plan ID - - - ESA level during sixth 1/6 second - - - Spare - - - PCEM count range during first 1/6-second: raw or compressed - - - SCEM count range during first 1/6-second: raw or compressed - - - Coincidence count range during first 1/6-second: raw or compressed - - - PCEM count range during second 1/6-second: raw or compressed - - - SCEM count range during second 1/6-second: raw or compressed - - - Coincidence count range during second 1/6-second: raw or compressed - - - PCEM count range during third 1/6-second: raw or compressed - - - SCEM count range during third 1/6-second: raw or compressed - - - Coincidence count range during third 1/6-second: raw or compressed - - - PCEM count range during fourth 1/6-second: raw or compressed - - - SCEM count range during fourth 1/6-second: raw or compressed - - - Coincidence count range during fourth 1/6-second: raw or compressed - - - PCEM count range during fifth 1/6-second: raw or compressed - - - SCEM count range during fifth 1/6-second: raw or compressed - - - Coincidence count range during fifth 1/6-second: raw or compressed - - - PCEM count range during sixth 1/6-second: raw or compressed - - - SCEM count range during sixth 1/6-second: raw or compressed - - - Coincidence count range during sixth 1/6-second: raw or compressed - - - 1st Primary CEM count: LS 16 bits if RAW, MS 16 bits if compressed - - - 1st Secondary CEM count: LS 16 bits if RAW, MS 16 bits if compressed - - - 1st Coincidence count: LS 16 bits if RAW, MS 16 bits if compressed - - - 2nd Primary CEM count: LS 16 bits if RAW, MS 16 bits if compressed - - - 2nd Secondary CEM count: LS 16 bits if RAW, MS 16 bits if compressed - - - 2nd Coincidence count: LS 16 bits if RAW, MS 16 bits if compressed - - - 3rd Primary CEM count: LS 16 bits if RAW, MS 16 bits if compressed - - - 3rd Secondary CEM count: LS 16 bits if RAW, MS 16 bits if compressed - - - 3rd Coincidence count: LS 16 bits if RAW, MS 16 bits if compressed - - - 4th Primary CEM count: LS 16 bits if RAW, MS 16 bits if compressed - - - 4th Primary CEM count: LS 16 bits if RAW, MS 16 bits if compressed - - - 4th Primary CEM count: LS 16 bits if RAW, MS 16 bits if compressed - - - 5th Primary CEM count: LS 16 bits if RAW, MS 16 bits if compressed - - - 5th Secondary CEM count: LS 16 bits if RAW, MS 16 bits if compressed - - - 5th Coincidence count: LS 16 bits if RAW, MS 16 bits if compressed - - - 6th Primary CEM count: LS 16 bits if RAW, MS 16 bits if compressed - - - 6th Secondary CEM count: LS 16 bits if RAW, MS 16 bits if compressed - - - 6th Coincidence count: LS 16 bits if RAW, MS 16 bits if compressed - - - - + Cumulative modulo-256 count of successfully executed commands - + Cumulative modulo-256 count of rejected commands - + Which LUT is in use - + If the CEM interrupt occurs and dipping the supplies does not help within 3 (TBR) consecutive samples then SWAPI is safed. - + If the CEM interrupt occurs and dipping the supplies does not help within 3 (TBR) consecutive samples then SWAPI is safed. - + SWAPI has rebooted due to a watchdog expiration. - + SWAPI has received safe command from S/C - + SWAPI has safed itself. The cause would be due to one of the following status bits. - + The count rate threshold for the PCEM counter has been exceeded once and has continued to be exceeded despite measures by SWAPIFW. The PCEM count rate threshold defaults to TBD but can be updated with command TBD. - + The count rate threshold for the SCEM counter has been exceeded once and has continued to be exceeded despite measures by SWAPIFW. The SCEM count rate threshold defaults to TBD but can be updated with command TBD. - + The current threshold for the PCEM has been exceeded once and has continued to be exceeded despite measures by SWAPIFW. An overcurrent on the CEM's collectively trips an interrupt which the SWAPIFW handles. The SWAPIFW also collects the PCEM current monitor for comparison at 1 Hz. The CEM current rate threshold defaults to TBD but can be updated with command TBD. - + The current threshold for the SCEM has been exceeded once and has continued to be exceeded despite measures by SWAPIFW. An overcurrent on the CEMs collectively trips an interrupt which the SWAPIFW handles. The SWAPIFW also collects the SCEM current monitor for comparison at 1 Hz. The CEM current rate threshold defaults to TBD but can be updated with command TBD. - + The voltage tolerance for the PCEM for its current setting has been exceeded. The PCEM voltage tolerance is set by SWAPIFW and cannot be altered by command. - + The voltage tolerance for the SCEM for its current setting has been exceeded. The SCEM voltage tolerance is set by SWAPIFW and cannot be altered by command. - + The voltage tolerance for +5 V or -5 V supply has been exceeded. - + The current tolerance for +5 V or -5 V supply has been exceeded. - + The upper temperature limit of any one of the thermistors has been exceeded. - + The lower temperature limit of any one of the thermistors has been exceeded. The lower temperature limit is set by SWAPIFW and cannot be altered by command. - + + Enumerated type representing each of the modes + + MEMDP state - + Temperature of sensor detector. AD MUX = 0x10 - + Temperature of hvps. AD MUX = 0x11 - + Temperature of controller. AD MUX = 0x12 - + Volt mon of primary channel electron multiplier high-voltage power supply. AD MUX = 0x02 - + Volt mon of secondary channel electron multiplier high-voltage power supply. AD MUX = 0x03 - + Strip current monitor of primary electron multiplier high-voltage power supply. AD MUX = 0x04 - + Strip current monitor of secondary electron multiplier high-voltage power supply. AD MUX = 0x05 - + Volt mon of +5 V power supply. AD MUX = 0x0C - + Volt mon of -5 V power supply. AD MUX = 0x0D - + Current monitor of +5 V power supply. AD MUX = 0x0E - + Current monitor of -5 V power supply. AD MUX = 0x0F - + Revision number for the SWAPI software - + Opcode of last executed command - + DAC level of PHD LLD - + MEMLD state - + Indicates which application is running (Boot or App) - + State of ESA Enable - + State of primary channel electron multiplier disable/enable - + State of secondary channel electron disable/enable - + + Spare + + The PCEM count rate was tripped but handled by SWAPIFW - + The SCEM count rate was tripped but handled by SWAPIFW - + The level at which safety algorithms for the PCEM monitor are tripped due to overcurrent in the PCEM monitor - + The level at which safety algorithms for the SCEM monitor are tripped due to overcurrent in the SCEM monitor - + PCEM DAC level - + SCEM DAC level - + Volt mon of analog ground. - + The current limit monitor for the PCEM and SCEM at which the SWAPI software brings down the PCEM and SCEM by TBD volts for TBD seconds before they are brought back up. - + Volt mon of electrostatic analyzer high-voltage power supply. - + Volt mon of +2.5 V reference. - + Volt mon of PHD LLD. - + Spare - + The count value at which the primary CEM safety limit is set. If this limit is exceeded twice SWAPI is safed - + The count value at which the secondary CEM safety limit is set. If this limit is exceeded twice SWAPI is safed - + State of whether the stim pulsers are enabled or disabled. - + Count of missed PPS signals (i.e. a PPS was not received when expected). This is a 2-bit counter that will freeze when it reaches 3. Can be cleared via the CLR_LATCHED command. - + The limit at which the PCEM and SCEM current triggers an interrupt to the SWAPI software. - + A value that represents whether command echo is enabled or disabled - + State of disable/safe/arm plug - + State of high-voltage software disable/enable - + Spare - + The number of counts to dip the CEM supplies when a CEM current interrupt occurs - + The PLAN ID used for the current science sweeping mode if any. - + Sweep table ID within the PLAN ID that is being used for the current science sweeping mode. - + If the PCEM voltage is out of tolerance for only 0.5 second this bit is asserted. - + If the PCEM current is out of tolerance for only 0.5 second this bit is asserted. - + If the SCEM voltage is out of tolerance for only 0.5 second this bit is asserted. - + If the SCEM current is out of tolerance for only 0.5 second this bit is asserted. - + The PCEM current interrupt was tripped but handled by SWAPIFW - + The SCEM current interrupt was tripped but handled by SWAPIFW - + EEPROM 2 is ready to be written - + EEPROM 1 is ready to be written - + Type number of the FPGA - + Revision number of the FPGA - + A value representing how often the I-ALiRT telemetry packet is output. - + A value representing how often all of the 12-second science packets are output. - + A value representing a choice for how often the housekeeping packet is output. - + Spare - + A status of the power on check of the FPGA initialization check - need to fix or make Spare - + A status of the power on check of the EEP_L2 checksum compared against a stored checksum - + A status of the power on check of EEP_L1 checksum compared against a stored checksum - + A status of the power on check of the RAM_D memory test - + A status of the power on check of the EEP_C2 checksum compared against a stored checksum - + A status of the power on check of the EEP_C1 checksum compared against a stored checksum - + A status of the power on check of the RAM_C memory test - + A status of the power on check of the PROM checksum compared against a stored checksum - - Lookup table version + Lookup table version - + 2.5V Regulator for FPVA Core - + 3.3V Regulator for LVDS TX/RX - + PCEM Voltage Level Command - + SCEM Voltage Level Command - + ESA Voltage Level Command - + SCEM LLD Threshold - + Spare - + DAC level of PHD LLD 2 - - - - DAC level of PHD LLD 2 + + XOR checksum starting with beginning of CCSDS header to the byte prior to this field - - DAC level of PHD LLD 2 + + CCSDS Packet Time (SCLK) at which C&DH software created packet) + + + Sequence number of set of steps in energy sweep - - - Packet checksum + + Sweep ID + + + Plan ID + + + LVENG, LVSCI, HVENG or HVSCI + + + Spare + + + ESA level during sixth 1/6 second + + + Spare + + + PCEM count range during first 1/6-second: raw or compressed + + + SCEM count range during first 1/6-second: raw or compressed + + + Coincidence count range during first 1/6-second: raw or compressed + + + PCEM count range during second 1/6-second: raw or compressed + + + SCEM count range during second 1/6-second: raw or compressed + + + Coincidence count range during second 1/6-second: raw or compressed + + + PCEM count range during third 1/6-second: raw or compressed + + + SCEM count range during third 1/6-second: raw or compressed + + + Coincidence count range during third 1/6-second: raw or compressed + + + PCEM count range during fourth 1/6-second: raw or compressed + + + SCEM count range during fourth 1/6-second: raw or compressed + + + Coincidence count range during fourth 1/6-second: raw or compressed + + + PCEM count range during fifth 1/6-second: raw or compressed + + + SCEM count range during fifth 1/6-second: raw or compressed + + + Coincidence count range during fifth 1/6-second: raw or compressed + + + PCEM count range during sixth 1/6-second: raw or compressed + + + SCEM count range during sixth 1/6-second: raw or compressed + + + Coincidence count range during sixth 1/6-second: raw or compressed + + + 1st Primary CEM count: LS 16 bits if RAW, MS 16 bits if compressed + + + 1st Secondary CEM count: LS 16 bits if RAW, MS 16 bits if compressed + + + 1st Coincidence count: LS 16 bits if RAW, MS 16 bits if compressed + + + 2nd Primary CEM count: LS 16 bits if RAW, MS 16 bits if compressed + + + 2nd Secondary CEM count: LS 16 bits if RAW, MS 16 bits if compressed + + + 2nd Coincidence count: LS 16 bits if RAW, MS 16 bits if compressed + + + 3rd Primary CEM count: LS 16 bits if RAW, MS 16 bits if compressed + + + 3rd Secondary CEM count: LS 16 bits if RAW, MS 16 bits if compressed + + + 3rd Coincidence count: LS 16 bits if RAW, MS 16 bits if compressed + + + 4th Primary CEM count: LS 16 bits if RAW, MS 16 bits if compressed + + + 4th Primary CEM count: LS 16 bits if RAW, MS 16 bits if compressed + + + 4th Primary CEM count: LS 16 bits if RAW, MS 16 bits if compressed + + + 5th Primary CEM count: LS 16 bits if RAW, MS 16 bits if compressed + + + 5th Secondary CEM count: LS 16 bits if RAW, MS 16 bits if compressed + + + 5th Coincidence count: LS 16 bits if RAW, MS 16 bits if compressed + + + 6th Primary CEM count: LS 16 bits if RAW, MS 16 bits if compressed + + + 6th Secondary CEM count: LS 16 bits if RAW, MS 16 bits if compressed + + + 6th Coincidence count: LS 16 bits if RAW, MS 16 bits if compressed + + XOR checksum starting with beginning of CCSDS header to the byte prior to this field - + @@ -530,179 +1632,166 @@ - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - \ No newline at end of file diff --git a/imap_processing/tests/swapi/test_swapi_decom.py b/imap_processing/tests/swapi/test_swapi_decom.py index f3aff0273..a9fff9ea4 100644 --- a/imap_processing/tests/swapi/test_swapi_decom.py +++ b/imap_processing/tests/swapi/test_swapi_decom.py @@ -2,43 +2,50 @@ import pytest from imap_processing import imap_module_directory -from imap_processing.decom import decom_packets from imap_processing.swapi.l1.swapi_l1 import ( SWAPIAPID, ) -from imap_processing.utils import group_by_apid +from imap_processing.utils import packet_file_to_datasets + + +@pytest.fixture(scope="session") +def decom_test_science_data(swapi_l0_test_data_path): + """Read test data from file with derived values""" + test_file = "imap_swapi_l0_raw_20231012_v001.pkts" + packet_file = imap_module_directory / swapi_l0_test_data_path / test_file + packet_definition = ( + f"{imap_module_directory}/swapi/packet_definitions/swapi_packet_definition.xml" + ) + return packet_file_to_datasets( + packet_file, packet_definition, use_derived_value=True + ) @pytest.fixture(scope="session") def decom_test_data(swapi_l0_test_data_path): - """Read test data from file""" + """Read test data from file as raw values""" test_file = "imap_swapi_l0_raw_20231012_v001.pkts" packet_file = imap_module_directory / swapi_l0_test_data_path / test_file packet_definition = ( f"{imap_module_directory}/swapi/packet_definitions/swapi_packet_definition.xml" ) - data_list = [] - data_list.extend(decom_packets(packet_file, packet_definition)) - return data_list + return packet_file_to_datasets( + packet_file, packet_definition, use_derived_value=False + ) def test_number_of_packets(decom_test_data): """This test and validate number of packets.""" - grouped_data = group_by_apid(decom_test_data) - sci_packets = grouped_data[SWAPIAPID.SWP_SCI] + sci_packets = decom_test_data[SWAPIAPID.SWP_SCI] expected_sci_packets = 54 - assert len(sci_packets) == expected_sci_packets + assert len(sci_packets["epoch"]) == expected_sci_packets - hk_packets = grouped_data[SWAPIAPID.SWP_HK] + hk_packets = decom_test_data[SWAPIAPID.SWP_HK] expected_hk_packets = 54 - assert len(hk_packets) == expected_hk_packets + assert len(hk_packets["epoch"]) == expected_hk_packets - aut_packets = grouped_data[SWAPIAPID.SWP_AUT] - expected_aut_packets = 54 - assert len(aut_packets) == expected_aut_packets - -def test_swapi_sci_data(decom_test_data, swapi_l0_validation_data_path): +def test_swapi_sci_data(decom_test_science_data, swapi_l0_validation_data_path): """This test and validate raw data of SWAPI raw science data.""" # read validation data raw_validation_data = pd.read_csv( @@ -46,33 +53,30 @@ def test_swapi_sci_data(decom_test_data, swapi_l0_validation_data_path): index_col="SHCOARSE", ) - grouped_data = group_by_apid(decom_test_data) - sci_packets = grouped_data[SWAPIAPID.SWP_SCI] - first_data = sci_packets[0] - validation_data = raw_validation_data.loc[first_data.data["SHCOARSE"].raw_value] + sci_packets = decom_test_science_data[SWAPIAPID.SWP_SCI] + first_data = sci_packets.isel(epoch=0) + validation_data = raw_validation_data.loc[first_data["shcoarse"].values] # compare raw values of validation data - for key, value in first_data.data.items(): - # check if the data is the same - if key == "PLAN_ID_SCIENCE": - # We had to work around this because HK and SCI packet uses - # PLAN_ID but they uses different length of bits. - assert value.raw_value == validation_data["PLAN_ID"] - elif key == "SPARE_2_SCIENCE": - # Same for this SPARE_2 as above case - assert value.raw_value == validation_data["SPARE_2"] - elif key == "MODE": - # Because validation data uses derived value instead of raw value - assert value.derived_value == validation_data[key] - elif "RNG" in key: - assert value.derived_value == validation_data[key] - else: - # for SHCOARSE we need the name of the column. - # This is done because pandas removed it from the - # main columns to make it the index. - assert value.raw_value == ( - validation_data[key] if key != "SHCOARSE" else validation_data.name - ) + for key in raw_validation_data.columns: + if key in [ + "PHAPID", + "timestamp", + "PHGROUPF", + "PHSHF", + "PHVERNO", + "PHSEQCNT", + "PHDLEN", + "PHTYPE", + ]: + continue + + # for SHCOARSE we need the name of the column. + # This is done because pandas removed it from the + # main columns to make it the index. + assert first_data[key.lower()].values == ( + validation_data[key] if key != "SHCOARSE" else validation_data.name + ) def test_swapi_hk_data(decom_test_data, swapi_l0_validation_data_path): @@ -83,53 +87,26 @@ def test_swapi_hk_data(decom_test_data, swapi_l0_validation_data_path): index_col="SHCOARSE", ) - grouped_data = group_by_apid(decom_test_data) - hk_packets = grouped_data[SWAPIAPID.SWP_HK] - first_data = hk_packets[0] - validation_data = raw_validation_data.loc[first_data.data["SHCOARSE"].raw_value] + hk_packets = decom_test_data[SWAPIAPID.SWP_HK] + first_data = hk_packets.isel(epoch=0) + validation_data = raw_validation_data.loc[first_data["shcoarse"].values] # compare raw values of validation data - for key, value in first_data.data.items(): - if key == "PLAN_ID_HK": - # We had to work around this because HK and SCI packet uses - # PLAN_ID but they uses different length of bits. - assert value.raw_value == validation_data["PLAN_ID"] - elif key == "SPARE_2_HK": - # Same for this SPARE_2 as PLAN_ID - assert value.raw_value == validation_data["SPARE_2"] - elif key == "SHCOARSE": - # for SHCOARSE we need the name of the column. - # This is done because pandas removed it from the main columns - # to make it the index. - assert value.raw_value == validation_data.name - elif key == "N5_V": - # TODO: remove this elif after getting good validation data - # Validation data has wrong value for N5_V + for key in raw_validation_data.columns: + if key in [ + "PHAPID", + "timestamp", + "PHGROUPF", + "PHSHF", + "PHVERNO", + "PHSEQCNT", + "PHDLEN", + "PHTYPE", + ]: continue - else: - assert value.raw_value == validation_data[key] - - -def test_swapi_aut_data(decom_test_data, swapi_l0_validation_data_path): - """This test and validate raw data of SWAPI raw autonomy data.""" - # read validation data - raw_validation_data = pd.read_csv( - swapi_l0_validation_data_path / "idle_export_raw.SWP_AUT_20231012_125245.csv", - index_col="SHCOARSE", - ) - - grouped_data = group_by_apid(decom_test_data) - aut_packets = grouped_data[SWAPIAPID.SWP_AUT] - first_data = aut_packets[0] - validation_data = raw_validation_data.loc[first_data.data["SHCOARSE"].raw_value] - - # compare raw values of science data - for key, value in first_data.data.items(): - if key == "SHCOARSE": - assert value.raw_value == validation_data.name - elif key == "SPARE_1_AUT": - # We had to work around this because HK and SCI packet uses - # SPARE_1 but they uses different length of bits. - assert value.raw_value == validation_data["SPARE_1"] - else: - assert value.raw_value == validation_data[key] + # for SHCOARSE we need the name of the column. + # This is done because pandas removed it from the + # main columns to make it the index. + assert first_data[key.lower()].values == ( + validation_data[key] if key != "SHCOARSE" else validation_data.name + ) diff --git a/imap_processing/tests/swapi/test_swapi_l1.py b/imap_processing/tests/swapi/test_swapi_l1.py index 5c221eef1..80d3e1e6d 100644 --- a/imap_processing/tests/swapi/test_swapi_l1.py +++ b/imap_processing/tests/swapi/test_swapi_l1.py @@ -37,7 +37,7 @@ def test_filter_good_data(): total_sweeps = 3 ds = xr.Dataset( { - "plan_id_science": xr.DataArray(np.full((total_sweeps * 12), 1)), + "plan_id": xr.DataArray(np.full((total_sweeps * 12), 1)), "sweep_table": xr.DataArray(np.repeat(np.arange(total_sweeps), 12)), "mode": xr.DataArray(np.full((total_sweeps * 12), SWAPIMODE.HVENG.value)), }, @@ -68,9 +68,9 @@ def test_filter_good_data(): expected = [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23] np.testing.assert_array_equal(filter_good_data(ds), expected) - # Check for bad plan_id_science data. + # Check for bad plan_id data. ds["sweep_table"] = xr.DataArray(np.repeat(np.arange(total_sweeps), 12)) - ds["plan_id_science"][24 : total_sweeps * 12] = np.arange(0, 12) + ds["plan_id"][24 : total_sweeps * 12] = np.arange(0, 12) np.testing.assert_array_equal(filter_good_data(ds), np.arange(0, 24)) @@ -302,7 +302,7 @@ def test_process_swapi_science(decom_test_data): ) # make PLAN_ID data incorrect - ds_data["plan_id_science"][:12] = np.arange(12) + ds_data["plan_id"][:12] = np.arange(12) processed_data = process_swapi_science(ds_data, data_version="001") # Test dataset dimensions