From c70e0241d9856b446f5835c1a5957cfd273135ea Mon Sep 17 00:00:00 2001 From: Greg Lucas Date: Fri, 23 Aug 2024 20:04:26 -0600 Subject: [PATCH] WIP/MNT: Update SWE to use packet_file_to_datasets This is switching SWE over to packet_file_to_datasets rather than iterating through all of the packets individually. --- docs/source/code-documentation/swe.rst | 9 - .../config/imap_swe_l1a_variable_attrs.yaml | 2 +- imap_processing/swe/l0/__init__.py | 0 imap_processing/swe/l0/decom_swe.py | 25 -- imap_processing/swe/l1a/swe_l1a.py | 33 +- imap_processing/swe/l1a/swe_science.py | 94 ++--- .../swe_packet_definition.xml | 397 ++++++++++-------- imap_processing/swe/utils/swe_utils.py | 28 -- imap_processing/tests/swe/conftest.py | 26 +- imap_processing/tests/swe/test_swe_l1a.py | 12 - .../tests/swe/test_swe_l1a_science.py | 46 +- imap_processing/tests/swe/test_swe_l1b.py | 45 +- .../tests/swe/test_swe_l1b_science.py | 17 +- 13 files changed, 346 insertions(+), 388 deletions(-) delete mode 100644 imap_processing/swe/l0/__init__.py delete mode 100644 imap_processing/swe/l0/decom_swe.py diff --git a/docs/source/code-documentation/swe.rst b/docs/source/code-documentation/swe.rst index 35a2a96a1..58c1875d3 100644 --- a/docs/source/code-documentation/swe.rst +++ b/docs/source/code-documentation/swe.rst @@ -8,15 +8,6 @@ SWE This is the SWE (Solar Wind Electrons) Instrument module, which contains the code for processing data from the SWE instrument. -The L0 code to decommutate the CCSDS packet data can be found below. - -.. autosummary:: - :toctree: generated/ - :template: autosummary.rst - :recursive: - - l0.decom_swe - The L1A code to unpack electron counts can be found below. .. autosummary:: diff --git a/imap_processing/cdf/config/imap_swe_l1a_variable_attrs.yaml b/imap_processing/cdf/config/imap_swe_l1a_variable_attrs.yaml index 93375f2ec..1d4bbc450 100644 --- a/imap_processing/cdf/config/imap_swe_l1a_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_swe_l1a_variable_attrs.yaml @@ -70,7 +70,7 @@ raw_counts: FIELDNAM: Raw Counts FORMAT: I3 UNITS: counts - VALIDMAX: 256 + VALIDMAX: 255 VALIDMIN: 0 VAR_TYPE: data diff --git a/imap_processing/swe/l0/__init__.py b/imap_processing/swe/l0/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/imap_processing/swe/l0/decom_swe.py b/imap_processing/swe/l0/decom_swe.py deleted file mode 100644 index 439b71c16..000000000 --- a/imap_processing/swe/l0/decom_swe.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Decommutates SWE CCSDS data packets.""" - -from imap_processing import decom, imap_module_directory - - -def decom_packets(packet_file: str) -> list: - """ - Decom SWE data packets using SWE packet definition. - - Parameters - ---------- - packet_file : str - Path to data packet path with filename. - - Returns - ------- - List - List of all the unpacked data. - """ - unpacked_data: list - xtce_document = ( - f"{imap_module_directory}/swe/packet_definitions/swe_packet_definition.xml" - ) - unpacked_data = decom.decom_packets(packet_file, xtce_document) - return unpacked_data diff --git a/imap_processing/swe/l1a/swe_l1a.py b/imap_processing/swe/l1a/swe_l1a.py index 4ccca7cdf..9198ef880 100644 --- a/imap_processing/swe/l1a/swe_l1a.py +++ b/imap_processing/swe/l1a/swe_l1a.py @@ -4,17 +4,17 @@ import xarray as xr -from imap_processing.swe.l0 import decom_swe +from imap_processing import imap_module_directory from imap_processing.swe.l1a.swe_science import swe_science from imap_processing.swe.utils.swe_utils import ( SWEAPID, ) -from imap_processing.utils import group_by_apid, sort_by_time +from imap_processing.utils import packet_file_to_datasets logger = logging.getLogger(__name__) -def swe_l1a(file_path: str, data_version: str) -> xr.Dataset: +def swe_l1a(packet_file: str, data_version: str) -> xr.Dataset: """ Will process SWE l0 data into l1a data. @@ -24,8 +24,8 @@ def swe_l1a(file_path: str, data_version: str) -> xr.Dataset: Parameters ---------- - file_path : str - Path where data is downloaded. + packet_file : str + Path where the raw packet file is stored. data_version : str Data version to write to CDF files and the Data_version CDF attribute. Should be in the format Vxxx. @@ -35,15 +35,14 @@ def swe_l1a(file_path: str, data_version: str) -> xr.Dataset: List List of xarray.Dataset. """ - packets = decom_swe.decom_packets(file_path) - - # group data by appId - grouped_data = group_by_apid(packets) - - # TODO: figure out how to handle non-science data error - # Process science data packets - # sort data by acquisition time - sorted_packets = sort_by_time(grouped_data[SWEAPID.SWE_SCIENCE], "ACQ_START_COARSE") - logger.debug("Processing science data for [%s] packets", len(sorted_packets)) - - return swe_science(decom_data=sorted_packets, data_version=data_version) + xtce_document = ( + f"{imap_module_directory}/swe/packet_definitions/swe_packet_definition.xml" + ) + datasets_by_apid = packet_file_to_datasets( + packet_file, xtce_document, use_derived_value=False + ) + + # TODO: figure out how to handle non-science data + return swe_science( + l0_dataset=datasets_by_apid[SWEAPID.SWE_SCIENCE], data_version=data_version + ) diff --git a/imap_processing/swe/l1a/swe_science.py b/imap_processing/swe/l1a/swe_science.py index 034226d94..123b296b5 100644 --- a/imap_processing/swe/l1a/swe_science.py +++ b/imap_processing/swe/l1a/swe_science.py @@ -1,16 +1,11 @@ """Contains code to perform SWE L1a science processing.""" -import collections import logging import numpy as np import xarray as xr from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes -from imap_processing.cdf.utils import met_to_j2000ns -from imap_processing.swe.utils.swe_utils import ( - add_metadata_to_array, -) logger = logging.getLogger(__name__) @@ -67,7 +62,7 @@ def decompressed_counts(cem_count: int) -> int: ) -def swe_science(decom_data: list, data_version: str) -> xr.Dataset: +def swe_science(l0_dataset: xr.Dataset, data_version: str) -> xr.Dataset: """ SWE L1a science processing. @@ -97,8 +92,8 @@ def swe_science(decom_data: list, data_version: str) -> xr.Dataset: Parameters ---------- - decom_data : list - Decompressed packet data. + l0_dataset : xarray.Dataset + Raw packet data from SWE stored as an xarray dataset. data_version : str Data version for the 'Data_version' CDF attribute. This is the version of the @@ -109,39 +104,30 @@ def swe_science(decom_data: list, data_version: str) -> xr.Dataset: dataset : xarray.Dataset The xarray dataset with data. """ - science_array = [] - raw_science_array = [] - - metadata_arrays: dict[list] = collections.defaultdict(list) - # We know we can only have 8 bit numbers input, so iterate over all # possibilities once up front decompression_table = np.array([decompressed_counts(i) for i in range(256)]) - for data_packet in decom_data: - # read raw data - binary_data = data_packet.data["SCIENCE_DATA"].raw_value - # read binary string to an int and then convert it to - # bytes. This is to convert the string to bytes. - # Eg. "0000000011110011" --> b'\x00\xf3' - # 1260 = 15 seconds x 12 energy steps x 7 CEMs - byte_data = int(binary_data, 2).to_bytes(1260, byteorder="big") - # convert bytes to numpy array of uint8 - raw_counts = np.frombuffer(byte_data, dtype=np.uint8) - - # Uncompress counts. Decompressed data is a list of 1260 - # where 1260 = 180 x 7 CEMs - # Take the "raw_counts" indices/counts mapping from - # decompression_table and then reshape the return - uncompress_data = np.take(decompression_table, raw_counts).reshape(180, 7) # type: ignore[attr-defined] - # Save raw counts data as well - raw_counts = raw_counts.reshape(180, 7) - - # Save data with its metadata field to attrs and DataArray of xarray. - # Save data as np.int64 to be complaint with ISTP' FILLVAL - science_array.append(uncompress_data.astype(np.int64)) - raw_science_array.append(raw_counts.astype(np.int64)) - metadata_arrays = add_metadata_to_array(data_packet, metadata_arrays) + # Loop through each packet individually with a list comprehension and + # perform the following steps: + # 1. Turn the binary string of 0s and 1s to an int + # 2. Convert the int into a bytes object of length 1260 (10080 / 8) + # Eg. "0000000011110011" --> b'\x00\xf3' + # 1260 = 15 seconds x 12 energy steps x 7 CEMs + # 3. Read that bytes data to a numpy array of uint8 through the buffer protocol + # 4. Reshape the data to 180 x 7 + raw_science_array = np.array( + [ + np.frombuffer( + int(binary_string, 2).to_bytes(1260, byteorder="big"), dtype=np.uint8 + ).reshape(180, 7) + for binary_string in l0_dataset["science_data"].values + ] + ) + + # Decompress the raw science data using numpy broadcasting logic + # science_array will be the same shape as raw_science_array (npackets, 180, 7) + science_array = decompression_table[raw_science_array] # Load CDF attrs cdf_attrs = ImapCdfAttributes() @@ -149,9 +135,8 @@ def swe_science(decom_data: list, data_version: str) -> xr.Dataset: cdf_attrs.add_instrument_variable_attrs("swe", "l1a") cdf_attrs.add_global_attribute("Data_version", data_version) - epoch_converted_time = met_to_j2000ns(metadata_arrays["SHCOARSE"]) epoch_time = xr.DataArray( - epoch_converted_time, + l0_dataset["epoch"], name="epoch", dims=["epoch"], attrs=cdf_attrs.get_variable_attributes("epoch"), @@ -202,7 +187,7 @@ def swe_science(decom_data: list, data_version: str) -> xr.Dataset: # Add APID to global attrs for following processing steps l1a_global_attrs = cdf_attrs.get_global_attributes("imap_swe_l1a_sci") # Formatting to string to be complaint with ISTP - l1a_global_attrs["packet_apid"] = f"{decom_data[0].header['PKT_APID'].raw_value}" + # l1a_global_attrs["packet_apid"] = SWEAPID.SWE_SCIENCE.value dataset = xr.Dataset( coords={ "epoch": epoch_time, @@ -215,16 +200,23 @@ def swe_science(decom_data: list, data_version: str) -> xr.Dataset: ) dataset["science_data"] = science_xarray dataset["raw_science_data"] = raw_science_xarray + # TODO: Remove the header in packet_file_to_datasets + # The science_data variable is also in the l1 dataset with different values + l0_dataset = l0_dataset.drop_vars( + [ + "science_data", + "version", + "type", + "sec_hdr_flg", + "pkt_apid", + "seq_flgs", + "src_seq_ctr", + "pkt_len", + ] + ) + for var_name, arr in l0_dataset.variables.items(): + arr.attrs = cdf_attrs.get_variable_attributes(var_name) + dataset = dataset.merge(l0_dataset) - # create xarray dataset for each metadata field - for key, value in metadata_arrays.items(): - # Lowercase the key to be complaint with ISTP's metadata field - metadata_field = key.lower() - dataset[metadata_field] = xr.DataArray( - value, - dims=["epoch"], - attrs=cdf_attrs.get_variable_attributes(metadata_field), - ) - - logger.info("SWE L1A science data process completed") + logger.info("SWE L1A science data processing completed.") return dataset diff --git a/imap_processing/swe/packet_definitions/swe_packet_definition.xml b/imap_processing/swe/packet_definitions/swe_packet_definition.xml index 759833205..edcde2eea 100644 --- a/imap_processing/swe/packet_definitions/swe_packet_definition.xml +++ b/imap_processing/swe/packet_definitions/swe_packet_definition.xml @@ -1,260 +1,303 @@ - + - + - - + + + + + - + + + + + + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - + + + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + - 10080 + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + CCSDS Packet Version Number (always 0) - + CCSDS Packet Type Indicator (0=telemetry) - + CCSDS Packet Secondary Header Flag (always 1) - + CCSDS Packet Application Process ID - + CCSDS Packet Grouping Flags (3=not part of group) - + CCSDS Packet Sequence Count (increments with each new packet) - + CCSDS Packet Length (number of bytes after Packet length minus 1) - - CCSDS Packet Time Stamp (coarse time) + + Mission elapsed time - - + Acquisition start time coarse in seconds - + Acquisition start time fine in microseconds - + CEM Nominal status bit - '1' -- nominal - '0' -- not nominal - +'1' -- nominal +'0' -- not nominal - + Spin period valid bit - '1' -- valid - '0' -- invalid - +'1' -- valid +'0' -- invalid - + Spin phase valid bit - '1' -- valid - '0' -- invalid - +'1' -- valid +'0' -- invalid - + Spin period source bit - '1' -- sun sensor (safing) - '0' -- star tracker (nominal) - +'1' -- sun sensor (safing) +'0' -- star tracker (nominal) - + HVPS settling duration - + Acquisition duration - + Spin phase - + Spin period - + Thruster fired during this quarter cycle - '1' -- yes, thruster fired - '0' -- no, thruster not fired - +'1' -- yes, warning received +'0' -- no, warning not received - + High CEM count observed for at least one CEM during this quarter cycle - '1' -- yes, high count occurred - '0' -- no, no high count occurred - +'1' -- yes, high count occurred +'0' -- no, no high count occurred - - ESA steps in packet + + Stim state +'1' -- enabled +'0' -- disabled - - ESA acquisition configuration + + Quarter cycle number - + ESA table identifier - + ESA acquisition configuration - + Threshold DAC value - + Stim pulse configuration register. Bits 0-3, period, 4-7, duration, 8-9 mode - + Data for a science acquisiton quarter cycle. (56 bit * 180 array) - Seven consecutive bytes represent data for a single step: - CEM1 - 8 bit counter - CEM2 - 8 bit counter - CEM3 - 8 bit counter - CEM4 - 8 bit counter - CEM5 - 8 bit counter - CEM6 - 8 bit counter - CEM7 - 8 bit counter - 180 steps are for the 180 ESA voltages used in a quarter cycle - +Seven consecutive bytes represent data for a single step: +CEM1 - 8 bit counter +CEM2 - 8 bit counter +CEM3 - 8 bit counter +CEM4 - 8 bit counter +CEM5 - 8 bit counter +CEM6 - 8 bit counter +CEM7 - 8 bit counter +180 steps are for the 180 ESA voltages used in a quarter cycle - - - + Checksum - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + - - + + \ No newline at end of file diff --git a/imap_processing/swe/utils/swe_utils.py b/imap_processing/swe/utils/swe_utils.py index b5e44d391..5b68c6cd9 100644 --- a/imap_processing/swe/utils/swe_utils.py +++ b/imap_processing/swe/utils/swe_utils.py @@ -2,36 +2,8 @@ from enum import IntEnum -import space_packet_parser - class SWEAPID(IntEnum): """Create ENUM for apid.""" SWE_SCIENCE = 1344 - - -def add_metadata_to_array( - data_packet: space_packet_parser.parser.Packet, metadata_arrays: dict -) -> dict: - """ - Add metadata to the metadata_arrays. - - Parameters - ---------- - data_packet : space_packet_parser.parser.Packet - SWE data packet. - metadata_arrays : dict - Metadata arrays. - - Returns - ------- - metadata_arrays : dict - The metadata_array with metadata from input data packet. - """ - for key, value in data_packet.data.items(): - if key == "SCIENCE_DATA": - continue - metadata_arrays.setdefault(key, []).append(value.raw_value) - - return metadata_arrays diff --git a/imap_processing/tests/swe/conftest.py b/imap_processing/tests/swe/conftest.py index df6226423..dd43d0bc2 100644 --- a/imap_processing/tests/swe/conftest.py +++ b/imap_processing/tests/swe/conftest.py @@ -1,7 +1,8 @@ import pytest from imap_processing import imap_module_directory -from imap_processing.swe.l0 import decom_swe +from imap_processing.swe.utils.swe_utils import SWEAPID +from imap_processing.utils import packet_file_to_datasets @pytest.fixture(scope="session") @@ -10,4 +11,25 @@ def decom_test_data(): packet_file = ( imap_module_directory / "tests/swe/l0_data/2024051010_SWE_SCIENCE_packet.bin" ) - return decom_swe.decom_packets(packet_file) + xtce_document = ( + imap_module_directory / "swe/packet_definitions/swe_packet_definition.xml" + ) + datasets_by_apid = packet_file_to_datasets( + packet_file, xtce_document, use_derived_value=False + ) + return datasets_by_apid[SWEAPID.SWE_SCIENCE] + + +@pytest.fixture(scope="session") +def decom_test_data_derived(): + """Read test data from file""" + packet_file = ( + imap_module_directory / "tests/swe/l0_data/2024051010_SWE_SCIENCE_packet.bin" + ) + xtce_document = ( + imap_module_directory / "swe/packet_definitions/swe_packet_definition.xml" + ) + datasets_by_apid = packet_file_to_datasets( + packet_file, xtce_document, use_derived_value=True + ) + return datasets_by_apid[SWEAPID.SWE_SCIENCE] diff --git a/imap_processing/tests/swe/test_swe_l1a.py b/imap_processing/tests/swe/test_swe_l1a.py index 96b65c1d5..d46275574 100644 --- a/imap_processing/tests/swe/test_swe_l1a.py +++ b/imap_processing/tests/swe/test_swe_l1a.py @@ -1,18 +1,6 @@ from imap_processing import imap_module_directory from imap_processing.cdf.utils import write_cdf from imap_processing.swe.l1a.swe_l1a import swe_l1a -from imap_processing.swe.utils.swe_utils import ( - SWEAPID, -) -from imap_processing.utils import group_by_apid - - -def test_group_by_apid(decom_test_data): - grouped_data = group_by_apid(decom_test_data) - - # check total dataset for swe science - total_science_data = grouped_data[SWEAPID.SWE_SCIENCE] - assert len(total_science_data) == 29 def test_cdf_creation(): diff --git a/imap_processing/tests/swe/test_swe_l1a_science.py b/imap_processing/tests/swe/test_swe_l1a_science.py index 44580218b..0293fac49 100644 --- a/imap_processing/tests/swe/test_swe_l1a_science.py +++ b/imap_processing/tests/swe/test_swe_l1a_science.py @@ -9,7 +9,7 @@ def test_number_of_packets(decom_test_data): """This test and validate number of packets.""" expected_number_of_packets = 29 - assert len(decom_test_data) == expected_number_of_packets + assert len(decom_test_data["epoch"]) == expected_number_of_packets def test_decompress_algorithm(): @@ -29,22 +29,20 @@ def test_swe_raw_science_data(decom_test_data): index_col="SHCOARSE", ) - first_data = decom_test_data[0] - validation_data = raw_validation_data.loc[first_data.data["SHCOARSE"].raw_value] + first_data = decom_test_data.isel(epoch=0) + validation_data = raw_validation_data.loc[first_data["shcoarse"].values] - # compare raw values of housekeeping data - for key, value in first_data.data.items(): - if key == "SHCOARSE": - # compare SHCOARSE value - assert value.raw_value == validation_data.name - continue - if key == "SCIENCE_DATA": - continue - # check if the data is the same - assert value.raw_value == validation_data[key] + # compare raw values of the packets + shared_keys = set([x.lower() for x in validation_data.keys()]).intersection( + first_data.keys() + ) + # TODO: Why are all the fields not the same between the two + assert len(shared_keys) == 19 + for key in shared_keys: + assert first_data[key] == validation_data[key.upper()] -def test_swe_derived_science_data(decom_test_data): +def test_swe_derived_science_data(decom_test_data_derived): """This test and validate raw and derived data of SWE science data.""" # read validation data test_data_path = imap_module_directory / "tests/swe/l0_validation_data" @@ -53,8 +51,8 @@ def test_swe_derived_science_data(decom_test_data): index_col="SHCOARSE", ) - first_data = decom_test_data[0] - validation_data = derived_validation_data.loc[first_data.data["SHCOARSE"].raw_value] + first_data = decom_test_data_derived.isel(epoch=0) + validation_data = derived_validation_data.loc[first_data["shcoarse"].values] enum_name_list = [ "CEM_NOMINAL_ONLY", @@ -68,24 +66,20 @@ def test_swe_derived_science_data(decom_test_data): ] # check ENUM values for enum_name in enum_name_list: - assert first_data.data[enum_name].derived_value == validation_data[enum_name] + assert first_data[enum_name.lower()] == validation_data[enum_name] def test_data_order(decom_test_data): # test that the data is in right order - assert decom_test_data[0].data["QUARTER_CYCLE"].derived_value == "FIRST" - assert decom_test_data[1].data["QUARTER_CYCLE"].derived_value == "SECOND" - assert decom_test_data[2].data["QUARTER_CYCLE"].derived_value == "THIRD" - assert decom_test_data[3].data["QUARTER_CYCLE"].derived_value == "FORTH" + np.testing.assert_array_equal( + decom_test_data.isel(epoch=slice(0, 4))["quarter_cycle"], [0, 1, 2, 3] + ) # Get unpacked science data processed_data = swe_science(decom_test_data, "001") - quarter_cycle = processed_data["quarter_cycle"].data - assert quarter_cycle[0] == 0 - assert quarter_cycle[1] == 1 - assert quarter_cycle[2] == 2 - assert quarter_cycle[3] == 3 + quarter_cycle = processed_data["quarter_cycle"].isel(epoch=slice(0, 4)) + np.testing.assert_array_equal(quarter_cycle, [0, 1, 2, 3]) def test_swe_science_algorithm(decom_test_data): diff --git a/imap_processing/tests/swe/test_swe_l1b.py b/imap_processing/tests/swe/test_swe_l1b.py index 105a63a54..6dc849db7 100644 --- a/imap_processing/tests/swe/test_swe_l1b.py +++ b/imap_processing/tests/swe/test_swe_l1b.py @@ -1,3 +1,4 @@ +import numpy as np import pandas as pd import pytest @@ -6,43 +7,17 @@ from imap_processing.swe.l1a.swe_l1a import swe_l1a from imap_processing.swe.l1a.swe_science import swe_science from imap_processing.swe.l1b.swe_l1b import swe_l1b -from imap_processing.swe.utils.swe_utils import ( - SWEAPID, -) -from imap_processing.utils import convert_raw_to_eu, group_by_apid -def test_swe_l1b(decom_test_data): - """Test that calculate engineering unit(EU) is - matches validation data. +def test_swe_l1b(decom_test_data_derived): + """Test that calculate engineering unit(EU) matches validation data. Parameters ---------- - decom_test_data : list - List of packets + decom_test_data_derived : xarray.dataset + Dataset with derived values """ - grouped_data = group_by_apid(decom_test_data) - # Process science to l1a. - # because of test data being in the wrong - # order, we need to manually re-sort data - # into order. - sorted_packets = sorted( - grouped_data[SWEAPID.SWE_SCIENCE], - key=lambda x: x.data["QUARTER_CYCLE"].raw_value, - ) - science_l1a_ds = swe_science(sorted_packets, "001") - # convert value from raw to engineering units as needed - conversion_table_path = ( - imap_module_directory / "swe/l1b/engineering_unit_convert_table.csv" - ) - # Look up packet name from APID - packet_name = SWEAPID.SWE_SCIENCE.name - # Convert raw data to engineering units as needed - science_l1b = convert_raw_to_eu( - science_l1a_ds, - conversion_table_path=conversion_table_path, - packet_name=packet_name, - ) + science_l1a_ds = swe_science(decom_test_data_derived, "001") # read science validation data test_data_path = imap_module_directory / "tests/swe/l0_validation_data" @@ -51,8 +26,8 @@ def test_swe_l1b(decom_test_data): index_col="SHCOARSE", ) - second_data = sorted_packets[1] - validation_data = eu_validation_data.loc[second_data.data["SHCOARSE"].raw_value] + second_data = science_l1a_ds.isel(epoch=1) + validation_data = eu_validation_data.loc[second_data["shcoarse"].values] science_eu_field_list = [ "SPIN_PHASE", @@ -62,8 +37,8 @@ def test_swe_l1b(decom_test_data): # Test EU values for science data for field in science_eu_field_list: - assert round(science_l1b[field.lower()].data[1], 5) == round( - validation_data[field], 5 + np.testing.assert_almost_equal( + second_data[field.lower()].values, validation_data[field], decimal=5 ) diff --git a/imap_processing/tests/swe/test_swe_l1b_science.py b/imap_processing/tests/swe/test_swe_l1b_science.py index e446489f5..1dbd491d2 100644 --- a/imap_processing/tests/swe/test_swe_l1b_science.py +++ b/imap_processing/tests/swe/test_swe_l1b_science.py @@ -2,11 +2,12 @@ import pytest from imap_processing import imap_module_directory -from imap_processing.swe.l0 import decom_swe from imap_processing.swe.l1a.swe_science import swe_science from imap_processing.swe.l1b.swe_l1b_science import ( get_indices_of_full_cycles, ) +from imap_processing.swe.utils.swe_utils import SWEAPID +from imap_processing.utils import packet_file_to_datasets @pytest.fixture(scope="session") @@ -29,11 +30,17 @@ def l1a_test_data(): imap_module_directory / "tests/swe/l0_data/20230927173238_SWE_SCIENCE_packet.bin", ] - data = [] - for packet_file in packet_files: - data.extend(decom_swe.decom_packets(packet_file)) + dataset = packet_file_to_datasets(packet_files[0], use_derived_value=False)[ + SWEAPID.SWE_SCIENCE + ] + for packet_file in packet_files[1:]: + dataset.merge( + packet_file_to_datasets(packet_file, use_derived_value=False)[ + SWEAPID.SWE_SCIENCE + ] + ) # Get unpacked science data - unpacked_data = swe_science(data, "001") + unpacked_data = swe_science(dataset, "001") return unpacked_data