diff --git a/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml b/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml index 54701c71c..9d7a535d9 100644 --- a/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml @@ -36,6 +36,28 @@ energy_attrs: VALIDMAX: 127 VAR_TYPE: support_data +inst_az_attrs: + <<: *default + CATDESC: Azimuth + FIELDNAM: Azimuth + FORMAT: I2 + LABLAXIS: Azimuth + UNITS: ' ' + VALIDMIN: 0 + VALIDMAX: 31 + VAR_TYPE: support_data + +spin_sector_attrs: + <<: *default + CATDESC: Spin sector indicating range of spin angles + FIELDNAM: Spin sector + FORMAT: I2 + LABLAXIS: spin sector + UNITS: ' ' + VALIDMIN: 0 + VALIDMAX: 11 + VAR_TYPE: support_data + # <=== Labels ===> energy_label: CATDESC: Energy per charge (E/q) sweeping step diff --git a/imap_processing/cdf/config/imap_codice_l1b_variable_attrs.yaml b/imap_processing/cdf/config/imap_codice_l1b_variable_attrs.yaml index 1d5d44eb5..cbb14205f 100644 --- a/imap_processing/cdf/config/imap_codice_l1b_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_l1b_variable_attrs.yaml @@ -32,6 +32,28 @@ energy_attrs: VALIDMAX: 127 VAR_TYPE: support_data +inst_az_attrs: + <<: *default + CATDESC: Azimuth + FIELDNAM: Azimuth + FORMAT: I2 + LABLAXIS: Azimuth + UNITS: ' ' + VALIDMIN: 0 + VALIDMAX: 31 + VAR_TYPE: support_data + +spin_sector_attrs: + <<: *default + CATDESC: Spin sector indicating range of spin angles + FIELDNAM: Spin sector + FORMAT: I4 + LABLAXIS: spin sector + UNITS: ' ' + VALIDMIN: 0 + VALIDMAX: 1152 + VAR_TYPE: support_data + # <=== Labels ===> energy_label: CATDESC: Energy per charge (E/q) sweeping step diff --git a/imap_processing/codice/codice_l1a.py b/imap_processing/codice/codice_l1a.py index 0c925fee7..bc59ace81 100644 --- a/imap_processing/codice/codice_l1a.py +++ b/imap_processing/codice/codice_l1a.py @@ -27,16 +27,22 @@ from imap_processing.cdf.utils import met_to_j2000ns from imap_processing.codice import constants from imap_processing.codice.codice_l0 import decom_packets +from imap_processing.codice.decompress import decompress from imap_processing.codice.utils import CODICEAPID, add_metadata_to_array from imap_processing.utils import group_by_apid, sort_by_time logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -# TODO: Decom data arrays need to be decompressed # TODO: In decommutation, how to have a variable length data and then a checksum # after it? (Might be fixed with new XTCE script updates) # TODO: Add support for decomming multiple APIDs from a single file +# TODO: Add these as variables in CDF: SPIN_PERIOD, ST_BIAS_GAIN_MODE, +# SW_BIAS_GAIN_MODE, RGFO_HALF_SPIN, NSO_HALF_SPIN, DATA_QUALITY +# TODO: Use new packet_file_to_dataset() function to simplify things +# TODO: Determine what should go in event data CDF and how it should be +# structured. +# TODO: Make sure CDF attributes match expected nomenclature class CoDICEL1aPipeline: @@ -70,7 +76,7 @@ class CoDICEL1aPipeline: get_esa_sweep_values() Retrieve the ESA sweep values. unpack_science_data() - Make 4D L1a data product from the decompressed science data. + Decompress, unpack, and restructure science data arrays. """ def __init__(self, table_id: int, plan_id: int, plan_step: int, view_id: int): @@ -92,8 +98,11 @@ def configure_data_products(self, apid: int) -> None: config = constants.DATA_PRODUCT_CONFIGURATIONS.get(apid) # type: ignore[call-overload] self.num_counters = config["num_counters"] self.num_energy_steps = config["num_energy_steps"] + self.num_spin_sectors = config["num_spin_sectors"] + self.num_positions = config["num_positions"] self.variable_names = config["variable_names"] self.dataset_name = config["dataset_name"] + self.instrument = config["instrument"] def create_science_dataset(self, met: np.int64, data_version: str) -> xr.Dataset: """ @@ -121,11 +130,23 @@ def create_science_dataset(self, met: np.int64, data_version: str) -> xr.Dataset # Define coordinates epoch = xr.DataArray( - met_to_j2000ns(met), # TODO: Fix after SIT-3 (see note below) + [met_to_j2000ns(met)], name="epoch", dims=["epoch"], attrs=cdf_attrs.get_variable_attributes("epoch"), ) + inst_az = xr.DataArray( + np.arange(self.num_positions), + name="inst_az", + dims=["inst_az"], + attrs=cdf_attrs.get_variable_attributes("inst_az_attrs"), + ) + spin_sector = xr.DataArray( + np.arange(self.num_spin_sectors), + name="spin_sector", + dims=["spin_sector"], + attrs=cdf_attrs.get_variable_attributes("spin_sector_attrs"), + ) energy_steps = xr.DataArray( np.arange(self.num_energy_steps), name="energy", @@ -145,6 +166,8 @@ def create_science_dataset(self, met: np.int64, data_version: str) -> xr.Dataset dataset = xr.Dataset( coords={ "epoch": epoch, + "inst_az": inst_az, + "spin_sector": spin_sector, "energy": energy_steps, "energy_label": energy_label, }, @@ -153,25 +176,34 @@ def create_science_dataset(self, met: np.int64, data_version: str) -> xr.Dataset # Create a data variable for each counter for variable_data, variable_name in zip(self.data, self.variable_names): - # TODO: Currently, cdflib doesn't properly write/read CDF files that - # have a single epoch value. To get around this for now, use - # two epoch values and reshape accordingly. Revisit this after - # SIT-3. See https://github.com/MAVENSDC/cdflib/issues/268 - variable_data_arr = np.array(list(variable_data) * 2, dtype=int).reshape( - 2, self.num_energy_steps - ) + # Data arrays are structured depending on the instrument + if self.instrument == "lo": + variable_data_arr = np.array(variable_data).reshape( + 1, self.num_positions, self.num_spin_sectors, self.num_energy_steps + ) + dims = ["epoch", "inst_az", "spin_sector", "energy"] + elif self.instrument == "hi": + variable_data_arr = np.array(variable_data).reshape( + 1, self.num_energy_steps, self.num_positions, self.num_spin_sectors + ) + dims = ["epoch", "energy", "inst_az", "spin_sector"] + + # Get the CDF attributes cdf_attrs_key = ( f"{self.dataset_name.split('imap_codice_l1a_')[-1]}-{variable_name}" ) + attrs = cdf_attrs.get_variable_attributes(cdf_attrs_key) + + # Create the CDF data variable dataset[variable_name] = xr.DataArray( variable_data_arr, name=variable_name, - dims=["epoch", "energy"], - attrs=cdf_attrs.get_variable_attributes(cdf_attrs_key), + dims=dims, + attrs=attrs, ) # Add ESA Sweep Values and acquisition times (lo only) - if "_lo_" in self.dataset_name: + if self.instrument == "lo": self.get_esa_sweep_values() self.get_acquisition_times() dataset["esa_sweep_values"] = xr.DataArray( @@ -264,12 +296,13 @@ def get_esa_sweep_values(self) -> None: def unpack_science_data(self, science_values: str) -> None: """ - Unpack the science data from the packet. + Decompress, unpack, and restructure science data arrays. - For LO SW Species Counts data, the science data within the packet is a - blob of compressed values of length 2048 bits (16 species * 128 energy - levels). These data need to be divided up by species so that each - species can have their own data variable in the L1A CDF file. + The science data within the packet is a compressed, binary string of + values. These data need to be divided up by species or priorities (or + what I am calling "counters" as a general term), and re-arranged into + 3D arrays representing spin sectors, positions, and energies (the order + of which depends on the instrument). Parameters ---------- @@ -277,18 +310,32 @@ def unpack_science_data(self, science_values: str) -> None: A string of binary data representing the science values of the data. """ self.compression_algorithm = constants.LO_COMPRESSION_ID_LOOKUP[self.view_id] - self.collapse_table_id = constants.LO_COLLAPSE_TABLE_ID_LOOKUP[self.view_id] - # TODO: Turn this back on after SIT-3 - # For SIT-3, just create appropriate length data arrays of all ones - # Divide up the data by the number of priorities or species - # science_values = packets[0].data["DATA"].raw_value - # num_bits = len(science_values) - # chunk_size = len(science_values) // self.num_counters - # self.data = [ - # science_values[i : i + chunk_size] for i in range(0, num_bits, chunk_size) - # ] - self.data = [["1"] * 128] * self.num_counters + # Decompress the binary string into a list of integers + science_values_decompressed = decompress( + science_values, self.compression_algorithm + ) + + # Re-arrange the counter data + # For CoDICE-lo, data are a 3D arrays with a shape representing + # [,,] + if self.instrument == "lo": + self.data = np.array(science_values_decompressed, dtype=np.uint32).reshape( + self.num_counters, + self.num_positions, + self.num_spin_sectors, + self.num_energy_steps, + ) + + # For CoDICE-hi, data are a 3D array with a shape representing + # [,,] + elif self.instrument == "hi": + self.data = np.array(science_values_decompressed, dtype=np.uint32).reshape( + self.num_counters, + self.num_energy_steps, + self.num_positions, + self.num_spin_sectors, + ) def create_event_dataset( @@ -334,9 +381,6 @@ def create_event_dataset( attrs=cdf_attrs.get_global_attributes(dataset_name), ) - # TODO: Determine what should go in event data CDF and how it should be - # structured. - return dataset @@ -385,13 +429,15 @@ def create_hskp_dataset( ) # TODO: Change 'TBD' catdesc and fieldname - # Once housekeeping packet definition file is re-generated with updated - # version of space_packet_parser, can get fieldname and catdesc info via: - # for key, value in (packet.header | packet.data).items(): - # fieldname = value.short_description - # catdesc = value.long_description - # I am holding off making this change until I acquire updated housekeeping - # packets/validation data that match the latest telemetry definitions + # Once housekeeping packet definition file is re-generated with + # updated version of space_packet_parser, can get fieldname and + # catdesc info via: + # for key, value in (packet.header | packet.data).items(): + # fieldname = value.short_description + # catdesc = value.long_description + # I am holding off making this change until I acquire updated + # housekeeping packets/validation data that match the latest telemetry + # definitions for key, value in metadata_arrays.items(): attrs = cdf_attrs.get_variable_attributes("codice_support_attrs") attrs["CATDESC"] = "TBD" @@ -457,8 +503,6 @@ def process_codice_l1a(file_path: Path, data_version: str) -> xr.Dataset: dataset : xarray.Dataset The ``xarray`` dataset containing the science data and supporting metadata. """ - # TODO: Use new packet_file_to_dataset() function to simplify things - # Decom the packets, group data by APID, and sort by time packets = decom_packets(file_path) grouped_data = group_by_apid(packets) @@ -496,7 +540,7 @@ def process_codice_l1a(file_path: Path, data_version: str) -> xr.Dataset: # Determine the start time of the packet met = packets[0].data["ACQ_START_SECONDS"].raw_value - met = [met, met + 1] # TODO: Remove after cdflib fix + # Extract the data science_values = packets[0].data["DATA"].raw_value @@ -510,4 +554,5 @@ def process_codice_l1a(file_path: Path, data_version: str) -> xr.Dataset: dataset = pipeline.create_science_dataset(met, data_version) logger.info(f"\nFinal data product:\n{dataset}\n") + return dataset diff --git a/imap_processing/codice/constants.py b/imap_processing/codice/constants.py index a8bcf3503..f063087f2 100644 --- a/imap_processing/codice/constants.py +++ b/imap_processing/codice/constants.py @@ -12,6 +12,9 @@ ESA = ElectroStatic Analyzer """ +# TODO: What to do in the case of a value of 255 in LOSSY_A and LOSSY_B +# compression? + from imap_processing.codice.utils import CODICEAPID, CoDICECompression APIDS_FOR_SCIENCE_PROCESSING = [ @@ -76,75 +79,111 @@ DATA_PRODUCT_CONFIGURATIONS = { CODICEAPID.COD_HI_INST_COUNTS_AGGREGATED: { "num_counters": 1, - "num_energy_steps": 128, + "num_energy_steps": 1, # TODO: Double check with Joey + "num_positions": 6, # TODO: Double check with Joey + "num_spin_sectors": 1, "variable_names": HI_INST_COUNTS_AGGREGATED_NAMES, "dataset_name": "imap_codice_l1a_hi_counters_aggregated", + "instrument": "hi", }, CODICEAPID.COD_HI_INST_COUNTS_SINGLES: { "num_counters": 3, - "num_energy_steps": 128, + "num_energy_steps": 1, # TODO: Double check with Joey + "num_positions": 16, # TODO: Double check with Joey + "num_spin_sectors": 1, "variable_names": HI_INST_COUNTS_SINGLES_NAMES, "dataset_name": "imap_codice_l1a_hi_counters_singles", + "instrument": "hi", }, CODICEAPID.COD_HI_OMNI_SPECIES_COUNTS: { "num_counters": 8, - "num_energy_steps": 128, + "num_energy_steps": 15, # TODO: Double check with Joey + "num_positions": 4, # TODO: Double check with Joey + "num_spin_sectors": 1, "variable_names": HI_OMNI_SPECIES_NAMES, "dataset_name": "imap_codice_l1a_hi_omni", + "instrument": "hi", }, CODICEAPID.COD_HI_SECT_SPECIES_COUNTS: { "num_counters": 4, - "num_energy_steps": 128, + "num_energy_steps": 8, + "num_positions": 12, + "num_spin_sectors": 12, "variable_names": HI_SECT_SPECIES_NAMES, "dataset_name": "imap_codice_l1a_hi_sectored", + "instrument": "hi", }, CODICEAPID.COD_LO_INST_COUNTS_AGGREGATED: { "num_counters": 1, "num_energy_steps": 128, + "num_positions": 6, + "num_spin_sectors": 6, "variable_names": LO_INST_COUNTS_AGGREGATED_NAMES, "dataset_name": "imap_codice_l1a_lo_counters_aggregated", + "instrument": "lo", }, CODICEAPID.COD_LO_INST_COUNTS_SINGLES: { "num_counters": 1, "num_energy_steps": 128, + "num_positions": 24, + "num_spin_sectors": 6, "variable_names": LO_INST_COUNTS_SINGLES_NAMES, "dataset_name": "imap_codice_l1a_lo_counters_singles", + "instrument": "lo", }, CODICEAPID.COD_LO_SW_ANGULAR_COUNTS: { "num_counters": 4, "num_energy_steps": 128, + "num_positions": 5, + "num_spin_sectors": 12, "variable_names": LO_SW_ANGULAR_NAMES, "dataset_name": "imap_codice_l1a_lo_sw_angular", + "instrument": "lo", }, CODICEAPID.COD_LO_NSW_ANGULAR_COUNTS: { "num_counters": 1, "num_energy_steps": 128, + "num_positions": 19, + "num_spin_sectors": 12, "variable_names": LO_NSW_ANGULAR_NAMES, "dataset_name": "imap_codice_l1a_lo_nsw_angular", + "instrument": "lo", }, CODICEAPID.COD_LO_SW_PRIORITY_COUNTS: { "num_counters": 5, "num_energy_steps": 128, + "num_positions": 1, + "num_spin_sectors": 12, "variable_names": LO_SW_PRIORITY_NAMES, "dataset_name": "imap_codice_l1a_lo_sw_priority", + "instrument": "lo", }, CODICEAPID.COD_LO_NSW_PRIORITY_COUNTS: { "num_counters": 2, "num_energy_steps": 128, + "num_positions": 1, + "num_spin_sectors": 12, "variable_names": LO_NSW_PRIORITY_NAMES, "dataset_name": "imap_codice_l1a_lo_nsw_priority", + "instrument": "lo", }, CODICEAPID.COD_LO_SW_SPECIES_COUNTS: { "num_counters": 16, "num_energy_steps": 128, + "num_positions": 1, + "num_spin_sectors": 1, "variable_names": LO_SW_SPECIES_NAMES, "dataset_name": "imap_codice_l1a_lo_sw_species", + "instrument": "lo", }, CODICEAPID.COD_LO_NSW_SPECIES_COUNTS: { "num_counters": 8, "num_energy_steps": 128, + "num_positions": 1, + "num_spin_sectors": 1, "variable_names": LO_NSW_SPECIES_NAMES, "dataset_name": "imap_codice_l1a_lo_nsw_species", + "instrument": "lo", }, } @@ -179,25 +218,6 @@ 9: CoDICECompression.LOSSY_A_LOSSLESS, } -# Collapse table ID lookup table for Lo data products -# The key is the view_id and the value is the ID for the collapse table -LO_COLLAPSE_TABLE_ID_LOOKUP = {0: 0, 1: 1, 2: 2, 3: 3, 4: 4, 5: 5, 6: 6, 7: 7, 8: 8} - -# Collapse table ID lookup table for Hi data products -# The key is the view_id and the value is the ID for the collapse table -Hi_COLLAPSE_TABLE_ID_LOOKUP = { - 0: 8, - 1: 9, - 2: 10, - 3: 0, - 4: 1, - 5: 2, - 6: 4, - 7: 5, - 8: 6, - 9: 7, -} - # ESA Sweep table ID lookup table # The combination of plan_id and plan_step determine the ESA sweep Table to use # Currently, ESA sweep table 0 is used for every plan_id/plan_step combination, @@ -538,6 +558,7 @@ 252: 475136, 253: 491520, 254: 507904, + 255: 999999, } LOSSY_B_TABLE = { @@ -796,4 +817,5 @@ 252: 6815744, 253: 7340032, 254: 7864320, + 255: 9999999, } diff --git a/imap_processing/codice/decompress.py b/imap_processing/codice/decompress.py index bbbad4b6c..9a48b72b2 100644 --- a/imap_processing/codice/decompress.py +++ b/imap_processing/codice/decompress.py @@ -26,18 +26,14 @@ This information was provided via email from Greg Dunn on Oct 23, 2023 """ -# TODO: Add support for performing decompression of a list of values instead of -# a single value - import lzma from enum import IntEnum -from typing import Union from imap_processing.codice.constants import LOSSY_A_TABLE, LOSSY_B_TABLE from imap_processing.codice.utils import CoDICECompression -def _apply_lossy_a(compressed_value: int) -> int: +def _apply_lossy_a(compressed_bytes: bytes) -> list[int]: """ Apply 8-bit to 32-bit Lossy A decompression algorithm. @@ -45,18 +41,20 @@ def _apply_lossy_a(compressed_value: int) -> int: Parameters ---------- - compressed_value : int - The compressed 8-bit value. + compressed_bytes : bytes + The compressed byte stream. Returns ------- - int - The 24- or 32-bit decompressed value. + decompressed_values : list[int] + The 24- or 32-bit decompressed values. """ - return LOSSY_A_TABLE[compressed_value] + compressed_values = list(compressed_bytes) + decompressed_values = [LOSSY_A_TABLE[item] for item in compressed_values] + return decompressed_values -def _apply_lossy_b(compressed_value: int) -> int: +def _apply_lossy_b(compressed_bytes: bytes) -> list[int]: """ Apply 8-bit to 32-bit Lossy B decompression algorithm. @@ -64,44 +62,41 @@ def _apply_lossy_b(compressed_value: int) -> int: Parameters ---------- - compressed_value : int - The compressed 8-bit value. + compressed_bytes : bytes + The compressed byte stream. Returns ------- - int - The 24- or 32-bit decompressed value. + decompressed_values : list[int] + The 24- or 32-bit decompressed values. """ - return LOSSY_B_TABLE[compressed_value] + compressed_values = list(compressed_bytes) + decompressed_values = [LOSSY_B_TABLE[item] for item in compressed_values] + return decompressed_values -def _apply_lzma_lossless(compressed_value: Union[int, bytes]) -> int: +def _apply_lzma_lossless(compressed_bytes: bytes) -> bytes: """ Apply LZMA lossless decompression algorithm. Parameters ---------- - compressed_value : int or bytes - The compressed 8-bit value. + compressed_bytes : bytes + The compressed byte stream. Returns ------- - decompressed_value : int - The 24- or 32-bit decompressed value. + lzma_decompressed_values : bytes + The 24- or 32-bit lzma decompressed values. """ - if isinstance(compressed_value, int): - bytes_compressed_value = compressed_value.to_bytes(compressed_value, "big") - else: - bytes_compressed_value = compressed_value - decompressed_value = lzma.decompress(bytes_compressed_value) - decompressed_value_int = int.from_bytes(decompressed_value, byteorder="big") + lzma_decompressed_values = lzma.decompress(compressed_bytes) - return decompressed_value_int + return lzma_decompressed_values -def decompress(compressed_value: int, algorithm: IntEnum) -> int: +def decompress(compressed_binary: str, algorithm: IntEnum) -> list[int]: """ - Will decompress the value. + Perform decompression on a binary string into a list of integers. Apply the appropriate decompression algorithm(s) based on the value of the ``algorithm`` attribute. One or more individual algorithms may be @@ -109,32 +104,39 @@ def decompress(compressed_value: int, algorithm: IntEnum) -> int: Parameters ---------- - compressed_value : int - The 8-bit compressed value to decompress. + compressed_binary : str + The compressed binary string. algorithm : int The algorithm to apply. Supported algorithms are provided in the ``codice_utils.CoDICECompression`` class. Returns ------- - decompressed_value : int - The 24- or 32-bit decompressed value. + decompressed_values : list[int] + The 24- or 32-bit decompressed values. """ + # Convert the binary string to a byte stream + compressed_bytes = int(compressed_binary, 2).to_bytes( + (len(compressed_binary) + 7) // 8, byteorder="big" + ) + + # Apply the appropriate decompression algorithm if algorithm == CoDICECompression.NO_COMPRESSION: - decompressed_value = compressed_value + decompressed_values = list(compressed_bytes) elif algorithm == CoDICECompression.LOSSY_A: - decompressed_value = _apply_lossy_a(compressed_value) + decompressed_values = _apply_lossy_a(compressed_bytes) elif algorithm == CoDICECompression.LOSSY_B: - decompressed_value = _apply_lossy_b(compressed_value) + decompressed_values = _apply_lossy_b(compressed_bytes) elif algorithm == CoDICECompression.LOSSLESS: - decompressed_value = _apply_lzma_lossless(compressed_value) + decompressed_bytes = _apply_lzma_lossless(compressed_bytes) + decompressed_values = list(decompressed_bytes) elif algorithm == CoDICECompression.LOSSY_A_LOSSLESS: - decompressed_value = _apply_lzma_lossless(compressed_value) - decompressed_value = _apply_lossy_a(decompressed_value) + decompressed_bytes = _apply_lzma_lossless(compressed_bytes) + decompressed_values = _apply_lossy_a(decompressed_bytes) elif algorithm == CoDICECompression.LOSSY_B_LOSSLESS: - decompressed_value = _apply_lzma_lossless(compressed_value) - decompressed_value = _apply_lossy_b(decompressed_value) + decompressed_bytes = _apply_lzma_lossless(compressed_bytes) + decompressed_values = _apply_lossy_b(decompressed_bytes) else: raise ValueError(f"{algorithm} is not supported") - return decompressed_value + return decompressed_values diff --git a/imap_processing/tests/codice/test_codice_l1a.py b/imap_processing/tests/codice/test_codice_l1a.py index 383a07b9b..d9fd7207c 100644 --- a/imap_processing/tests/codice/test_codice_l1a.py +++ b/imap_processing/tests/codice/test_codice_l1a.py @@ -19,19 +19,19 @@ EXPECTED_ARRAY_SHAPES = [ (99,), # hskp - (1, 128), # hi-counters-aggregated - (1, 128), # hi-counters-singles - (1, 128), # hi-omni - (1, 128), # hi-sectored - (1, 128), # hi-pha - (1, 128), # lo-counters-aggregated - (1, 128), # lo-counters-aggregated - (1, 128), # lo-sw-angular - (1, 128), # lo-nsw-angular - (1, 128), # lo-sw-priority - (1, 128), # lo-nsw-priority - (1, 128), # lo-sw-species - (1, 128), # lo-nsw-species + (1, 1, 6, 1), # hi-counters-aggregated # TODO: Double check with Joey + (1, 1, 16, 1), # hi-counters-singles # TODO: Double check with Joey + (1, 15, 4, 1), # hi-omni # TODO: Double check with Joey + (1, 8, 12, 12), # hi-sectored + (1, 1), # hi-pha + (1, 6, 6, 128), # lo-counters-aggregated + (1, 24, 6, 128), # lo-counters-singles + (1, 5, 12, 128), # lo-sw-angular + (1, 19, 12, 128), # lo-nsw-angular + (1, 1, 12, 128), # lo-sw-priority + (1, 1, 12, 128), # lo-nsw-priority + (1, 1, 1, 128), # lo-sw-species + (1, 1, 1, 128), # lo-nsw-species (1, 128), # lo-pha ] EXPECTED_ARRAY_SIZES = [ @@ -110,9 +110,6 @@ def test_l1a_cdf_filenames(test_l1a_data: xr.Dataset, expected_logical_source: s assert dataset.attrs["Logical_source"] == expected_logical_source -@pytest.mark.xfail( - reason="Currently failing due to cdflib/epoch issue. See https://github.com/MAVENSDC/cdflib/issues/268" -) @pytest.mark.parametrize( "test_l1a_data, expected_shape", list(zip(TEST_PACKETS, EXPECTED_ARRAY_SHAPES)), diff --git a/imap_processing/tests/codice/test_decompress.py b/imap_processing/tests/codice/test_decompress.py index e74f60d73..853a94ccc 100644 --- a/imap_processing/tests/codice/test_decompress.py +++ b/imap_processing/tests/codice/test_decompress.py @@ -9,34 +9,37 @@ from imap_processing.codice.utils import CoDICECompression # Test the algorithms using input value of 234 (picked randomly) -LZMA_EXAMPLE = lzma.compress((234).to_bytes(1, byteorder="big")) +lzma_bytes = lzma.compress((234).to_bytes(1, byteorder="big")) +LZMA_EXAMPLE = "".join(format(byte, "08b") for byte in lzma_bytes) TEST_DATA = [ - (234, CoDICECompression.NO_COMPRESSION, 234), - (234, CoDICECompression.LOSSY_A, 221184), - (234, CoDICECompression.LOSSY_B, 1441792), - (LZMA_EXAMPLE, CoDICECompression.LOSSLESS, 234), - (LZMA_EXAMPLE, CoDICECompression.LOSSY_A_LOSSLESS, 221184), - (LZMA_EXAMPLE, CoDICECompression.LOSSY_B_LOSSLESS, 1441792), + ("11101010", CoDICECompression.NO_COMPRESSION, [234]), + ("11101010", CoDICECompression.LOSSY_A, [221184]), + ("11101010", CoDICECompression.LOSSY_B, [1441792]), + (LZMA_EXAMPLE, CoDICECompression.LOSSLESS, [234]), + (LZMA_EXAMPLE, CoDICECompression.LOSSY_A_LOSSLESS, [221184]), + (LZMA_EXAMPLE, CoDICECompression.LOSSY_B_LOSSLESS, [1441792]), ] @pytest.mark.parametrize( - ("compressed_value", "algorithm", "expected_result"), TEST_DATA + ("compressed_binary", "algorithm", "expected_result"), TEST_DATA ) -def test_decompress(compressed_value: int, algorithm: IntEnum, expected_result: int): +def test_decompress( + compressed_binary: str, algorithm: IntEnum, expected_result: list[int] +): """Tests the ``decompress`` function Parameters ---------- - compressed_value : int - The compressed value to test decompression on + compressed_binary : str + The compressed binary string to test decompression on algorithm : IntEnum The algorithm to use in decompression - expected_result : int + expected_result : list[int] The expected, decompressed value """ - decompressed_value = decompress(compressed_value, algorithm) + decompressed_value = decompress(compressed_binary, algorithm) assert decompressed_value == expected_result @@ -44,4 +47,4 @@ def test_decompress_raises(): """Tests that the ``decompress`` function raises with an unknown algorithm""" with pytest.raises(ValueError, match="some_unsupported_algorithm"): - decompress(234, "some_unsupported_algorithm") + decompress("11101010", "some_unsupported_algorithm")