diff --git a/imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml b/imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml index 005085e52..fa8dc99e0 100644 --- a/imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml @@ -49,6 +49,13 @@ imap_codice_l1a_hi_sectored: Logical_source: imap_codice_l1a_hi-sectored Logical_source_description: IMAP Mission CoDICE Hi Level-1A Sectored Data. +imap_codice_l1a_hi_pha: + <<: *instrument_base + Data_level: 1A + Data_type: L1A_hi-pha->Level-1A Hi Event Data + Logical_source: imap_codice_l1a_hi-pha + Logical_source_description: IMAP Mission CoDICE Hi Level-1A Event Data. + imap_codice_l1a_lo_counters_aggregated: <<: *instrument_base Data_level: 1A @@ -105,6 +112,13 @@ imap_codice_l1a_lo_nsw_species: Logical_source: imap_codice_l1a_lo-nsw-species Logical_source_description: IMAP Mission CoDICE Lo Level-1A Non-Sunward Species Counts Data. +imap_codice_l1a_lo_pha: + <<: *instrument_base + Data_level: 1A + Data_type: L1A_lo-pha->Level-1A Lo Event Data + Logical_source: imap_codice_l1a_lo-pha + Logical_source_description: IMAP Mission CoDICE Lo Level-1A Event Data. + # L1b imap_codice_l1b_hskp: <<: *instrument_base diff --git a/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml b/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml index 2407e77aa..54701c71c 100644 --- a/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml @@ -1,14 +1,24 @@ +# TODO: Some currently blank attributes may be removed once +# cdf_attribute_manager.py is updated to only check required attributes. +# See comment here: +# https://github.com/IMAP-Science-Operations-Center/imap_processing/pull/711#discussion_r1690212527 # <=== Defaults ===> default_attrs: &default - # Assumed values for all variable attrs unless overwritten - DISPLAY_TYPE: no_plot - FILLVAL: -9223372036854775808 - FORMAT: I12 - UNITS: dN - VALIDMIN: -9223372036854775808 - VALIDMAX: 9223372036854775807 - VAR_TYPE: data - SCALETYP: linear + CATDESC: "" + DISPLAY_TYPE: no_plot + FIELDNAM: "" + FILLVAL: -9223372036854775808 + FORMAT: I12 + LABLAXIS: "" + REFERENCE_POSITION: "" + RESOLUTION: "" + SCALETYP: linear + TIME_BASE: "" + TIME_SCALE: "" + UNITS: dN + VALIDMIN: -9223372036854775808 + VALIDMAX: 9223372036854775807 + VAR_TYPE: data codice_support_attrs: &support_default <<: *default @@ -29,23 +39,32 @@ energy_attrs: # <=== Labels ===> energy_label: CATDESC: Energy per charge (E/q) sweeping step + DISPLAY_TYPE: "" FIELDNAM: Energy step + FILLVAL: "" FORMAT: A3 + LABLAXIS: "" + REFERENCE_POSITION: "" + RESOLUTION: "" + TIME_BASE: "" + TIME_SCALE: "" + UNITS: "" + VALIDMAX: "" + VALIDMIN: "" VAR_TYPE: metadata # <=== Dataset Attributes ===> -# TODO: resolve ISTP warning: Width of F10 FORMAT may be insufficient for -# values. Adjust FORMAT or VALIDMIN/VALIDMAX value to resolve. acquisition_times_attrs: <<: *default CATDESC: Time of acquisition for the energy step + DEPEND_1: energy FIELDNAM: Acquisition Time - FILLVAL: 1.7976931348623157e+308 + FILLVAL: .NAN FORMAT: F10.3 LABLAXIS: Acq Time UNITS: ms VALIDMIN: 0 - VALIDMAX: 1.7976931348623157e+308 + VALIDMAX: 1000 VAR_TYPE: support_data counters_attrs: &counters @@ -63,6 +82,7 @@ counters_attrs: &counters esa_sweep_attrs: <<: *default CATDESC: ElectroStatic Analyzer Energy Values + DEPEND_1: energy FIELDNAM: ESA Voltage FORMAT: I19 LABLAXIS: ESA V diff --git a/imap_processing/cdf/config/imap_codice_l1b_variable_attrs.yaml b/imap_processing/cdf/config/imap_codice_l1b_variable_attrs.yaml index 90c79664f..1d5d44eb5 100644 --- a/imap_processing/cdf/config/imap_codice_l1b_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_l1b_variable_attrs.yaml @@ -1,14 +1,20 @@ # <=== Defaults ===> default_attrs: &default - # Assumed values for all variable attrs unless overwritten - DISPLAY_TYPE: no_plot - FILLVAL: -9223372036854775808 - FORMAT: I12 - UNITS: dN - VALIDMIN: -9223372036854775808 - VALIDMAX: 9223372036854775807 - VAR_TYPE: data - SCALETYP: linear + CATDESC: "" + DISPLAY_TYPE: no_plot + FIELDNAM: "" + FILLVAL: -9223372036854775808 + FORMAT: I12 + LABLAXIS: "" + REFERENCE_POSITION: "" + RESOLUTION: "" + SCALETYP: linear + TIME_BASE: "" + TIME_SCALE: "" + UNITS: dN + VALIDMIN: -9223372036854775808 + VALIDMAX: 9223372036854775807 + VAR_TYPE: data codice_support_attrs: &support_default <<: *default @@ -29,23 +35,32 @@ energy_attrs: # <=== Labels ===> energy_label: CATDESC: Energy per charge (E/q) sweeping step + DISPLAY_TYPE: "" FIELDNAM: Energy step + FILLVAL: "" FORMAT: A3 + LABLAXIS: "" + REFERENCE_POSITION: "" + RESOLUTION: "" + TIME_BASE: "" + TIME_SCALE: "" + UNITS: "" + VALIDMAX: "" + VALIDMIN: "" VAR_TYPE: metadata # <=== Dataset Attributes ===> -# TODO: resolve ISTP warning: Width of F10 FORMAT may be insufficient for -# values. Adjust FORMAT or VALIDMIN/VALIDMAX value to resolve. acquisition_times_attrs: <<: *default CATDESC: Time of acquisition for the energy step + DEPEND_1: energy FIELDNAM: Acquisition Time - FILLVAL: 1.7976931348623157e+308 + FILLVAL: .NAN FORMAT: F10.3 LABLAXIS: Acq Time UNITS: ms VALIDMIN: 0 - VALIDMAX: 1.7976931348623157e+308 + VALIDMAX: 1000 VAR_TYPE: support_data counters_attrs: &counters @@ -55,16 +70,15 @@ counters_attrs: &counters DEPEND_1: energy DISPLAY_TYPE: time_series FIELDNAM: Fill in at creation - FILLVAL: 1.7976931348623157e+308 - FORMAT: F10.3 LABL_PTR_1: energy_label - UNITS: rates + UNITS: counts VALIDMIN: 0 - VALIDMAX: 1.7976931348623157e+308 + VALIDMAX: 8388607 # max value for a signed 24-bit integer esa_sweep_attrs: <<: *default CATDESC: ElectroStatic Analyzer Energy Values + DEPEND_1: energy FIELDNAM: ESA Voltage FORMAT: I19 LABLAXIS: ESA V diff --git a/imap_processing/codice/codice_l0.py b/imap_processing/codice/codice_l0.py index 71a5db5bb..4515af8fb 100644 --- a/imap_processing/codice/codice_l0.py +++ b/imap_processing/codice/codice_l0.py @@ -38,6 +38,7 @@ def decom_packets(packet_file: Path) -> list: "imap_codice_l0_hi-counters-singles_20240429_v001.pkts": "P_COD_HI_INST_COUNTS_SINGLES.xml", # noqa "imap_codice_l0_hi-omni_20240429_v001.pkts": "P_COD_HI_OMNI_SPECIES_COUNTS.xml", "imap_codice_l0_hi-sectored_20240429_v001.pkts": "P_COD_HI_SECT_SPECIES_COUNTS.xml", # noqa + "imap_codice_l0_hi-pha_20240429_v001.pkts": "P_COD_HI_PHA.xml", "imap_codice_l0_hskp_20100101_v001.pkts": "P_COD_NHK.xml", "imap_codice_l0_lo-counters-aggregated_20240429_v001.pkts": "P_COD_LO_INST_COUNTS_AGGREGATED.xml", # noqa "imap_codice_l0_lo-counters-singles_20240429_v001.pkts": "P_COD_LO_INST_COUNTS_SINGLES.xml", # noqa @@ -47,6 +48,7 @@ def decom_packets(packet_file: Path) -> list: "imap_codice_l0_lo-nsw-priority_20240429_v001.pkts": "P_COD_LO_NSW_PRIORITY_COUNTS.xml", # noqa "imap_codice_l0_lo-sw-species_20240429_v001.pkts": "P_COD_LO_SW_SPECIES_COUNTS.xml", # noqa "imap_codice_l0_lo-nsw-species_20240429_v001.pkts": "P_COD_LO_NSW_SPECIES_COUNTS.xml", # noqa + "imap_codice_l0_lo-pha_20240429_v001.pkts": "P_COD_LO_PHA.xml", } xtce_document = Path( diff --git a/imap_processing/codice/codice_l1a.py b/imap_processing/codice/codice_l1a.py index 94760fa6d..0c925fee7 100644 --- a/imap_processing/codice/codice_l1a.py +++ b/imap_processing/codice/codice_l1a.py @@ -13,6 +13,7 @@ from __future__ import annotations +import collections import logging from pathlib import Path @@ -23,19 +24,19 @@ from imap_processing import imap_module_directory from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes -from imap_processing.cdf.utils import IMAP_EPOCH, met_to_j2000ns +from imap_processing.cdf.utils import met_to_j2000ns from imap_processing.codice import constants from imap_processing.codice.codice_l0 import decom_packets -from imap_processing.codice.utils import CODICEAPID, create_hskp_dataset +from imap_processing.codice.utils import CODICEAPID, add_metadata_to_array from imap_processing.utils import group_by_apid, sort_by_time logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) # TODO: Decom data arrays need to be decompressed -# TODO: Add metadata attrs to science dataset? # TODO: In decommutation, how to have a variable length data and then a checksum -# after it? +# after it? (Might be fixed with new XTCE script updates) +# TODO: Add support for decomming multiple APIDs from a single file class CoDICEL1aPipeline: @@ -60,12 +61,12 @@ class CoDICEL1aPipeline: Methods ------- + configure_data_products() + Set the various settings for defining the data products. create_science_dataset() Create an ``xarray`` dataset for the unpacked science data. get_acquisition_times() Retrieve the acquisition times via the Lo stepping table. - get_data_products() - Retrieve the lo data products. get_esa_sweep_values() Retrieve the ESA sweep values. unpack_science_data() @@ -79,6 +80,21 @@ def __init__(self, table_id: int, plan_id: int, plan_step: int, view_id: int): self.plan_step = plan_step self.view_id = view_id + def configure_data_products(self, apid: int) -> None: + """ + Set the various settings for defining the data products. + + Parameters + ---------- + apid : int + The APID of interest. + """ + config = constants.DATA_PRODUCT_CONFIGURATIONS.get(apid) # type: ignore[call-overload] + self.num_counters = config["num_counters"] + self.num_energy_steps = config["num_energy_steps"] + self.variable_names = config["variable_names"] + self.dataset_name = config["dataset_name"] + def create_science_dataset(self, met: np.int64, data_version: str) -> xr.Dataset: """ Create an ``xarray`` dataset for the unpacked science data. @@ -140,7 +156,7 @@ def create_science_dataset(self, met: np.int64, data_version: str) -> xr.Dataset # TODO: Currently, cdflib doesn't properly write/read CDF files that # have a single epoch value. To get around this for now, use # two epoch values and reshape accordingly. Revisit this after - # SIT-3. + # SIT-3. See https://github.com/MAVENSDC/cdflib/issues/268 variable_data_arr = np.array(list(variable_data) * 2, dtype=int).reshape( 2, self.num_energy_steps ) @@ -156,6 +172,8 @@ def create_science_dataset(self, met: np.int64, data_version: str) -> xr.Dataset # Add ESA Sweep Values and acquisition times (lo only) if "_lo_" in self.dataset_name: + self.get_esa_sweep_values() + self.get_acquisition_times() dataset["esa_sweep_values"] = xr.DataArray( self.esa_sweep_values, dims=["energy"], @@ -213,23 +231,6 @@ def get_acquisition_times(self) -> None: row_number = np.argmax(energy_steps == str(step_number), axis=1).argmax() self.acquisition_times.append(lo_stepping_values.acq_time[row_number]) - def get_data_products(self, apid: int) -> None: - """ - Retrieve various settings for defining the data products. - - Parameters - ---------- - apid : int - The APID of interest. - """ - config = constants.DATA_PRODUCT_CONFIGURATIONS.get(apid) # type: ignore[call-overload] - # TODO Change, No overload variant of "get" of - # "dict" matches argument type "str". - self.num_counters = config["num_counters"] - self.num_energy_steps = config["num_energy_steps"] - self.variable_names = config["variable_names"] - self.dataset_name = config["dataset_name"] - def get_esa_sweep_values(self) -> None: """ Retrieve the ESA sweep values. @@ -290,6 +291,119 @@ def unpack_science_data(self, science_values: str) -> None: self.data = [["1"] * 128] * self.num_counters +def create_event_dataset( + met: list[int], event_data: str, dataset_name: str, data_version: str +) -> xr.Dataset: + """ + Create dataset for event data. + + Parameters + ---------- + met : list[int] + The Mission Elapsed Time of the data. + event_data : str + A string of binary numbers representing the event data. + dataset_name : str + The name for the dataset. + data_version : str + Version of the data product being created. + + Returns + ------- + dataset : xarray.Dataset + Xarray dataset containing the event data. + """ + cdf_attrs = ImapCdfAttributes() + cdf_attrs.add_instrument_global_attrs("codice") + cdf_attrs.add_instrument_variable_attrs("codice", "l1a") + cdf_attrs.add_global_attribute("Data_version", data_version) + + # Define coordinates + epoch = xr.DataArray( + met_to_j2000ns(met), # TODO: Fix after SIT-3 (see note below) + name="epoch", + dims=["epoch"], + attrs=cdf_attrs.get_variable_attributes("epoch"), + ) + + # Create the dataset to hold the data variables + dataset = xr.Dataset( + coords={ + "epoch": epoch, + }, + attrs=cdf_attrs.get_global_attributes(dataset_name), + ) + + # TODO: Determine what should go in event data CDF and how it should be + # structured. + + return dataset + + +def create_hskp_dataset( + packets: list[space_packet_parser.parser.Packet], + data_version: str, +) -> xr.Dataset: + """ + Create dataset for each metadata field for housekeeping data. + + Parameters + ---------- + packets : list[space_packet_parser.parser.Packet] + The list of packets to process. + data_version : str + Version of the data product being created. + + Returns + ------- + dataset : xarray.Dataset + Xarray dataset containing the metadata. + """ + cdf_attrs = ImapCdfAttributes() + cdf_attrs.add_instrument_global_attrs("codice") + cdf_attrs.add_instrument_variable_attrs("codice", "l1a") + cdf_attrs.add_global_attribute("Data_version", data_version) + + metadata_arrays: dict = collections.defaultdict(list) + + for packet in packets: + add_metadata_to_array(packet, metadata_arrays) + + epoch = xr.DataArray( + met_to_j2000ns( + metadata_arrays["SHCOARSE"], + reference_epoch=np.datetime64("2010-01-01T00:01:06.184", "ns"), + ), + name="epoch", + dims=["epoch"], + attrs=cdf_attrs.get_variable_attributes("epoch"), + ) + + dataset = xr.Dataset( + coords={"epoch": epoch}, + attrs=cdf_attrs.get_global_attributes("imap_codice_l1a_hskp"), + ) + + # TODO: Change 'TBD' catdesc and fieldname + # Once housekeeping packet definition file is re-generated with updated + # version of space_packet_parser, can get fieldname and catdesc info via: + # for key, value in (packet.header | packet.data).items(): + # fieldname = value.short_description + # catdesc = value.long_description + # I am holding off making this change until I acquire updated housekeeping + # packets/validation data that match the latest telemetry definitions + for key, value in metadata_arrays.items(): + attrs = cdf_attrs.get_variable_attributes("codice_support_attrs") + attrs["CATDESC"] = "TBD" + attrs["DEPEND_0"] = "epoch" + attrs["FIELDNAM"] = "TBD" + attrs["LABLAXIS"] = key + + dataset[key] = xr.DataArray(value, dims=["epoch"], attrs=attrs) + + return dataset + + def get_params(packet: space_packet_parser.parser.Packet) -> tuple[int, int, int, int]: """ Return the four 'main' parameters used for l1a processing. @@ -343,88 +457,57 @@ def process_codice_l1a(file_path: Path, data_version: str) -> xr.Dataset: dataset : xarray.Dataset The ``xarray`` dataset containing the science data and supporting metadata. """ - # TODO: Once simulated data for codice-hi is acquired, there shouldn't be a - # need to split the processing based on the file_path, so this function can - # be simplified. - - apids_for_lo_science_processing = [ - CODICEAPID.COD_LO_INST_COUNTS_AGGREGATED, - CODICEAPID.COD_LO_INST_COUNTS_SINGLES, - CODICEAPID.COD_LO_SW_ANGULAR_COUNTS, - CODICEAPID.COD_LO_NSW_ANGULAR_COUNTS, - CODICEAPID.COD_LO_SW_PRIORITY_COUNTS, - CODICEAPID.COD_LO_NSW_PRIORITY_COUNTS, - CODICEAPID.COD_LO_SW_SPECIES_COUNTS, - CODICEAPID.COD_LO_NSW_SPECIES_COUNTS, - ] - - # TODO: Temporary workaround in order to create hi data products in absence - # of simulated data - if file_path.name.startswith(("imap_codice_l0_lo", "imap_codice_l0_hskp")): - # Decom the packets, group data by APID, and sort by time - packets = decom_packets(file_path) - grouped_data = group_by_apid(packets) - - for apid in grouped_data: - logger.info(f"\nProcessing {CODICEAPID(apid).name} packet") - - if apid == CODICEAPID.COD_NHK: - packets = grouped_data[apid] - sorted_packets = sort_by_time(packets, "SHCOARSE") - dataset = create_hskp_dataset(sorted_packets, data_version) - - elif apid in apids_for_lo_science_processing: - # Sort the packets by time - packets = sort_by_time(grouped_data[apid], "SHCOARSE") - - # Determine the start time of the packet - met = packets[0].data["ACQ_START_SECONDS"].raw_value - met = [met, met + 1] # TODO: Remove after SIT-3 - # Extract the data - science_values = packets[0].data["DATA"].raw_value - - # Get the four "main" parameters for processing - table_id, plan_id, plan_step, view_id = get_params(packets[0]) - - # Run the pipeline to create a dataset for the product - pipeline = CoDICEL1aPipeline(table_id, plan_id, plan_step, view_id) - pipeline.get_esa_sweep_values() - pipeline.get_acquisition_times() - pipeline.get_data_products(apid) - pipeline.unpack_science_data(science_values) - dataset = pipeline.create_science_dataset(met, data_version) - - # TODO: Temporary workaround in order to create hi data products in absence - # of simulated data. This is essentially the same process as is for - # lo, but don't try to decom any packets, just define the data - # outright. - elif file_path.name.startswith("imap_codice_l0_hi"): - if file_path.name.startswith("imap_codice_l0_hi-counters-aggregated"): - apid = CODICEAPID.COD_HI_INST_COUNTS_AGGREGATED - table_id, plan_id, plan_step, view_id = (1, 0, 0, 3) - elif file_path.name.startswith("imap_codice_l0_hi-counters-singles"): - apid = CODICEAPID.COD_HI_INST_COUNTS_SINGLES - table_id, plan_id, plan_step, view_id = (1, 0, 0, 4) - elif file_path.name.startswith("imap_codice_l0_hi-omni"): - apid = CODICEAPID.COD_HI_OMNI_SPECIES_COUNTS - table_id, plan_id, plan_step, view_id = (1, 0, 0, 5) - elif file_path.name.startswith("imap_codice_l0_hi-sectored"): - apid = CODICEAPID.COD_HI_SECT_SPECIES_COUNTS - table_id, plan_id, plan_step, view_id = (1, 0, 0, 6) - - met0: np.timedelta64 = (np.datetime64("2024-04-29T00:00") - IMAP_EPOCH).astype( - "timedelta64[s]" - ) - met0 = met0.astype(np.int64) - met = [met0, met0 + 1] # Using this to match the other data products - science_values = "" # Currently don't have simulated data for this + # TODO: Use new packet_file_to_dataset() function to simplify things - pipeline = CoDICEL1aPipeline(table_id, plan_id, plan_step, view_id) - pipeline.get_data_products(apid) - pipeline.unpack_science_data(science_values) - dataset = pipeline.create_science_dataset(met, data_version) + # Decom the packets, group data by APID, and sort by time + packets = decom_packets(file_path) + grouped_data = group_by_apid(packets) - # Write dataset to CDF - logger.info(f"\nFinal data product:\n{dataset}\n") + for apid in grouped_data: + logger.info(f"\nProcessing {CODICEAPID(apid).name} packet") + + if apid == CODICEAPID.COD_NHK: + packets = grouped_data[apid] + sorted_packets = sort_by_time(packets, "SHCOARSE") + dataset = create_hskp_dataset(sorted_packets, data_version) + + elif apid in [CODICEAPID.COD_LO_PHA, CODICEAPID.COD_HI_PHA]: + if apid == CODICEAPID.COD_LO_PHA: + dataset_name = "imap_codice_l1a_lo_pha" + elif apid == CODICEAPID.COD_HI_PHA: + dataset_name = "imap_codice_l1a_hi_pha" + + # Sort the packets by time + packets = sort_by_time(grouped_data[apid], "SHCOARSE") + # Determine the start time of the packet + met = packets[0].data["ACQ_START_SECONDS"].raw_value + met = [met, met + 1] # TODO: Remove after cdflib fix + + # Extract the data + event_data = packets[0].data["EVENT_DATA"].raw_value + + # Create the dataset + dataset = create_event_dataset(met, event_data, dataset_name, data_version) + + elif apid in constants.APIDS_FOR_SCIENCE_PROCESSING: + # Sort the packets by time + packets = sort_by_time(grouped_data[apid], "SHCOARSE") + + # Determine the start time of the packet + met = packets[0].data["ACQ_START_SECONDS"].raw_value + met = [met, met + 1] # TODO: Remove after cdflib fix + # Extract the data + science_values = packets[0].data["DATA"].raw_value + + # Get the four "main" parameters for processing + table_id, plan_id, plan_step, view_id = get_params(packets[0]) + + # Run the pipeline to create a dataset for the product + pipeline = CoDICEL1aPipeline(table_id, plan_id, plan_step, view_id) + pipeline.configure_data_products(apid) + pipeline.unpack_science_data(science_values) + dataset = pipeline.create_science_dataset(met, data_version) + + logger.info(f"\nFinal data product:\n{dataset}\n") return dataset diff --git a/imap_processing/codice/constants.py b/imap_processing/codice/constants.py index 425b6b57d..a8bcf3503 100644 --- a/imap_processing/codice/constants.py +++ b/imap_processing/codice/constants.py @@ -14,6 +14,22 @@ from imap_processing.codice.utils import CODICEAPID, CoDICECompression +APIDS_FOR_SCIENCE_PROCESSING = [ + CODICEAPID.COD_HI_INST_COUNTS_AGGREGATED, + CODICEAPID.COD_HI_INST_COUNTS_SINGLES, + CODICEAPID.COD_HI_OMNI_SPECIES_COUNTS, + CODICEAPID.COD_HI_SECT_SPECIES_COUNTS, + CODICEAPID.COD_LO_INST_COUNTS_AGGREGATED, + CODICEAPID.COD_LO_INST_COUNTS_SINGLES, + CODICEAPID.COD_LO_SW_ANGULAR_COUNTS, + CODICEAPID.COD_LO_NSW_ANGULAR_COUNTS, + CODICEAPID.COD_LO_SW_PRIORITY_COUNTS, + CODICEAPID.COD_LO_NSW_PRIORITY_COUNTS, + CODICEAPID.COD_LO_SW_SPECIES_COUNTS, + CODICEAPID.COD_LO_NSW_SPECIES_COUNTS, +] + + # CDF-friendly names for lo data products LO_INST_COUNTS_AGGREGATED_NAMES = ["aggregated"] LO_INST_COUNTS_SINGLES_NAMES = ["apd_singles"] diff --git a/imap_processing/codice/packet_definitions/P_COD_HI_INST_COUNTS_AGGREGATED.xml b/imap_processing/codice/packet_definitions/P_COD_HI_INST_COUNTS_AGGREGATED.xml index 64bce2a39..4ac8bc9c1 100644 --- a/imap_processing/codice/packet_definitions/P_COD_HI_INST_COUNTS_AGGREGATED.xml +++ b/imap_processing/codice/packet_definitions/P_COD_HI_INST_COUNTS_AGGREGATED.xml @@ -1,6 +1,6 @@ - + @@ -43,7 +43,7 @@ - 4325376 + 512 diff --git a/imap_processing/codice/packet_definitions/P_COD_HI_INST_COUNTS_SINGLES.xml b/imap_processing/codice/packet_definitions/P_COD_HI_INST_COUNTS_SINGLES.xml index 80f0cec4d..1319d645f 100644 --- a/imap_processing/codice/packet_definitions/P_COD_HI_INST_COUNTS_SINGLES.xml +++ b/imap_processing/codice/packet_definitions/P_COD_HI_INST_COUNTS_SINGLES.xml @@ -1,6 +1,6 @@ - + @@ -43,7 +43,7 @@ - 4325376 + 768 diff --git a/imap_processing/codice/packet_definitions/P_COD_HI_OMNI_SPECIES_COUNTS.xml b/imap_processing/codice/packet_definitions/P_COD_HI_OMNI_SPECIES_COUNTS.xml index 92743c372..53bae3cde 100644 --- a/imap_processing/codice/packet_definitions/P_COD_HI_OMNI_SPECIES_COUNTS.xml +++ b/imap_processing/codice/packet_definitions/P_COD_HI_OMNI_SPECIES_COUNTS.xml @@ -1,6 +1,6 @@ - + @@ -39,11 +39,11 @@ - + - 28704768 + 1376 @@ -124,7 +124,7 @@ NUMBER OF BYTES IN THE DATA ARRAY. IF COMPRESSED, THIS VALUE REPRESENTS THE LENGTH OF THE COMPRESSED DATA. - + COUNTER DATA VARIABLE LENGTH; MAXIMUM (BASED ON UNCOLLAPSED, UNCOMPRESSED DATA, AND ASSUMING ALL 146 SQRT(2) SPECEIS COUNTERS INCLUDED): diff --git a/imap_processing/codice/packet_definitions/P_COD_HI_PHA.xml b/imap_processing/codice/packet_definitions/P_COD_HI_PHA.xml index cc28a6896..b75581c0a 100644 --- a/imap_processing/codice/packet_definitions/P_COD_HI_PHA.xml +++ b/imap_processing/codice/packet_definitions/P_COD_HI_PHA.xml @@ -1,6 +1,6 @@ - + @@ -12,12 +12,12 @@ + + + - - - @@ -27,17 +27,14 @@ - - - - + - 276480 + 5760 @@ -64,73 +61,64 @@ CCSDS Packet Length (number of bytes after Packet length minus 1) - - S/C Time - Seconds - Secondary Header - Whole-seconds part of SCLK + + SECONDARY HEADER - WHOLE-SECONDS PART OF SCLK - - Packet Version - Packet version - this will be incremented each time the format of the packet changesCOD_LO_PHA. + + PACKET VERSION - THIS WILL BE INCREMENTED EACH TIME THE FORMAT OF THE PACKET CHANGESCOD_LO_PHA. - - Spin Period reported by the Spacecraft - Spin period reported by the Spacecraft in the Time and Status message. Reported period is the period that was active when the 16-spin acquisition cycle started. + + SPIN PERIOD REPORTED BY THE SPACECRAFT IN THE TIME AND STATUS MESSAGE. REPORTED PERIOD IS THE PERIOD THAT WAS ACTIVE WHEN THE 16-SPIN ACQUISITION CYCLE STARTED. - - Acquisition Start Time (Seconds) - Full-seconds portion of the time at which the 16-spin cycle started + + FULL-SECONDS PORTION OF THE TIME AT WHICH THE 16-SPIN CYCLE STARTED - - Acquisition Start Time (Subseconds) - Sub-seconds portion of the time at which the 16-spin cycle started + + SUB-SECONDS PORTION OF THE TIME AT WHICH THE 16-SPIN CYCLE STARTED - - Spare for alignment - Spare for alignment + + SPARE FOR ALIGNMENT - - Total number of TCR events - Total number of TCR events that occurred during the collection cycle + + BIAS GAIN MODE FOR THE SUPRATHERMAL SECTOR - - Total number of DCR events - Total number of DCR events that occurred during the collection cycle + + BIAS GAIN MODE FOR THE SOLARWIND SECTOR - - Total number of APD-Only events - Total number of APD-Only events that occurred during the collection cycle + + + INDICATES THAT THERE WAS SOME ERROR DETECTED DURING ACQUISITION OR PROCESSING OF THE DATA. ERRORS COULD INCLUDE CORRUPTED ACQUISITION MEMORY (I.E. EDAC ERRORS), TIMING VIOLATIONS, OR OTHER EVENTS THAT INTERRUPTED OR OTHERWISE AFFECTED DATA COLLECTION. - - Number of events included in this packet - Number of events selected for downlink (i.e. number of events in the Event_Data array) + + WHETHER THE EVENT DATA IS COMPRESSED. IF 1/YES, EVENT_DATA ARRAY IS COMPRESSED USING THE RICE COMPRESSION ALGORITHM. - - Whether the event data is compressed - Whether the event data is compressed. If 1/Yes, Event_Data array is compressed using the Rice compression algorithm. + + NUMBER OF EVENTS SELECTED FOR DOWNLINK (I.E. NUMBER OF EVENTS IN THE EVENT_DATA ARRAY) - - Number of bytes in the Event Data array - Number of bytes in the Event_Data array. If compressed, this value represents the length of the compressed data. + + NUMBER OF BYTES IN THE EVENT_DATA ARRAY. IF COMPRESSED, THIS VALUE REPRESENTS THE LENGTH OF THE COMPRESSED DATA. - - Event Data - Optionally compressed array of Event Data + + OPTIONALLY COMPRESSED ARRAY OF EVENT DATA -Format is TBD; some considerations/options: -- Full events have a lot of redundant data (e.g. will have many events with the same priority/spin/spin phase information). How well does compression to deal with the redundancy? -- Could include mini-headers for each (priority,spin, spin-phase) group and strip the redundant data from the events -- Should events be tightly packed, or can we pad out to 64-bit word boundaries? How well does compression compensate for the extra bits? +FORMAT IS TBD; SOME CONSIDERATIONS/OPTIONS: +- FULL EVENTS HAVE A LOT OF REDUNDANT DATA (E.G. WILL HAVE MANY EVENTS WITH THE SAME PRIORITY/SPIN/SPIN PHASE INFORMATION). HOW WELL DOES COMPRESSION TO DEAL WITH THE REDUNDANCY? +- COULD INCLUDE MINI-HEADERS FOR EACH (PRIORITY,SPIN, SPIN-PHASE) GROUP AND STRIP THE REDUNDANT DATA FROM THE EVENTS +- SHOULD EVENTS BE TIGHTLY PACKED, OR CAN WE PAD OUT TO 64-BIT WORD BOUNDARIES? HOW WELL DOES COMPRESSION COMPENSATE FOR THE EXTRA BITS? -Each event consists of: -- 10-bit TOF -- 9-bit SSD Energy -- 2-bit Energy Range -- 7-bit Spin Angle -- 4-bit SSD Position -- 4-bit Spin Number -- 2-bit PHA Type +EACH EVENT CONSISTS OF: +- 10-BIT TOF +- 9-BIT SSD ENERGY +- 2-BIT ENERGY RANGE +- 7-BIT SPIN ANGLE +- 4-BIT SSD POSITION +- 4-BIT SPIN NUMBER +- 2-BIT PHA TYPE -TBD: Events may be tightly packed, or may have spares added to keep each event byte-aligned. In either case, there may be up to 1 byte of padding to keep the total size of the packet even. +TBD: EVENTS MAY BE TIGHTLY PACKED, OR MAY HAVE SPARES ADDED TO KEEP EACH EVENT BYTE-ALIGNED. IN EITHER CASE, THERE MAY BE UP TO 1 BYTE OF PADDING TO KEEP THE TOTAL SIZE OF THE PACKET EVEN. + + + PACKET CHECKSUM @@ -158,13 +146,15 @@ TBD: Events may be tightly packed, or may have spares added to keep each event b - - - - + + + + + + diff --git a/imap_processing/codice/packet_definitions/P_COD_HI_SECT_SPECIES_COUNTS.xml b/imap_processing/codice/packet_definitions/P_COD_HI_SECT_SPECIES_COUNTS.xml index 397b3c841..176c12ff7 100644 --- a/imap_processing/codice/packet_definitions/P_COD_HI_SECT_SPECIES_COUNTS.xml +++ b/imap_processing/codice/packet_definitions/P_COD_HI_SECT_SPECIES_COUNTS.xml @@ -1,6 +1,6 @@ - + @@ -39,11 +39,11 @@ - + - 9043968 + 992 @@ -124,7 +124,7 @@ NUMBER OF BYTES IN THE DATA ARRAY. IF COMPRESSED, THIS VALUE REPRESENTS THE LENGTH OF THE COMPRESSED DATA. - + COUNTER DATA VARIABLE LENGTH; MAXIMUM (BASED ON UNCOLLAPSED, UNCOMPRESSED DATA, AND ASSUMING ALL X2 SPECEIS COUNTERS INCLUDED): diff --git a/imap_processing/codice/packet_definitions/P_COD_LO_PHA.xml b/imap_processing/codice/packet_definitions/P_COD_LO_PHA.xml index e0e28109d..fc8cd2d85 100644 --- a/imap_processing/codice/packet_definitions/P_COD_LO_PHA.xml +++ b/imap_processing/codice/packet_definitions/P_COD_LO_PHA.xml @@ -1,6 +1,6 @@ - + @@ -12,12 +12,12 @@ + + + - - - @@ -27,17 +27,14 @@ - - - - + - 276480 + 640 @@ -64,75 +61,66 @@ CCSDS Packet Length (number of bytes after Packet length minus 1) - - S/C Time - Seconds - Secondary Header - Whole-seconds part of SCLK + + SECONDARY HEADER - WHOLE-SECONDS PART OF SCLK - - Packet Version - Packet version - this will be incremented each time the format of the packet changesCOD_LO_PHA. + + PACKET VERSION - THIS WILL BE INCREMENTED EACH TIME THE FORMAT OF THE PACKET CHANGESCOD_LO_PHA. - - Spin Period reported by the Spacecraft - Spin period reported by the Spacecraft in the Time and Status message. Reported period is the period that was active when the 16-spin acquisition cycle started. + + SPIN PERIOD REPORTED BY THE SPACECRAFT IN THE TIME AND STATUS MESSAGE. REPORTED PERIOD IS THE PERIOD THAT WAS ACTIVE WHEN THE 16-SPIN ACQUISITION CYCLE STARTED. - - Acquisition Start Time (Seconds) - Full-seconds portion of the time at which the 16-spin cycle started + + FULL-SECONDS PORTION OF THE TIME AT WHICH THE 16-SPIN CYCLE STARTED - - Acquisition Start Time (Subseconds) - Sub-seconds portion of the time at which the 16-spin cycle started + + SUB-SECONDS PORTION OF THE TIME AT WHICH THE 16-SPIN CYCLE STARTED - - Spare for alignment - Spare for alignment + + SPARE FOR ALIGNMENT - - Total number of TCR events - Total number of TCR events that occurred during the collection cycle + + BIAS GAIN MODE FOR THE SUPRATHERMAL SECTOR - - Total number of DCR events - Total number of DCR events that occurred during the collection cycle + + BIAS GAIN MODE FOR THE SOLARWIND SECTOR - - Total number of APD-Only events - Total number of APD-Only events that occurred during the collection cycle + + + INDICATES THAT THERE WAS SOME ERROR DETECTED DURING ACQUISITION OR PROCESSING OF THE DATA. ERRORS COULD INCLUDE CORRUPTED ACQUISITION MEMORY (I.E. EDAC ERRORS), TIMING VIOLATIONS, OR OTHER EVENTS THAT INTERRUPTED OR OTHERWISE AFFECTED DATA COLLECTION. - - Number of events included in this packet - Number of events selected for downlink (i.e. number of events in the Event_Data array) + + WHETHER THE EVENT DATA IS COMPRESSED. IF 1/YES, EVENT_DATA ARRAY IS COMPRESSED USING THE LZMA COMPRESSION ALGORITHM. - - Whether the event data is compressed - Whether the event data is compressed. If 1/Yes, Event_Data array is compressed using the LZMA compression algorithm. + + NUMBER OF EVENTS SELECTED FOR DOWNLINK (I.E. NUMBER OF EVENTS IN THE EVENT_DATA ARRAY) - - Number of bytes in the Event Data array - Number of bytes in the Event_Data array. If compressed, this value represents the length of the compressed data. + + NUMBER OF BYTES IN THE EVENT_DATA ARRAY. IF COMPRESSED, THIS VALUE REPRESENTS THE LENGTH OF THE COMPRESSED DATA. - - Event Data - Optionally compressed array of Event Data + + OPTIONALLY COMPRESSED ARRAY OF EVENT DATA -Format is TBD; some considerations/options: -- Full events have a lot of redundant data (e.g. will have many events with the same priority/e-step/spin phase information). How well does compression to deal with the redundancy? -- Could include mini-headers for each (priority,e-step, spin-phase) group and strip the redundant data from the events -- Should events be tightly packed, or can we pad out to 64-bit word boundaries? How well does compression compensate for the extra bits? +FORMAT IS TBD; SOME CONSIDERATIONS/OPTIONS: +- FULL EVENTS HAVE A LOT OF REDUNDANT DATA (E.G. WILL HAVE MANY EVENTS WITH THE SAME PRIORITY/E-STEP/SPIN PHASE INFORMATION). HOW WELL DOES COMPRESSION TO DEAL WITH THE REDUNDANCY? +- COULD INCLUDE MINI-HEADERS FOR EACH (PRIORITY,E-STEP, SPIN-PHASE) GROUP AND STRIP THE REDUNDANT DATA FROM THE EVENTS +- SHOULD EVENTS BE TIGHTLY PACKED, OR CAN WE PAD OUT TO 64-BIT WORD BOUNDARIES? HOW WELL DOES COMPRESSION COMPENSATE FOR THE EXTRA BITS? -Each event consists of: -- 7-bit E-step -- 10-bit TOF -- 9-bit APD Energy -- 7-bit Spin Angle -- 5-bit Position -- 5-bit APD-ID -- 1-bit APD-Gain -- 2-bit PHA Type -- 3-bit Priority Range +EACH EVENT CONSISTS OF: +- 7-BIT E-STEP +- 10-BIT TOF +- 9-BIT APD ENERGY +- 7-BIT SPIN ANGLE +- 5-BIT POSITION +- 5-BIT APD-ID +- 1-BIT APD-GAIN +- 2-BIT PHA TYPE +- 3-BIT PRIORITY RANGE -TBD: Events may be tightly packed, or may have spares added to keep each event byte-aligned. In either case, there may be up to 1 byte of padding to keep the total size of the packet even. +TBD: EVENTS MAY BE TIGHTLY PACKED, OR MAY HAVE SPARES ADDED TO KEEP EACH EVENT BYTE-ALIGNED. IN EITHER CASE, THERE MAY BE UP TO 1 BYTE OF PADDING TO KEEP THE TOTAL SIZE OF THE PACKET EVEN. + + + PACKET CHECKSUM @@ -160,13 +148,15 @@ TBD: Events may be tightly packed, or may have spares added to keep each event b - - - - + + + + + + diff --git a/imap_processing/codice/utils.py b/imap_processing/codice/utils.py index a4a7e8560..934656146 100644 --- a/imap_processing/codice/utils.py +++ b/imap_processing/codice/utils.py @@ -5,15 +5,9 @@ other CoDICE processing modules. """ -import collections from enum import IntEnum -import numpy as np import space_packet_parser -import xarray as xr - -from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes -from imap_processing.cdf.utils import met_to_j2000ns class CODICEAPID(IntEnum): @@ -99,70 +93,3 @@ def add_metadata_to_array(packet: space_packet_parser, metadata_arrays: dict) -> metadata_arrays.setdefault(key, []).append(value.raw_value) return metadata_arrays - - -# TODO: Correct the type of "packets" -def create_hskp_dataset( # type: ignore[no-untyped-def] - packets, - data_version: str, -) -> xr.Dataset: - """ - Create dataset for each metadata field for housekeeping data. - - Parameters - ---------- - packets : list[space_packet_parser.parser.Packet] - The list of packets to process. - data_version : str - Version of the data product being created. - - Returns - ------- - dataset : xarray.Dataset - Xarray dataset containing the metadata. - """ - cdf_attrs = ImapCdfAttributes() - cdf_attrs.add_instrument_global_attrs("codice") - cdf_attrs.add_instrument_variable_attrs("codice", "l1a") - cdf_attrs.add_global_attribute("Data_version", data_version) - - metadata_arrays: dict = collections.defaultdict(list) - - for packet in packets: - add_metadata_to_array(packet, metadata_arrays) - - # TODO: Is there a way to get the attrs from the YAML-based method? - epoch = xr.DataArray( - met_to_j2000ns( - metadata_arrays["SHCOARSE"], - reference_epoch=np.datetime64("2010-01-01T00:01:06.184", "ns"), - ), - name="epoch", - dims=["epoch"], - attrs=cdf_attrs.get_variable_attributes("epoch"), - ) - - dataset = xr.Dataset( - coords={"epoch": epoch}, - attrs=cdf_attrs.get_global_attributes("imap_codice_l1a_hskp"), - ) - - # TODO: Change 'TBD' catdesc and fieldname - # Once packet definition files are re-generated, can get this info from - # something like this: - # for key, value in (packet.header | packet.data).items(): - # fieldname = value.short_description - # catdesc = value.short_description - # I am holding off making this change until I acquire updated housekeeping - # packets/validation data that match the latest telemetry definitions - # I may also be able to replace this function with utils.create_dataset(?) - for key, value in metadata_arrays.items(): - attrs = cdf_attrs.get_variable_attributes("codice_support_attrs") - attrs["CATDESC"] = "TBD" - attrs["DEPEND_0"] = "epoch" - attrs["FIELDNAM"] = "TBD" - attrs["LABLAXIS"] = key - - dataset[key] = xr.DataArray(value, dims=["epoch"], attrs=attrs) - - return dataset diff --git a/imap_processing/tests/codice/conftest.py b/imap_processing/tests/codice/conftest.py new file mode 100644 index 000000000..66f36a697 --- /dev/null +++ b/imap_processing/tests/codice/conftest.py @@ -0,0 +1,56 @@ +from imap_processing import imap_module_directory + +TEST_DATA_PATH = imap_module_directory / "tests" / "codice" / "data" + +TEST_PACKETS = [ + TEST_DATA_PATH / "imap_codice_l0_hskp_20100101_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_hi-counters-aggregated_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_hi-counters-singles_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_hi-omni_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_hi-sectored_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_hi-pha_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_lo-counters-aggregated_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_lo-counters-singles_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_lo-sw-angular_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_lo-nsw-angular_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_lo-sw-priority_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_lo-nsw-priority_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_lo-sw-species_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_lo-nsw-species_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_lo-pha_20240429_v001.pkts", +] + +TEST_L1A_FILES = [ + TEST_DATA_PATH / "imap_codice_l1a_hskp_20100101_v001.pkts", + TEST_DATA_PATH / "imap_codice_l1a_hi-counters-aggregated_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l1a_hi-counters-singles_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l1a_hi-omni_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l1a_hi-sectored_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l1a_lo-counters-aggregated_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l1a_lo-counters-singles_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l1a_lo-sw-angular_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l1a_lo-nsw-angular_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l1a_lo-sw-priority_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l1a_lo-nsw-priority_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l1a_lo-sw-species_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l1a_lo-nsw-species_20240429_v001.pkts", +] + +# Placeholder for validation data files +VALIDATION_DATA = [ + TEST_DATA_PATH / "validation_hskp.cdf", + TEST_DATA_PATH / "validation_hi-counters-aggregated.cdf", + TEST_DATA_PATH / "validation_hi-counters-singles.cdf", + TEST_DATA_PATH / "validation_hi-omni.cdf", + TEST_DATA_PATH / "validation_hi-sectored.cdf", + TEST_DATA_PATH / "validation_hi-pha.cdf", + TEST_DATA_PATH / "validation_lo-counters-aggregated.cdf", + TEST_DATA_PATH / "validation_lo-counters-singles.cdf", + TEST_DATA_PATH / "validation_lo-sw-angular.cdf", + TEST_DATA_PATH / "validation_lo-nsw-angular.cdf", + TEST_DATA_PATH / "validation_lo-sw-priority.cdf", + TEST_DATA_PATH / "validation_lo-nsw-priority.cdf", + TEST_DATA_PATH / "validation_lo-sw-species.cdf", + TEST_DATA_PATH / "validation_lo-nsw-species.cdf", + TEST_DATA_PATH / "validation_lo-pha.cdf", +] diff --git a/imap_processing/tests/codice/data/imap_codice_l0_hi-counters-aggregated_20240429_v001.pkts b/imap_processing/tests/codice/data/imap_codice_l0_hi-counters-aggregated_20240429_v001.pkts index e69de29bb..53d675033 100644 Binary files a/imap_processing/tests/codice/data/imap_codice_l0_hi-counters-aggregated_20240429_v001.pkts and b/imap_processing/tests/codice/data/imap_codice_l0_hi-counters-aggregated_20240429_v001.pkts differ diff --git a/imap_processing/tests/codice/data/imap_codice_l0_hi-counters-singles_20240429_v001.pkts b/imap_processing/tests/codice/data/imap_codice_l0_hi-counters-singles_20240429_v001.pkts index e69de29bb..b7a783fed 100644 Binary files a/imap_processing/tests/codice/data/imap_codice_l0_hi-counters-singles_20240429_v001.pkts and b/imap_processing/tests/codice/data/imap_codice_l0_hi-counters-singles_20240429_v001.pkts differ diff --git a/imap_processing/tests/codice/data/imap_codice_l0_hi-omni_20240429_v001.pkts b/imap_processing/tests/codice/data/imap_codice_l0_hi-omni_20240429_v001.pkts index e69de29bb..bb1be93f6 100644 Binary files a/imap_processing/tests/codice/data/imap_codice_l0_hi-omni_20240429_v001.pkts and b/imap_processing/tests/codice/data/imap_codice_l0_hi-omni_20240429_v001.pkts differ diff --git a/imap_processing/tests/codice/data/imap_codice_l0_hi-pha_20240429_v001.pkts b/imap_processing/tests/codice/data/imap_codice_l0_hi-pha_20240429_v001.pkts new file mode 100644 index 000000000..b2167acc9 Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l0_hi-pha_20240429_v001.pkts differ diff --git a/imap_processing/tests/codice/data/imap_codice_l0_hi-sectored_20240429_v001.pkts b/imap_processing/tests/codice/data/imap_codice_l0_hi-sectored_20240429_v001.pkts index e69de29bb..976d17255 100644 Binary files a/imap_processing/tests/codice/data/imap_codice_l0_hi-sectored_20240429_v001.pkts and b/imap_processing/tests/codice/data/imap_codice_l0_hi-sectored_20240429_v001.pkts differ diff --git a/imap_processing/tests/codice/data/imap_codice_l0_lo-pha_20240429_v001.pkts b/imap_processing/tests/codice/data/imap_codice_l0_lo-pha_20240429_v001.pkts new file mode 100644 index 000000000..2dfc6c374 Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l0_lo-pha_20240429_v001.pkts differ diff --git a/imap_processing/tests/codice/test_codice_l0.py b/imap_processing/tests/codice/test_codice_l0.py index 47a1c9dd3..6920fdd74 100644 --- a/imap_processing/tests/codice/test_codice_l0.py +++ b/imap_processing/tests/codice/test_codice_l0.py @@ -9,7 +9,7 @@ from imap_processing import imap_module_directory from imap_processing.codice import codice_l0 -from imap_processing.codice.utils import create_hskp_dataset +from imap_processing.codice.codice_l1a import create_hskp_dataset from imap_processing.utils import convert_raw_to_eu @@ -62,7 +62,7 @@ def validation_data() -> pd.core.frame.DataFrame: return validation_data -def test_eu_hk_data( +def test_eu_hskp_data( decom_test_data: list[space_packet_parser.parser.Packet], validation_data: pd.core.frame.DataFrame, ): @@ -105,7 +105,7 @@ def test_eu_hk_data( assert round(eu_val, 5) == round(validation_val, 5) -def test_raw_hk_data( +def test_raw_hskp_data( decom_test_data: list[space_packet_parser.parser.Packet], validation_data: pd.core.frame.DataFrame, ): diff --git a/imap_processing/tests/codice/test_codice_l1a.py b/imap_processing/tests/codice/test_codice_l1a.py index b1f2bd962..383a07b9b 100644 --- a/imap_processing/tests/codice/test_codice_l1a.py +++ b/imap_processing/tests/codice/test_codice_l1a.py @@ -1,15 +1,20 @@ """Tests the L1a processing for decommutated CoDICE data""" +import logging from pathlib import Path import numpy as np import pytest import xarray as xr -from imap_processing import imap_module_directory -from imap_processing.cdf.utils import load_cdf +from imap_processing.cdf.utils import load_cdf, write_cdf from imap_processing.codice.codice_l1a import process_codice_l1a +from .conftest import TEST_PACKETS, VALIDATION_DATA + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + # TODO: Add test that processes a file with multiple APIDs EXPECTED_ARRAY_SHAPES = [ @@ -18,6 +23,7 @@ (1, 128), # hi-counters-singles (1, 128), # hi-omni (1, 128), # hi-sectored + (1, 128), # hi-pha (1, 128), # lo-counters-aggregated (1, 128), # lo-counters-aggregated (1, 128), # lo-sw-angular @@ -26,6 +32,7 @@ (1, 128), # lo-nsw-priority (1, 128), # lo-sw-species (1, 128), # lo-nsw-species + (1, 128), # lo-pha ] EXPECTED_ARRAY_SIZES = [ 123, # hskp @@ -33,6 +40,7 @@ 3, # hi-counters-singles 8, # hi-omni 4, # hi-sectored + 0, # hi-pha 3, # lo-counters-aggregated 3, # lo-counters-singles 6, # lo-sw-angular @@ -41,6 +49,7 @@ 4, # lo-nsw-priority 18, # lo-sw-species 10, # lo-nsw-species + 0, # lo-pha ] EXPECTED_LOGICAL_SOURCE = [ "imap_codice_l1a_hskp", @@ -48,6 +57,7 @@ "imap_codice_l1a_hi-counters-singles", "imap_codice_l1a_hi-omni", "imap_codice_l1a_hi-sectored", + "imap_codice_l1a_hi-pha", "imap_codice_l1a_lo-counters-aggregated", "imap_codice_l1a_lo-counters-singles", "imap_codice_l1a_lo-sw-angular", @@ -56,64 +66,7 @@ "imap_codice_l1a_lo-nsw-priority", "imap_codice_l1a_lo-sw-species", "imap_codice_l1a_lo-nsw-species", -] -TEST_PACKETS = [ - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_hskp_20100101_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_hi-counters-aggregated_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_hi-counters-singles_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_hi-omni_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_hi-sectored_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-counters-aggregated_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-counters-singles_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-sw-angular_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-nsw-angular_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-sw-priority_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-nsw-priority_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-sw-species_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-nsw-species_20240429_v001.pkts" - ), -] - -# Placeholder for validation data files -VALIDATION_DATA = [ - f"{imap_module_directory}/tests/codice/data/validation_hskp.cdf", - f"{imap_module_directory}/tests/codice/data/validation_hi-counters-aggregated.cdf", - f"{imap_module_directory}/tests/codice/data/validation_hi-counters-singles.cdf", - f"{imap_module_directory}/tests/codice/data/validation_hi-omni.cdf", - f"{imap_module_directory}/tests/codice/data/validation_hi-sectored.cdf", - f"{imap_module_directory}/tests/codice/data/validation_lo-counters-aggregated.cdf", - f"{imap_module_directory}/tests/codice/data/validation_lo-counters-singles.cdf", - f"{imap_module_directory}/tests/codice/data/validation_lo-sw-angular.cdf", - f"{imap_module_directory}/tests/codice/data/validation_lo-nsw-angular.cdf", - f"{imap_module_directory}/tests/codice/data/validation_lo-sw-priority.cdf", - f"{imap_module_directory}/tests/codice/data/validation_lo-nsw-priority.cdf", - f"{imap_module_directory}/tests/codice/data/validation_lo-sw-species.cdf", - f"{imap_module_directory}/tests/codice/data/validation_lo-nsw-species.cdf", + "imap_codice_l1a_lo-pha", ] @@ -128,6 +81,11 @@ def test_l1a_data(request) -> xr.Dataset: """ dataset = process_codice_l1a(file_path=request.param, data_version="001") + + # Write the dataset to a CDF so it can be manually inspected as well + file_path = write_cdf(dataset) + logger.info(f"CDF file written to {file_path}") + return dataset @@ -153,7 +111,7 @@ def test_l1a_cdf_filenames(test_l1a_data: xr.Dataset, expected_logical_source: s @pytest.mark.xfail( - reason="Currently failing due to cdflib/epoch issue. Revisit after SIT-3" + reason="Currently failing due to cdflib/epoch issue. See https://github.com/MAVENSDC/cdflib/issues/268" ) @pytest.mark.parametrize( "test_l1a_data, expected_shape", diff --git a/imap_processing/tests/codice/test_codice_l1b.py b/imap_processing/tests/codice/test_codice_l1b.py index 7b36ec251..685f88dde 100644 --- a/imap_processing/tests/codice/test_codice_l1b.py +++ b/imap_processing/tests/codice/test_codice_l1b.py @@ -1,14 +1,13 @@ """Tests the L1b processing for CoDICE L1a data""" -from pathlib import Path - import pytest import xarray as xr -from imap_processing import imap_module_directory from imap_processing.cdf.utils import load_cdf from imap_processing.codice.codice_l1b import process_codice_l1b +from .conftest import TEST_L1A_FILES + EXPECTED_LOGICAL_SOURCE = [ "imap_codice_l1b_hskp", "imap_codice_l1b_hi-counters-aggregated", @@ -24,50 +23,9 @@ "imap_codice_l1b_lo-sw-species", "imap_codice_l1b_lo-nsw-species", ] -TEST_FILES = [ - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_hskp_20100101_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_hi-counters-aggregated_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_hi-counters-singles_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_hi-omni_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_hi-sectored_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-counters-aggregated_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-counters-singles_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-sw-angular_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-nsw-angular_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-sw-priority_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-nsw-priority_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-sw-species_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-nsw-species_20240429_v001.pkts" - ), -] -@pytest.fixture(params=TEST_FILES) +@pytest.fixture(params=TEST_L1A_FILES) def test_l1b_data(request) -> xr.Dataset: """Return a ``xarray`` dataset containing test data. @@ -83,7 +41,7 @@ def test_l1b_data(request) -> xr.Dataset: @pytest.mark.parametrize( "test_l1b_data, expected_logical_source", - list(zip(TEST_FILES, EXPECTED_LOGICAL_SOURCE)), + list(zip(TEST_L1A_FILES, EXPECTED_LOGICAL_SOURCE)), indirect=["test_l1b_data"], ) def test_l1b_cdf_filenames(test_l1b_data: xr.Dataset, expected_logical_source: str):