From c7ecb632cd1c8c4de337f23843ef9867d862d237 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Wed, 17 Jul 2024 14:32:44 -0600 Subject: [PATCH 01/22] Moved create_hskp_dataset to within l1a module to be consistent with how science datasets are created --- imap_processing/codice/codice_l1a.py | 71 +++++++++++++++++- imap_processing/codice/utils.py | 74 ------------------- .../tests/codice/test_codice_l0.py | 4 +- .../tests/codice/test_codice_l1a.py | 3 + 4 files changed, 76 insertions(+), 76 deletions(-) diff --git a/imap_processing/codice/codice_l1a.py b/imap_processing/codice/codice_l1a.py index f38e9cc55..d1089f86c 100644 --- a/imap_processing/codice/codice_l1a.py +++ b/imap_processing/codice/codice_l1a.py @@ -13,6 +13,7 @@ from __future__ import annotations +import collections import logging from pathlib import Path @@ -22,11 +23,12 @@ import xarray as xr from imap_processing import imap_module_directory +from imap_processing.cdf.global_attrs import ConstantCoordinates from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes from imap_processing.cdf.utils import IMAP_EPOCH, met_to_j2000ns from imap_processing.codice import constants from imap_processing.codice.codice_l0 import decom_packets -from imap_processing.codice.utils import CODICEAPID, create_hskp_dataset +from imap_processing.codice.utils import CODICEAPID, add_metadata_to_array from imap_processing.utils import group_by_apid, sort_by_time logger = logging.getLogger(__name__) @@ -36,6 +38,7 @@ # TODO: Add metadata attrs to science dataset? # TODO: In decommutation, how to have a variable length data and then a checksum # after it? +# TODO: Add support for decomming multiple APIDs from a single file class CoDICEL1aPipeline: @@ -290,6 +293,72 @@ def unpack_science_data(self, science_values: str) -> None: self.data = [["1"] * 128] * self.num_counters +def create_hskp_dataset( + packets: list[space_packet_parser.parser.Packet], + data_version: str, +) -> xr.Dataset: + """ + Create dataset for each metadata field for housekeeping data. + + Parameters + ---------- + packets : list[space_packet_parser.parser.Packet] + The list of packets to process. + data_version : str + Version of the data product being created. + + Returns + ------- + dataset : xarray.Dataset + Xarray dataset containing the metadata. + """ + cdf_attrs = ImapCdfAttributes() + cdf_attrs.add_instrument_global_attrs("codice") + cdf_attrs.add_instrument_variable_attrs("codice", "l1a") + cdf_attrs.add_global_attribute("Data_version", data_version) + + metadata_arrays: dict = collections.defaultdict(list) + + for packet in packets: + add_metadata_to_array(packet, metadata_arrays) + + # TODO: Is there a way to get the attrs from the YAML-based method? + epoch = xr.DataArray( + met_to_j2000ns( + metadata_arrays["SHCOARSE"], + reference_epoch=np.datetime64("2010-01-01T00:01:06.184", "ns"), + ), + name="epoch", + dims=["epoch"], + attrs=ConstantCoordinates.EPOCH, + ) + + dataset = xr.Dataset( + coords={"epoch": epoch}, + attrs=cdf_attrs.get_global_attributes("imap_codice_l1a_hskp"), + ) + + # TODO: Change 'TBD' catdesc and fieldname + # Once packet definition files are re-generated, can get this info from + # something like this: + # for key, value in (packet.header | packet.data).items(): + # fieldname = value.short_description + # catdesc = value.short_description + # I am holding off making this change until I acquire updated housekeeping + # packets/validation data that match the latest telemetry definitions + # I may also be able to replace this function with utils.create_dataset(?) + for key, value in metadata_arrays.items(): + attrs = cdf_attrs.get_variable_attributes("codice_support_attrs") + attrs["CATDESC"] = "TBD" + attrs["DEPEND_0"] = "epoch" + attrs["FIELDNAM"] = "TBD" + attrs["LABLAXIS"] = key + + dataset[key] = xr.DataArray(value, dims=["epoch"], attrs=attrs) + + return dataset + + def get_params(packet: space_packet_parser.parser.Packet) -> tuple[int, int, int, int]: """ Return the four 'main' parameters used for l1a processing. diff --git a/imap_processing/codice/utils.py b/imap_processing/codice/utils.py index 2c9be744b..934656146 100644 --- a/imap_processing/codice/utils.py +++ b/imap_processing/codice/utils.py @@ -5,16 +5,9 @@ other CoDICE processing modules. """ -import collections from enum import IntEnum -import numpy as np import space_packet_parser -import xarray as xr - -from imap_processing.cdf.global_attrs import ConstantCoordinates -from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes -from imap_processing.cdf.utils import met_to_j2000ns class CODICEAPID(IntEnum): @@ -100,70 +93,3 @@ def add_metadata_to_array(packet: space_packet_parser, metadata_arrays: dict) -> metadata_arrays.setdefault(key, []).append(value.raw_value) return metadata_arrays - - -# TODO: Correct the type of "packets" -def create_hskp_dataset( # type: ignore[no-untyped-def] - packets, - data_version: str, -) -> xr.Dataset: - """ - Create dataset for each metadata field for housekeeping data. - - Parameters - ---------- - packets : list[space_packet_parser.parser.Packet] - The list of packets to process. - data_version : str - Version of the data product being created. - - Returns - ------- - dataset : xarray.Dataset - Xarray dataset containing the metadata. - """ - cdf_attrs = ImapCdfAttributes() - cdf_attrs.add_instrument_global_attrs("codice") - cdf_attrs.add_instrument_variable_attrs("codice", "l1a") - cdf_attrs.add_global_attribute("Data_version", data_version) - - metadata_arrays: dict = collections.defaultdict(list) - - for packet in packets: - add_metadata_to_array(packet, metadata_arrays) - - # TODO: Is there a way to get the attrs from the YAML-based method? - epoch = xr.DataArray( - met_to_j2000ns( - metadata_arrays["SHCOARSE"], - reference_epoch=np.datetime64("2010-01-01T00:01:06.184", "ns"), - ), - name="epoch", - dims=["epoch"], - attrs=ConstantCoordinates.EPOCH, - ) - - dataset = xr.Dataset( - coords={"epoch": epoch}, - attrs=cdf_attrs.get_global_attributes("imap_codice_l1a_hskp"), - ) - - # TODO: Change 'TBD' catdesc and fieldname - # Once packet definition files are re-generated, can get this info from - # something like this: - # for key, value in (packet.header | packet.data).items(): - # fieldname = value.short_description - # catdesc = value.short_description - # I am holding off making this change until I acquire updated housekeeping - # packets/validation data that match the latest telemetry definitions - # I may also be able to replace this function with utils.create_dataset(?) - for key, value in metadata_arrays.items(): - attrs = cdf_attrs.get_variable_attributes("codice_support_attrs") - attrs["CATDESC"] = "TBD" - attrs["DEPEND_0"] = "epoch" - attrs["FIELDNAM"] = "TBD" - attrs["LABLAXIS"] = key - - dataset[key] = xr.DataArray(value, dims=["epoch"], attrs=attrs) - - return dataset diff --git a/imap_processing/tests/codice/test_codice_l0.py b/imap_processing/tests/codice/test_codice_l0.py index 47a1c9dd3..8fcac359f 100644 --- a/imap_processing/tests/codice/test_codice_l0.py +++ b/imap_processing/tests/codice/test_codice_l0.py @@ -9,9 +9,11 @@ from imap_processing import imap_module_directory from imap_processing.codice import codice_l0 -from imap_processing.codice.utils import create_hskp_dataset +from imap_processing.codice.codice_l1a import create_hskp_dataset from imap_processing.utils import convert_raw_to_eu +# TODO: Move tests for housekeeping data to test_codice_l1a.py + @pytest.fixture(scope="session") def decom_test_data() -> list: diff --git a/imap_processing/tests/codice/test_codice_l1a.py b/imap_processing/tests/codice/test_codice_l1a.py index b1f2bd962..57101e153 100644 --- a/imap_processing/tests/codice/test_codice_l1a.py +++ b/imap_processing/tests/codice/test_codice_l1a.py @@ -99,6 +99,9 @@ ), ] + +# Test data + # Placeholder for validation data files VALIDATION_DATA = [ f"{imap_module_directory}/tests/codice/data/validation_hskp.cdf", From 0155067511f69fd52446cb1c4059198c6e0c884d Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Wed, 17 Jul 2024 14:54:51 -0600 Subject: [PATCH 02/22] Now using epoch attrs defined in yaml files instead of dataclasses --- imap_processing/codice/codice_l1a.py | 11 ++++------- imap_processing/tests/codice/test_codice_l0.py | 6 ++---- imap_processing/tests/codice/test_codice_l1a.py | 3 --- 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/imap_processing/codice/codice_l1a.py b/imap_processing/codice/codice_l1a.py index d1089f86c..f3c2a1a78 100644 --- a/imap_processing/codice/codice_l1a.py +++ b/imap_processing/codice/codice_l1a.py @@ -23,7 +23,6 @@ import xarray as xr from imap_processing import imap_module_directory -from imap_processing.cdf.global_attrs import ConstantCoordinates from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes from imap_processing.cdf.utils import IMAP_EPOCH, met_to_j2000ns from imap_processing.codice import constants @@ -322,7 +321,6 @@ def create_hskp_dataset( for packet in packets: add_metadata_to_array(packet, metadata_arrays) - # TODO: Is there a way to get the attrs from the YAML-based method? epoch = xr.DataArray( met_to_j2000ns( metadata_arrays["SHCOARSE"], @@ -330,7 +328,7 @@ def create_hskp_dataset( ), name="epoch", dims=["epoch"], - attrs=ConstantCoordinates.EPOCH, + attrs=cdf_attrs.get_variable_attributes("epoch_attrs"), ) dataset = xr.Dataset( @@ -339,14 +337,13 @@ def create_hskp_dataset( ) # TODO: Change 'TBD' catdesc and fieldname - # Once packet definition files are re-generated, can get this info from - # something like this: + # Once housekeeping packet definition file is re-generated with updated + # version of space_packet_parser, can get fieldname and catdesc info via: # for key, value in (packet.header | packet.data).items(): # fieldname = value.short_description - # catdesc = value.short_description + # catdesc = value.long_description # I am holding off making this change until I acquire updated housekeeping # packets/validation data that match the latest telemetry definitions - # I may also be able to replace this function with utils.create_dataset(?) for key, value in metadata_arrays.items(): attrs = cdf_attrs.get_variable_attributes("codice_support_attrs") attrs["CATDESC"] = "TBD" diff --git a/imap_processing/tests/codice/test_codice_l0.py b/imap_processing/tests/codice/test_codice_l0.py index 8fcac359f..6920fdd74 100644 --- a/imap_processing/tests/codice/test_codice_l0.py +++ b/imap_processing/tests/codice/test_codice_l0.py @@ -12,8 +12,6 @@ from imap_processing.codice.codice_l1a import create_hskp_dataset from imap_processing.utils import convert_raw_to_eu -# TODO: Move tests for housekeeping data to test_codice_l1a.py - @pytest.fixture(scope="session") def decom_test_data() -> list: @@ -64,7 +62,7 @@ def validation_data() -> pd.core.frame.DataFrame: return validation_data -def test_eu_hk_data( +def test_eu_hskp_data( decom_test_data: list[space_packet_parser.parser.Packet], validation_data: pd.core.frame.DataFrame, ): @@ -107,7 +105,7 @@ def test_eu_hk_data( assert round(eu_val, 5) == round(validation_val, 5) -def test_raw_hk_data( +def test_raw_hskp_data( decom_test_data: list[space_packet_parser.parser.Packet], validation_data: pd.core.frame.DataFrame, ): diff --git a/imap_processing/tests/codice/test_codice_l1a.py b/imap_processing/tests/codice/test_codice_l1a.py index 57101e153..b1f2bd962 100644 --- a/imap_processing/tests/codice/test_codice_l1a.py +++ b/imap_processing/tests/codice/test_codice_l1a.py @@ -99,9 +99,6 @@ ), ] - -# Test data - # Placeholder for validation data files VALIDATION_DATA = [ f"{imap_module_directory}/tests/codice/data/validation_hskp.cdf", From f5ca632f63cf9480582c47c641281d417efe2b9a Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Wed, 17 Jul 2024 15:45:51 -0600 Subject: [PATCH 03/22] Fixed warnings given off from CdfAttributeManager --- .../imap_codice_l1a_variable_attrs.yaml | 65 ++++++++++++------- imap_processing/codice/codice_l1a.py | 6 +- .../tests/codice/test_codice_l1a.py | 9 ++- 3 files changed, 50 insertions(+), 30 deletions(-) diff --git a/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml b/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml index 20c18384a..cb749b163 100644 --- a/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml @@ -1,14 +1,20 @@ # <=== Defaults ===> default_attrs: &default - # Assumed values for all variable attrs unless overwritten - DISPLAY_TYPE: no_plot - FILLVAL: -9223372036854775808 - FORMAT: I12 - UNITS: dN - VALIDMIN: -9223372036854775808 - VALIDMAX: 9223372036854775807 - VAR_TYPE: data - SCALETYP: linear + CATDESC: "" + DISPLAY_TYPE: no_plot + FIELDNAM: "" + FILLVAL: -9223372036854775808 + FORMAT: I12 + LABLAXIS: "" + REFERENCE_POSITION: "" + RESOLUTION: "" + SCALETYP: linear + TIME_BASE: "" + TIME_SCALE: "" + UNITS: dN + VALIDMIN: -9223372036854775808 + VALIDMAX: 9223372036854775807 + VAR_TYPE: data codice_support_attrs: &support_default <<: *default @@ -27,26 +33,37 @@ energy_attrs: VAR_TYPE: support_data epoch_attrs: - CATDESC: Time, number of nanoseconds since J2000 with leap seconds included - FIELDNAM: epoch - LABLAXIS: epoch - FILLVAL: -9223372036854775808 - FORMAT: " " # Supposedly not required, fails in xarray_to_cdf - VALIDMIN: -9223372036854775808 - VALIDMAX: 9223372036854775807 - UNITS: ns - VAR_TYPE: support_data - SCALETYP: linear - MONOTON: INCREASE - TIME_BASE: J2000 - TIME_SCALE: Terrestrial Time - REFERENCE_POSITION: Rotating Earth Geoid + <<: *default + CATDESC: Time, number of nanoseconds since J2000 with leap seconds included + FIELDNAM: epoch + LABLAXIS: epoch + FILLVAL: -9223372036854775808 + FORMAT: " " # Supposedly not required, fails in xarray_to_cdf + VALIDMIN: -9223372036854775808 + VALIDMAX: 9223372036854775807 + UNITS: ns + VAR_TYPE: support_data + SCALETYP: linear + MONOTON: INCREASE + TIME_BASE: J2000 + TIME_SCALE: Terrestrial Time + REFERENCE_POSITION: Rotating Earth Geoid # <=== Labels ===> energy_label: CATDESC: Energy per charge (E/q) sweeping step + DISPLAY_TYPE: "" FIELDNAM: Energy step + FILLVAL: "" FORMAT: A3 + LABLAXIS: "" + REFERENCE_POSITION: "" + RESOLUTION: "" + TIME_BASE: "" + TIME_SCALE: "" + UNITS: "" + VALIDMAX: "" + VALIDMIN: "" VAR_TYPE: metadata # <=== Dataset Attributes ===> @@ -55,6 +72,7 @@ energy_label: acquisition_times_attrs: <<: *default CATDESC: Time of acquisition for the energy step + DEPEND_1: energy FIELDNAM: Acquisition Time FILLVAL: 1.7976931348623157e+308 FORMAT: F10.3 @@ -79,6 +97,7 @@ counters_attrs: &counters esa_sweep_attrs: <<: *default CATDESC: ElectroStatic Analyzer Energy Values + DEPEND_1: energy FIELDNAM: ESA Voltage FORMAT: I19 LABLAXIS: ESA V diff --git a/imap_processing/codice/codice_l1a.py b/imap_processing/codice/codice_l1a.py index f3c2a1a78..2179f6fcf 100644 --- a/imap_processing/codice/codice_l1a.py +++ b/imap_processing/codice/codice_l1a.py @@ -142,7 +142,7 @@ def create_science_dataset(self, met: np.int64, data_version: str) -> xr.Dataset # TODO: Currently, cdflib doesn't properly write/read CDF files that # have a single epoch value. To get around this for now, use # two epoch values and reshape accordingly. Revisit this after - # SIT-3. + # SIT-3. See https://github.com/MAVENSDC/cdflib/issues/268 variable_data_arr = np.array(list(variable_data) * 2, dtype=int).reshape( 2, self.num_energy_steps ) @@ -424,8 +424,6 @@ def process_codice_l1a(file_path: Path, data_version: str) -> xr.Dataset: CODICEAPID.COD_LO_NSW_SPECIES_COUNTS, ] - # TODO: Temporary workaround in order to create hi data products in absence - # of simulated data if file_path.name.startswith(("imap_codice_l0_lo", "imap_codice_l0_hskp")): # Decom the packets, group data by APID, and sort by time packets = decom_packets(file_path) @@ -488,7 +486,5 @@ def process_codice_l1a(file_path: Path, data_version: str) -> xr.Dataset: pipeline.unpack_science_data(science_values) dataset = pipeline.create_science_dataset(met, data_version) - # Write dataset to CDF logger.info(f"\nFinal data product:\n{dataset}\n") - return dataset diff --git a/imap_processing/tests/codice/test_codice_l1a.py b/imap_processing/tests/codice/test_codice_l1a.py index b1f2bd962..8b381a409 100644 --- a/imap_processing/tests/codice/test_codice_l1a.py +++ b/imap_processing/tests/codice/test_codice_l1a.py @@ -7,7 +7,7 @@ import xarray as xr from imap_processing import imap_module_directory -from imap_processing.cdf.utils import load_cdf +from imap_processing.cdf.utils import load_cdf, write_cdf from imap_processing.codice.codice_l1a import process_codice_l1a # TODO: Add test that processes a file with multiple APIDs @@ -128,6 +128,11 @@ def test_l1a_data(request) -> xr.Dataset: """ dataset = process_codice_l1a(file_path=request.param, data_version="001") + + # Write the dataset to a CDF so it can be manually inspected as well + file_path = write_cdf(dataset) + print(f"CDF file written to {file_path}") + return dataset @@ -153,7 +158,7 @@ def test_l1a_cdf_filenames(test_l1a_data: xr.Dataset, expected_logical_source: s @pytest.mark.xfail( - reason="Currently failing due to cdflib/epoch issue. Revisit after SIT-3" + reason="Currently failing due to cdflib/epoch issue. See https://github.com/MAVENSDC/cdflib/issues/268" ) @pytest.mark.parametrize( "test_l1a_data, expected_shape", From a78ddb6502f3bcbb6d36f3bb962ee73675ff1263 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Wed, 17 Jul 2024 15:59:25 -0600 Subject: [PATCH 04/22] Improvements to some CDF attribute definitions --- .../imap_codice_l1a_variable_attrs.yaml | 6 +- .../imap_codice_l1b_variable_attrs.yaml | 77 +++++++++++-------- 2 files changed, 48 insertions(+), 35 deletions(-) diff --git a/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml b/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml index cb749b163..a52786a7f 100644 --- a/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml @@ -67,19 +67,17 @@ energy_label: VAR_TYPE: metadata # <=== Dataset Attributes ===> -# TODO: resolve ISTP warning: Width of F10 FORMAT may be insufficient for -# values. Adjust FORMAT or VALIDMIN/VALIDMAX value to resolve. acquisition_times_attrs: <<: *default CATDESC: Time of acquisition for the energy step DEPEND_1: energy FIELDNAM: Acquisition Time - FILLVAL: 1.7976931348623157e+308 + FILLVAL: .NAN FORMAT: F10.3 LABLAXIS: Acq Time UNITS: ms VALIDMIN: 0 - VALIDMAX: 1.7976931348623157e+308 + VALIDMAX: 1000 VAR_TYPE: support_data counters_attrs: &counters diff --git a/imap_processing/cdf/config/imap_codice_l1b_variable_attrs.yaml b/imap_processing/cdf/config/imap_codice_l1b_variable_attrs.yaml index 268a5b848..2db92bc1a 100644 --- a/imap_processing/cdf/config/imap_codice_l1b_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_l1b_variable_attrs.yaml @@ -1,14 +1,20 @@ # <=== Defaults ===> default_attrs: &default - # Assumed values for all variable attrs unless overwritten - DISPLAY_TYPE: no_plot - FILLVAL: -9223372036854775808 - FORMAT: I12 - UNITS: dN - VALIDMIN: -9223372036854775808 - VALIDMAX: 9223372036854775807 - VAR_TYPE: data - SCALETYP: linear + CATDESC: "" + DISPLAY_TYPE: no_plot + FIELDNAM: "" + FILLVAL: -9223372036854775808 + FORMAT: I12 + LABLAXIS: "" + REFERENCE_POSITION: "" + RESOLUTION: "" + SCALETYP: linear + TIME_BASE: "" + TIME_SCALE: "" + UNITS: dN + VALIDMIN: -9223372036854775808 + VALIDMAX: 9223372036854775807 + VAR_TYPE: data codice_support_attrs: &support_default <<: *default @@ -27,41 +33,51 @@ energy_attrs: VAR_TYPE: support_data epoch_attrs: - CATDESC: Time, number of nanoseconds since J2000 with leap seconds included - FIELDNAM: epoch - LABLAXIS: epoch - FILLVAL: -9223372036854775808 - FORMAT: " " # Supposedly not required, fails in xarray_to_cdf - VALIDMIN: -9223372036854775808 - VALIDMAX: 9223372036854775807 - UNITS: ns - VAR_TYPE: support_data - SCALETYP: linear - MONOTON: INCREASE - TIME_BASE: J2000 - TIME_SCALE: Terrestrial Time - REFERENCE_POSITION: Rotating Earth Geoid + <<: *default + CATDESC: Time, number of nanoseconds since J2000 with leap seconds included + FIELDNAM: epoch + LABLAXIS: epoch + FILLVAL: -9223372036854775808 + FORMAT: " " # Supposedly not required, fails in xarray_to_cdf + VALIDMIN: -9223372036854775808 + VALIDMAX: 9223372036854775807 + UNITS: ns + VAR_TYPE: support_data + SCALETYP: linear + MONOTON: INCREASE + TIME_BASE: J2000 + TIME_SCALE: Terrestrial Time + REFERENCE_POSITION: Rotating Earth Geoid # <=== Labels ===> energy_label: CATDESC: Energy per charge (E/q) sweeping step + DISPLAY_TYPE: "" FIELDNAM: Energy step + FILLVAL: "" FORMAT: A3 + LABLAXIS: "" + REFERENCE_POSITION: "" + RESOLUTION: "" + TIME_BASE: "" + TIME_SCALE: "" + UNITS: "" + VALIDMAX: "" + VALIDMIN: "" VAR_TYPE: metadata # <=== Dataset Attributes ===> -# TODO: resolve ISTP warning: Width of F10 FORMAT may be insufficient for -# values. Adjust FORMAT or VALIDMIN/VALIDMAX value to resolve. acquisition_times_attrs: <<: *default CATDESC: Time of acquisition for the energy step + DEPEND_1: energy FIELDNAM: Acquisition Time - FILLVAL: 1.7976931348623157e+308 + FILLVAL: .NAN FORMAT: F10.3 LABLAXIS: Acq Time UNITS: ms VALIDMIN: 0 - VALIDMAX: 1.7976931348623157e+308 + VALIDMAX: 1000 VAR_TYPE: support_data counters_attrs: &counters @@ -71,16 +87,15 @@ counters_attrs: &counters DEPEND_1: energy DISPLAY_TYPE: time_series FIELDNAM: Fill in at creation - FILLVAL: 1.7976931348623157e+308 - FORMAT: F10.3 LABL_PTR_1: energy_label - UNITS: rates + UNITS: counts VALIDMIN: 0 - VALIDMAX: 1.7976931348623157e+308 + VALIDMAX: 8388607 # max value for a signed 24-bit integer esa_sweep_attrs: <<: *default CATDESC: ElectroStatic Analyzer Energy Values + DEPEND_1: energy FIELDNAM: ESA Voltage FORMAT: I19 LABLAXIS: ESA V From adffee26b7209e545f3a38f0f327e53ee8e95d1b Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Thu, 18 Jul 2024 13:30:32 -0600 Subject: [PATCH 05/22] Renamed method for condifuring data products --- imap_processing/codice/codice_l1a.py | 40 +++++++++++++--------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/imap_processing/codice/codice_l1a.py b/imap_processing/codice/codice_l1a.py index 2179f6fcf..df939348f 100644 --- a/imap_processing/codice/codice_l1a.py +++ b/imap_processing/codice/codice_l1a.py @@ -62,12 +62,12 @@ class CoDICEL1aPipeline: Methods ------- + configure_data_products() + Set the various settings for defining the data products. create_science_dataset() Create an ``xarray`` dataset for the unpacked science data. get_acquisition_times() Retrieve the acquisition times via the Lo stepping table. - get_data_products() - Retrieve the lo data products. get_esa_sweep_values() Retrieve the ESA sweep values. unpack_science_data() @@ -81,6 +81,21 @@ def __init__(self, table_id: int, plan_id: int, plan_step: int, view_id: int): self.plan_step = plan_step self.view_id = view_id + def configure_data_products(self, apid: int) -> None: + """ + Set the various settings for defining the data products. + + Parameters + ---------- + apid : int + The APID of interest. + """ + config = constants.DATA_PRODUCT_CONFIGURATIONS.get(apid) # type: ignore[call-overload] + self.num_counters = config["num_counters"] + self.num_energy_steps = config["num_energy_steps"] + self.variable_names = config["variable_names"] + self.dataset_name = config["dataset_name"] + def create_science_dataset(self, met: np.int64, data_version: str) -> xr.Dataset: """ Create an ``xarray`` dataset for the unpacked science data. @@ -215,23 +230,6 @@ def get_acquisition_times(self) -> None: row_number = np.argmax(energy_steps == str(step_number), axis=1).argmax() self.acquisition_times.append(lo_stepping_values.acq_time[row_number]) - def get_data_products(self, apid: int) -> None: - """ - Retrieve various settings for defining the data products. - - Parameters - ---------- - apid : int - The APID of interest. - """ - config = constants.DATA_PRODUCT_CONFIGURATIONS.get(apid) # type: ignore[call-overload] - # TODO Change, No overload variant of "get" of - # "dict" matches argument type "str". - self.num_counters = config["num_counters"] - self.num_energy_steps = config["num_energy_steps"] - self.variable_names = config["variable_names"] - self.dataset_name = config["dataset_name"] - def get_esa_sweep_values(self) -> None: """ Retrieve the ESA sweep values. @@ -454,7 +452,7 @@ def process_codice_l1a(file_path: Path, data_version: str) -> xr.Dataset: pipeline = CoDICEL1aPipeline(table_id, plan_id, plan_step, view_id) pipeline.get_esa_sweep_values() pipeline.get_acquisition_times() - pipeline.get_data_products(apid) + pipeline.configure_data_products(apid) pipeline.unpack_science_data(science_values) dataset = pipeline.create_science_dataset(met, data_version) @@ -482,7 +480,7 @@ def process_codice_l1a(file_path: Path, data_version: str) -> xr.Dataset: science_values = "" # Currently don't have simulated data for this pipeline = CoDICEL1aPipeline(table_id, plan_id, plan_step, view_id) - pipeline.get_data_products(apid) + pipeline.configure_data_products(apid) pipeline.unpack_science_data(science_values) dataset = pipeline.create_science_dataset(met, data_version) From 391fcc6c2adaf4411cbf3df2d87d214782fc0c6c Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Wed, 24 Jul 2024 09:53:13 -0600 Subject: [PATCH 06/22] Added initial support for lo and hi event data processing --- .../config/imap_codice_global_cdf_attrs.yaml | 14 ++ imap_processing/codice/codice_l0.py | 2 + imap_processing/codice/codice_l1a.py | 78 ++++++++++- imap_processing/codice/constants.py | 9 ++ .../packet_definitions/P_COD_HI_PHA.xml | 120 ++++++++--------- .../packet_definitions/P_COD_LO_PHA.xml | 124 ++++++++---------- .../imap_codice_l0_hi-pha_20240429_v001.pkts | Bin 0 -> 8528 bytes .../imap_codice_l0_lo-pha_20240429_v001.pkts | Bin 0 -> 10812 bytes .../tests/codice/test_codice_l1a.py | 14 ++ 9 files changed, 228 insertions(+), 133 deletions(-) create mode 100644 imap_processing/tests/codice/data/imap_codice_l0_hi-pha_20240429_v001.pkts create mode 100644 imap_processing/tests/codice/data/imap_codice_l0_lo-pha_20240429_v001.pkts diff --git a/imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml b/imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml index 005085e52..fa8dc99e0 100644 --- a/imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml @@ -49,6 +49,13 @@ imap_codice_l1a_hi_sectored: Logical_source: imap_codice_l1a_hi-sectored Logical_source_description: IMAP Mission CoDICE Hi Level-1A Sectored Data. +imap_codice_l1a_hi_pha: + <<: *instrument_base + Data_level: 1A + Data_type: L1A_hi-pha->Level-1A Hi Event Data + Logical_source: imap_codice_l1a_hi-pha + Logical_source_description: IMAP Mission CoDICE Hi Level-1A Event Data. + imap_codice_l1a_lo_counters_aggregated: <<: *instrument_base Data_level: 1A @@ -105,6 +112,13 @@ imap_codice_l1a_lo_nsw_species: Logical_source: imap_codice_l1a_lo-nsw-species Logical_source_description: IMAP Mission CoDICE Lo Level-1A Non-Sunward Species Counts Data. +imap_codice_l1a_lo_pha: + <<: *instrument_base + Data_level: 1A + Data_type: L1A_lo-pha->Level-1A Lo Event Data + Logical_source: imap_codice_l1a_lo-pha + Logical_source_description: IMAP Mission CoDICE Lo Level-1A Event Data. + # L1b imap_codice_l1b_hskp: <<: *instrument_base diff --git a/imap_processing/codice/codice_l0.py b/imap_processing/codice/codice_l0.py index 71a5db5bb..4515af8fb 100644 --- a/imap_processing/codice/codice_l0.py +++ b/imap_processing/codice/codice_l0.py @@ -38,6 +38,7 @@ def decom_packets(packet_file: Path) -> list: "imap_codice_l0_hi-counters-singles_20240429_v001.pkts": "P_COD_HI_INST_COUNTS_SINGLES.xml", # noqa "imap_codice_l0_hi-omni_20240429_v001.pkts": "P_COD_HI_OMNI_SPECIES_COUNTS.xml", "imap_codice_l0_hi-sectored_20240429_v001.pkts": "P_COD_HI_SECT_SPECIES_COUNTS.xml", # noqa + "imap_codice_l0_hi-pha_20240429_v001.pkts": "P_COD_HI_PHA.xml", "imap_codice_l0_hskp_20100101_v001.pkts": "P_COD_NHK.xml", "imap_codice_l0_lo-counters-aggregated_20240429_v001.pkts": "P_COD_LO_INST_COUNTS_AGGREGATED.xml", # noqa "imap_codice_l0_lo-counters-singles_20240429_v001.pkts": "P_COD_LO_INST_COUNTS_SINGLES.xml", # noqa @@ -47,6 +48,7 @@ def decom_packets(packet_file: Path) -> list: "imap_codice_l0_lo-nsw-priority_20240429_v001.pkts": "P_COD_LO_NSW_PRIORITY_COUNTS.xml", # noqa "imap_codice_l0_lo-sw-species_20240429_v001.pkts": "P_COD_LO_SW_SPECIES_COUNTS.xml", # noqa "imap_codice_l0_lo-nsw-species_20240429_v001.pkts": "P_COD_LO_NSW_SPECIES_COUNTS.xml", # noqa + "imap_codice_l0_lo-pha_20240429_v001.pkts": "P_COD_LO_PHA.xml", } xtce_document = Path( diff --git a/imap_processing/codice/codice_l1a.py b/imap_processing/codice/codice_l1a.py index df939348f..1f19f0a47 100644 --- a/imap_processing/codice/codice_l1a.py +++ b/imap_processing/codice/codice_l1a.py @@ -290,6 +290,55 @@ def unpack_science_data(self, science_values: str) -> None: self.data = [["1"] * 128] * self.num_counters +def create_event_dataset( + met: list[int], event_data: str, dataset_name: str, data_version: str +) -> xr.Dataset: + """ + Create dataset for event data. + + Parameters + ---------- + met : list[int] + The Mission Elapsed Time of the data. + event_data : str + A string of binary numbers representing the event data. + dataset_name : str + The name for the dataset. + data_version : str + Version of the data product being created. + + Returns + ------- + dataset : xarray.Dataset + Xarray dataset containing the event data. + """ + cdf_attrs = ImapCdfAttributes() + cdf_attrs.add_instrument_global_attrs("codice") + cdf_attrs.add_instrument_variable_attrs("codice", "l1a") + cdf_attrs.add_global_attribute("Data_version", data_version) + + # Define coordinates + epoch = xr.DataArray( + met_to_j2000ns(met), # TODO: Fix after SIT-3 (see note below) + name="epoch", + dims=["epoch"], + attrs=cdf_attrs.get_variable_attributes("epoch_attrs"), + ) + + # Create the dataset to hold the data variables + dataset = xr.Dataset( + coords={ + "epoch": epoch, + }, + attrs=cdf_attrs.get_global_attributes(dataset_name), + ) + + # TODO: Determine what should go in event data CDF and how it should be + # structured. + + return dataset + + def create_hskp_dataset( packets: list[space_packet_parser.parser.Packet], data_version: str, @@ -411,6 +460,10 @@ def process_codice_l1a(file_path: Path, data_version: str) -> xr.Dataset: # need to split the processing based on the file_path, so this function can # be simplified. + apids_for_event_data_processing = [ + CODICEAPID.COD_LO_PHA, + CODICEAPID.COD_HI_PHA, + ] apids_for_lo_science_processing = [ CODICEAPID.COD_LO_INST_COUNTS_AGGREGATED, CODICEAPID.COD_LO_INST_COUNTS_SINGLES, @@ -422,7 +475,9 @@ def process_codice_l1a(file_path: Path, data_version: str) -> xr.Dataset: CODICEAPID.COD_LO_NSW_SPECIES_COUNTS, ] - if file_path.name.startswith(("imap_codice_l0_lo", "imap_codice_l0_hskp")): + if file_path.name.startswith( + ("imap_codice_l0_lo", "imap_codice_l0_hskp", "imap_codice_l0_hi-pha") + ): # Decom the packets, group data by APID, and sort by time packets = decom_packets(file_path) grouped_data = group_by_apid(packets) @@ -456,6 +511,27 @@ def process_codice_l1a(file_path: Path, data_version: str) -> xr.Dataset: pipeline.unpack_science_data(science_values) dataset = pipeline.create_science_dataset(met, data_version) + elif apid in apids_for_event_data_processing: + if apid == CODICEAPID.COD_LO_PHA: + dataset_name = "imap_codice_l1a_lo_pha" + elif apid == CODICEAPID.COD_HI_PHA: + dataset_name = "imap_codice_l1a_hi_pha" + + # Sort the packets by time + packets = sort_by_time(grouped_data[apid], "SHCOARSE") + + # Determine the start time of the packet + met = packets[0].data["ACQ_START_SECONDS"].raw_value + met = [met, met + 1] # TODO: Remove after SIT-3 + + # Extract the data + event_data = packets[0].data["EVENT_DATA"].raw_value + + # Create the dataset + dataset = create_event_dataset( + met, event_data, dataset_name, data_version + ) + # TODO: Temporary workaround in order to create hi data products in absence # of simulated data. This is essentially the same process as is for # lo, but don't try to decom any packets, just define the data diff --git a/imap_processing/codice/constants.py b/imap_processing/codice/constants.py index 425b6b57d..f104b24f9 100644 --- a/imap_processing/codice/constants.py +++ b/imap_processing/codice/constants.py @@ -49,6 +49,9 @@ "heplus", "cnoplus", ] +LO_PHA_NAMES = [ + "foo", +] # CDF-friendly names for hi data products HI_INST_COUNTS_AGGREGATED_NAMES = ["aggregated"] @@ -130,6 +133,12 @@ "variable_names": LO_NSW_SPECIES_NAMES, "dataset_name": "imap_codice_l1a_lo_nsw_species", }, + CODICEAPID.COD_LO_PHA: { + "num_counters": 1, + "num_energy_steps": 128, + "variable_names": LO_PHA_NAMES, + "dataset_name": "imap_codice_l1a_lo_pha", + }, } # Compression ID lookup table for Lo data products diff --git a/imap_processing/codice/packet_definitions/P_COD_HI_PHA.xml b/imap_processing/codice/packet_definitions/P_COD_HI_PHA.xml index cc28a6896..b75581c0a 100644 --- a/imap_processing/codice/packet_definitions/P_COD_HI_PHA.xml +++ b/imap_processing/codice/packet_definitions/P_COD_HI_PHA.xml @@ -1,6 +1,6 @@ - + @@ -12,12 +12,12 @@ + + + - - - @@ -27,17 +27,14 @@ - - - - + - 276480 + 5760 @@ -64,73 +61,64 @@ CCSDS Packet Length (number of bytes after Packet length minus 1) - - S/C Time - Seconds - Secondary Header - Whole-seconds part of SCLK + + SECONDARY HEADER - WHOLE-SECONDS PART OF SCLK - - Packet Version - Packet version - this will be incremented each time the format of the packet changesCOD_LO_PHA. + + PACKET VERSION - THIS WILL BE INCREMENTED EACH TIME THE FORMAT OF THE PACKET CHANGESCOD_LO_PHA. - - Spin Period reported by the Spacecraft - Spin period reported by the Spacecraft in the Time and Status message. Reported period is the period that was active when the 16-spin acquisition cycle started. + + SPIN PERIOD REPORTED BY THE SPACECRAFT IN THE TIME AND STATUS MESSAGE. REPORTED PERIOD IS THE PERIOD THAT WAS ACTIVE WHEN THE 16-SPIN ACQUISITION CYCLE STARTED. - - Acquisition Start Time (Seconds) - Full-seconds portion of the time at which the 16-spin cycle started + + FULL-SECONDS PORTION OF THE TIME AT WHICH THE 16-SPIN CYCLE STARTED - - Acquisition Start Time (Subseconds) - Sub-seconds portion of the time at which the 16-spin cycle started + + SUB-SECONDS PORTION OF THE TIME AT WHICH THE 16-SPIN CYCLE STARTED - - Spare for alignment - Spare for alignment + + SPARE FOR ALIGNMENT - - Total number of TCR events - Total number of TCR events that occurred during the collection cycle + + BIAS GAIN MODE FOR THE SUPRATHERMAL SECTOR - - Total number of DCR events - Total number of DCR events that occurred during the collection cycle + + BIAS GAIN MODE FOR THE SOLARWIND SECTOR - - Total number of APD-Only events - Total number of APD-Only events that occurred during the collection cycle + + + INDICATES THAT THERE WAS SOME ERROR DETECTED DURING ACQUISITION OR PROCESSING OF THE DATA. ERRORS COULD INCLUDE CORRUPTED ACQUISITION MEMORY (I.E. EDAC ERRORS), TIMING VIOLATIONS, OR OTHER EVENTS THAT INTERRUPTED OR OTHERWISE AFFECTED DATA COLLECTION. - - Number of events included in this packet - Number of events selected for downlink (i.e. number of events in the Event_Data array) + + WHETHER THE EVENT DATA IS COMPRESSED. IF 1/YES, EVENT_DATA ARRAY IS COMPRESSED USING THE RICE COMPRESSION ALGORITHM. - - Whether the event data is compressed - Whether the event data is compressed. If 1/Yes, Event_Data array is compressed using the Rice compression algorithm. + + NUMBER OF EVENTS SELECTED FOR DOWNLINK (I.E. NUMBER OF EVENTS IN THE EVENT_DATA ARRAY) - - Number of bytes in the Event Data array - Number of bytes in the Event_Data array. If compressed, this value represents the length of the compressed data. + + NUMBER OF BYTES IN THE EVENT_DATA ARRAY. IF COMPRESSED, THIS VALUE REPRESENTS THE LENGTH OF THE COMPRESSED DATA. - - Event Data - Optionally compressed array of Event Data + + OPTIONALLY COMPRESSED ARRAY OF EVENT DATA -Format is TBD; some considerations/options: -- Full events have a lot of redundant data (e.g. will have many events with the same priority/spin/spin phase information). How well does compression to deal with the redundancy? -- Could include mini-headers for each (priority,spin, spin-phase) group and strip the redundant data from the events -- Should events be tightly packed, or can we pad out to 64-bit word boundaries? How well does compression compensate for the extra bits? +FORMAT IS TBD; SOME CONSIDERATIONS/OPTIONS: +- FULL EVENTS HAVE A LOT OF REDUNDANT DATA (E.G. WILL HAVE MANY EVENTS WITH THE SAME PRIORITY/SPIN/SPIN PHASE INFORMATION). HOW WELL DOES COMPRESSION TO DEAL WITH THE REDUNDANCY? +- COULD INCLUDE MINI-HEADERS FOR EACH (PRIORITY,SPIN, SPIN-PHASE) GROUP AND STRIP THE REDUNDANT DATA FROM THE EVENTS +- SHOULD EVENTS BE TIGHTLY PACKED, OR CAN WE PAD OUT TO 64-BIT WORD BOUNDARIES? HOW WELL DOES COMPRESSION COMPENSATE FOR THE EXTRA BITS? -Each event consists of: -- 10-bit TOF -- 9-bit SSD Energy -- 2-bit Energy Range -- 7-bit Spin Angle -- 4-bit SSD Position -- 4-bit Spin Number -- 2-bit PHA Type +EACH EVENT CONSISTS OF: +- 10-BIT TOF +- 9-BIT SSD ENERGY +- 2-BIT ENERGY RANGE +- 7-BIT SPIN ANGLE +- 4-BIT SSD POSITION +- 4-BIT SPIN NUMBER +- 2-BIT PHA TYPE -TBD: Events may be tightly packed, or may have spares added to keep each event byte-aligned. In either case, there may be up to 1 byte of padding to keep the total size of the packet even. +TBD: EVENTS MAY BE TIGHTLY PACKED, OR MAY HAVE SPARES ADDED TO KEEP EACH EVENT BYTE-ALIGNED. IN EITHER CASE, THERE MAY BE UP TO 1 BYTE OF PADDING TO KEEP THE TOTAL SIZE OF THE PACKET EVEN. + + + PACKET CHECKSUM @@ -158,13 +146,15 @@ TBD: Events may be tightly packed, or may have spares added to keep each event b - - - - + + + + + + diff --git a/imap_processing/codice/packet_definitions/P_COD_LO_PHA.xml b/imap_processing/codice/packet_definitions/P_COD_LO_PHA.xml index e0e28109d..fc8cd2d85 100644 --- a/imap_processing/codice/packet_definitions/P_COD_LO_PHA.xml +++ b/imap_processing/codice/packet_definitions/P_COD_LO_PHA.xml @@ -1,6 +1,6 @@ - + @@ -12,12 +12,12 @@ + + + - - - @@ -27,17 +27,14 @@ - - - - + - 276480 + 640 @@ -64,75 +61,66 @@ CCSDS Packet Length (number of bytes after Packet length minus 1) - - S/C Time - Seconds - Secondary Header - Whole-seconds part of SCLK + + SECONDARY HEADER - WHOLE-SECONDS PART OF SCLK - - Packet Version - Packet version - this will be incremented each time the format of the packet changesCOD_LO_PHA. + + PACKET VERSION - THIS WILL BE INCREMENTED EACH TIME THE FORMAT OF THE PACKET CHANGESCOD_LO_PHA. - - Spin Period reported by the Spacecraft - Spin period reported by the Spacecraft in the Time and Status message. Reported period is the period that was active when the 16-spin acquisition cycle started. + + SPIN PERIOD REPORTED BY THE SPACECRAFT IN THE TIME AND STATUS MESSAGE. REPORTED PERIOD IS THE PERIOD THAT WAS ACTIVE WHEN THE 16-SPIN ACQUISITION CYCLE STARTED. - - Acquisition Start Time (Seconds) - Full-seconds portion of the time at which the 16-spin cycle started + + FULL-SECONDS PORTION OF THE TIME AT WHICH THE 16-SPIN CYCLE STARTED - - Acquisition Start Time (Subseconds) - Sub-seconds portion of the time at which the 16-spin cycle started + + SUB-SECONDS PORTION OF THE TIME AT WHICH THE 16-SPIN CYCLE STARTED - - Spare for alignment - Spare for alignment + + SPARE FOR ALIGNMENT - - Total number of TCR events - Total number of TCR events that occurred during the collection cycle + + BIAS GAIN MODE FOR THE SUPRATHERMAL SECTOR - - Total number of DCR events - Total number of DCR events that occurred during the collection cycle + + BIAS GAIN MODE FOR THE SOLARWIND SECTOR - - Total number of APD-Only events - Total number of APD-Only events that occurred during the collection cycle + + + INDICATES THAT THERE WAS SOME ERROR DETECTED DURING ACQUISITION OR PROCESSING OF THE DATA. ERRORS COULD INCLUDE CORRUPTED ACQUISITION MEMORY (I.E. EDAC ERRORS), TIMING VIOLATIONS, OR OTHER EVENTS THAT INTERRUPTED OR OTHERWISE AFFECTED DATA COLLECTION. - - Number of events included in this packet - Number of events selected for downlink (i.e. number of events in the Event_Data array) + + WHETHER THE EVENT DATA IS COMPRESSED. IF 1/YES, EVENT_DATA ARRAY IS COMPRESSED USING THE LZMA COMPRESSION ALGORITHM. - - Whether the event data is compressed - Whether the event data is compressed. If 1/Yes, Event_Data array is compressed using the LZMA compression algorithm. + + NUMBER OF EVENTS SELECTED FOR DOWNLINK (I.E. NUMBER OF EVENTS IN THE EVENT_DATA ARRAY) - - Number of bytes in the Event Data array - Number of bytes in the Event_Data array. If compressed, this value represents the length of the compressed data. + + NUMBER OF BYTES IN THE EVENT_DATA ARRAY. IF COMPRESSED, THIS VALUE REPRESENTS THE LENGTH OF THE COMPRESSED DATA. - - Event Data - Optionally compressed array of Event Data + + OPTIONALLY COMPRESSED ARRAY OF EVENT DATA -Format is TBD; some considerations/options: -- Full events have a lot of redundant data (e.g. will have many events with the same priority/e-step/spin phase information). How well does compression to deal with the redundancy? -- Could include mini-headers for each (priority,e-step, spin-phase) group and strip the redundant data from the events -- Should events be tightly packed, or can we pad out to 64-bit word boundaries? How well does compression compensate for the extra bits? +FORMAT IS TBD; SOME CONSIDERATIONS/OPTIONS: +- FULL EVENTS HAVE A LOT OF REDUNDANT DATA (E.G. WILL HAVE MANY EVENTS WITH THE SAME PRIORITY/E-STEP/SPIN PHASE INFORMATION). HOW WELL DOES COMPRESSION TO DEAL WITH THE REDUNDANCY? +- COULD INCLUDE MINI-HEADERS FOR EACH (PRIORITY,E-STEP, SPIN-PHASE) GROUP AND STRIP THE REDUNDANT DATA FROM THE EVENTS +- SHOULD EVENTS BE TIGHTLY PACKED, OR CAN WE PAD OUT TO 64-BIT WORD BOUNDARIES? HOW WELL DOES COMPRESSION COMPENSATE FOR THE EXTRA BITS? -Each event consists of: -- 7-bit E-step -- 10-bit TOF -- 9-bit APD Energy -- 7-bit Spin Angle -- 5-bit Position -- 5-bit APD-ID -- 1-bit APD-Gain -- 2-bit PHA Type -- 3-bit Priority Range +EACH EVENT CONSISTS OF: +- 7-BIT E-STEP +- 10-BIT TOF +- 9-BIT APD ENERGY +- 7-BIT SPIN ANGLE +- 5-BIT POSITION +- 5-BIT APD-ID +- 1-BIT APD-GAIN +- 2-BIT PHA TYPE +- 3-BIT PRIORITY RANGE -TBD: Events may be tightly packed, or may have spares added to keep each event byte-aligned. In either case, there may be up to 1 byte of padding to keep the total size of the packet even. +TBD: EVENTS MAY BE TIGHTLY PACKED, OR MAY HAVE SPARES ADDED TO KEEP EACH EVENT BYTE-ALIGNED. IN EITHER CASE, THERE MAY BE UP TO 1 BYTE OF PADDING TO KEEP THE TOTAL SIZE OF THE PACKET EVEN. + + + PACKET CHECKSUM @@ -160,13 +148,15 @@ TBD: Events may be tightly packed, or may have spares added to keep each event b - - - - + + + + + + diff --git a/imap_processing/tests/codice/data/imap_codice_l0_hi-pha_20240429_v001.pkts b/imap_processing/tests/codice/data/imap_codice_l0_hi-pha_20240429_v001.pkts new file mode 100644 index 0000000000000000000000000000000000000000..b2167acc932fa303a586c201d7a7a5573e1f0a48 GIT binary patch literal 8528 zcmZ{nMNk|J)ys)V)Zt;JYtCyssuA0002x|1T5(0HXr{AfLT)c&KXv0I&xO4dT!M5-7}nLQV|( z`CGV$$QA!2%)RN~jXknJqnqvIIB4x2&nb>7wZt$`TP8TGLk1S8!6o({iq z<3bTq1*W3Enz06?ANnt?ddcXJY{`JJTlef$Ut@Cz(O7i7SnYbe##jbCmoCC?><23Z zpV=3gY4f1!qGVLlRQ#!Kc)}HQQo5rW3nqHpcnr3*tS_*LE#G@oNfjgpq4emUxkyU zNn=PAIZ^|(BuBGH1zz@CgeNV9T^lXTP&-rVS(a$N?A>p3IbpS0syy^M-oPg$rsFd3$Cz6l>&(a(Ay+xk2Ec8p;+U z;gtf58xpLg7MSfP+T^HL*%De4MK!?qljy$n;O|6LwY8IGe_;#R2`%S=c^pu7M^Ckv zV#pdrViIIjokuiVwMTVzwXVp|i`wd#Oy93lkZRZ-IIW1%h0X@l%-m7UvAmNux?H4j z%UATjOI;3PlMCkRWefheUr<^sTKa;9C@?Z;lidFoRk<(J3R z1E@=gfYmd>n(b-4M8+)C0)`=;g_vRB(ErB}riel$+z_vwJD>*olfSu=hRWEdgb4F~ zoN4#Qc(`?!4A+{Mw#Mifro&FwBBj%WEQm;|qM`UM6zwZ?UUFpH z#IE4o3RwZTvHR41Y8s+#aipH`0&AT7mCaO+7Ox`31gT$_yTFkO&eD{DQM~N%lGd8n z4-4KV+$FhTovsWtpE|OhlKl0)I26+td$MOEdOp%pU5>gtx5=xxVD-NW1|7iT?NL45 zOZ0hGU+BlLaw$2!(oWNsMqAeE&k*KjRgDSAg)5NQ$56pc&1?Y86fZjuLQjK}OTTNN zUvZp@0v*kf;i+(hNkjWMdqmW>G(jMx3Ew|gER-$ShE8l-d}+9N;rC)(!R;Lkf~8N3i_;-36R){^L86 zowKy5f)DZFon&yVLiulm6w*3gmLG4_29k4Dd%=ey@E);Aw68(*pwE9<<^n-ucNq02f8KJm%f-QD0Xa zUs~nYF>?wBy++3a*xFr;#W^bQ4yKPe;0sm4Cjt#BzXV`FX|(K%^^ll@9n&=wIP61X z+mPVLRLCOWpij5hHoHhKiD-A;~7GYkS21>R-BD4J>g;4Yz6dXEo z;}^a-2Y3F_gt<1oUmFSv8uq-F&Q0@&SavT%Hed0A*0A$w>f}A3?tZQ4{Zz<~KJp`9 zmy^OsaG9a0v;Xt-5d_>n4v;BsMD(|k5PP~;A7H$Zr+Hb;rPk00)nL_gU1c}B*7RC~8LODI&%E21!>BR?Ml5lrMBy2ZesZARl4>DCxg)&eB&wRK6a_JU zuco?^-A9NHEx}i#m-=AKJJPvHT)PZej}3kL#`(3JB&_~Q9iVCKwjpZfh=*^aeIr)_zZ==iv$ z1rpGn7P-Ff=Ta0xO@6Yy3$-jQ*K@T-C#h}^hVC-fJNDCzW7g|Z*GC8$I?sd2;e2xZ zh@6w5yk5o4gI?NUuG8?v9sAVcRd+1+a(p$&Q~!YFSt#M2%+-juvjO0bD18lz-E!M{ z`7qo-w3&Ux2satFlob#>$i0*9-vA&gs_}AiEch!_okYIZH~bFCA{zkGMcIKLBr03S zubbJ~y{t`x#qYP2Klf2RBaFkhLD4Q-0;5Ur*xk`u3<$sv&jHX(>is9}d}57X#F zCx_w7HO?B;9>Cfc4j#D$|H{)F#8ME`$lDm{u7T%x4Nay@#mb@PQXF=#!>gzckHj|J zij~E&%g$Mc*6y1~P6{pjbGdjqV{U4iXQwsCp0qwyG38oc02Z~?lJs$vv#U8i%Zgf> z+PQP>G<57oDtSoUHn)<;mVj)B=k}uoC_CCbFuR`VQp!$zeNnQo#RVK+VOthYUh$Tl z)#In3M?;+?s_)fBGY0;08t4`mjj^f_dnU&Az%lH^ejgp)q*kA-_@mL8>FNhMj2@(x zMW)vC$Wc(l+io+(fF%1o#Tc*NY!3;Z7BLntv}VDso=3tRH9dU)$Y{^<%$#@dG1G#f z{1@<2U*=)BSJv^vu$q}ZG-2)L?-{c#2#GQh-7Ju1!UhsQM{`y3a7B;(ZJ(e4*Tn;Q zXDu@N7avNas|~4-sayLa*&_KA-IJseU&4oY>}reRd^l-rl9$%-1fvpM1pMXo zLl0>16RN-6WG03WYv7l9DoInrV4cA00AbxG7cKRj51q^xCqGe0?XT{tMuaU9z(vZ) zi{e<$fn*@U(-R@7JaRu{lXQRFOL>!L*se(AXgy3k&rU0vb9`v z@~g-du5QWL4nu^wl~<3ScRpTo@~c*3l2gitS$=kO#63!N=2KG|w|KSn(dI!h@$OBW zQdC;jD5sAYq=er{p?t9{T&5yKNxvmX!Z;y)z&#G{Y@jf*7ycUa#I0z~!cZ|>C*kyn zA6%*Bn1A^_bhXmj1$e(z)VZTl8o)eyg3__xQQa1FxX&^ilqDbGUZfs!M!{-L!SYD+ zv?UQt<_-imPx)IrcJl=R`=dN5#?t(-AcCD>;rJCoe+$^6fIZGtTM%xA0rjNw`!Ix? zzkCp1%x09pfBI_S9%0`+S2^k4L0Ak4!VnW^NcG&Z@n-O9O~24$71UkJjpov~rF8c& z>6)`Neg=RzM%f}cCiE>LJl30W=lyZI$bK~q(APz12*q)6BrYuL>4#tGSCP*)sGJoz z%aG?_qE9Z0qGd&LeR1|yp{*#?3pE>a;sVeX;4zyPbgP2z>g+Y{F=c*^dt;&o>&=fT zE0?%3sAfLEWy$7o3P4Lmy(1{fN0v&>{8^pa)qO8n)MEw8cYJ@Xb%3!Q|&5J1P++GX^5 zS&_BHDR{JcxoAC4>+;LsU#Oww7BBL9tbBjN0Aus>#A9rCn2P1lTH}f)G1y(cJ;ZDpZH2T~7f@ zzP;#cI5O*t>yS)MYo6SVU6!5D-*l{Lxh6v|(7+hq-yU1zepp5&dAo*zdML5l_yh~> zeMLdsb_pz5C+Tuyo1H|-bd<;y0q|z_*r6h&BK!m2b<+}Xe6X`cA){vFU0C&%8akDQ z_FqR{$n#gVEE@?W7i6_Z>8Y$*v=pA|)0QjWHDI{Y4aRY!F!aHIE0C;j5v1|LxUZDTk zbO2ZlP4Nh%SnxeC_P@3uG5`QY1polu{^u6_EC6KtXAA1X1fQ|y3VtaKWOtIdE?_O? zb(~29gYCj7-PY$--FIPEt53TyHNFrL2dGuYwF@jRf4QVB5<*89u zZO+?<&)=C>nD}-6_Km7lF6%8wf%bn(-n@ydH$C&@KKSvt!FwyYZ;^^am5j=!XZ4H1 zsELgB{A4Zyxe8fcBm$7+oa^Gn%P!&bmDQ$dtS~m%v`QVHI^IIOW3N}#dUl4I2m@)v zQaj?Jb?6Sez0Mx^EzD4hQ~X5RA#^h*mDL*2lM-m?cc)0;ub;brfT{S&_O#4Kx!@g&ddXSec#Ae5|7`WcJwJGhmNd0x3xj zHt6I_tw`xUdzBnBlta3BW?r+Ujj9`4Q0drZQ&WCUwt1%0p=UC*ABLCUHdS{P zlzxg2qcA5q@*Smmz?kcWsY}wL9FS|4e3<4F*9kdt+?i20UbM9WY6c@zsx}SL*j3Qs zu`2@&{|!Dv>z?ZGzGdZ1mUR}ccJlG4VdSJ&i_bs9&&^@fQeNI4#Jy56MBlwPOH8TN zs!92ES+zu6>mXc{!CXCi^n6qI99?emqMihmi_ga?hN4s4{SPnRb?xm?K>05m>D`!J z1N4IMwBK1@lN;2d7tbHYASPG|WY<+Emd2pqvsdBU-M?Vixt?y0J-|+g2dlb!wp)`Y zvvo3ubgO>>4*3HYHt0;5gMZp@eiDfcn^0c;wHhm5r`4DF6-^xyeo4wk*38P$0b(;NbIu4YMA3sE-ldF<_twrlgpX55G zHG{0C&jxMn8Qd3D;A(w7g}0OsgeAgx6z{+#AJ@iULYT}DhIy4MLnySPXGrL+32#Gn zERt4}^6<7{K0nHfY144#Q3kah3n0e%C8m^?T-dF7s?i);hKy;d{fJ^D;$^7qbhOLm zKmHurcO$o7{ZklOuT7g%*O{d zy!?*5?>6`gNV^ghjuR?^a=Lb~sZ91k;C@*=4Vy02eoHAG!RlPV>3fqPbH3I~wIM$t zIg9aF$F`X~noeavWuW}Ifb5(-uwlkF3@wxfsk03R-(1?%;uG1U(3w8wt)jR+IYTSX z4dmEWkZA{*OT2DxVqFGw=|asolzP9)HDM4D*$kjX3q7WUiN4C$IB1U$N%MJ zw0~ax0RZT!|LJ8|Ea=F8d-;>kHoW%m&000NfyH0(C38;((aJ$0-22W${S0n9!hK{x zHskn^Fb{|dn#3KPx8-u}PE!(XbCph^jI-|wxm-IZLM--_v@MJu)xPUJJqI7JnzZhn zJT$@b02~clWcf#Q+yq~I?U7+DbOTw|C6SSOM;I*9L=t{yxC@r)!>}tdF&#xI>z_i^ z3YJvD(V@iwmzx3T^uGN2PM=u3YYV&I?$-v zVpG@mxX}tcNM(;il9-lfm+ErvOT#c|Z?3_Wfl}h&iCH>GXWZj@zH}Z;Az0j~c0I0N z*GuH!jM8B}n-1hsL_D*jkV>6$yXJV)usi!UE>AHp)7wA?68BGLgI}r@ohZS}x7fAi zqy90<;K0#3|588&n!{S7Qe{axR)pT{{$5v_g!#r}j8T zcv3$y(#{ABDqsSS$AwTVrHyq`UV{!Jp@(9?a>RR^gOGMGD0p8;D8Be=vy4YFBuMtB zFq9W(UP;(T^?d7|MxJzZUC$oLSIs}+@q5Ed(1iBx(RV2|?+0UnKJIn@$2*`3`qOd% zcpU&$3QU0q0NnhAME;X^D)fJNe?(~fFWxc!VciV?K*;!?ynpLL(EK;=aKYP@Y23S~ z=;FkBcMj9)cs{|&ftZfoM;R*g)$`$JPqK`5uh2CjFPqx%1qLr(g22CuvdHbo7zFib zsk$?#!5J^ja;IJieec$C@miK)5+sR1JBmXp8VzA8#;%&BNDYSx6OUer`BKsuTwo0u zi8Tcs7a=K%T;<;N^(V$f;&+Kfy054d-DPm})8V+ZqGz_j)x=(KV^aH@;h6JyWvQ^VGN>*FKu{6y!ofzwER?;OOl)UBS}z~ zhAT@ZAG}{?FG^2;+I^&f93+Nd_z;wf0mjLFpy1ZCJ_+HWE!RWSFYbT+7Vf(Y!o>8*z;$rx9+Tff)N zJYUO985Xc_Y@5YFd%@oEjmNNooYKNc%e>;Uij?o!xlO8Q(BSVc?9Lfuo2Rf6D=Y^m z{Cy6gh|55a{2K~oqZCbgGq@u+EP9E)FNo6nQL@Ush}FXVmr#>UqhiZdrtLj!s>8c~ zH65cc(w6HbCLjzI)*yxjq-{<5ujMD&H)Lm1{WIcBjpQw1V_KH1+NaJn7Z`HBM*5za zd8R{K1(?`ORgbOBp+aT{+XDzqGk0cshK}({bJ)^u$T6{bj)*!X!DP52y?D(?Kl`{1 z431RA1J8)5-?EW71r6xG1vcd9W#Pt8Z2mtVZUSB!Bc)0AHY0|xz| zU_Vh;mx3n?hpgA)gg3~=dKkE%gB$$+1%Gs`ZY$MVZ$ikneAiFCSVBqU&MULI`7^8 zOv=vs_OfQVR+u^dEa>7n-wHuh@~i#>frXpx{{O!lnz>s$M|w%6*H1O?FkQSfwx+4Q)|2z{&PGm^y5 zcGg(<;y|VXe2x68n}o_`Gu8p(&Y0SeKy-Y3pT%!}Otl6Mc_ppXOl!sOV@~y2=qi_& zOd}65M3}}DIPG=M&-qq(n8Lkv&si>qTOQ1Dru#O2K{d2%4`10)Hfc!pJzf5!7OJWv zj1TTx+ox|jdYBALuVi;^zV>Id_h{1#q@~*C-W9bni#wtrAKV1R2GvT`VI^=Wl0*in zM?T?i_~IJqo%LH%v#ULsG|+#G)AEs{LKjg~Rrwp2SJkD#eyUjEc|T4_RK|W@Hg=pa ziWG~^_gc;*Pcr`79RGScWGjK9@Y1-TAFO-&*KZUHD_Ac|%f12aek!R=$SqT%Q%gkK z#Ph5*xq-Col2NYF% zaEns*7=pG$uVGQ)XL??`My3Q8t+9^NRzupD2JrsnoG1m({F*E`G@TtftjcRTxV3!#UTSCfqu=s1z!1Kl$6fS> z+9DBQ+W5hJ3x$i>gQY!s2XN70pDgkNngbKXt@BGs2CqXvGi-gBk`pOItDyp(rw?f1 zog*!JiuESaxJR6CxC^SP8P#nNMJ>-xiB*waA|p#kEOIzBBPJp^U^qWqftVl!`ct39 zdhwvD-&zHw+r4wQ8Oc~qUYIkor%pvP)A^1u7E$JtKSwuC1We$%KdSRZ8|rWR3^;Oe z_U?cTyTiU1IZTqyq-3nLxbE)=rI;?l~ri(HR~I}2pi>({%5 zV=X`DwoH=z7p)ZY9M;)0Ut(k$y&y2X>4FqT{z0E(&hv{Wf(_4tV+IP5iS44suAs&9zO^g;9pS9dVUbuSoHmRA zn^RII5@6?n2DgE+m4}srfN852^j6$SEhF-~iA8rZbZO)8erYwAoZ_m{Y9ROa4EHqS zCe0sj6mcQ=<1Z60!Jjf;oh@sR^(<4I0zO$u=g3NOoVF4I;W|Zd=i0Z!(#0bgzn4Oc z@7%Np`pEqfor~iLpJjWCAVkO_$47l5jhd+#YM1Xl>7_#|>gb9K(NoO4Vr%renWLW< z#-@G08ccO%%PhWi4RFX@y1}X&QQ?OGBgA}ioF(W=(9*qhD^MMl(|}~-!T4V10&N&| znbkCr6tK~KisRU8iQ~aeJwUO>6}dT8!folUIgM`Zej$ZfL&DhY=Dk{-W|b%2!_2ir z(QXqhN-l7|q~+hH>%rYBIzqOLOZv-+YM498A<^u6BPlrrD_R%W9oXi}=kaf35jfjH zghrlp!F_`nFh; zMv3WmS$8o>jsg=-RCERV{DegXJ9o+cJe2-gkoQB$HU02*^D7 z?nwDp?C;w}?go~^m7FbJ)z4t6xdf=3N?Au0gc@k|38=%%hn9q`6Zro9TjQBDucLVR zp1Z9rm-FJCQG2Tiz39)2&PKykO`O8}78iv{Zft3h21D%HxWgylEASs>OOzicS*f0T z<oXLgG0WaU-A9qFn>ouXhNzzbN*PlCBt>&eRkW{Ag(}*(^5b`_pwC zwD4(g)kcd?ET5~Q`&%{z4Oi6FHUQ%lj(>tE!faGD`BOdn@OOl6CE{eM6p{9}LVGb+ zuY`&eVsCTiHSCJYc*-G$*>A(PH5SfSOcH<8sNazcLN6}ME?B?$lt*ObOq;m`Zcoy9 zG;E7`BgX0AeaWSD`gzyficrr>Soenp;NJrDpX6P*>B28*l1<%W26DEXwkdlu`VonBLEhV}+21HZxliWZ Xk@o`%T7`}3 literal 0 HcmV?d00001 diff --git a/imap_processing/tests/codice/data/imap_codice_l0_lo-pha_20240429_v001.pkts b/imap_processing/tests/codice/data/imap_codice_l0_lo-pha_20240429_v001.pkts new file mode 100644 index 0000000000000000000000000000000000000000..2dfc6c3741c2691275852cf54dc279c716ff6a63 GIT binary patch literal 10812 zcmZ{~V{m2B)&+V_Y}>Zk>7-+$JGO1xwrzK8t7F?%$37jOB%S2-ef56a`d-~1tM-q% zXU!U8jk#;DF&9#hFc|fPkOnso005x=e@`F)07(V_pqak&dTVL}08l5ZEs_ubav;vX zmt0woOZM?T(J1r)0Lz%Yb_~0;*yPaH+d~mQw3MLMcG4OS_r&khP5b8Zr9{&?^-#z= z3Kn&+ln@MJix(~l19s*|Hx)Ju_mS-4z4d8FoU+9uW9gUo@%0+8(_BTe$ZKkPu*N)L z*UoI4=!zE()UgKGP7OTbZ-MhWxcSpyD6uOkZnxpsc ze1H8c3zzr-q(A-H$zV*qg092Aq{ujgdc>Z6`w85eijis4)>P?=KHH{Ge*W0 z70yD-e;ewQugh{(>vw7`Gbn3Qc+KYE8G~lkDI=Aj;hV6@*66ACMu_p%W!TXCCwS$y(khbGi7WiySA`YG{XjWi+xx@cpT&g^9Irs!`_P&nbL(KxB%z)NyYNFp4gCMGkh}G~gaZRAi@()H$bH@kHv3e&ZJj z(}^XSOC@Wy&qGmPdtx@(OBaYvL630(&Po6VH|S|M`te(7NuPBBUzP+&Nt*+iDtQh0 zUnpfk84;ne!tk1ls7 zzfUx#zQ+pn^0z74R?5T#BC zIAL1&h@DI#nA_;8)z)l%Xvo6lztT^IQgK1t<&5lOr*nMd^Tx5kV~aV9Y>R2sZ$}0} zzHXs3L${fQ4Ax1P%RIbq&`)Wp;+EG5pV70e43r(9Y(eH~?qiJhM976)LB6QR1{)q` zph9Hv>@Lvj)xDyh#{Jw`m}vElv1lUTd)n#khU@JX zQ;&2rref_L0uFx@G?4syYO*w}#RAh@ut0DLkwb~}JY--LWM})l+q&>NID`zWI;)LU zR&?fLY@8n+vV$5B^A%*LK>jL$_?c%OThKyOPidd-_jUEVSs(N9Jx1K*LqkeNmR{Q& zeeFZpi;Hnzdh?<-875dnvX==VQMtub7D(^+=eV!IV-IvXV8(=i@v^{A`pdUFjt2GT znnHJ1c?bJ}p^Qv}x#{@JUuSe4vE>cqcX0nm*xAlU=8t$SXClNjT`U3bqbVeg^6FH9 zL|=p5yWq#za{~*OZ2qDF)YnYCSokg&UAZK9M$udl3LJMC-@WKl9v++*0rC1 zABP(mNm$aWQ-)Y=mUY9$N_Ow5xt5u%M=g4`{~Zk7cx*6!SheC6j?^VAvMa$S8v_Q? zXESNACwioRwrred(q^Dbj{;@_D!U(fdDWmq$febKS3wpORV(vwN&t!Zz4{drZ6WYu z$LXp1Sf0suxmfE3#=n)(i=Oq6Ijh@D=X+ma=PHXO($?{Q6TJ_I3;!kM9c<8etvTJM zP=!Z1dIep%SvA%l=fyuHNw}sYwNq9nF69E&$@&eu#lk< zj$?N<5cz~#F`0q7GOhJxpbVt8Gu~B$X9bbrCG9P9^3}oa5CuYg>7+-%$I|=c6M*vB7&8U;=51kni z&#f?wfwR&)jhs4FeNfDYk(CPgdvR#5``cqjpT-a7QfQcQJ^XQ{vXF%%HsLE~o_Sa;=A zpmms1uc&XdW5#Q0K%RhhF@+CMU+l~|#%dYUreLkgP= z01)R%&i*l8cp@u&zdPA6pFi#&@4;FR!C8ccsIVMEon5a{2--q56CJ<6IWuu9Y-@gA zN&6Gd6~)&SMDe{XH^Z@=IV-<(aCmO}yBTW;I&dxC&yLN+EWK1zzo9@~F3!2owiqrv zi<-&sF;ILpA*k^|>_&GH8wcpAF`a&mEFpYU8(QUP1m7}$Qjz*Y`*gNPz{{Z;J)WO5 z+F2$Lxu3$k9*M9PPP9@UJN5>)Qnlz0{`nzB*KActA(&RFEhKeY4%|2>3Kr{zD@wkI75egGPQU^$n|YzlySowebQ*aqWaj*blpt73v-+p|kkQt-qrYwJeku zLbt(ZiQz?=Y(Sl$&}@XBD&f!pGhO6dyC4U&ZE4w28JQ-+6?4N!?Dtq-9`cuCRk%WJAqi+SS>6I|R-R<4)#Ew4bmA}0b(%Uw-?x3VdOXPdEj zx^3SzX>52K-ETMV!huxN5NAT_y(L&F-tsb?u{)P8+1hrjpp~LCS8?-cpVT71Ds--n z;F;Tlxr+*VBwTG4%z`~dQpt7OH%UmA1BvWB0QpI6uH8mdJX|Xl1e5OizPnBfS*GP; zRzBV-8zP+SG)&y=V_m!RmZ#Kax57^&l&pZE?RvstczXJiCYy4j*}~~xv(W}EhCZ$J zyWB-6$YzVhZ*%R%s$lqL9%H4Snb1TrI0Z7Kh@!QI)8seg&A4WbhjwW{KZ6Q|eobDytqO;96+@;+ZSve+uoOK;H-<2 zErWp3o;Lii3=as_vH&c{SM7D73@`iqdlxM;luo?d$u_P6_DcL9mqZ>|mC{N)Ijx2%ZF zD|+&?`iRZ9o;=BxNDGtn0@KfAUq3okBsyz%53o66v`AW=)t2#O znKE`c9s={R@90yvH)Yf@bww#>lt%JJOZP9?C4Mrq8gjM0)PPo!@c0-`YPKx+@9a$N zjw*Ud`6r5g52u6Y*Wg>mw07uQkeI?;)KRj<4_$d4Rjjbr*!X}HO8o+DLmw<=Gxrs> zm7!vh$G&{b<~|gqPG+R?;=K05%2~%5kkq3)y$>Aox>pkUDBahC0yyIfrZWSk!i7pL zjjI|3MW$JwXFuf6*5;a^0gTgZKUUq;T8I+9B|}Pq>MrxbbBN}@ZDZwCCZfIF5ioo` zuqeB{BjcA@q4S0w<0Jw_X2UTvD|{KP<+M#Iu%Fu%&JMUJ;6h)j#m&0W{~Yp&!%n3h z=w{H`6^79VtBU;I!{BXjK?3LS8!lw7FwvWZeNi%%w1sF@8P-mz%IZ|wMHzL)r%`~Q zmN*j~u&ROEI46ZHpb=q1UAvAs$z5%r{xP%D{)%8bVPMmFL;@2FGopyI=A+%deL?2( zgbgx3L=~T8z=q@$p7_&+J>NlR3n?*J2~e>`9Fy_kC3N9qMwF~b>sFC5S=@>1G1H`* z8@>cqCT5nj`^3OS*^)5UW8bt0xa51KRgHEN_BNHyk>{Im5_yNfT%CI*j+CtOG=*Wa z(_F5ZN_9q5=}w~1{LzTllqnECppiGJI@1ITvV9FyG8{v-Y=u%2g2|<0ceYgd2sdL3 z3;WHbvX)ubxWj5gvSsTO3k4d0UwoDAq~du>R47$-(~cX|X8`5kMX%CqK-eIxVN~9&%61e;gH%U-CbQlCiUyCO{|v?Ci9hsZPv9Pml6*4XUF4I#Pe_P z{6{gVwH2AeZ>+8eDXKoNeRu8ThEf7B71`+{YA*&CP=S6n@eSE>)p@ z9a`3`K8-I?7=|aQ$Ufb z+tlz^Ypjxe*jZc5*nw$`p2FQ!msCbU{z*KIi%cc-i7Olq8lUm#I&hvfw1&d+`*ItD z|9~f1_uSn{`81fu{fH=pCZJS9cCs=o#3|9h6QhLjE zweICq9LV3&iic+^*mHQj-35vr)lhki_fe7QN8R20ql88#C?mEOr+IKqCyC+GILnqi z7FV(xxjFByP@@G0Sw4O#TGSj{V?>kqZr9;k5Gfjv1VbPU@0XF$5w+#2KG}sp19G-) zkuu_t-GokzYR$T!zgRj{?NqNIHbuyOK~N0&KpN@ib(njp|y4Z9keq9 zGwy$l(Zc=g8j3SD?@bO#S=isby0MeaOJ;Im8?P+-{yyn0Fs2zv$98^pgD07^^S2pcI#_~rlSM_~U8QUL&%bpO+jAScB* z{^v&+`@0jPwLzO+n)5Ia1EMZW?;<2iwmf+d8~i#Q;&aF>L?;5c2D;wH9=8ab*9NO< zc&eLvr@1f$uIFj6ynd>RMA7of{=G{RT0NX@B1IHC>^ceT!hx%(pv3RGHM&UgZ@PkQ z-3e*AeQapiAFRK5`AD)t59w;Rw)!n_*FvVV6dS99^Qy)~yTkcVXI(e^nLs-nlczua zgqXsU_ovuG+k#k{?juVNBGu8y|E>%RqV^6!9Io9(e7fjX^Fr{c@doSG%_O}`zcoYH zl?A@x>_$Qr5#dLMG_Jp+wjk@~1bd+42edZ5tmOT!r5&bB!XHW7BmI0iRR-t%vmU8G zcAu@d+X)n>k(#k|&yDJu`9x>o3z~#>9Cf!$W8VvXYi*Zo8N>g3|(TJ4o z#Jq0`YO_UqpsP}%C@F(7zL5uMP1xgksHQcv8Oe0yAGGkP&{v~I4wqG!SHq&KS%aQK zZgm~p81Wm#5U$0ftK#hAro$VTR5Lwld9=TUP(x+@RE#-1HV^MLk@Ys#l_`e|OuLTX zRvCb%0%J13I4!wHELfRT?M#9(H~NxE3kz1BR^b0e?_ zG#lUfmqB_f9dhY~47H)5vJ(l%DwDIC_#y=&_qKe^Enfv@vht(T3WRz!Hlb-bIgu7= zoX>cwd;Il$fqCw6ergU-n$2RAAkWzuu^~5Okgr)un1>Zn+J-YlC8&RT@$VUYBNIKz zP#)5o)0PhFSgM!q(dph1>n&ma0-TaojVssT@w;%Edz#7B0ea-z#Y5&OXe%c29YhNC z66JApo*DXg+@;n?{4=WIPog??sA6iiP5>qZQ`n3IXwEuOh=pn13$ANBn7Oo@2~oY+ z>2GBTH;zNXz}8n0S<9drwNaehStR-!3KN1}c!+I1vK@(vahkC~s0BfRa3{F;FW`8c6ES+Bqz!2Hv*54kIO#APsi6?ki3*H~T5Z68SCs_FtdK z{WRt}hrdFDgcC<*DyyX3yQ)^caMm2_DTyXq1t8xIZiPz6$(OJ5b7YVc(8|up38%G$ zq(|+kL=q@wB47k`h9X%)bR^fQ`Y9}xIxlZZ!wBBt+ypOu{rlYBUXQ;i+kShOWZT<0%ofIdI(zt6UM%kL_7B4!Rau-af-j$k>xSG?00iU+OU)Sv@vV++ ze=E{pS#z$}@mLfFDR`pT4%ELZdMhnotOU8P7J$bAXx&EDo4XgF-XrLjshVmJiT-{Z z3PWAGy5^IG=A8>OFY!Td9y6o`&+Fv){fLD^+C${M;8{4$VI^KoM;Z}Y7Nv?Du zk+wR{yX2ZuCX(7PGVr~_Fdxu4yCfdC5z+6ZGcg=T(-$Jm|KKXN66vdW*rfnwEWq1B zrpXsQT}jd1v z++##N8F(PuR*y?lew-FM18QeGZK=onT*G+eXD%YFW4i0jEQ$Qz3+D5 z%7qY5b)V*CxJ;z$gIJf2k)pz1n^6O9CE96mE&!68otX^eJ6zfVp{FqLJR(;u>&YVS zVp3BnO*(oNN+eAb^He1j&KhMSs1i4Ow9d!V!xX9mlSKWLr;ibxg z;s#v>BiSyWFIbH#xj5_mm7BkYScX1BAjOxA%1Y`lDubof@=gT%Hs~z#=fzDuR5nxv z4Hnk0@8(EasyY`$Pc-P>T4W2S%O33~Elcc!*kZ4wr-&r7V@|@HH)l)GzEhBkh2BSi z=j+HIA2TpXP;xSLQ=GGrz9j;v+h@=ms@}PA-2|8>mM#K>YKy@QZ_-NarCLKKcTHgJ zq9muZC^(k}ZX`yWVW)#^KEDcKn`YQBU%VG~VWZf{U)vw<12+)-hq6(`-6^^!?6Z_ zSxH4Zb$9ZiE%789vEI!J(hpNJoM8~tW_3YAa73X&bhH)aF@^>TquP|1rY7Ac$I|7O zE9GihTZS6#`(K0^%cp5N^%`FG(o}_a?gVkKw`QCaU2->#2z0xK_ zRrX(vyvsw=sE8&R8+?qVwMx$%yiJ^GC5 znkMR-BKu0aeF~ithJmro!T3~V7Y9#9i(h77?k?gp^ zSyTQt7cTqumpZQIJj}7}CvzL=LhIICQFazVTc7eb1aT!;n^!pbdneFk@(39_Nq><$ zu{t;Q#+rK|G1pZpw=H)NA6CQXp zo+t#qW?yo)Iw{E;^~`VhNaMfV#9Ag-Mp#$tbYLt+kcQSX@q#jfxj4y4>hlmPf!CY; zZ?&|9nCeX$%M43qC0QBI)(Dz$eH!H`$8bk&`Ea`Sz@{}Pcx-EAVs1G38lH>E{7%xv z6Z~qmEF47jZBxZS{Fc%YW`#6tbdKPStQ4IThJ`M3NVAc(6c*zA*x8HQ ziMV;0G#D2mNMou2fjJoA${}pz=|w-z#4|gQHtKw0C$U#DD8(#3{9BpburA z71JlysyB&U;J|AMZ~fu#Li#HUgk52#7m{8QtBk-2yW2fk$}ZvHBFU%kZc(W8rC&Mv zA}q_Azc&0&0pFot>+8>n-c%8EQ;tqplv`%lQ!?1{Pq0R|)Y{m&9D0e65j2@zsjc8f zc$7PMWzIwXbgJ&SBm>CehO+mOKd;~k3k-m|_dw~%?ZeK-G=g#ee1TG>PhDf$QdOxI^{6 z?UmTJ4)H4v)4-K52)mV}fHUsf@gGZ4xqY;f(}J;8yO-g_eHwj%YV+$kqZ6_~0;1WU zp}22lX`578aD%_4!}I%t8{-g-+eL3z{Lf2RTbs@oCb= zA;uNW^bRvic5l8YNOcQaFg;`^Vz{Wv_&s5dTt}5lNDUpt(P76At5)b67NFQ22phPP z4Jb49$(^UBR~4FJHjid(5+azJo4b6(k_-kw2|#tjVUX-gAE3AJb}brvVu=4TiRc392-XyfE36v*vYW2-IU%++b))&)!kL9p zq8s+X2tN!l++`HKmJDJavvY}T%q6WN8!8Olk%6;gjH_rW6}S&+{Or73OyV^O@6Y)H z5^aCFrX2u;KgRp@U_6D6oMbC`$A6tSF@fX z#r>Cg*`DyP^FpvaSDKsr{S~)Q38;S5Hx=#Dg!AhJ|`ZH$zt23PeV1k!qm`5(2nw!ZDoOv5m zxQcu$swk2Wm^^&i4sQU_0(r6YwNXSS#z_OPMN7kmvZ%_Ix%XV=oBT58`TlN{SZ2NK z!MB`|361ZYd0$Kzy?4Rd(nU=cF@I$JgKlLRoU&*`zm!hndx=icG}i{^vlfKV4ZV=# z^?q$3BRaB4NQ~M^;C61xCC)f*YCOcm2r$;3jx;N9oFj<=uosctms;X7a1zud@0@ZZ zI?5%iXCaWxc&`_}5tT_5>Xh^gWf}2>QnH3uqgB0*w>onIr#G>Yt7sMT|BmPH*V9CB>M4_Vbfs9xYz(#} zdvb`3%vZED;Se}@`AVst!ypd1j~g7%TL$4#jYH`6`>VSY6|^k}0`WB4o;DALe#@}M zhSIBjkMama)mN{gXJ#)^7k~xZ(W9$zVIp$;qI;HaLT2Ks97wFpK`OpF;vUDr*9)Ge zWgAioh;DCToNoDkDyqWBvx1P8?_4r)z(hI7-zPi)vz%@GRfFSJM)zu{l2iAxWVw-( znJRc%QO%A~(Nuufxd6pDdV{sx!FWK`N*)Tz$zV@rrf`ufO`SI&BruK&Qr`1~A9hWe zr6f5^`jg9^0%F8>!eZ|f(tD#%?0iU!HM*WBe$TL+d|(b(RoKSg#c@trE)~k8A8*RP z_mORLRWFOh649}^+EXSHQB$ZVQG^85z1R@W?=G8T{+LBD@z0NVfmHa9A0g8tzm;TO zEnaYR@I44h5kFD2)3=v-qtDkTVcr4i_f8>xd-l&o?L4)qWM@>%%Mg0w*#L%o(Uvgo zRyQ~FeduQUB)=-Iu5iSWS+nLpt+$K}K{SkBT>r#;4+b3IfQ8tqrWlEsH*ud+9USGPwdi zlT?ug(oc(|L3ZcDF)?L{NgQAtKV? zGvV=}3X{RJVUE8!;{H9e3-63UZXpz z4}9;NrbItHtwJZ2bSS3h53G?@2w40Pzp{p0Q_YKJ1#V75BFZO?>u_>~JoVbfz<;`Js%i zE55riZbvazGK|027KZ;mIZHHX&Dc*_QPDfD&kG(Pv+q^O4*LWXs#aBJNru9B5uQoc z6CT`(ZvF#`NNY?{@HNC80Newv|GV9^S`qO1-yCT|{Nv~iuwMM5?xelcYWuP<=tH;erE`njj0ilNjfJ00CRzf|L~0#V$%ILJ}tF> z`2Ipn{2zSy|FGEr01yrT6Cb1og#CXs@DU4KJ&et+9c?0^t<$H`A7=>r$3AG;h3BqiaUUJslqDKatpMhit=R6;e|R2=RAI)bIx{n`kb>?2W0IQ-0#QHk;i3Q)Q5s1$lQ39m0e_fN!tGxst?VU0~xlj!c_* zI4mn&co!_<~NxARgsA4gWQHeF(fLF*BlhcTBwRlF8j%E)tVq$>Ic1B}U%~7m8JyJLg$OfoVs*c-O6|~?jVbSG z5Von`QV>(6Y^l2$?6b#~asB(In^G=pa2YBP7qo*Snk3!%2<}v?pu>!%z%A5WunYa| z=NQEJYecTDIT%Fb<~45M8gz4Rh5<*dY%-?`%dY`(-c(a6!eNG$TY{EMz!0sz?)v7@ zUfnlD!|PD4zGQ_B!H@6#0zy^rw{Gtq4r+cC>f`c5tQ@Y)=|8eBcaIAs{`SAe;AgT1 z#4O)#!}C$c+oTbHk-*NpC!?5w{Wlf>)X-D#e0Hdx0f5oa>o5SoTF|}9f9u)U_n)5M z0apJ<&+z|JA^$DK|D|Uj02crd#(-E0yZSeb2!u}T2nGN&BH?};P6CqMD))@wO_KmX zHoSPknk&+_d%u6J^#9H!)O7yU`v4;UpL)>$qW|fkhyPV?A20@J{MSE*p$cb<=fl5y zN$O5+gEW4Q+_AiH4WlZB@ggZ>7CD@qx-S6_=L z`+kaOyFHG4Q;X3Q`l|1{DtAM?km31f(V8&m)|+Gu^nPs*Vesi&>v%{+XBpb8mtRbF zsR>)-YmEn@d-AHIiN1w7)JY25HuBHF(BS)yA!}_jiYTn&sm+PHaDt3U=-L}|J5*>8 r=Y?>kteN-Nt;F^p)Btv=-T2@?UTFZ}3~>A(5gWRmzyEs=B#ZtZS0ZDK literal 0 HcmV?d00001 diff --git a/imap_processing/tests/codice/test_codice_l1a.py b/imap_processing/tests/codice/test_codice_l1a.py index 8b381a409..a7464328f 100644 --- a/imap_processing/tests/codice/test_codice_l1a.py +++ b/imap_processing/tests/codice/test_codice_l1a.py @@ -18,6 +18,7 @@ (1, 128), # hi-counters-singles (1, 128), # hi-omni (1, 128), # hi-sectored + (1, 128), # hi-pha (1, 128), # lo-counters-aggregated (1, 128), # lo-counters-aggregated (1, 128), # lo-sw-angular @@ -26,6 +27,7 @@ (1, 128), # lo-nsw-priority (1, 128), # lo-sw-species (1, 128), # lo-nsw-species + (1, 128), # lo-pha ] EXPECTED_ARRAY_SIZES = [ 123, # hskp @@ -33,6 +35,7 @@ 3, # hi-counters-singles 8, # hi-omni 4, # hi-sectored + 0, # hi-pha 3, # lo-counters-aggregated 3, # lo-counters-singles 6, # lo-sw-angular @@ -41,6 +44,7 @@ 4, # lo-nsw-priority 18, # lo-sw-species 10, # lo-nsw-species + 0, # lo-pha ] EXPECTED_LOGICAL_SOURCE = [ "imap_codice_l1a_hskp", @@ -48,6 +52,7 @@ "imap_codice_l1a_hi-counters-singles", "imap_codice_l1a_hi-omni", "imap_codice_l1a_hi-sectored", + "imap_codice_l1a_hi-pha", "imap_codice_l1a_lo-counters-aggregated", "imap_codice_l1a_lo-counters-singles", "imap_codice_l1a_lo-sw-angular", @@ -56,6 +61,7 @@ "imap_codice_l1a_lo-nsw-priority", "imap_codice_l1a_lo-sw-species", "imap_codice_l1a_lo-nsw-species", + "imap_codice_l1a_lo-pha", ] TEST_PACKETS = [ Path( @@ -73,6 +79,9 @@ Path( f"{imap_module_directory}/tests/codice/data/imap_codice_l0_hi-sectored_20240429_v001.pkts" ), + Path( + f"{imap_module_directory}/tests/codice/data/imap_codice_l0_hi-pha_20240429_v001.pkts" + ), Path( f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-counters-aggregated_20240429_v001.pkts" ), @@ -97,6 +106,9 @@ Path( f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-nsw-species_20240429_v001.pkts" ), + Path( + f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-pha_20240429_v001.pkts" + ), ] # Placeholder for validation data files @@ -106,6 +118,7 @@ f"{imap_module_directory}/tests/codice/data/validation_hi-counters-singles.cdf", f"{imap_module_directory}/tests/codice/data/validation_hi-omni.cdf", f"{imap_module_directory}/tests/codice/data/validation_hi-sectored.cdf", + f"{imap_module_directory}/tests/codice/data/validation_hi-pha.cdf", f"{imap_module_directory}/tests/codice/data/validation_lo-counters-aggregated.cdf", f"{imap_module_directory}/tests/codice/data/validation_lo-counters-singles.cdf", f"{imap_module_directory}/tests/codice/data/validation_lo-sw-angular.cdf", @@ -114,6 +127,7 @@ f"{imap_module_directory}/tests/codice/data/validation_lo-nsw-priority.cdf", f"{imap_module_directory}/tests/codice/data/validation_lo-sw-species.cdf", f"{imap_module_directory}/tests/codice/data/validation_lo-nsw-species.cdf", + f"{imap_module_directory}/tests/codice/data/validation_lo-pha.cdf", ] From 3fe12ed029ea11bd2fa13fcefd092eb6f58f2cee Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Wed, 24 Jul 2024 09:57:24 -0600 Subject: [PATCH 07/22] Improved formatting for readability --- .../tests/codice/test_codice_l1a.py | 62 +++++-------------- 1 file changed, 16 insertions(+), 46 deletions(-) diff --git a/imap_processing/tests/codice/test_codice_l1a.py b/imap_processing/tests/codice/test_codice_l1a.py index a7464328f..bc9d87e05 100644 --- a/imap_processing/tests/codice/test_codice_l1a.py +++ b/imap_processing/tests/codice/test_codice_l1a.py @@ -64,52 +64,22 @@ "imap_codice_l1a_lo-pha", ] TEST_PACKETS = [ - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_hskp_20100101_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_hi-counters-aggregated_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_hi-counters-singles_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_hi-omni_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_hi-sectored_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_hi-pha_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-counters-aggregated_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-counters-singles_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-sw-angular_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-nsw-angular_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-sw-priority_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-nsw-priority_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-sw-species_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-nsw-species_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-pha_20240429_v001.pkts" - ), -] + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_hskp_20100101_v001.pkts"), + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_hi-counters-aggregated_20240429_v001.pkts"), + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_hi-counters-singles_20240429_v001.pkts"), + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_hi-omni_20240429_v001.pkts"), + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_hi-sectored_20240429_v001.pkts"), + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_hi-pha_20240429_v001.pkts"), + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-counters-aggregated_20240429_v001.pkts"), + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-counters-singles_20240429_v001.pkts"), + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-sw-angular_20240429_v001.pkts"), + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-nsw-angular_20240429_v001.pkts"), + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-sw-priority_20240429_v001.pkts"), + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-nsw-priority_20240429_v001.pkts"), + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-sw-species_20240429_v001.pkts"), + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-nsw-species_20240429_v001.pkts"), + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-pha_20240429_v001.pkts"), +] # fmt: skip # Placeholder for validation data files VALIDATION_DATA = [ From 42e347b9d758183fbdc3be44a34ffd43cd67839b Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Wed, 24 Jul 2024 10:13:47 -0600 Subject: [PATCH 08/22] Changes to support actual hi-sectored simulated data --- imap_processing/codice/codice_l1a.py | 32 +++++++++++++++--- .../P_COD_HI_SECT_SPECIES_COUNTS.xml | 8 ++--- ...p_codice_l0_hi-sectored_20240429_v001.pkts | Bin 0 -> 160 bytes 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/imap_processing/codice/codice_l1a.py b/imap_processing/codice/codice_l1a.py index 1f19f0a47..df8e4505d 100644 --- a/imap_processing/codice/codice_l1a.py +++ b/imap_processing/codice/codice_l1a.py @@ -464,6 +464,9 @@ def process_codice_l1a(file_path: Path, data_version: str) -> xr.Dataset: CODICEAPID.COD_LO_PHA, CODICEAPID.COD_HI_PHA, ] + apids_for_hi_science_processing = [ + CODICEAPID.COD_HI_SECT_SPECIES_COUNTS, + ] apids_for_lo_science_processing = [ CODICEAPID.COD_LO_INST_COUNTS_AGGREGATED, CODICEAPID.COD_LO_INST_COUNTS_SINGLES, @@ -476,7 +479,12 @@ def process_codice_l1a(file_path: Path, data_version: str) -> xr.Dataset: ] if file_path.name.startswith( - ("imap_codice_l0_lo", "imap_codice_l0_hskp", "imap_codice_l0_hi-pha") + ( + "imap_codice_l0_lo", + "imap_codice_l0_hskp", + "imap_codice_l0_hi-pha", + "imap_codice_l0_hi-sectored", + ) ): # Decom the packets, group data by APID, and sort by time packets = decom_packets(file_path) @@ -532,6 +540,25 @@ def process_codice_l1a(file_path: Path, data_version: str) -> xr.Dataset: met, event_data, dataset_name, data_version ) + elif apid in apids_for_hi_science_processing: + # Sort the packets by time + packets = sort_by_time(grouped_data[apid], "SHCOARSE") + + # Determine the start time of the packet + met = packets[0].data["ACQ_START_SECONDS"].raw_value + met = [met, met + 1] # TODO: Remove after SIT-3 + # Extract the data + science_values = packets[0].data["DATA"].raw_value + + # Get the four "main" parameters for processing + table_id, plan_id, plan_step, view_id = get_params(packets[0]) + + # Run the pipeline to create a dataset for the product + pipeline = CoDICEL1aPipeline(table_id, plan_id, plan_step, view_id) + pipeline.configure_data_products(apid) + pipeline.unpack_science_data(science_values) + dataset = pipeline.create_science_dataset(met, data_version) + # TODO: Temporary workaround in order to create hi data products in absence # of simulated data. This is essentially the same process as is for # lo, but don't try to decom any packets, just define the data @@ -546,9 +573,6 @@ def process_codice_l1a(file_path: Path, data_version: str) -> xr.Dataset: elif file_path.name.startswith("imap_codice_l0_hi-omni"): apid = CODICEAPID.COD_HI_OMNI_SPECIES_COUNTS table_id, plan_id, plan_step, view_id = (1, 0, 0, 5) - elif file_path.name.startswith("imap_codice_l0_hi-sectored"): - apid = CODICEAPID.COD_HI_SECT_SPECIES_COUNTS - table_id, plan_id, plan_step, view_id = (1, 0, 0, 6) met0 = (np.datetime64("2024-04-29T00:00") - IMAP_EPOCH).astype("timedelta64[s]") met0 = met0.astype(np.int64) diff --git a/imap_processing/codice/packet_definitions/P_COD_HI_SECT_SPECIES_COUNTS.xml b/imap_processing/codice/packet_definitions/P_COD_HI_SECT_SPECIES_COUNTS.xml index 397b3c841..176c12ff7 100644 --- a/imap_processing/codice/packet_definitions/P_COD_HI_SECT_SPECIES_COUNTS.xml +++ b/imap_processing/codice/packet_definitions/P_COD_HI_SECT_SPECIES_COUNTS.xml @@ -1,6 +1,6 @@ - + @@ -39,11 +39,11 @@ - + - 9043968 + 992 @@ -124,7 +124,7 @@ NUMBER OF BYTES IN THE DATA ARRAY. IF COMPRESSED, THIS VALUE REPRESENTS THE LENGTH OF THE COMPRESSED DATA. - + COUNTER DATA VARIABLE LENGTH; MAXIMUM (BASED ON UNCOLLAPSED, UNCOMPRESSED DATA, AND ASSUMING ALL X2 SPECEIS COUNTERS INCLUDED): diff --git a/imap_processing/tests/codice/data/imap_codice_l0_hi-sectored_20240429_v001.pkts b/imap_processing/tests/codice/data/imap_codice_l0_hi-sectored_20240429_v001.pkts index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..976d1725550f7c337e5cff3b6f06655d86757a32 100644 GIT binary patch literal 160 zcmd;?df+p|OlhH1X9kAt(h!;f1Q;0@*#3WFV5s?PUKJ6=z`*kC+7>q^21Q0Opk#^u z)5Q-2|1(&{GBD)-{Smt|bX{;y#jjf%7{gayPl?*l&U4mw>W01BwA)%Djx;B_h)+Fp z+N9&M@T%wQ{h1l=mCQVT#SCOgPQ Date: Wed, 24 Jul 2024 10:18:36 -0600 Subject: [PATCH 09/22] Changes to support actual hi-omni simulated data --- imap_processing/codice/codice_l1a.py | 8 +++++--- .../P_COD_HI_OMNI_SPECIES_COUNTS.xml | 8 ++++---- .../imap_codice_l0_hi-omni_20240429_v001.pkts | Bin 0 -> 208 bytes 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/imap_processing/codice/codice_l1a.py b/imap_processing/codice/codice_l1a.py index df8e4505d..b7fda4b0e 100644 --- a/imap_processing/codice/codice_l1a.py +++ b/imap_processing/codice/codice_l1a.py @@ -465,6 +465,7 @@ def process_codice_l1a(file_path: Path, data_version: str) -> xr.Dataset: CODICEAPID.COD_HI_PHA, ] apids_for_hi_science_processing = [ + CODICEAPID.COD_HI_OMNI_SPECIES_COUNTS, CODICEAPID.COD_HI_SECT_SPECIES_COUNTS, ] apids_for_lo_science_processing = [ @@ -484,6 +485,7 @@ def process_codice_l1a(file_path: Path, data_version: str) -> xr.Dataset: "imap_codice_l0_hskp", "imap_codice_l0_hi-pha", "imap_codice_l0_hi-sectored", + "imap_codice_l0_hi-omni", ) ): # Decom the packets, group data by APID, and sort by time @@ -570,9 +572,9 @@ def process_codice_l1a(file_path: Path, data_version: str) -> xr.Dataset: elif file_path.name.startswith("imap_codice_l0_hi-counters-singles"): apid = CODICEAPID.COD_HI_INST_COUNTS_SINGLES table_id, plan_id, plan_step, view_id = (1, 0, 0, 4) - elif file_path.name.startswith("imap_codice_l0_hi-omni"): - apid = CODICEAPID.COD_HI_OMNI_SPECIES_COUNTS - table_id, plan_id, plan_step, view_id = (1, 0, 0, 5) + # elif file_path.name.startswith("imap_codice_l0_hi-omni"): + # apid = CODICEAPID.COD_HI_OMNI_SPECIES_COUNTS + # table_id, plan_id, plan_step, view_id = (1, 0, 0, 5) met0 = (np.datetime64("2024-04-29T00:00") - IMAP_EPOCH).astype("timedelta64[s]") met0 = met0.astype(np.int64) diff --git a/imap_processing/codice/packet_definitions/P_COD_HI_OMNI_SPECIES_COUNTS.xml b/imap_processing/codice/packet_definitions/P_COD_HI_OMNI_SPECIES_COUNTS.xml index 92743c372..53bae3cde 100644 --- a/imap_processing/codice/packet_definitions/P_COD_HI_OMNI_SPECIES_COUNTS.xml +++ b/imap_processing/codice/packet_definitions/P_COD_HI_OMNI_SPECIES_COUNTS.xml @@ -1,6 +1,6 @@ - + @@ -39,11 +39,11 @@ - + - 28704768 + 1376 @@ -124,7 +124,7 @@ NUMBER OF BYTES IN THE DATA ARRAY. IF COMPRESSED, THIS VALUE REPRESENTS THE LENGTH OF THE COMPRESSED DATA. - + COUNTER DATA VARIABLE LENGTH; MAXIMUM (BASED ON UNCOLLAPSED, UNCOMPRESSED DATA, AND ASSUMING ALL 146 SQRT(2) SPECEIS COUNTERS INCLUDED): diff --git a/imap_processing/tests/codice/data/imap_codice_l0_hi-omni_20240429_v001.pkts b/imap_processing/tests/codice/data/imap_codice_l0_hi-omni_20240429_v001.pkts index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..bb1be93f68a8880e26d2600206c7582f0deedc56 100644 GIT binary patch literal 208 zcmd;?a^MrgNok=}X9kAt(h!;f1Q;0@SpR=wU|93lyecA!fq~`OwJmN;42q0mK*7^Zk!CeoR*%vZ~}f>R(#5Eh6r^*eu_~`i!Z Date: Wed, 24 Jul 2024 10:26:36 -0600 Subject: [PATCH 10/22] Changes to support actual hi-counters simulated data --- imap_processing/codice/codice_l1a.py | 48 +++++++++--------- .../P_COD_HI_INST_COUNTS_AGGREGATED.xml | 4 +- .../P_COD_HI_INST_COUNTS_SINGLES.xml | 4 +- ..._hi-counters-aggregated_20240429_v001.pkts | Bin 0 -> 100 bytes ..._l0_hi-counters-singles_20240429_v001.pkts | Bin 0 -> 132 bytes 5 files changed, 28 insertions(+), 28 deletions(-) diff --git a/imap_processing/codice/codice_l1a.py b/imap_processing/codice/codice_l1a.py index b7fda4b0e..15c832e06 100644 --- a/imap_processing/codice/codice_l1a.py +++ b/imap_processing/codice/codice_l1a.py @@ -24,7 +24,7 @@ from imap_processing import imap_module_directory from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes -from imap_processing.cdf.utils import IMAP_EPOCH, met_to_j2000ns +from imap_processing.cdf.utils import met_to_j2000ns from imap_processing.codice import constants from imap_processing.codice.codice_l0 import decom_packets from imap_processing.codice.utils import CODICEAPID, add_metadata_to_array @@ -465,6 +465,8 @@ def process_codice_l1a(file_path: Path, data_version: str) -> xr.Dataset: CODICEAPID.COD_HI_PHA, ] apids_for_hi_science_processing = [ + CODICEAPID.COD_HI_INST_COUNTS_AGGREGATED, + CODICEAPID.COD_HI_INST_COUNTS_SINGLES, CODICEAPID.COD_HI_OMNI_SPECIES_COUNTS, CODICEAPID.COD_HI_SECT_SPECIES_COUNTS, ] @@ -483,9 +485,7 @@ def process_codice_l1a(file_path: Path, data_version: str) -> xr.Dataset: ( "imap_codice_l0_lo", "imap_codice_l0_hskp", - "imap_codice_l0_hi-pha", - "imap_codice_l0_hi-sectored", - "imap_codice_l0_hi-omni", + "imap_codice_l0_hi", ) ): # Decom the packets, group data by APID, and sort by time @@ -565,26 +565,26 @@ def process_codice_l1a(file_path: Path, data_version: str) -> xr.Dataset: # of simulated data. This is essentially the same process as is for # lo, but don't try to decom any packets, just define the data # outright. - elif file_path.name.startswith("imap_codice_l0_hi"): - if file_path.name.startswith("imap_codice_l0_hi-counters-aggregated"): - apid = CODICEAPID.COD_HI_INST_COUNTS_AGGREGATED - table_id, plan_id, plan_step, view_id = (1, 0, 0, 3) - elif file_path.name.startswith("imap_codice_l0_hi-counters-singles"): - apid = CODICEAPID.COD_HI_INST_COUNTS_SINGLES - table_id, plan_id, plan_step, view_id = (1, 0, 0, 4) - # elif file_path.name.startswith("imap_codice_l0_hi-omni"): - # apid = CODICEAPID.COD_HI_OMNI_SPECIES_COUNTS - # table_id, plan_id, plan_step, view_id = (1, 0, 0, 5) - - met0 = (np.datetime64("2024-04-29T00:00") - IMAP_EPOCH).astype("timedelta64[s]") - met0 = met0.astype(np.int64) - met = [met0, met0 + 1] # Using this to match the other data products - science_values = "" # Currently don't have simulated data for this - - pipeline = CoDICEL1aPipeline(table_id, plan_id, plan_step, view_id) - pipeline.configure_data_products(apid) - pipeline.unpack_science_data(science_values) - dataset = pipeline.create_science_dataset(met, data_version) + # elif file_path.name.startswith("imap_codice_l0_hi"): + # if file_path.name.startswith("imap_codice_l0_hi-counters-aggregated"): + # apid = CODICEAPID.COD_HI_INST_COUNTS_AGGREGATED + # table_id, plan_id, plan_step, view_id = (1, 0, 0, 3) + # elif file_path.name.startswith("imap_codice_l0_hi-counters-singles"): + # apid = CODICEAPID.COD_HI_INST_COUNTS_SINGLES + # table_id, plan_id, plan_step, view_id = (1, 0, 0, 4) + # elif file_path.name.startswith("imap_codice_l0_hi-omni"): + # apid = CODICEAPID.COD_HI_OMNI_SPECIES_COUNTS + # table_id, plan_id, plan_step, view_id = (1, 0, 0, 5) + + # met0 = (np.datetime64("2024-04-29T00:00") - IMAP_EPOCH).astype("timedelta64[s]") + # met0 = met0.astype(np.int64) + # met = [met0, met0 + 1] # Using this to match the other data products + # science_values = "" # Currently don't have simulated data for this + # + # pipeline = CoDICEL1aPipeline(table_id, plan_id, plan_step, view_id) + # pipeline.configure_data_products(apid) + # pipeline.unpack_science_data(science_values) + # dataset = pipeline.create_science_dataset(met, data_version) logger.info(f"\nFinal data product:\n{dataset}\n") return dataset diff --git a/imap_processing/codice/packet_definitions/P_COD_HI_INST_COUNTS_AGGREGATED.xml b/imap_processing/codice/packet_definitions/P_COD_HI_INST_COUNTS_AGGREGATED.xml index 64bce2a39..4ac8bc9c1 100644 --- a/imap_processing/codice/packet_definitions/P_COD_HI_INST_COUNTS_AGGREGATED.xml +++ b/imap_processing/codice/packet_definitions/P_COD_HI_INST_COUNTS_AGGREGATED.xml @@ -1,6 +1,6 @@ - + @@ -43,7 +43,7 @@ - 4325376 + 512 diff --git a/imap_processing/codice/packet_definitions/P_COD_HI_INST_COUNTS_SINGLES.xml b/imap_processing/codice/packet_definitions/P_COD_HI_INST_COUNTS_SINGLES.xml index 80f0cec4d..1319d645f 100644 --- a/imap_processing/codice/packet_definitions/P_COD_HI_INST_COUNTS_SINGLES.xml +++ b/imap_processing/codice/packet_definitions/P_COD_HI_INST_COUNTS_SINGLES.xml @@ -1,6 +1,6 @@ - + @@ -43,7 +43,7 @@ - 4325376 + 768 diff --git a/imap_processing/tests/codice/data/imap_codice_l0_hi-counters-aggregated_20240429_v001.pkts b/imap_processing/tests/codice/data/imap_codice_l0_hi-counters-aggregated_20240429_v001.pkts index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..53d675033c55dfd784b66ab2b758124a1063e72d 100644 GIT binary patch literal 100 zcmd;?bl?L+th7+7GXukRX$Z{#0*nj{%>O?%FgW}*uZoCbU|@N6ZHpTdgCe6CP_jh- t>0(9(*1!M%{0A$}FMpzA(x$bXfl-d_p#BnB`E8$Tfd&9|MMkYy2LNvt9RmOW literal 0 HcmV?d00001 diff --git a/imap_processing/tests/codice/data/imap_codice_l0_hi-counters-singles_20240429_v001.pkts b/imap_processing/tests/codice/data/imap_codice_l0_hi-counters-singles_20240429_v001.pkts index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..b7a783fed4cb7c22e458d7019dc6d37a2f43197f 100644 GIT binary patch literal 132 zcmd;?eBdKPt+Y_8GXukRX$Z{#0*nj{EdM_>FeLmnuZoBQa-Ln=;>N_F$S4MsEYW|u z_yL1HgGwwzseSYbmZi}hVb%Ni{(i{)!1 Date: Wed, 24 Jul 2024 10:49:21 -0600 Subject: [PATCH 11/22] Simplified the code for event/hskp/science processing --- imap_processing/codice/codice_l1a.py | 185 ++++++++------------------- imap_processing/codice/constants.py | 16 +++ 2 files changed, 70 insertions(+), 131 deletions(-) diff --git a/imap_processing/codice/codice_l1a.py b/imap_processing/codice/codice_l1a.py index 15c832e06..171497b5c 100644 --- a/imap_processing/codice/codice_l1a.py +++ b/imap_processing/codice/codice_l1a.py @@ -34,9 +34,8 @@ logger.setLevel(logging.INFO) # TODO: Decom data arrays need to be decompressed -# TODO: Add metadata attrs to science dataset? # TODO: In decommutation, how to have a variable length data and then a checksum -# after it? +# after it? (Might be fixed with new XTCE script updates) # TODO: Add support for decomming multiple APIDs from a single file @@ -173,6 +172,8 @@ def create_science_dataset(self, met: np.int64, data_version: str) -> xr.Dataset # Add ESA Sweep Values and acquisition times (lo only) if "_lo_" in self.dataset_name: + self.get_esa_sweep_values() + self.get_acquisition_times() dataset["esa_sweep_values"] = xr.DataArray( self.esa_sweep_values, dims=["energy"], @@ -456,135 +457,57 @@ def process_codice_l1a(file_path: Path, data_version: str) -> xr.Dataset: dataset : xarray.Dataset The ``xarray`` dataset containing the science data and supporting metadata. """ - # TODO: Once simulated data for codice-hi is acquired, there shouldn't be a - # need to split the processing based on the file_path, so this function can - # be simplified. - - apids_for_event_data_processing = [ - CODICEAPID.COD_LO_PHA, - CODICEAPID.COD_HI_PHA, - ] - apids_for_hi_science_processing = [ - CODICEAPID.COD_HI_INST_COUNTS_AGGREGATED, - CODICEAPID.COD_HI_INST_COUNTS_SINGLES, - CODICEAPID.COD_HI_OMNI_SPECIES_COUNTS, - CODICEAPID.COD_HI_SECT_SPECIES_COUNTS, - ] - apids_for_lo_science_processing = [ - CODICEAPID.COD_LO_INST_COUNTS_AGGREGATED, - CODICEAPID.COD_LO_INST_COUNTS_SINGLES, - CODICEAPID.COD_LO_SW_ANGULAR_COUNTS, - CODICEAPID.COD_LO_NSW_ANGULAR_COUNTS, - CODICEAPID.COD_LO_SW_PRIORITY_COUNTS, - CODICEAPID.COD_LO_NSW_PRIORITY_COUNTS, - CODICEAPID.COD_LO_SW_SPECIES_COUNTS, - CODICEAPID.COD_LO_NSW_SPECIES_COUNTS, - ] - - if file_path.name.startswith( - ( - "imap_codice_l0_lo", - "imap_codice_l0_hskp", - "imap_codice_l0_hi", - ) - ): - # Decom the packets, group data by APID, and sort by time - packets = decom_packets(file_path) - grouped_data = group_by_apid(packets) - - for apid in grouped_data: - logger.info(f"\nProcessing {CODICEAPID(apid).name} packet") - - if apid == CODICEAPID.COD_NHK: - packets = grouped_data[apid] - sorted_packets = sort_by_time(packets, "SHCOARSE") - dataset = create_hskp_dataset(sorted_packets, data_version) - - elif apid in apids_for_lo_science_processing: - # Sort the packets by time - packets = sort_by_time(grouped_data[apid], "SHCOARSE") - - # Determine the start time of the packet - met = packets[0].data["ACQ_START_SECONDS"].raw_value - met = [met, met + 1] # TODO: Remove after SIT-3 - # Extract the data - science_values = packets[0].data["DATA"].raw_value - - # Get the four "main" parameters for processing - table_id, plan_id, plan_step, view_id = get_params(packets[0]) - - # Run the pipeline to create a dataset for the product - pipeline = CoDICEL1aPipeline(table_id, plan_id, plan_step, view_id) - pipeline.get_esa_sweep_values() - pipeline.get_acquisition_times() - pipeline.configure_data_products(apid) - pipeline.unpack_science_data(science_values) - dataset = pipeline.create_science_dataset(met, data_version) - - elif apid in apids_for_event_data_processing: - if apid == CODICEAPID.COD_LO_PHA: - dataset_name = "imap_codice_l1a_lo_pha" - elif apid == CODICEAPID.COD_HI_PHA: - dataset_name = "imap_codice_l1a_hi_pha" - - # Sort the packets by time - packets = sort_by_time(grouped_data[apid], "SHCOARSE") - - # Determine the start time of the packet - met = packets[0].data["ACQ_START_SECONDS"].raw_value - met = [met, met + 1] # TODO: Remove after SIT-3 - - # Extract the data - event_data = packets[0].data["EVENT_DATA"].raw_value - - # Create the dataset - dataset = create_event_dataset( - met, event_data, dataset_name, data_version - ) - - elif apid in apids_for_hi_science_processing: - # Sort the packets by time - packets = sort_by_time(grouped_data[apid], "SHCOARSE") - - # Determine the start time of the packet - met = packets[0].data["ACQ_START_SECONDS"].raw_value - met = [met, met + 1] # TODO: Remove after SIT-3 - # Extract the data - science_values = packets[0].data["DATA"].raw_value - - # Get the four "main" parameters for processing - table_id, plan_id, plan_step, view_id = get_params(packets[0]) - - # Run the pipeline to create a dataset for the product - pipeline = CoDICEL1aPipeline(table_id, plan_id, plan_step, view_id) - pipeline.configure_data_products(apid) - pipeline.unpack_science_data(science_values) - dataset = pipeline.create_science_dataset(met, data_version) - - # TODO: Temporary workaround in order to create hi data products in absence - # of simulated data. This is essentially the same process as is for - # lo, but don't try to decom any packets, just define the data - # outright. - # elif file_path.name.startswith("imap_codice_l0_hi"): - # if file_path.name.startswith("imap_codice_l0_hi-counters-aggregated"): - # apid = CODICEAPID.COD_HI_INST_COUNTS_AGGREGATED - # table_id, plan_id, plan_step, view_id = (1, 0, 0, 3) - # elif file_path.name.startswith("imap_codice_l0_hi-counters-singles"): - # apid = CODICEAPID.COD_HI_INST_COUNTS_SINGLES - # table_id, plan_id, plan_step, view_id = (1, 0, 0, 4) - # elif file_path.name.startswith("imap_codice_l0_hi-omni"): - # apid = CODICEAPID.COD_HI_OMNI_SPECIES_COUNTS - # table_id, plan_id, plan_step, view_id = (1, 0, 0, 5) - - # met0 = (np.datetime64("2024-04-29T00:00") - IMAP_EPOCH).astype("timedelta64[s]") - # met0 = met0.astype(np.int64) - # met = [met0, met0 + 1] # Using this to match the other data products - # science_values = "" # Currently don't have simulated data for this - # - # pipeline = CoDICEL1aPipeline(table_id, plan_id, plan_step, view_id) - # pipeline.configure_data_products(apid) - # pipeline.unpack_science_data(science_values) - # dataset = pipeline.create_science_dataset(met, data_version) + # TODO: Use new packet_file_to_dataset() function to simplify things + + # Decom the packets, group data by APID, and sort by time + packets = decom_packets(file_path) + grouped_data = group_by_apid(packets) + + for apid in grouped_data: + logger.info(f"\nProcessing {CODICEAPID(apid).name} packet") + + if apid == CODICEAPID.COD_NHK: + packets = grouped_data[apid] + sorted_packets = sort_by_time(packets, "SHCOARSE") + dataset = create_hskp_dataset(sorted_packets, data_version) + + elif apid in [CODICEAPID.COD_LO_PHA, CODICEAPID.COD_HI_PHA]: + if apid == CODICEAPID.COD_LO_PHA: + dataset_name = "imap_codice_l1a_lo_pha" + elif apid == CODICEAPID.COD_HI_PHA: + dataset_name = "imap_codice_l1a_hi_pha" + + # Sort the packets by time + packets = sort_by_time(grouped_data[apid], "SHCOARSE") + + # Determine the start time of the packet + met = packets[0].data["ACQ_START_SECONDS"].raw_value + met = [met, met + 1] # TODO: Remove after cdflib fix + + # Extract the data + event_data = packets[0].data["EVENT_DATA"].raw_value + + # Create the dataset + dataset = create_event_dataset(met, event_data, dataset_name, data_version) + + elif apid in constants.APIDS_FOR_SCIENCE_PROCESSING: + # Sort the packets by time + packets = sort_by_time(grouped_data[apid], "SHCOARSE") + + # Determine the start time of the packet + met = packets[0].data["ACQ_START_SECONDS"].raw_value + met = [met, met + 1] # TODO: Remove after cdflib fix + # Extract the data + science_values = packets[0].data["DATA"].raw_value + + # Get the four "main" parameters for processing + table_id, plan_id, plan_step, view_id = get_params(packets[0]) + + # Run the pipeline to create a dataset for the product + pipeline = CoDICEL1aPipeline(table_id, plan_id, plan_step, view_id) + pipeline.configure_data_products(apid) + pipeline.unpack_science_data(science_values) + dataset = pipeline.create_science_dataset(met, data_version) logger.info(f"\nFinal data product:\n{dataset}\n") return dataset diff --git a/imap_processing/codice/constants.py b/imap_processing/codice/constants.py index f104b24f9..b41fa55a0 100644 --- a/imap_processing/codice/constants.py +++ b/imap_processing/codice/constants.py @@ -14,6 +14,22 @@ from imap_processing.codice.utils import CODICEAPID, CoDICECompression +APIDS_FOR_SCIENCE_PROCESSING = [ + CODICEAPID.COD_HI_INST_COUNTS_AGGREGATED, + CODICEAPID.COD_HI_INST_COUNTS_SINGLES, + CODICEAPID.COD_HI_OMNI_SPECIES_COUNTS, + CODICEAPID.COD_HI_SECT_SPECIES_COUNTS, + CODICEAPID.COD_LO_INST_COUNTS_AGGREGATED, + CODICEAPID.COD_LO_INST_COUNTS_SINGLES, + CODICEAPID.COD_LO_SW_ANGULAR_COUNTS, + CODICEAPID.COD_LO_NSW_ANGULAR_COUNTS, + CODICEAPID.COD_LO_SW_PRIORITY_COUNTS, + CODICEAPID.COD_LO_NSW_PRIORITY_COUNTS, + CODICEAPID.COD_LO_SW_SPECIES_COUNTS, + CODICEAPID.COD_LO_NSW_SPECIES_COUNTS, +] + + # CDF-friendly names for lo data products LO_INST_COUNTS_AGGREGATED_NAMES = ["aggregated"] LO_INST_COUNTS_SINGLES_NAMES = ["apd_singles"] From ec810a0b4d21d9627c50a6e231760174df3cf5e4 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Wed, 24 Jul 2024 10:51:55 -0600 Subject: [PATCH 12/22] Improved readability --- .../tests/codice/test_codice_l1b.py | 54 +++++-------------- 1 file changed, 14 insertions(+), 40 deletions(-) diff --git a/imap_processing/tests/codice/test_codice_l1b.py b/imap_processing/tests/codice/test_codice_l1b.py index 7b36ec251..8f71040b9 100644 --- a/imap_processing/tests/codice/test_codice_l1b.py +++ b/imap_processing/tests/codice/test_codice_l1b.py @@ -25,46 +25,20 @@ "imap_codice_l1b_lo-nsw-species", ] TEST_FILES = [ - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_hskp_20100101_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_hi-counters-aggregated_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_hi-counters-singles_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_hi-omni_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_hi-sectored_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-counters-aggregated_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-counters-singles_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-sw-angular_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-nsw-angular_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-sw-priority_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-nsw-priority_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-sw-species_20240429_v001.pkts" - ), - Path( - f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-nsw-species_20240429_v001.pkts" - ), -] + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_hskp_20100101_v001.pkts"), + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_hi-counters-aggregated_20240429_v001.pkts"), + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_hi-counters-singles_20240429_v001.pkts"), + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_hi-omni_20240429_v001.pkts"), + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_hi-sectored_20240429_v001.pkts"), + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-counters-aggregated_20240429_v001.pkts"), + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-counters-singles_20240429_v001.pkts"), + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-sw-angular_20240429_v001.pkts"), + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-nsw-angular_20240429_v001.pkts"), + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-sw-priority_20240429_v001.pkts"), + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-nsw-priority_20240429_v001.pkts"), + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-sw-species_20240429_v001.pkts"), + Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-nsw-species_20240429_v001.pkts"), +] # fmt: skip @pytest.fixture(params=TEST_FILES) From cbe0b0a4de2b3746c5d3d6d1c851a7468a41d3bf Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Wed, 24 Jul 2024 11:27:37 -0600 Subject: [PATCH 13/22] Added placeholder for HI_PHA_NAMES --- imap_processing/codice/constants.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/imap_processing/codice/constants.py b/imap_processing/codice/constants.py index b41fa55a0..2f0e9d716 100644 --- a/imap_processing/codice/constants.py +++ b/imap_processing/codice/constants.py @@ -66,7 +66,7 @@ "cnoplus", ] LO_PHA_NAMES = [ - "foo", + "TBD", ] # CDF-friendly names for hi data products @@ -74,6 +74,9 @@ HI_INST_COUNTS_SINGLES_NAMES = ["tcr", "ssdo", "stssd"] HI_OMNI_SPECIES_NAMES = ["h", "he3", "he4", "c", "o", "ne_mg_si", "fe", "uh"] HI_SECT_SPECIES_NAMES = ["h", "he3he4", "cno", "fe"] +HI_PHA_NAMES = [ + "TBD", +] # TODO: Hi products shape should be energy x ssd index x spin sector (8*12*12) DATA_PRODUCT_CONFIGURATIONS = { From 3811061b9a765b2edf31feaab79d79099f31f6f8 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Wed, 24 Jul 2024 11:30:10 -0600 Subject: [PATCH 14/22] Removed PHA variable name placeholders --- imap_processing/codice/constants.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/imap_processing/codice/constants.py b/imap_processing/codice/constants.py index 2f0e9d716..a8bcf3503 100644 --- a/imap_processing/codice/constants.py +++ b/imap_processing/codice/constants.py @@ -65,18 +65,12 @@ "heplus", "cnoplus", ] -LO_PHA_NAMES = [ - "TBD", -] # CDF-friendly names for hi data products HI_INST_COUNTS_AGGREGATED_NAMES = ["aggregated"] HI_INST_COUNTS_SINGLES_NAMES = ["tcr", "ssdo", "stssd"] HI_OMNI_SPECIES_NAMES = ["h", "he3", "he4", "c", "o", "ne_mg_si", "fe", "uh"] HI_SECT_SPECIES_NAMES = ["h", "he3he4", "cno", "fe"] -HI_PHA_NAMES = [ - "TBD", -] # TODO: Hi products shape should be energy x ssd index x spin sector (8*12*12) DATA_PRODUCT_CONFIGURATIONS = { @@ -152,12 +146,6 @@ "variable_names": LO_NSW_SPECIES_NAMES, "dataset_name": "imap_codice_l1a_lo_nsw_species", }, - CODICEAPID.COD_LO_PHA: { - "num_counters": 1, - "num_energy_steps": 128, - "variable_names": LO_PHA_NAMES, - "dataset_name": "imap_codice_l1a_lo_pha", - }, } # Compression ID lookup table for Lo data products From 89d3bbda39197d61d703b63ad05164bd16e6aac0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 26 Jul 2024 20:18:16 +0000 Subject: [PATCH 15/22] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- imap_processing/codice/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/imap_processing/codice/utils.py b/imap_processing/codice/utils.py index 22780f390..934656146 100644 --- a/imap_processing/codice/utils.py +++ b/imap_processing/codice/utils.py @@ -9,6 +9,7 @@ import space_packet_parser + class CODICEAPID(IntEnum): """Create ENUM for CoDICE APIDs.""" From 18962a209e04b2e8b08ddefa62f46a1d788c7359 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Fri, 26 Jul 2024 14:46:42 -0600 Subject: [PATCH 16/22] Moved hard-coded test file paths to conftest --- imap_processing/tests/codice/conftest.py | 56 +++++++++++++++++++ .../tests/codice/test_codice_l1a.py | 39 +------------ .../tests/codice/test_codice_l1b.py | 24 ++------ 3 files changed, 62 insertions(+), 57 deletions(-) create mode 100644 imap_processing/tests/codice/conftest.py diff --git a/imap_processing/tests/codice/conftest.py b/imap_processing/tests/codice/conftest.py new file mode 100644 index 000000000..66f36a697 --- /dev/null +++ b/imap_processing/tests/codice/conftest.py @@ -0,0 +1,56 @@ +from imap_processing import imap_module_directory + +TEST_DATA_PATH = imap_module_directory / "tests" / "codice" / "data" + +TEST_PACKETS = [ + TEST_DATA_PATH / "imap_codice_l0_hskp_20100101_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_hi-counters-aggregated_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_hi-counters-singles_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_hi-omni_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_hi-sectored_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_hi-pha_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_lo-counters-aggregated_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_lo-counters-singles_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_lo-sw-angular_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_lo-nsw-angular_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_lo-sw-priority_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_lo-nsw-priority_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_lo-sw-species_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_lo-nsw-species_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l0_lo-pha_20240429_v001.pkts", +] + +TEST_L1A_FILES = [ + TEST_DATA_PATH / "imap_codice_l1a_hskp_20100101_v001.pkts", + TEST_DATA_PATH / "imap_codice_l1a_hi-counters-aggregated_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l1a_hi-counters-singles_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l1a_hi-omni_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l1a_hi-sectored_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l1a_lo-counters-aggregated_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l1a_lo-counters-singles_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l1a_lo-sw-angular_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l1a_lo-nsw-angular_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l1a_lo-sw-priority_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l1a_lo-nsw-priority_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l1a_lo-sw-species_20240429_v001.pkts", + TEST_DATA_PATH / "imap_codice_l1a_lo-nsw-species_20240429_v001.pkts", +] + +# Placeholder for validation data files +VALIDATION_DATA = [ + TEST_DATA_PATH / "validation_hskp.cdf", + TEST_DATA_PATH / "validation_hi-counters-aggregated.cdf", + TEST_DATA_PATH / "validation_hi-counters-singles.cdf", + TEST_DATA_PATH / "validation_hi-omni.cdf", + TEST_DATA_PATH / "validation_hi-sectored.cdf", + TEST_DATA_PATH / "validation_hi-pha.cdf", + TEST_DATA_PATH / "validation_lo-counters-aggregated.cdf", + TEST_DATA_PATH / "validation_lo-counters-singles.cdf", + TEST_DATA_PATH / "validation_lo-sw-angular.cdf", + TEST_DATA_PATH / "validation_lo-nsw-angular.cdf", + TEST_DATA_PATH / "validation_lo-sw-priority.cdf", + TEST_DATA_PATH / "validation_lo-nsw-priority.cdf", + TEST_DATA_PATH / "validation_lo-sw-species.cdf", + TEST_DATA_PATH / "validation_lo-nsw-species.cdf", + TEST_DATA_PATH / "validation_lo-pha.cdf", +] diff --git a/imap_processing/tests/codice/test_codice_l1a.py b/imap_processing/tests/codice/test_codice_l1a.py index bc9d87e05..3ba38bf5a 100644 --- a/imap_processing/tests/codice/test_codice_l1a.py +++ b/imap_processing/tests/codice/test_codice_l1a.py @@ -6,10 +6,11 @@ import pytest import xarray as xr -from imap_processing import imap_module_directory from imap_processing.cdf.utils import load_cdf, write_cdf from imap_processing.codice.codice_l1a import process_codice_l1a +from .conftest import TEST_PACKETS, VALIDATION_DATA + # TODO: Add test that processes a file with multiple APIDs EXPECTED_ARRAY_SHAPES = [ @@ -63,42 +64,6 @@ "imap_codice_l1a_lo-nsw-species", "imap_codice_l1a_lo-pha", ] -TEST_PACKETS = [ - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_hskp_20100101_v001.pkts"), - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_hi-counters-aggregated_20240429_v001.pkts"), - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_hi-counters-singles_20240429_v001.pkts"), - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_hi-omni_20240429_v001.pkts"), - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_hi-sectored_20240429_v001.pkts"), - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_hi-pha_20240429_v001.pkts"), - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-counters-aggregated_20240429_v001.pkts"), - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-counters-singles_20240429_v001.pkts"), - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-sw-angular_20240429_v001.pkts"), - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-nsw-angular_20240429_v001.pkts"), - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-sw-priority_20240429_v001.pkts"), - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-nsw-priority_20240429_v001.pkts"), - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-sw-species_20240429_v001.pkts"), - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-nsw-species_20240429_v001.pkts"), - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l0_lo-pha_20240429_v001.pkts"), -] # fmt: skip - -# Placeholder for validation data files -VALIDATION_DATA = [ - f"{imap_module_directory}/tests/codice/data/validation_hskp.cdf", - f"{imap_module_directory}/tests/codice/data/validation_hi-counters-aggregated.cdf", - f"{imap_module_directory}/tests/codice/data/validation_hi-counters-singles.cdf", - f"{imap_module_directory}/tests/codice/data/validation_hi-omni.cdf", - f"{imap_module_directory}/tests/codice/data/validation_hi-sectored.cdf", - f"{imap_module_directory}/tests/codice/data/validation_hi-pha.cdf", - f"{imap_module_directory}/tests/codice/data/validation_lo-counters-aggregated.cdf", - f"{imap_module_directory}/tests/codice/data/validation_lo-counters-singles.cdf", - f"{imap_module_directory}/tests/codice/data/validation_lo-sw-angular.cdf", - f"{imap_module_directory}/tests/codice/data/validation_lo-nsw-angular.cdf", - f"{imap_module_directory}/tests/codice/data/validation_lo-sw-priority.cdf", - f"{imap_module_directory}/tests/codice/data/validation_lo-nsw-priority.cdf", - f"{imap_module_directory}/tests/codice/data/validation_lo-sw-species.cdf", - f"{imap_module_directory}/tests/codice/data/validation_lo-nsw-species.cdf", - f"{imap_module_directory}/tests/codice/data/validation_lo-pha.cdf", -] @pytest.fixture(params=TEST_PACKETS) diff --git a/imap_processing/tests/codice/test_codice_l1b.py b/imap_processing/tests/codice/test_codice_l1b.py index 8f71040b9..685f88dde 100644 --- a/imap_processing/tests/codice/test_codice_l1b.py +++ b/imap_processing/tests/codice/test_codice_l1b.py @@ -1,14 +1,13 @@ """Tests the L1b processing for CoDICE L1a data""" -from pathlib import Path - import pytest import xarray as xr -from imap_processing import imap_module_directory from imap_processing.cdf.utils import load_cdf from imap_processing.codice.codice_l1b import process_codice_l1b +from .conftest import TEST_L1A_FILES + EXPECTED_LOGICAL_SOURCE = [ "imap_codice_l1b_hskp", "imap_codice_l1b_hi-counters-aggregated", @@ -24,24 +23,9 @@ "imap_codice_l1b_lo-sw-species", "imap_codice_l1b_lo-nsw-species", ] -TEST_FILES = [ - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_hskp_20100101_v001.pkts"), - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_hi-counters-aggregated_20240429_v001.pkts"), - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_hi-counters-singles_20240429_v001.pkts"), - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_hi-omni_20240429_v001.pkts"), - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_hi-sectored_20240429_v001.pkts"), - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-counters-aggregated_20240429_v001.pkts"), - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-counters-singles_20240429_v001.pkts"), - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-sw-angular_20240429_v001.pkts"), - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-nsw-angular_20240429_v001.pkts"), - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-sw-priority_20240429_v001.pkts"), - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-nsw-priority_20240429_v001.pkts"), - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-sw-species_20240429_v001.pkts"), - Path(f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-nsw-species_20240429_v001.pkts"), -] # fmt: skip -@pytest.fixture(params=TEST_FILES) +@pytest.fixture(params=TEST_L1A_FILES) def test_l1b_data(request) -> xr.Dataset: """Return a ``xarray`` dataset containing test data. @@ -57,7 +41,7 @@ def test_l1b_data(request) -> xr.Dataset: @pytest.mark.parametrize( "test_l1b_data, expected_logical_source", - list(zip(TEST_FILES, EXPECTED_LOGICAL_SOURCE)), + list(zip(TEST_L1A_FILES, EXPECTED_LOGICAL_SOURCE)), indirect=["test_l1b_data"], ) def test_l1b_cdf_filenames(test_l1b_data: xr.Dataset, expected_logical_source: str): From 33d9f3a5efbcbc7022eefeece7e2e4ac53024975 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Fri, 26 Jul 2024 14:52:41 -0600 Subject: [PATCH 17/22] Removed print statement in favor of logging --- imap_processing/tests/codice/test_codice_l1a.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/imap_processing/tests/codice/test_codice_l1a.py b/imap_processing/tests/codice/test_codice_l1a.py index 3ba38bf5a..383a07b9b 100644 --- a/imap_processing/tests/codice/test_codice_l1a.py +++ b/imap_processing/tests/codice/test_codice_l1a.py @@ -1,5 +1,6 @@ """Tests the L1a processing for decommutated CoDICE data""" +import logging from pathlib import Path import numpy as np @@ -11,6 +12,9 @@ from .conftest import TEST_PACKETS, VALIDATION_DATA +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + # TODO: Add test that processes a file with multiple APIDs EXPECTED_ARRAY_SHAPES = [ @@ -80,7 +84,7 @@ def test_l1a_data(request) -> xr.Dataset: # Write the dataset to a CDF so it can be manually inspected as well file_path = write_cdf(dataset) - print(f"CDF file written to {file_path}") + logger.info(f"CDF file written to {file_path}") return dataset From 1788befeb754f9199232b9465656c7393750b143 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Wed, 14 Aug 2024 15:45:46 -0600 Subject: [PATCH 18/22] Added TODO to fix atttributes in the future --- .../cdf/config/imap_codice_l1a_variable_attrs.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml b/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml index a52786a7f..dfff23a9d 100644 --- a/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml @@ -1,3 +1,7 @@ +# TODO: Some currently blank attributes may be removed once +# cdf_attribute_manager.py is updated to only check required attributes. +# See comment here: +# https://github.com/IMAP-Science-Operations-Center/imap_processing/pull/711#discussion_r1690212527 # <=== Defaults ===> default_attrs: &default CATDESC: "" From 553c22009fbee1e3e5cba91bef3acc678e703d88 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Wed, 14 Aug 2024 15:56:39 -0600 Subject: [PATCH 19/22] Fixed bug in epoch attrs name --- imap_processing/codice/codice_l1a.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/imap_processing/codice/codice_l1a.py b/imap_processing/codice/codice_l1a.py index 5fc6e2811..8c652cdee 100644 --- a/imap_processing/codice/codice_l1a.py +++ b/imap_processing/codice/codice_l1a.py @@ -89,7 +89,7 @@ def configure_data_products(self, apid: int) -> None: apid : int The APID of interest. """ - config = constants.DATA_PRODUCT_CONFIGURATIONS.get(apid) # type: ignore[call-overload] + config = constants.DATA_PRODUCT_CONFIGURATIONS.get(apid) self.num_counters = config["num_counters"] self.num_energy_steps = config["num_energy_steps"] self.variable_names = config["variable_names"] @@ -323,7 +323,7 @@ def create_event_dataset( met_to_j2000ns(met), # TODO: Fix after SIT-3 (see note below) name="epoch", dims=["epoch"], - attrs=cdf_attrs.get_variable_attributes("epoch_attrs"), + attrs=cdf_attrs.get_variable_attributes("epoch"), ) # Create the dataset to hold the data variables @@ -376,7 +376,7 @@ def create_hskp_dataset( ), name="epoch", dims=["epoch"], - attrs=cdf_attrs.get_variable_attributes("epoch_attrs"), + attrs=cdf_attrs.get_variable_attributes("epoch"), ) dataset = xr.Dataset( From 3cb7736247444caec7cef04b5191ed7a2c05a78e Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Thu, 15 Aug 2024 11:33:18 -0600 Subject: [PATCH 20/22] Fixed mypy error --- imap_processing/codice/codice_l1a.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imap_processing/codice/codice_l1a.py b/imap_processing/codice/codice_l1a.py index 8c652cdee..fff901cfc 100644 --- a/imap_processing/codice/codice_l1a.py +++ b/imap_processing/codice/codice_l1a.py @@ -80,7 +80,7 @@ def __init__(self, table_id: int, plan_id: int, plan_step: int, view_id: int): self.plan_step = plan_step self.view_id = view_id - def configure_data_products(self, apid: int) -> None: + def configure_data_products(self, apid: str) -> None: """ Set the various settings for defining the data products. From e7f44c94f2347cc630f597c62016be4da1c25bde Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Thu, 15 Aug 2024 11:33:37 -0600 Subject: [PATCH 21/22] Fixed mypy error --- imap_processing/codice/codice_l1a.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imap_processing/codice/codice_l1a.py b/imap_processing/codice/codice_l1a.py index fff901cfc..bfe04db87 100644 --- a/imap_processing/codice/codice_l1a.py +++ b/imap_processing/codice/codice_l1a.py @@ -86,7 +86,7 @@ def configure_data_products(self, apid: str) -> None: Parameters ---------- - apid : int + apid : str The APID of interest. """ config = constants.DATA_PRODUCT_CONFIGURATIONS.get(apid) From d8e057bc24e51a4c49cc8460ea16e17f821e524c Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Thu, 15 Aug 2024 11:36:30 -0600 Subject: [PATCH 22/22] Fixed mypy error --- imap_processing/codice/codice_l1a.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/imap_processing/codice/codice_l1a.py b/imap_processing/codice/codice_l1a.py index bfe04db87..0c925fee7 100644 --- a/imap_processing/codice/codice_l1a.py +++ b/imap_processing/codice/codice_l1a.py @@ -80,16 +80,16 @@ def __init__(self, table_id: int, plan_id: int, plan_step: int, view_id: int): self.plan_step = plan_step self.view_id = view_id - def configure_data_products(self, apid: str) -> None: + def configure_data_products(self, apid: int) -> None: """ Set the various settings for defining the data products. Parameters ---------- - apid : str + apid : int The APID of interest. """ - config = constants.DATA_PRODUCT_CONFIGURATIONS.get(apid) + config = constants.DATA_PRODUCT_CONFIGURATIONS.get(apid) # type: ignore[call-overload] self.num_counters = config["num_counters"] self.num_energy_steps = config["num_energy_steps"] self.variable_names = config["variable_names"]