From 422cfef92beec2d867764c71d3dd630c5e76bc9d Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Thu, 5 Sep 2024 13:27:29 -0600 Subject: [PATCH 1/4] Updated various CDF attributes/variable names to match expected nomenclature --- .../imap_codice_l1a_variable_attrs.yaml | 83 ++++++++----------- imap_processing/codice/codice_l1a.py | 43 ++++------ 2 files changed, 53 insertions(+), 73 deletions(-) diff --git a/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml b/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml index 9d7a535d9..9f79dd3d2 100644 --- a/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml @@ -25,94 +25,83 @@ codice_support_attrs: &support_default VAR_TYPE: support_data # <=== Coordinates ===> -energy_attrs: +esa_step: <<: *default CATDESC: Energy per charge (E/q) sweeping step - FIELDNAM: Energy step + FIELDNAM: Energy Index FORMAT: I3 - LABLAXIS: energy step + LABLAXIS: Energy Index + SI_CONVERSION: " > " UNITS: ' ' VALIDMIN: 0 - VALIDMAX: 127 + VALIDMAX: 128 VAR_TYPE: support_data -inst_az_attrs: +inst_az: <<: *default - CATDESC: Azimuth - FIELDNAM: Azimuth + CATDESC: Azimuth Index + FIELDNAM: Azimuth Index FORMAT: I2 - LABLAXIS: Azimuth - UNITS: ' ' + LABLAXIS: " " + SI_CONVERSION: " > " + UNITS: " " VALIDMIN: 0 VALIDMAX: 31 VAR_TYPE: support_data -spin_sector_attrs: +spin_sector: <<: *default CATDESC: Spin sector indicating range of spin angles - FIELDNAM: Spin sector + FIELDNAM: Spin Sector Index FORMAT: I2 - LABLAXIS: spin sector - UNITS: ' ' + LABLAXIS: " " + SI_CONVERSION: " > " + UNITS: " " VALIDMIN: 0 - VALIDMAX: 11 + VALIDMAX: 12 VAR_TYPE: support_data -# <=== Labels ===> -energy_label: - CATDESC: Energy per charge (E/q) sweeping step - DISPLAY_TYPE: "" - FIELDNAM: Energy step - FILLVAL: "" - FORMAT: A3 - LABLAXIS: "" - REFERENCE_POSITION: "" - RESOLUTION: "" - TIME_BASE: "" - TIME_SCALE: "" - UNITS: "" - VALIDMAX: "" - VALIDMIN: "" - VAR_TYPE: metadata - -# <=== Dataset Attributes ===> -acquisition_times_attrs: +# <=== Dataset Variable Attributes ===> +# The following are set in multiple data products +acquisition_time_per_step: <<: *default - CATDESC: Time of acquisition for the energy step - DEPEND_1: energy + CATDESC: Acquisition time for each step of energy FIELDNAM: Acquisition Time - FILLVAL: .NAN + FILLVAL: -1.0e+30 FORMAT: F10.3 - LABLAXIS: Acq Time + LABLAXIS: " " + SI_CONVERSION: " > " UNITS: ms - VALIDMIN: 0 - VALIDMAX: 1000 + VALIDMIN: 0.000000 + VALIDMAX: 625.000000 VAR_TYPE: support_data counters_attrs: &counters <<: *default CATDESC: Fill in at creation DEPEND_0: epoch - DEPEND_1: energy + DEPEND_1: esa_step DISPLAY_TYPE: time_series FIELDNAM: Fill in at creation - LABL_PTR_1: energy_label UNITS: counts VALIDMIN: 0 VALIDMAX: 8388607 # max value for a signed 24-bit integer -esa_sweep_attrs: +energy_table: <<: *default CATDESC: ElectroStatic Analyzer Energy Values - DEPEND_1: energy + DEPEND_1: esa_step FIELDNAM: ESA Voltage - FORMAT: I19 + FORMAT: I5 LABLAXIS: ESA V - UNITS: V - VALIDMIN: 0 + SCALETYP: log + SI_CONVERSION: " > " + UNITS: eV + VALIDMIN: 1 + VALIDMAX: 14100 VAR_TYPE: support_data -# <=== Data Variable Attributes ===> +# The following are data product-specific # hi-counters-aggregated hi_counters_aggregated-aggregated: <<: *counters diff --git a/imap_processing/codice/codice_l1a.py b/imap_processing/codice/codice_l1a.py index f1776a4fa..4234967ee 100644 --- a/imap_processing/codice/codice_l1a.py +++ b/imap_processing/codice/codice_l1a.py @@ -136,27 +136,19 @@ def create_science_dataset( np.arange(self.num_positions), name="inst_az", dims=["inst_az"], - attrs=cdf_attrs.get_variable_attributes("inst_az_attrs"), + attrs=cdf_attrs.get_variable_attributes("inst_az"), ) spin_sector = xr.DataArray( np.arange(self.num_spin_sectors), name="spin_sector", dims=["spin_sector"], - attrs=cdf_attrs.get_variable_attributes("spin_sector_attrs"), + attrs=cdf_attrs.get_variable_attributes("spin_sector"), ) - energy_steps = xr.DataArray( + esa_step = xr.DataArray( np.arange(self.num_energy_steps), - name="energy", - dims=["energy"], - attrs=cdf_attrs.get_variable_attributes("energy_attrs"), - ) - - # Define labels - energy_label = xr.DataArray( - energy_steps.values.astype(str), - name="energy_label", - dims=["energy_label"], - attrs=cdf_attrs.get_variable_attributes("energy_label"), + name="esa_step", + dims=["esa_step"], + attrs=cdf_attrs.get_variable_attributes("esa_step"), ) # Create the dataset to hold the data variables @@ -165,8 +157,7 @@ def create_science_dataset( "epoch": epoch, "inst_az": inst_az, "spin_sector": spin_sector, - "energy": energy_steps, - "energy_label": energy_label, + "esa_step": esa_step, }, attrs=cdf_attrs.get_global_attributes(self.dataset_name), ) @@ -183,7 +174,7 @@ def create_science_dataset( self.num_energy_steps, ) ) - dims = ["epoch", "inst_az", "spin_sector", "energy"] + dims = ["epoch", "inst_az", "spin_sector", "esa_step"] elif self.instrument == "hi": variable_data_arr = np.array(variable_data).reshape( ( @@ -193,7 +184,7 @@ def create_science_dataset( self.num_spin_sectors, ) ) - dims = ["epoch", "energy", "inst_az", "spin_sector"] + dims = ["epoch", "esa_step", "inst_az", "spin_sector"] # Get the CDF attributes cdf_attrs_key = ( @@ -211,17 +202,17 @@ def create_science_dataset( # Add ESA Sweep Values and acquisition times (lo only) if self.instrument == "lo": - self.get_esa_sweep_values() + self.get_energy_table() self.get_acquisition_times() dataset["esa_sweep_values"] = xr.DataArray( - self.esa_sweep_values, - dims=["energy"], - attrs=cdf_attrs.get_variable_attributes("esa_sweep_attrs"), + self.energy_table, + dims=["esa_step"], + attrs=cdf_attrs.get_variable_attributes("energy_table"), ) dataset["acquisition_times"] = xr.DataArray( self.acquisition_times, - dims=["energy"], - attrs=cdf_attrs.get_variable_attributes("acquisition_times_attrs"), + dims=["esa_step"], + attrs=cdf_attrs.get_variable_attributes("acquisition_time_per_step"), ) return dataset @@ -270,7 +261,7 @@ def get_acquisition_times(self) -> None: row_number = np.argmax(energy_steps == str(step_number), axis=1).argmax() self.acquisition_times.append(lo_stepping_values.acq_time[row_number]) - def get_esa_sweep_values(self) -> None: + def get_energy_table(self) -> None: """ Retrieve the ESA sweep values. @@ -299,7 +290,7 @@ def get_esa_sweep_values(self) -> None: # Get the appropriate values sweep_table = sweep_data[sweep_data["table_idx"] == sweep_table_id] - self.esa_sweep_values = sweep_table["esa_v"].values + self.energy_table = sweep_table["esa_v"].values def unpack_science_data(self, science_values: str) -> None: """ From 59d5b21317ffbe3d1b4c107b3b88097dec78c8b8 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Thu, 5 Sep 2024 13:39:24 -0600 Subject: [PATCH 2/4] Fixed doc build error --- imap_processing/codice/codice_l1a.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imap_processing/codice/codice_l1a.py b/imap_processing/codice/codice_l1a.py index 4234967ee..1bcc75d1a 100644 --- a/imap_processing/codice/codice_l1a.py +++ b/imap_processing/codice/codice_l1a.py @@ -68,7 +68,7 @@ class CoDICEL1aPipeline: Create an ``xarray`` dataset for the unpacked science data. get_acquisition_times() Retrieve the acquisition times via the Lo stepping table. - get_esa_sweep_values() + get_energy_table() Retrieve the ESA sweep values. unpack_science_data() Decompress, unpack, and restructure science data arrays. From f6949173e516a29101bffe413911c11b33660fec Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Mon, 9 Sep 2024 15:47:50 -0600 Subject: [PATCH 3/4] Added LABL_PTR_* attrs in anticipation of cdflib 1.3 upgrade --- .../imap_codice_l1a_variable_attrs.yaml | 166 +++++++++++++++++- 1 file changed, 164 insertions(+), 2 deletions(-) diff --git a/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml b/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml index 9f79dd3d2..2ec096577 100644 --- a/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml @@ -39,7 +39,7 @@ esa_step: inst_az: <<: *default - CATDESC: Azimuth Index + CATDESC: Instrument Azimuth Index FIELDNAM: Azimuth Index FORMAT: I2 LABLAXIS: " " @@ -107,278 +107,440 @@ hi_counters_aggregated-aggregated: <<: *counters CATDESC: Aggregated Rates FIELDNAM: Rates - Aggregated + LABL_PTR_1: esa_step + LABL_PTR_2: inst_az + LABL_PTR_3: spin_sector # hi-counters-singles hi_counters_singles-tcr: <<: *counters CATDESC: TCR Rates FIELDNAM: Rates - Event A (TCR) + LABL_PTR_1: esa_step + LABL_PTR_2: inst_az + LABL_PTR_3: spin_sector hi_counters_singles-ssdo: <<: *counters CATDESC: SSDO Rates FIELDNAM: Rates - Event E (SSDO) + LABL_PTR_1: esa_step + LABL_PTR_2: inst_az + LABL_PTR_3: spin_sector hi_counters_singles-stssd: <<: *counters CATDESC: STSSD Rates FIELDNAM: Rates - Event G (STSSD) + LABL_PTR_1: esa_step + LABL_PTR_2: inst_az + LABL_PTR_3: spin_sector # hi-omni hi_omni-h: <<: *counters CATDESC: Omnidirectional H Counts FIELDNAM: H + LABL_PTR_1: esa_step + LABL_PTR_2: inst_az + LABL_PTR_3: spin_sector hi_omni-he3: <<: *counters CATDESC: Omnidirectional He3 Counts FIELDNAM: He3 + LABL_PTR_1: esa_step + LABL_PTR_2: inst_az + LABL_PTR_3: spin_sector hi_omni-he4: <<: *counters CATDESC: Omnidirectional He4 Counts FIELDNAM: He4 + LABL_PTR_1: esa_step + LABL_PTR_2: inst_az + LABL_PTR_3: spin_sector hi_omni-c: <<: *counters CATDESC: Omnidirectional C Counts FIELDNAM: C + LABL_PTR_1: esa_step + LABL_PTR_2: inst_az + LABL_PTR_3: spin_sector hi_omni-o: <<: *counters CATDESC: Omnidirectional O Counts FIELDNAM: O + LABL_PTR_1: esa_step + LABL_PTR_2: inst_az + LABL_PTR_3: spin_sector hi_omni-ne_mg_si: <<: *counters CATDESC: Omnidirectional Ne_Mg_Si Counts FIELDNAM: Ne_Mg_Si + LABL_PTR_1: esa_step + LABL_PTR_2: inst_az + LABL_PTR_3: spin_sector hi_omni-fe: <<: *counters CATDESC: Omnidirectional Fe Counts FIELDNAM: Fe + LABL_PTR_1: esa_step + LABL_PTR_2: inst_az + LABL_PTR_3: spin_sector hi_omni-uh: <<: *counters CATDESC: Omnidirectional UH Counts FIELDNAM: UH + LABL_PTR_1: esa_step + LABL_PTR_2: inst_az + LABL_PTR_3: spin_sector # hi-sectored hi_sectored-h: <<: *counters CATDESC: Sectored H Counts FIELDNAM: H + LABL_PTR_1: esa_step + LABL_PTR_2: inst_az + LABL_PTR_3: spin_sector hi_sectored-he3he4: <<: *counters CATDESC: Sectored He3He4 Counts FIELDNAM: He3He4 + LABL_PTR_1: esa_step + LABL_PTR_2: inst_az + LABL_PTR_3: spin_sector hi_sectored-cno: <<: *counters CATDESC: Sectored CNO Counts FIELDNAM: CNO + LABL_PTR_1: esa_step + LABL_PTR_2: inst_az + LABL_PTR_3: spin_sector hi_sectored-fe: <<: *counters CATDESC: Sectored Fe Counts FIELDNAM: Fe + LABL_PTR_1: esa_step + LABL_PTR_2: inst_az + LABL_PTR_3: spin_sector # lo-counters-aggregated lo_counters_aggregated-aggregated: <<: *counters CATDESC: Aggregated Rates FIELDNAM: Rates - Aggregated + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step # lo-counters-singles lo_counters_singles-apd_singles: <<: *counters CATDESC: Single Rates (APD) FIELDNAM: Rates - Single (APD) + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step # lo-sw-angular lo_sw_angular-hplus: <<: *counters CATDESC: Sunward H+ Species FIELDNAM: SW - H+ + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_sw_angular-heplusplus: <<: *counters CATDESC: Sunward He++ Species FIELDNAM: SW - He++ + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_sw_angular-oplus6: <<: *counters CATDESC: Sunward O+6 Species FIELDNAM: SW - O+6 + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_sw_angular-fe_loq: <<: *counters CATDESC: Sunward Fe lowQ Species FIELDNAM: SW - Fe lowQ + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_nsw_angular-heplusplus: <<: *counters CATDESC: Non-sunward He++ Species FIELDNAM: NSW - He++ + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step # lo-sw-priority lo_sw_priority-p0_tcrs: <<: *counters CATDESC: Sunward Sector Triple Coincidence Pickup Ions Priority FIELDNAM: SW Sector Triple Coincidence PUI's + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_sw_priority-p1_hplus: <<: *counters CATDESC: Sunward Sector H+ Priority FIELDNAM: SW Sector H+ + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_sw_priority-p2_heplusplus: <<: *counters CATDESC: Sunward Sector He++ Priority FIELDNAM: SW Sector He++ + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_sw_priority-p3_heavies: <<: *counters CATDESC: Sunward Sector High Charge State Heavies Priority FIELDNAM: SW Sector High Charge State Heavies + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_sw_priority-p4_dcrs: <<: *counters CATDESC: Sunward Sector Double Coincidence Pickup Ions Priority FIELDNAM: SW Sector Double Coincidence PUI's + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step # lo-nsw-priority lo_nsw_priority-p5_heavies: <<: *counters CATDESC: Non-sunward Sector Heavies Priority FIELDNAM: NSW Sector Heavies + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_nsw_priority-p6_hplus_heplusplus: <<: *counters CATDESC: Non-sunward H+ and He++ Priority FIELDNAM: NSW H+ and He++ + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step # lo-sw-species lo_sw_species-hplus: <<: *counters CATDESC: H+ Sunward Species FIELDNAM: SW - H+ + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_sw_species-heplusplus: <<: *counters CATDESC: He++ Sunward Species FIELDNAM: SW - He++ + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_sw_species-cplus4: <<: *counters CATDESC: C+4 Sunward Species FIELDNAM: SW - C+4 + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_sw_species-cplus5: <<: *counters CATDESC: C+5 Sunward Species FIELDNAM: SW - C+5 + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_sw_species-cplus6: <<: *counters CATDESC: C+6 Sunward Species FIELDNAM: SW - C+6 + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_sw_species-oplus5: <<: *counters CATDESC: O+5 Sunward Species FIELDNAM: SW - O+5 + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_sw_species-oplus6: <<: *counters CATDESC: O+6 Sunward Species FIELDNAM: SW - O+6 + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_sw_species-oplus7: <<: *counters CATDESC: O+7 Sunward Species FIELDNAM: SW - O+7 + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_sw_species-oplus8: <<: *counters CATDESC: O+8 Sunward Species FIELDNAM: SW - O+8 + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_sw_species-ne: <<: *counters CATDESC: Ne Sunward Species FIELDNAM: SW - Ne + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_sw_species-mg: <<: *counters CATDESC: Mg Sunward Species FIELDNAM: SW - Mg + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_sw_species-si: <<: *counters CATDESC: Si Sunward Species FIELDNAM: SW - Si + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_sw_species-fe_loq: <<: *counters CATDESC: Fe lowQ Sunward Species FIELDNAM: SW - Fe lowQ + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_sw_species-fe_hiq: <<: *counters CATDESC: Fe highQ Sunward Species FIELDNAM: SW - Fe highQ + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_sw_species-heplus: <<: *counters CATDESC: He+ Pickup Ion Sunward Species FIELDNAM: SW - He+ (PUI) + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_sw_species-cnoplus: <<: *counters CATDESC: CNO+ Pickup Ion Sunward Species FIELDNAM: SW - CNO+ (PUI) + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step # lo-nsw-species lo_nsw_species-hplus: <<: *counters CATDESC: H+ Non-sunward Species FIELDNAM: NSW - H+ + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_nsw_species-heplusplus: <<: *counters CATDESC: He++ Non-sunward Species FIELDNAM: NSW - He++ + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_nsw_species-c: <<: *counters CATDESC: C Non-sunward Species FIELDNAM: NSW - C + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_nsw_species-o: <<: *counters CATDESC: O Non-sunward Species FIELDNAM: NSW - O + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_nsw_species-ne_si_mg: <<: *counters CATDESC: Ne-Si-Mg Non-sunward Species FIELDNAM: NSW - Ne_Si_Mg + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_nsw_species-fe: <<: *counters CATDESC: Fe Non-sunward Species FIELDNAM: NSW - Fe + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_nsw_species-heplus: <<: *counters CATDESC: He+ Non-sunward Species FIELDNAM: NSW - He+ + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step lo_nsw_species-cnoplus: <<: *counters CATDESC: CNO+ Non-sunward Species - FIELDNAM: NSW - CNO+ \ No newline at end of file + FIELDNAM: NSW - CNO+ + LABL_PTR_1: inst_az + LABL_PTR_2: spin_sector + LABL_PTR_3: esa_step \ No newline at end of file From 466ab285fdef059707aa8452c504157576cb812b Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Wed, 11 Sep 2024 11:14:01 -0600 Subject: [PATCH 4/4] Switched esa_step VALIDMAX back to 127 --- imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml b/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml index 2ec096577..c2e5158b0 100644 --- a/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml @@ -34,7 +34,7 @@ esa_step: SI_CONVERSION: " > " UNITS: ' ' VALIDMIN: 0 - VALIDMAX: 128 + VALIDMAX: 127 VAR_TYPE: support_data inst_az: