Ultra l1b extended events

imap_processing/cdf/config/imap_ultra_l1b_variable_attrs.yaml

@@ -26,113 +26,157 @@ x_front:
   CATDESC: x front position
   FIELDNAM: Event x-position on front foil
   LABLAXIS: x front position
-  # TODO: come back to format
-  UNITS: mm
+  VALIDMIN: -1e6
+  FORMAT: F12.6
+  UNITS: mm / 100
 
 y_front:
   <<: *default
   CATDESC: y front position
   FIELDNAM: Event y-position on front foil
   LABLAXIS: y front position
-  # TODO: come back to format
-  UNITS: mm
+  VALIDMIN: -1e6
+  FORMAT: F12.6
+  UNITS: mm / 100
 
 x_back:
   <<: *default
   CATDESC: x back position
   FIELDNAM: x_back
   LABLAXIS: x back position
-  # TODO: come back to format
-  UNITS: mm
+  VALIDMIN: -1e6
+  FORMAT: F12.6
+  UNITS: mm / 100
 
 y_back:
   <<: *default
   CATDESC: y back position
   FIELDNAM: y_back
   LABLAXIS: y back position
-  # TODO: come back to format
-  UNITS: mm
+  VALIDMIN: -1e6
+  FORMAT: F12.6
+  UNITS: mm / 100
 
 x_coin:
   <<: *default
   CATDESC: x coincidence position
   FIELDNAM: x_coin
   LABLAXIS: x coincidence position
-  # TODO: come back to format
-  UNITS: mm
+  VALIDMIN: -1e6
+  FORMAT: F12.6
+  UNITS: mm / 100
 
 tof_start_stop:
   <<: *default
   CATDESC: Particle time of flight from start to stop
   FIELDNAM: tof_start_stop
   LABLAXIS: tof start stop
-  # TODO: come back to format
-  UNITS: ns
+  VALIDMIN: -1e6
+  FORMAT: F12.6
+  UNITS: nanosecond / 10
 
 tof_stop_coin:
   <<: *default
   CATDESC: Particle time of flight from stop to coincidence
   FIELDNAM: tof_stop_coin
   LABLAXIS: tof stop coin
-  # TODO: come back to format
-  UNITS: ns
+  VALIDMIN: -1e6
+  FORMAT: F12.6
+  UNITS: nanosecond / 10
 
 tof_corrected:
   <<: *default
   CATDESC: Corrected time of flight
   FIELDNAM: tof_corrected
   LABLAXIS: tof corrected
-  # TODO: come back to format
-  UNITS: ns
+  VALIDMIN: -1e6
+  FORMAT: F12.6
+  UNITS: nanosecond / 10
 
-eventtype:
+event_type:
   <<: *default
+  VALIDMAX: 15
   CATDESC: Eventtype (1-2 top and bottom stop types; 8-15 ssd stop types)
-  FIELDNAM: eventtype
-  LABLAXIS: eventtype
-  # TODO: come back to format
-  UNITS: " "
+  FIELDNAM: Event type
+  FORMAT: I2
+  LABLAXIS: event type
+
+coincidence_type:
+  <<: *default
+  VALIDMAX: 2
+  CATDESC: Coincidence type for the direct event (1-2 top and bottom)
+  FIELDNAM: Coincidence type
+  FORMAT: I1
+  LABLAXIS: coincidence type
+
+start_type:
+  <<: *default
+  VALIDMAX: 2
+  CATDESC: Start type for the direct event (1-2 left and right)
+  FIELDNAM: Start type
+  FORMAT: I1
+  LABLAXIS: start type
 
 vx_ultra:
   <<: *default
   CATDESC: Normalized component of the velocity vector in x direction in the instrument frame.
   FIELDNAM: vx_ultra
   LABLAXIS: vx ultra
-  # TODO: come back to format
+  VALIDMIN: -1e6
+  FORMAT: F12.6
   UNITS: " "
 
 vy_ultra:
   <<: *default
   CATDESC: Normalized component of the velocity vector in y direction in the instrument frame.
   FIELDNAM: vy_ultra
   LABLAXIS: vy ultra
-  # TODO: come back to format
+  VALIDMIN: -1e6
+  FORMAT: F12.6
   UNITS: " "
 
 vz_ultra:
   <<: *default
   CATDESC: Normalized component of the velocity vector in z direction in the instrument frame.
   FIELDNAM: vz_ultra
   LABLAXIS: vz ultra
-  # TODO: come back to format
+  VALIDMIN: -1e6
+  FORMAT: F12.6
   UNITS: " "
 
 energy:
   <<: *default
   CATDESC: Energy measured using the pulse height from the stop anode (DN) or Energy measured using the SSD (keV) depending on eventtype.
   FIELDNAM: energy
   LABLAXIS: energy
-  # TODO: come back to format
+  VALIDMIN: -1e6
+  FORMAT: F12.6
   UNITS: keV
 
 species:
   <<: *default
-  CATDESC: Species bin.
+  CATDESC: Species bin (0-255).
   FIELDNAM: species
   LABLAXIS: species
-  # TODO: come back to format
+  FORMAT: I2
   UNITS: " "
 
+front_back_distance:
+  <<: *default
+  CATDESC: Distance front to back.
+  FIELDNAM: front_back_distance
+  LABLAXIS: front_back_distance
+  FORMAT: F12.6
+  UNITS: mm / 100
+
+path_length:
+  <<: *default
+  CATDESC: Path length.
+  FIELDNAM: path_length
+  LABLAXIS: path_length
+  FORMAT: F12.6
+  UNITS: mm / 100
+
 event_efficiency:
   <<: *default
   CATDESC: Estimated event efficiency for this path through the instrument.
@@ -218,8 +262,17 @@ eventtimes:
   CATDESC: Event times calculated from event and universal spin table.
   FIELDNAM: eventtimes
   LABLAXIS: event times
-  # TODO: come back to format
-  UNITS: " "
+  FORMAT: F20.6
+  UNITS: seconds
+
+de_event_met:
+  <<: *default
+  CATDESC: Mission Elapsed Time (MET) of Direct Event
+  DISPLAY_TYPE: no_plot
+  FIELDNAM: Mission Elapsed Time
+  LABLAXIS: DE MET
+  FORMAT: I20
+  UNITS: seconds
 
 spin_number:
   <<: *default

imap_processing/tests/ultra/unit/conftest.py

@@ -6,7 +6,13 @@
 import pytest
 
 from imap_processing import decom
+from imap_processing.cdf.defaults import GlobalConstants
 from imap_processing.ultra.l0.decom_ultra import process_ultra_apids
+from imap_processing.ultra.l0.ultra_utils import (
+    ULTRA_AUX,
+    ULTRA_EVENTS,
+)
+from imap_processing.ultra.l1a import ultra_l1a
 from imap_processing.utils import group_by_apid
 
 
@@ -36,7 +42,7 @@ def ccsds_path_events():
     )
 
 
-@pytest.fixture()
+@pytest.fixture(scope="session")
 def ccsds_path_theta_0():
     """Returns the ccsds directory."""
     return (
@@ -63,7 +69,7 @@ def ccsds_path_tof():
     )
 
 
-@pytest.fixture()
+@pytest.fixture(scope="session")
 def xtce_path():
     """Returns the xtce image rates directory."""
     return (
@@ -161,3 +167,44 @@ def decom_test_data(request, xtce_path):
 
     data_packet_list = process_ultra_apids(grouped_data[apid], apid)
     return data_packet_list, packets
+
+
+@pytest.fixture()
+def events_fsw_comparison_theta_0():
+    """Returns the xtce auxiliary test data directory."""
+    filename = (
+        "FM45_40P_Phi28p5_BeamCal_LinearScan_phi28.50_theta-0.00"
+        "_Ultra_Image_Raw_Event_20240207T102746_withFSWcalcs.csv"
+    )
+    return (
+        Path(sys.modules[__name__.split(".")[0]].__file__).parent
+        / "tests"
+        / "ultra"
+        / "test_data"
+        / "l0"
+        / filename
+    )
+
+
+@pytest.fixture()
+def de_dataset(ccsds_path_theta_0, xtce_path):
+    """L1A test data"""
+    packets = decom.decom_packets(ccsds_path_theta_0, xtce_path)
+    grouped_data = group_by_apid(packets)
+    decom_ultra_events = process_ultra_apids(
+        grouped_data[ULTRA_EVENTS.apid[0]], ULTRA_EVENTS.apid[0]
+    )
+    decom_ultra_aux = process_ultra_apids(
+        grouped_data[ULTRA_AUX.apid[0]], ULTRA_AUX.apid[0]
+    )
+    dataset = ultra_l1a.create_dataset(
+        {
+            ULTRA_EVENTS.apid[0]: decom_ultra_events,
+            ULTRA_AUX.apid[0]: decom_ultra_aux,
+        }
+    )
+    # Remove start_type with fill values
+    l1a_de_dataset = dataset.where(
+        dataset["START_TYPE"] != GlobalConstants.INT_FILLVAL, drop=True
+    )
+    return l1a_de_dataset

imap_processing/tests/ultra/unit/test_lookup_utils.py

@@ -0,0 +1,68 @@
+import numpy as np
+import pandas as pd
+
+from imap_processing import imap_module_directory
+from imap_processing.ultra.l1b.lookup_utils import (
+    get_back_position,
+    get_energy_norm,
+    get_image_params,
+    get_norm,
+    get_y_adjust,
+)
+
+base_path = imap_module_directory / "ultra" / "lookup_tables"
+
+
+def test_get_y_adjust():
+    """Tests function get_y_adjust."""
+
+    yadjust_path = base_path / "yadjust.csv"
+    yadjust_df = pd.read_csv(yadjust_path).set_index("dYLUT")
+
+    array = np.array([8])
+    res = get_y_adjust(array)
+
+    assert res == yadjust_df["dYAdj"][8]
+
+
+def test_get_stop_norm():
+    """Tests function get_stop_norm."""
+
+    tdc_norm_path = base_path / "ultra45_tdc_norm.csv"
+    tdc_norm_df = pd.read_csv(tdc_norm_path, header=1)
+
+    array = np.array([378])
+    stop_norm = get_norm(array, "SpE", "ultra45")
+
+    assert stop_norm == tdc_norm_df["SpE"][378]
+
+
+def test_get_back_position():
+    """Tests function get_back_position."""
+
+    back_pos_path = base_path / "ultra45_back-pos-luts.csv"
+    back_pos_df = pd.read_csv(back_pos_path, index_col="Index_offset")
+
+    array = np.array([-2000])
+    dn_converted = get_back_position(array, "XBkBt", "ultra45")
+
+    assert dn_converted == back_pos_df["XBkBt"].iloc[-2000]
+
+
+def test_get_egy_norm():
+    """Tests function get_egy_norm."""
+
+    egy_norm_path = base_path / "EgyNorm.mem.csv"
+    egy_norm_df = pd.read_csv(egy_norm_path)
+
+    norm_composite_energy = get_energy_norm(np.array([2]), np.array([2]))
+
+    assert norm_composite_energy == egy_norm_df.iloc[2 * 4096 + 2]["NormEnergy"]
+
+
+def test_get_image_params():
+    """Tests function get_image_params."""
+
+    image_params = get_image_params("XFTLTOFF")
+
+    assert image_params == 49.3

imap_processing/tests/ultra/unit/test_ultra_l1b.py

@@ -73,7 +73,7 @@ def test_create_dataset(mock_data_l1b_dict):
     assert "epoch" in dataset.coords
     assert dataset.coords["epoch"].dtype == "datetime64[ns]"
     assert dataset.attrs["Logical_source"] == "imap_ultra_l1b_45sensor-de"
-    assert dataset["x_front"].attrs["UNITS"] == "mm"
+    assert dataset["x_front"].attrs["UNITS"] == "mm / 100"
     np.testing.assert_array_equal(dataset["x_front"], np.zeros(3))
 
 
@@ -99,9 +99,12 @@ def test_ultra_l1b_rates(mock_data_l1a_rates_dict):
     )
 
 
-def test_ultra_l1b_de(mock_data_l1a_de_aux_dict):
+def test_ultra_l1b_de(de_dataset):
     """Tests that L1b data is created."""
-    output_datasets = ultra_l1b(mock_data_l1a_de_aux_dict, data_version="001")
+    data_dict = {}
+    data_dict[de_dataset.attrs["Logical_source"]] = de_dataset
+    data_dict["imap_ultra_l1a_45sensor-aux"] = de_dataset
+    output_datasets = ultra_l1b(data_dict, data_version="001")
 
     assert len(output_datasets) == 1
     assert output_datasets[0].attrs["Logical_source"] == "imap_ultra_l1b_45sensor-de"