Updating Branch.

Merge branch 'dev' of github.com:IMAP-Science-Operations-Center/imap_processing into data-access-api-README

imap_processing/cdf/config/imap_default_global_cdf_attrs.yaml

@@ -1,8 +1,17 @@
-Project: STP>Solar-Terrestrial Physics
+# Reference for sections: https://spdf.gsfc.nasa.gov/istp_guide/gattributes.html
+# Reference from IBEX: https://spdf.gsfc.nasa.gov/pub/software/cdawlib/0SKELTABLES/ibex_h3_ena_lo_r08_omni_f3-gdf-maps_5yr_00000000_v01.skt
+Project: STP>Solar Terrestrial Probes
 Source_name: IMAP>Interstellar Mapping and Acceleration Probe
 Discipline: Solar Physics>Heliospheric Physics
-# TODO: CDF docs say this value should be IMAP
-Mission_group: IMAP>Interstellar Mapping and Acceleration Probe
-PI_name: Dr. David J. McComas
-PI_affiliation: Princeton Plasma Physics Laboratory, 100 Stellarator Road, Princeton, NJ 08540
+Mission_group: IMAP
+PI_name: Prof. David J. McComas
+PI_affiliation: Princeton University
 File_naming_convention: source_descriptor_datatype_yyyyMMdd_vNNN
+Acknowledgement: >
+  Please acknowledge the IMAP Mission Principal Investigator, Prof. David J. McComas of Princeton University.
+Rules_of_use: >
+  All IMAP data products are publicly released and citable for use in publications.
+  Please consult the IMAP team publications and personnel for further details
+  on production, processing, and usage of these data.
+LINK_TITLE: IMAP The Interstellar Mapping and Acceleration Probe
+HTTP_LINK: https://imap.princeton.edu/

imap_processing/tests/cdf/test_imap_cdf_manager.py

@@ -14,15 +14,15 @@ def test_add_instrument_global_attrs():
     # Testing data loaded in
     imap_instrument = imap_cdf_manager.get_global_attributes("imap_test_T1_test")
     assert imap_instrument["Data_type"] == "T1_test-one>Test-1 test one"
-    assert imap_instrument["Project"] == "STP>Solar-Terrestrial Physics"
+    assert imap_instrument["Project"] == "STP>Solar Terrestrial Probes"
 
     # Testing reloading data
     imap_cdf_manager.add_instrument_global_attrs("instrument2")
 
     # Testing data carried over, and overwritten
     instrument2_instrument = imap_cdf_manager.get_global_attributes("imap_swe_l1a_sci")
     assert instrument2_instrument["Data_type"] == "L1A_SCI>Level-1A Science data"
-    assert instrument2_instrument["Project"] == "STP>Solar-Terrestrial Physics"
+    assert instrument2_instrument["Project"] == "STP>Solar Terrestrial Probes"
 
 
 @mock.patch(

imap_processing/tests/ultra/unit/conftest.py

@@ -3,10 +3,16 @@
 import sys
 from pathlib import Path
 
+import numpy as np
 import pytest
 
 from imap_processing import decom
 from imap_processing.ultra.l0.decom_ultra import process_ultra_apids
+from imap_processing.ultra.l0.ultra_utils import (
+    ULTRA_AUX,
+    ULTRA_EVENTS,
+)
+from imap_processing.ultra.l1a import ultra_l1a
 from imap_processing.utils import group_by_apid
 
 
@@ -161,3 +167,44 @@ def decom_test_data(request, xtce_path):
 
     data_packet_list = process_ultra_apids(grouped_data[apid], apid)
     return data_packet_list, packets
+
+
+@pytest.fixture()
+def events_fsw_comparison_theta_0():
+    """FSW test data."""
+    filename = (
+        "FM45_40P_Phi28p5_BeamCal_LinearScan_phi28.50_theta-0.00"
+        "_Ultra_Image_Raw_Event_20240207T102746_withFSWcalcs.csv"
+    )
+    return (
+        Path(sys.modules[__name__.split(".")[0]].__file__).parent
+        / "tests"
+        / "ultra"
+        / "test_data"
+        / "l0"
+        / filename
+    )
+
+
+@pytest.fixture()
+def de_dataset(ccsds_path_theta_0, xtce_path):
+    """L1A test data"""
+    packets = decom.decom_packets(ccsds_path_theta_0, xtce_path)
+    grouped_data = group_by_apid(packets)
+    decom_ultra_events = process_ultra_apids(
+        grouped_data[ULTRA_EVENTS.apid[0]], ULTRA_EVENTS.apid[0]
+    )
+    decom_ultra_aux = process_ultra_apids(
+        grouped_data[ULTRA_AUX.apid[0]], ULTRA_AUX.apid[0]
+    )
+    dataset = ultra_l1a.create_dataset(
+        {
+            ULTRA_EVENTS.apid[0]: decom_ultra_events,
+            ULTRA_AUX.apid[0]: decom_ultra_aux,
+        }
+    )
+    # Remove start_type with fill values
+    l1a_de_dataset = dataset.where(
+        dataset["START_TYPE"] != np.iinfo(np.int64).min, drop=True
+    )
+    return l1a_de_dataset

imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py

@@ -0,0 +1,61 @@
+"""Tests Extended Raw Events for ULTRA L1b."""
+
+import pandas as pd
+import pytest
+
+from imap_processing.ultra.l1b.ultra_l1b_extended import (
+    get_front_x_position,
+    get_front_y_position,
+    get_path_length,
+)
+
+
+@pytest.fixture()
+def yf_fixture(de_dataset, events_fsw_comparison_theta_0):
+    """Fixture to compute and return yf and related data."""
+    df = pd.read_csv(events_fsw_comparison_theta_0)
+    df_filt = df[df["StartType"] != -1]
+
+    d, yf = get_front_y_position(
+        de_dataset["START_TYPE"].data, df_filt.Yb.values.astype("float")
+    )
+
+    return df_filt, d, yf
+
+
+def test_get_front_x_position(
+    de_dataset,
+    events_fsw_comparison_theta_0,
+):
+    """Tests get_front_x_position function."""
+
+    df = pd.read_csv(events_fsw_comparison_theta_0)
+    df_filt = df[df["StartType"] != -1]
+
+    xf = get_front_x_position(
+        de_dataset["START_TYPE"].data,
+        de_dataset["START_POS_TDC"].data,
+    )
+
+    assert xf == pytest.approx(df_filt["Xf"].astype("float"), 1e-5)
+
+
+def test_get_front_y_position(yf_fixture):
+    """Tests get_front_y_position function."""
+    df_filt, d, yf = yf_fixture
+
+    assert yf == pytest.approx(df_filt["Yf"].astype("float"), abs=1e-5)
+
+
+def test_get_path_length(de_dataset, yf_fixture):
+    """Tests get_path_length function."""
+
+    df_filt, d, yf = yf_fixture
+
+    test_xf = df_filt["Xf"].astype("float").values
+    test_yf = df_filt["Yf"].astype("float").values
+
+    test_xb = df_filt["Xb"].astype("float").values
+    test_yb = df_filt["Yb"].astype("float").values
+    r = get_path_length((test_xf, test_yf), (test_xb, test_yb), d)
+    assert r == pytest.approx(df_filt["r"].astype("float"), abs=1e-5)

imap_processing/ultra/l1b/lookup_utils.py

@@ -43,7 +43,7 @@ def get_y_adjust(dy_lut: np.ndarray) -> npt.NDArray:
     yadj : np.ndarray
         Y adjustment (mm).
     """
-    return _YADJUST_DF["dYAdj"].values[dy_lut]
+    return _YADJUST_DF["dYAdj"].iloc[dy_lut].values
 
 
 def get_norm(dn: np.ndarray, key: str, file_label: str) -> npt.NDArray:

imap_processing/ultra/l1b/ultra_l1b_extended.py

@@ -0,0 +1,138 @@
+"""Calculates Extended Raw Events for ULTRA L1b."""
+
+import numpy as np
+from numpy import ndarray
+
+from imap_processing.ultra.l1b.lookup_utils import (
+    get_image_params,
+    get_y_adjust,
+)
+
+# Constants in IMAP-Ultra Flight Software Specification document.
+D_SLIT_FOIL = 3.39  # shortest distance from slit to foil (mm)
+SLIT_Z = 44.89  # position of slit on Z axis (mm)
+YF_ESTIMATE_LEFT = 40.0  # front position of particle for left shutter (mm)
+YF_ESTIMATE_RIGHT = -40  # front position of particle for right shutter (mm)
+N_ELEMENTS = 256  # number of elements in lookup table
+TRIG_CONSTANT = 81.92  # trigonometric constant (mm)
+# TODO: make lookup tables into config files.
+
+
+def get_front_x_position(start_type: ndarray, start_position_tdc: ndarray) -> ndarray:
+    """
+    Calculate the front xf position.
+
+    Converts Start Position Time to Digital Converter (TDC)
+    values into units of hundredths of a millimeter using a scale factor and offsets.
+    Further description is available on pages 30 of
+    IMAP-Ultra Flight Software Specification document (7523-9009_Rev_-.pdf).
+
+    Parameters
+    ----------
+    start_type : ndarray
+        Start Type: 1=Left, 2=Right.
+    start_position_tdc : ndarray
+        Start Position Time to Digital Converter (TDC).
+
+    Returns
+    -------
+    xf : ndarray
+        X front position (hundredths of a millimeter).
+    """
+    # Left and right start types.
+    indices = np.nonzero((start_type == 1) | (start_type == 2))
+
+    xftsc = get_image_params("XFTSC")
+    xft_lt_off = get_image_params("XFTLTOFF")
+    xft_rt_off = get_image_params("XFTRTOFF")
+    xft_off = np.where(start_type[indices] == 1, xft_lt_off, xft_rt_off)
+
+    # Calculate xf and convert to hundredths of a millimeter
+    xf: ndarray = (xftsc * -start_position_tdc[indices] + xft_off) * 100
+
+    return xf
+
+
+def get_front_y_position(start_type: ndarray, yb: ndarray) -> tuple[ndarray, ndarray]:
+    """
+    Compute the adjustments for the front y position and distance front to back.
+
+    This function utilizes lookup tables and trigonometry based on
+    the angle of the foil. Further description is available in the
+    IMAP-Ultra Flight Software Specification document pg 30.
+
+    Parameters
+    ----------
+    start_type : np.array
+        Start Type: 1=Left, 2=Right.
+    yb : np.array
+        Y back position in hundredths of a millimeter.
+
+    Returns
+    -------
+    d : np.array
+        Distance front to back in hundredths of a millimeter.
+    yf : np.array
+        Front y position in hundredths of a millimeter.
+    """
+    # Determine start types
+    index_left = np.nonzero(start_type == 1)
+    index_right = np.nonzero(start_type == 2)
+
+    yf = np.zeros(len(start_type))
+    d = np.zeros(len(start_type))
+
+    # Compute adjustments for left start type
+    dy_lut_left = np.floor(
+        (YF_ESTIMATE_LEFT - yb[index_left] / 100) * N_ELEMENTS / TRIG_CONSTANT + 0.5
+    )
+    # y adjustment in mm
+    y_adjust_left = get_y_adjust(dy_lut_left) / 100
+    # hundredths of a millimeter
+    yf[index_left] = (YF_ESTIMATE_LEFT - y_adjust_left) * 100
+    # distance adjustment in mm
+    distance_adjust_left = np.sqrt(2) * D_SLIT_FOIL - y_adjust_left
+    # hundredths of a millimeter
+    d[index_left] = (SLIT_Z - distance_adjust_left) * 100
+
+    # Compute adjustments for right start type
+    dy_lut_right = np.floor(
+        (yb[index_right] / 100 - YF_ESTIMATE_RIGHT) * N_ELEMENTS / TRIG_CONSTANT + 0.5
+    )
+    # y adjustment in mm
+    y_adjust_right = get_y_adjust(dy_lut_right) / 100
+    # hundredths of a millimeter
+    yf[index_right] = (YF_ESTIMATE_RIGHT + y_adjust_right) * 100
+    # distance adjustment in mm
+    distance_adjust_right = np.sqrt(2) * D_SLIT_FOIL - y_adjust_right
+    # hundredths of a millimeter
+    d[index_right] = (SLIT_Z - distance_adjust_right) * 100
+
+    return np.array(d), np.array(yf)
+
+
+def get_path_length(front_position: tuple, back_position: tuple, d: float) -> float:
+    """
+    Calculate the path length.
+
+    Parameters
+    ----------
+    front_position : tuple of floats
+        Front position (xf,yf) (hundredths of a millimeter).
+    back_position : tuple of floats
+        Back position (xb,yb) (hundredths of a millimeter).
+    d : float
+        Distance from slit to foil (hundredths of a millimeter).
+
+    Returns
+    -------
+    r : float
+        Path length (hundredths of a millimeter).
+    """
+    r: float = np.sqrt(
+        (front_position[0] - back_position[0]) ** 2
+        + (front_position[1] - back_position[1]) ** 2
+        + (d) ** 2
+    )
+
+    return r