L1A SIT-3 Preparation (IMAP-Science-Operations-Center#556)

* L1A SIT-3 Preparation

.gitignore

@@ -168,3 +168,6 @@ cython_debug/
 
 # VSCode
 .vscode/
+
+# Data that is downloaded
+data/

imap_processing/__init__.py

@@ -44,7 +44,7 @@
     "mag": ["l0", "l1a", "l1b", "l1c", "l2pre", "l2"],
     "swapi": ["l0", "l1", "l2"],
     "swe": ["l0", "l1a", "l1b", "l2"],
-    "ultra": ["l0", "l1a", "l1b", "l1c", "l1d", "l2"],
+    "ultra": ["l0", "l1a", "l1b", "l1c", "l2"],
 }
 
 # Reference start time (launch time or epoch)

imap_processing/cli.py

@@ -36,6 +36,7 @@
 from imap_processing.mag.l1a.mag_l1a import mag_l1a
 from imap_processing.swe.l1a.swe_l1a import swe_l1a
 from imap_processing.swe.l1b.swe_l1b import swe_l1b
+from imap_processing.ultra.l1a import ultra_l1a
 
 logger = logging.getLogger(__name__)
 
@@ -451,10 +452,29 @@ def do_processing(self, dependencies):
 class Ultra(ProcessInstrument):
     """Process IMAP-Ultra."""
 
-    def do_processing(self, dependencies):
-        """Perform IMAP-Ultra specific processing."""
+    def do_processing(self, dependencies: list):
+        """
+        Perform IMAP-Ultra specific processing.
+
+        Attributes
+        ----------
+        dependencies: list
+        List of dependencies to process
+        """
         print(f"Processing IMAP-Ultra {self.data_level}")
 
+        if self.data_level == "l1a":
+            # File path is expected output file path
+            if len(dependencies) > 1:
+                raise ValueError(
+                    f"Unexpected dependencies found for ULTRA L1A:"
+                    f"{dependencies}. Expected only one dependency."
+                )
+
+            datasets = ultra_l1a.ultra_l1a(dependencies[0])
+            products = [write_cdf(dataset) for dataset in datasets]
+            return products
+
 
 def main():
     """Run the processing for a specific instrument & data level.

imap_processing/tests/test_cli.py

@@ -5,7 +5,24 @@
 
 import pytest
 
-from imap_processing.cli import Hi, _validate_args, main
+from imap_processing.cli import Hi, Ultra, _validate_args, main
+
+
+@pytest.fixture()
+def mock_instrument_dependencies():
+    with (
+        mock.patch("imap_processing.cli.imap_data_access.query") as mock_query,
+        mock.patch("imap_processing.cli.imap_data_access.download") as mock_download,
+        mock.patch("imap_processing.cli.imap_data_access.upload") as mock_upload,
+        mock.patch("imap_processing.cli.write_cdf") as mock_write_cdf,
+    ):
+        mocks = {
+            "mock_query": mock_query,
+            "mock_download": mock_download,
+            "mock_upload": mock_upload,
+            "mock_write_cdf": mock_write_cdf,
+        }
+        yield mocks
 
 
 @mock.patch("imap_processing.cli.Mag")
@@ -60,17 +77,14 @@ def test_validate_args(instrument, data_level, raises_value_error):
         _validate_args(args)
 
 
-@mock.patch("imap_processing.cli.imap_data_access.query")
-@mock.patch("imap_processing.cli.imap_data_access.download")
-@mock.patch("imap_processing.cli.imap_data_access.upload")
 @mock.patch("imap_processing.cli.hi_l1a.hi_l1a")
-@mock.patch("imap_processing.cli.write_cdf")
-def test_hi(mock_write_cdf, mock_hi_l1a, mock_upload, mock_download, mock_query):
+def test_hi(mock_hi_l1a, mock_instrument_dependencies):
     """Test coverage for cli.Hi class"""
-    mock_query.return_value = [{"file_path": "/path/to/file0"}]
-    mock_download.return_value = "file0"
+    mocks = mock_instrument_dependencies
+    mocks["mock_query"].return_value = [{"file_path": "/path/to/file0"}]
+    mocks["mock_download"].return_value = "file0"
     mock_hi_l1a.return_value = ["l1a_file0", "l1a_file1"]
-    mock_write_cdf.side_effect = ["/path/to/file0", "/path/to/file1"]
+    mocks["mock_write_cdf"].side_effect = ["/path/to/file0", "/path/to/file1"]
 
     dependency_str = (
         "[{"
@@ -83,7 +97,33 @@ def test_hi(mock_write_cdf, mock_hi_l1a, mock_upload, mock_download, mock_query)
     )
     instrument = Hi("l1a", dependency_str, "20231212", "20231213", "v005", True)
     instrument.process()
-    assert mock_query.call_count == 1
-    assert mock_download.call_count == 1
+    assert mocks["mock_query"].call_count == 1
+    assert mocks["mock_download"].call_count == 1
     assert mock_hi_l1a.call_count == 1
-    assert mock_upload.call_count == 2
+    assert mocks["mock_upload"].call_count == 2
+
+
+@mock.patch("imap_processing.cli.ultra_l1a.ultra_l1a")
+def test_ultra(mock_ultra_l1a, mock_instrument_dependencies):
+    """Test coverage for cli.Ultra class"""
+    mocks = mock_instrument_dependencies
+    mocks["mock_query"].return_value = [{"file_path": "/path/to/file0"}]
+    mocks["mock_download"].return_value = "dependency0"
+    mock_ultra_l1a.return_value = ["l1a_dataset0", "l1a_dataset1"]
+    mocks["mock_write_cdf"].side_effect = ["/path/to/product0", "/path/to/product1"]
+
+    dependency_str = (
+        "[{"
+        "'instrument': 'ultra',"
+        "'data_level': 'l0',"
+        "'descriptor': 'raw',"
+        "'version': 'v001',"
+        "'start_date': '20240207'"
+        "}]"
+    )
+    instrument = Ultra("l1a", dependency_str, "20240207", "20240208", "v001", True)
+    instrument.process()
+    assert mocks["mock_query"].call_count == 1
+    assert mocks["mock_download"].call_count == 1
+    assert mock_ultra_l1a.call_count == 1
+    assert mocks["mock_upload"].call_count == 2

imap_processing/tests/ultra/unit/conftest.py

@@ -5,6 +5,10 @@
 
 import pytest
 
+from imap_processing import decom
+from imap_processing.ultra.l0.decom_ultra import process_ultra_apids
+from imap_processing.utils import group_by_apid
+
 
 @pytest.fixture()
 def ccsds_path():
@@ -32,6 +36,20 @@ def ccsds_path_events():
     )
 
 
+@pytest.fixture()
+def ccsds_path_theta_0():
+    """Returns the ccsds directory."""
+    return (
+        Path(sys.modules[__name__.split(".")[0]].__file__).parent
+        / "tests"
+        / "ultra"
+        / "test_data"
+        / "l0"
+        / "FM45_40P_Phi28p5_BeamCal_LinearScan_phi28.50_theta-0.00"
+        "_20240207T102740.CCSDS"
+    )
+
+
 @pytest.fixture()
 def ccsds_path_tof():
     """Returns the ccsds directory."""
@@ -122,3 +140,24 @@ def tof_test_path():
         / "l0"
         / filename
     )
+
+
+@pytest.fixture()
+def decom_test_data(request, xtce_path):
+    """Read test data from file"""
+    apid = request.param["apid"]
+    filename = request.param["filename"]
+    ccsds_path = (
+        Path(sys.modules[__name__.split(".")[0]].__file__).parent
+        / "tests"
+        / "ultra"
+        / "test_data"
+        / "l0"
+        / filename
+    )
+
+    packets = decom.decom_packets(ccsds_path, xtce_path)
+    grouped_data = group_by_apid(packets)
+
+    data_packet_list = process_ultra_apids(grouped_data[apid], apid)
+    return data_packet_list, packets

imap_processing/tests/ultra/unit/test_decom_apid_880.py

@@ -2,26 +2,32 @@
 import pandas as pd
 import pytest
 
-from imap_processing import decom
-from imap_processing.ultra.l0.decom_ultra import decom_ultra_apids
 from imap_processing.ultra.l0.ultra_utils import ULTRA_AUX
 from imap_processing.utils import group_by_apid
 
 
-@pytest.fixture()
-def decom_test_data(ccsds_path, xtce_path):
-    """Read test data from file"""
-    data_packet_list = decom.decom_packets(ccsds_path, xtce_path)
-    return data_packet_list
-
-
+@pytest.mark.parametrize(
+    "decom_test_data",
+    [
+        pytest.param(
+            {
+                "apid": ULTRA_AUX.apid[0],
+                "filename": "Ultra45_EM_SwRI_Cal_Run7_"
+                "ThetaScan_20220530T225054.CCSDS",
+            }
+        )
+    ],
+    indirect=True,
+)
 def test_aux_enumerated(decom_test_data):
     """Test if enumerated values derived correctly"""
 
+    _, packets = decom_test_data
+
     count = 0  # count number of packets with APID 880
     total_packets = 23
 
-    grouped_data = group_by_apid(decom_test_data)
+    grouped_data = group_by_apid(packets)
     apid_data = grouped_data[880]
 
     for packet in apid_data:
@@ -34,22 +40,51 @@ def test_aux_enumerated(decom_test_data):
     assert count == total_packets
 
 
+@pytest.mark.parametrize(
+    "decom_test_data",
+    [
+        pytest.param(
+            {
+                "apid": ULTRA_AUX.apid[0],
+                "filename": "Ultra45_EM_SwRI_Cal_Run7_"
+                "ThetaScan_20220530T225054.CCSDS",
+            }
+        )
+    ],
+    indirect=True,
+)
 def test_aux_mode(decom_test_data):
     """Test if enumerated values derived correctly"""
 
-    for packet in decom_test_data:
+    _, packets = decom_test_data
+
+    for packet in packets:
         if packet.header["PKT_APID"].derived_value == 880:
             assert packet.data["HWMODE"].derived_value == "MODE0"
             assert packet.data["IMCENB"].derived_value == "MODE0"
             assert packet.data["LEFTDEFLECTIONCHARGE"].derived_value == "MODE0"
             assert packet.data["RIGHTDEFLECTIONCHARGE"].derived_value == "MODE0"
 
 
-def test_aux_decom(ccsds_path, xtce_path, aux_test_path):
+@pytest.mark.parametrize(
+    "decom_test_data",
+    [
+        pytest.param(
+            {
+                "apid": ULTRA_AUX.apid[0],
+                "filename": "Ultra45_EM_SwRI_Cal_Run7_"
+                "ThetaScan_20220530T225054.CCSDS",
+            }
+        )
+    ],
+    indirect=True,
+)
+def test_aux_decom(decom_test_data, aux_test_path):
     """This function reads validation data and checks that
     decom data matches validation data for auxiliary packet"""
 
-    decom_ultra = decom_ultra_apids(ccsds_path, xtce_path, ULTRA_AUX.apid[0])
+    decom_ultra, _ = decom_test_data
+
     df = pd.read_csv(aux_test_path, index_col="MET")
 
     np.testing.assert_array_equal(df.SpinStartSeconds, decom_ultra["TIMESPINSTART"])

imap_processing/tests/ultra/unit/test_decom_apid_881.py

@@ -4,28 +4,26 @@
 import pandas as pd
 import pytest
 
-from imap_processing import decom
-from imap_processing.ultra.l0.decom_ultra import decom_ultra_apids
 from imap_processing.ultra.l0.ultra_utils import RATES_KEYS, ULTRA_RATES
 
 
-@pytest.fixture()
-def decom_test_data(ccsds_path, xtce_path):
-    """Data for decom"""
-    data_packet_list = decom.decom_packets(ccsds_path, xtce_path)
-    return data_packet_list
-
-
-@pytest.fixture()
-def decom_ultra(ccsds_path, xtce_path):
-    """Data for decom_ultra"""
-    data_packets = decom_ultra_apids(ccsds_path, xtce_path, ULTRA_RATES.apid[0])
-    return data_packets
-
-
-def test_image_rate_decom(decom_ultra, rates_test_path):
+@pytest.mark.parametrize(
+    "decom_test_data",
+    [
+        pytest.param(
+            {
+                "apid": ULTRA_RATES.apid[0],
+                "filename": "Ultra45_EM_SwRI_Cal_Run7_"
+                "ThetaScan_20220530T225054.CCSDS",
+            }
+        )
+    ],
+    indirect=True,
+)
+def test_image_rate_decom(decom_test_data, rates_test_path):
     """This function reads validation data and checks that decom data
     matches validation data for image rate packet"""
+    decom_ultra, _ = decom_test_data
 
     df = pd.read_csv(rates_test_path, index_col="MET")
     total_packets = 23

imap_processing/tests/ultra/unit/test_decom_apid_883.py

@@ -4,21 +4,28 @@
 import pandas as pd
 import pytest
 
-from imap_processing.ultra.l0.decom_ultra import decom_ultra_apids
 from imap_processing.ultra.l0.ultra_utils import ULTRA_TOF
 
-
-@pytest.fixture()
-def decom_ultra(ccsds_path_tof, xtce_path):
-    """Data for decom_ultra"""
-    data_packet_list = decom_ultra_apids(ccsds_path_tof, xtce_path, ULTRA_TOF.apid[0])
-    return data_packet_list
-
-
-def test_tof_decom(decom_ultra, tof_test_path):
+# TODO: discuss with instrument team incomplete set of SIDs
+
+
+@pytest.mark.parametrize(
+    "decom_test_data",
+    [
+        pytest.param(
+            {
+                "apid": ULTRA_TOF.apid[0],
+                "filename": "FM45_TV_Cycle6_Hot_Ops_" "Front212_20240124T063837.CCSDS",
+            }
+        )
+    ],
+    indirect=True,
+)
+def test_tof_decom(decom_test_data, tof_test_path):
     """This function reads validation data and checks that decom data
     matches validation data for image rate packet"""
 
+    decom_ultra, _ = decom_test_data
     df = pd.read_csv(tof_test_path, index_col="SequenceCount")
 
     np.testing.assert_array_equal(df.Spin, decom_ultra["SPIN"].flatten())