Hi test data and fixture refactor (#709)

* Move common and hi fixtures into plugins directory
Add pytest_plugins definition to root conftest.py
Refactor hi test_data structure
Add fixtures for hi test data

* Refactor to match typical pytest usage of nested conftest.py files for fixtures

imap_processing/tests/conftest.py

@@ -3,6 +3,8 @@
 import imap_data_access
 import pytest
 
+from imap_processing import imap_module_directory
+
 
 @pytest.fixture(autouse=True)
 def _set_global_config(monkeypatch, tmp_path):
@@ -13,19 +15,6 @@ def _set_global_config(monkeypatch, tmp_path):
     )
 
 
-def ccsds_header_data(apid, pkt_len):
-    """Create binary data for CCSDS header with apid provided."""
-    # CCSDS primary header
-    # 3 bits - Version number
-    # 1 bit - Packet type
-    # 1 bit - Secondary header flag
-    # 16 bits - APID
-    # 2 bits - Sequence flag
-    # 14 bits - Packet sequence count
-    # 16 bits - Packet length
-    return f"{0:03b}{0:01b}{1:01b}{apid:011b}{1:02b}{0:014b}{pkt_len:016b}"
-
-
-def check_sum(bits_size):
-    """Create check test sum."""
-    return f"{0:0{bits_size}b}"
+@pytest.fixture(scope="session")
+def imap_tests_path():
+    return imap_module_directory / "tests"

imap_processing/tests/hi/conftest.py

@@ -2,7 +2,15 @@
 
 import pytest
 
-from imap_processing.tests.conftest import ccsds_header_data, check_sum
+
+@pytest.fixture(scope="session")
+def hi_test_data_path(imap_tests_path):
+    return imap_tests_path / "hi" / "test_data"
+
+
+@pytest.fixture(scope="session")
+def hi_l0_test_data_path(hi_test_data_path):
+    return hi_test_data_path / "l0"
 
 
 def create_metaevent(esa_step, met_subseconds, met_seconds):
@@ -17,6 +25,24 @@ def create_directevent(tof_1, tof_2, tof_3, de_tag):
     return f"{start_bitmask_data:02b}{tof_1:010b}{tof_2:010b}{tof_3:010b}{de_tag:016b}"
 
 
+def ccsds_header_data(apid, pkt_len):
+    """Create binary data for CCSDS header with apid provided."""
+    # CCSDS primary header
+    # 3 bits - Version number
+    # 1 bit - Packet type
+    # 1 bit - Secondary header flag
+    # 16 bits - APID
+    # 2 bits - Sequence flag
+    # 14 bits - Packet sequence count
+    # 16 bits - Packet length
+    return f"{0:03b}{0:01b}{1:01b}{apid:011b}{1:02b}{0:014b}{pkt_len:016b}"
+
+
+def check_sum(bits_size):
+    """Create check test sum."""
+    return f"{0:0{bits_size}b}"
+
+
 @pytest.fixture()
 def create_de_data(tmp_path):
     """Fixture to create fake direct event data. Note that there has been no

imap_processing/tests/hi/test_decom.py

@@ -1,43 +1,38 @@
 import pandas as pd
 import pytest
 
-from imap_processing import imap_module_directory
 from imap_processing.hi.l0.decom_hi import decom_packets
 
 
 @pytest.fixture(scope="session")
-def decom_nhk_data():
+def decom_nhk_data(hi_l0_test_data_path):
     """Read test housekeeping data from test folder"""
-    test_folder_path = "tests/hi/l0_test_data"
-    packet_file = imap_module_directory / f"{test_folder_path}/20231030_H45_APP_NHK.bin"
-    return decom_packets(packet_file)
+    packet_file = hi_l0_test_data_path / "20231030_H45_APP_NHK.bin"
+    return list(decom_packets(packet_file))
 
 
 @pytest.fixture(scope="session")
-def nhk_validation_data():
+def nhk_validation_data(hi_l0_test_data_path):
     """Read in validation data from the CSV file"""
-    test_data_path = imap_module_directory / "tests/hi/l0_validation_data"
     raw_validation_data = pd.read_csv(
-        test_data_path / "20231030_H45_APP_NHK.csv",
+        hi_l0_test_data_path / "20231030_H45_APP_NHK.csv",
         index_col="CCSDS_MET",
     )
     return raw_validation_data
 
 
 @pytest.fixture(scope="session")
-def decom_sci_de_data():
+def decom_sci_de_data(hi_l0_test_data_path):
     """Read science direct event data from test folder"""
-    test_folder_path = "tests/hi/l0_test_data"
-    packet_file = imap_module_directory / f"{test_folder_path}/20231030_H45_SCI_DE.bin"
-    return decom_packets(packet_file)
+    packet_file = hi_l0_test_data_path / "20231030_H45_SCI_DE.bin"
+    return list(decom_packets(packet_file))
 
 
 @pytest.fixture(scope="session")
-def decom_sci_cnt_data():
+def decom_sci_cnt_data(hi_l0_test_data_path):
     """Read science count data from test folder"""
-    test_folder_path = "tests/hi/l0_test_data"
-    packet_file = imap_module_directory / f"{test_folder_path}/20231030_H45_SCI_CNT.bin"
-    return decom_packets(packet_file)
+    packet_file = hi_l0_test_data_path / "20231030_H45_SCI_CNT.bin"
+    return list(decom_packets(packet_file))
 
 
 def test_app_nhk_decom(decom_nhk_data):

imap_processing/tests/hi/test_hi_l1b.py

@@ -1,17 +1,15 @@
 """Test coverage for imap_processing.hi.l1b.hi_l1b.py"""
 
-from imap_processing import imap_module_directory
 from imap_processing.hi.l1a.hi_l1a import hi_l1a
 from imap_processing.hi.l1b.hi_l1b import hi_l1b
 from imap_processing.hi.utils import HIAPID
 
 
-def test_hi_l1b_hk():
+def test_hi_l1b_hk(hi_l0_test_data_path):
     """Test coverage for imap_processing.hi.hi_l1b.hi_l1b() with
     housekeeping L1A as input"""
     # TODO: once things are more stable, check in an L1A HK file as test data
-    test_path = imap_module_directory / "tests/hi/l0_test_data"
-    bin_data_path = test_path / "20231030_H45_APP_NHK.bin"
+    bin_data_path = hi_l0_test_data_path / "20231030_H45_APP_NHK.bin"
     data_version = "001"
     processed_data = hi_l1a(packet_file_path=bin_data_path, data_version=data_version)
 

imap_processing/tests/hi/test_l1a.py

@@ -1,6 +1,5 @@
 import numpy as np
 
-from imap_processing import imap_module_directory
 from imap_processing.cdf.utils import write_cdf
 from imap_processing.hi.l1a import histogram as hist
 from imap_processing.hi.l1a.hi_l1a import hi_l1a
@@ -43,12 +42,11 @@ def test_sci_de_decom(create_de_data):
     assert cdf_filepath.name == cdf_filename
 
 
-def test_app_nhk_decom():
+def test_app_nhk_decom(hi_l0_test_data_path):
     """Test housekeeping data"""
 
     # Unpack housekeeping data
-    test_path = imap_module_directory / "tests/hi/l0_test_data"
-    bin_data_path = test_path / "20231030_H45_APP_NHK.bin"
+    bin_data_path = hi_l0_test_data_path / "20231030_H45_APP_NHK.bin"
     processed_data = hi_l1a(packet_file_path=bin_data_path, data_version="001")
 
     assert np.unique(processed_data[0]["pkt_apid"].values) == HIAPID.H45_APP_NHK.value
@@ -64,10 +62,9 @@ def test_app_nhk_decom():
     assert cem_raw_cdf_filepath.name == "imap_hi_l1a_45sensor-hk_20100313_v001.cdf"
 
 
-def test_app_hist_decom():
+def test_app_hist_decom(hi_l0_test_data_path):
     """Test histogram (SCI_CNT) data"""
-    test_path = imap_module_directory / "tests/hi/l0_test_data"
-    bin_data_path = test_path / "20231030_H45_SCI_CNT.bin"
+    bin_data_path = hi_l0_test_data_path / "20231030_H45_SCI_CNT.bin"
     processed_data = hi_l1a(packet_file_path=bin_data_path, data_version="001")
 
     assert processed_data[0].attrs["Logical_source"] == "imap_hi_l1a_45sensor-hist"