diff --git a/docs/source/code-documentation/tools/xtce-generator.rst b/docs/source/code-documentation/tools/xtce-generator.rst index cc16f5066..27e386222 100644 --- a/docs/source/code-documentation/tools/xtce-generator.rst +++ b/docs/source/code-documentation/tools/xtce-generator.rst @@ -124,6 +124,13 @@ the conversion details. - ANALOG - Apply an analog conversion - + * - MY_INSTRUMENT_HK + - VARIABLE_ENUMERATED + - 1 + - UINT + - STATE + - Apply an enumeration state + - * - MY_INSTRUMENT_HK - VARIABLE_LENGTH_BINARY_SCIENCE - 100 @@ -164,3 +171,26 @@ coefficients defined from ``c0`` to ``c7`` to define the order of the polynomial - - - + +States tab (optional) +~~~~~~~~~~~~~~~~~~~~~ + +Packet parsing can also apply enumeration/state conversions to the data being read in. +For example, to change from a raw unsigned integer value to a "VALID" / "INVALID" string. +The ``States`` tab is used to define these enumerations. + +.. list-table:: States + :header-rows: 1 + + * - packetName + - mnemonic + - value + - state + * - MY_INSTRUMENT_HK + - VARIABLE_ENUMERATED + - 0 + - INVALID + * - MY_INSTRUMENT_HK + - VARIABLE_ENUMERATED + - 1 + - VALID diff --git a/imap_processing/ccsds/excel_to_xtce.py b/imap_processing/ccsds/excel_to_xtce.py index d34aa6d3a..e1521dea1 100644 --- a/imap_processing/ccsds/excel_to_xtce.py +++ b/imap_processing/ccsds/excel_to_xtce.py @@ -251,7 +251,6 @@ def _add_parameter(self, row: pd.Series, total_packet_bits: int) -> None: # Combine the packet name and mnemonic to create a unique parameter name name = f"{row['packetName']}.{row['mnemonic']}" parameter.attrib["name"] = name - # UINT8, ... parameter.attrib["parameterTypeRef"] = name # Add descriptions if they exist @@ -329,6 +328,10 @@ def _add_parameter(self, row: pd.Series, total_packet_bits: int) -> None: # Go look up the conversion in the AnalogConversions tab # and add it to the encoding self._add_analog_conversion(row, encoding) + elif row["convertAs"] == "STATE": + # Go look up the states in the States tab + # and add them to the parameter type + self._add_state_conversion(row, parameter_type) def _add_analog_conversion(self, row: pd.Series, encoding: Et.Element) -> None: """ @@ -363,6 +366,34 @@ def _add_analog_conversion(self, row: pd.Series, encoding: Et.Element) -> None: term.attrib["coefficient"] = str(conversion[col]) term.attrib["exponent"] = str(i) + def _add_state_conversion(self, row: pd.Series, parameter_type: Et.Element) -> None: + """ + Add a state conversion to the parameter type. + + Changing from an IntegerParameterType to an EnumeratedParameterType. Adding + the list of state mappings to the parameter type. + + Parameters + ---------- + row : pandas.Row + Row to be added to the XTCE file, containing mnemonic, packetName. + parameter_type : Element + The parameter type element to add the conversion to. + """ + # It is an EnumeratedParameterType rather than an IntegerParameterType + parameter_type.tag = "xtce:EnumeratedParameterType" + enumeration_list = Et.SubElement(parameter_type, "xtce:EnumerationList") + # Lookup the enumeration states for this parameter from the States sheet + state_sheet = self.sheets["States"] + state_sheet = state_sheet.loc[ + (state_sheet["packetName"] == row["packetName"]) + & (state_sheet["mnemonic"] == row["mnemonic"]) + ] + for _, state_row in state_sheet.iterrows(): + enumeration = Et.SubElement(enumeration_list, "xtce:Enumeration") + enumeration.attrib["value"] = str(state_row["value"]) + enumeration.attrib["label"] = str(state_row["state"]) + def to_xml(self, output_xml_path: Path) -> None: """ Create and output an XTCE file from the Element Tree representation. diff --git a/imap_processing/tests/ccsds/test_data/excel_to_xtce_test_file.xlsx b/imap_processing/tests/ccsds/test_data/excel_to_xtce_test_file.xlsx deleted file mode 100644 index 29f0910da..000000000 Binary files a/imap_processing/tests/ccsds/test_data/excel_to_xtce_test_file.xlsx and /dev/null differ diff --git a/imap_processing/tests/ccsds/test_data/expected_output.xml b/imap_processing/tests/ccsds/test_data/expected_output.xml index 187953349..ea6d378cf 100644 --- a/imap_processing/tests/ccsds/test_data/expected_output.xml +++ b/imap_processing/tests/ccsds/test_data/expected_output.xml @@ -48,7 +48,7 @@ - + @@ -59,6 +59,13 @@ + + + + + + + @@ -109,11 +116,14 @@ Float data + + State data + Mission elapsed time - - Variable 1 - long desc + + Variable 1 long description @@ -142,6 +152,7 @@ + diff --git a/imap_processing/tests/ccsds/test_excel_to_xtce.py b/imap_processing/tests/ccsds/test_excel_to_xtce.py index 062a1074c..bbd9b4c6c 100644 --- a/imap_processing/tests/ccsds/test_excel_to_xtce.py +++ b/imap_processing/tests/ccsds/test_excel_to_xtce.py @@ -4,6 +4,7 @@ from pathlib import Path from unittest import mock +import pandas as pd import pytest from imap_processing.ccsds import excel_to_xtce @@ -12,36 +13,245 @@ @pytest.fixture() -def excel_file(): - p = Path(__file__).parent / "test_data" / "excel_to_xtce_test_file.xlsx" - return p +def xtce_excel_file(tmp_path): + """Create an excel file for testing. + + Dataframes for each tab of the spreadsheet that then get written to an excel file. + """ + # Create a pandas DataFrame for global attributes + subsystem = { + "infoField": ["subsystem", "sheetReleaseDate", "sheetReleaseRev"], + "infoValue": ["Test Instrument", "2024-07-26 00:00:00", "v1.2"], + } + + packets = {"packetName": ["TEST_PACKET", "TEST_PACKET2"], "apIdHex": ["0x1", "0xF"]} + + test_packet1 = { + "packetName": ["TEST_PACKET"] * 15, + "mnemonic": [ + "PHVERNO", + "PHTYPE", + "PHSHF", + "PHAPID", + "PHGROUPF", + "PHSEQCNT", + "PHDLEN", + "SHCOARSE", + "VAR_UINT", + "VAR_INT", + "VAR_SINT", + "VAR_BYTE", + "VAR_FILL", + "VAR_FLOAT", + "VAR_STATE", + ], + "lengthInBits": [3, 1, 1, 11, 2, 14, 16, 32, 2, 4, 5, 10000, 3, 32, 1], + "dataType": [ + "UINT", + "UINT", + "UINT", + "UINT", + "UINT", + "UINT", + "UINT", + "UINT", + "UINT", + "INT", + "SINT", + "BYTE", + "FILL", + "FLOAT", + "UINT", + ], + "convertAs": [ + "NONE", + "NONE", + "NONE", + "NONE", + "NONE", + "NONE", + "NONE", + "NONE", + "ANALOG", + "NONE", + "NONE", + "NONE", + "NONE", + "NONE", + "STATE", + ], + "units": [ + "DN", + "DN", + "DN", + "DN", + "DN", + "DN", + "DN", + "DN", + "DN", + "DN", + "DN", + "DN", + "DN", + "DN", + "DN", + ], + "longDescription": [ + "CCSDS Packet Version Number", + "CCSDS Packet Type Indicator", + "CCSDS Packet Secondary Header Flag", + "CCSDS Packet Application Process ID", + "CCSDS Packet Grouping Flags", + "CCSDS Packet Sequence Count", + "CCSDS Packet Length", + "Mission elapsed time", + "Unsgned integer data with conversion", + "Integer data", + "Signed integer data", + "Binary data - variable length", + "Fill data", + "Float data", + "State data", + ], + } + + test_packet2 = { + "packetName": ["TEST_PACKET2"] * 9, + "mnemonic": [ + "PHVERNO", + "PHTYPE", + "PHSHF", + "PHAPID", + "PHGROUPF", + "PHSEQCNT", + "PHDLEN", + "SHCOARSE", + "VAR1", + ], + "lengthInBits": [3, 1, 1, 11, 2, 14, 16, 32, 2], + "dataType": [ + "UINT", + "UINT", + "UINT", + "UINT", + "UINT", + "UINT", + "UINT", + "UINT", + "UINT", + ], + "convertAs": [ + "NONE", + "NONE", + "NONE", + "NONE", + "NONE", + "NONE", + "NONE", + "NONE", + "NONE", + ], + "units": ["DN", "DN", "DN", "DN", "DN", "DN", "DN", "DN", "DN"], + "longDescription": [ + "CCSDS Packet Version Number", + "CCSDS Packet Type Indicator", + "CCSDS Packet Secondary Header Flag", + "CCSDS Packet Application Process ID", + "CCSDS Packet Grouping Flags", + "CCSDS Packet Sequence Count", + "CCSDS Packet Length", + "Mission elapsed time", + "Variable 1 long description", + ], + "shortDescription": [ + "", + "", + "", + "", + "", + "", + "", + "", + "Variable 1 short description", + ], + } + + analog_conversions = { + "packetName": ["TEST_PACKET"], + "mnemonic": ["VAR_UINT"], + "convertAs": ["UNSEGMENTED_POLY"], + "segNumber": [1], + "lowValue": [0], + "highValue": [100], + "c0": [1.5], + "c1": [2.5], + "c2": [0], + "c3": [0], + "c4": [0], + "c5": [0], + "c6": [0], + "c7": [0], + } + + states = { + "packetName": ["TEST_PACKET"] * 2, + "mnemonic": ["VAR_STATE"] * 2, + "value": [0, 1], + "state": ["OFF", "ON"], + } + + # Write the DataFrame to an excel file + excel_path = tmp_path / "excel_to_xtce_test_file.xlsx" + excel_file = pd.ExcelWriter(excel_path, engine="openpyxl") + + pd.DataFrame(subsystem).to_excel(excel_file, sheet_name="Subsystem", index=False) + pd.DataFrame(packets).to_excel(excel_file, sheet_name="Packets", index=False) + pd.DataFrame(test_packet1).to_excel( + excel_file, sheet_name="TEST_PACKET", index=False + ) + # Test P_ version of sheet name as well + pd.DataFrame(test_packet2).to_excel( + excel_file, sheet_name="P_TEST_PACKET2", index=False + ) + pd.DataFrame(analog_conversions).to_excel( + excel_file, sheet_name="AnalogConversions", index=False + ) + pd.DataFrame(states).to_excel(excel_file, sheet_name="States", index=False) + + # Write the file to disk + excel_file.close() + + return excel_path -def test_generated_xml(excel_file, tmp_path): +def test_generated_xml(xtce_excel_file): """Make sure we are producing the expected contents within the XML file. To produce a new expected output file the following command can be used. imap_xtce imap_processing/tests/ccsds/test_data/excel_to_xtce_test_file.xlsx --output imap_processing/tests/ccsds/test_data/expected_output.xml """ - generator = excel_to_xtce.XTCEGenerator(excel_file) - output_file = tmp_path / "output.xml" + generator = excel_to_xtce.XTCEGenerator(xtce_excel_file) + output_file = xtce_excel_file.parent / "output.xml" generator.to_xml(output_file) - expected_file = excel_file.parent / "expected_output.xml" + expected_file = Path(__file__).parent / "test_data/expected_output.xml" + # Uncomment this line if you want to re-create the expected output file + # generator.to_xml(expected_file) with open(output_file) as f, open(expected_file) as f_expected: assert f.read() == f_expected.read() # General test @mock.patch("imap_processing.ccsds.excel_to_xtce.XTCEGenerator") -def test_main_general(mock_input, excel_file): +def test_main_general(mock_input, xtce_excel_file): """Testing base main function.""" test_args = [ "test_script", "--output", "swe.xml", - f"{excel_file}", + f"{xtce_excel_file}", ] with mock.patch.object(sys, "argv", test_args): excel_to_xtce.main()