diff --git a/imap_processing/ccsds/excel_to_xtce.py b/imap_processing/ccsds/excel_to_xtce.py new file mode 100644 index 000000000..69ca2c93b --- /dev/null +++ b/imap_processing/ccsds/excel_to_xtce.py @@ -0,0 +1,433 @@ +""" +Convert an Excel file of packet definitions into the XTCE format. + +This script reads in an Excel file containing packet definitions and converts +them into an XTCE file. + +.. code:: + imap_xtce /path/to/excel_file.xlsx --output /path/to/output.xml +""" + +import argparse +import xml.etree.ElementTree as Et +from importlib.util import find_spec +from pathlib import Path + +import pandas as pd + +_CCSDS_PARAMETERS = [ + { + "name": "VERSION", + "lengthInBits": 3, + "description": "CCSDS Packet Version Number (always 0)", + }, + { + "name": "TYPE", + "lengthInBits": 1, + "description": "CCSDS Packet Type Indicator (0=telemetry)", + }, + { + "name": "SEC_HDR_FLG", + "lengthInBits": 1, + "description": "CCSDS Packet Secondary Header Flag (always 1)", + }, + { + "name": "PKT_APID", + "lengthInBits": 11, + "description": "CCSDS Packet Application Process ID", + }, + { + "name": "SEQ_FLGS", + "lengthInBits": 2, + "description": "CCSDS Packet Grouping Flags (3=not part of group)", + }, + { + "name": "SRC_SEQ_CTR", + "lengthInBits": 14, + "description": "CCSDS Packet Sequence Count " + "(increments with each new packet)", + }, + { + "name": "PKT_LEN", + "lengthInBits": 16, + "description": "CCSDS Packet Length " + "(number of bytes after Packet length minus 1)", + }, +] + + +class XTCEGenerator: + """ + Automatically generate XTCE files from excel definition files. + + The excel file should have the following columns: mnemonic, sequence, lengthInBits, + startBit, dataType, convertAs, units, source, and either shortDescription or + longDescription. + + This class will correctly generate a CCSDS header and the provided data types. + + It is intended for use as a first pass of XTCE generation, and most cases, the + packet definitions will require manual updates. + + Use ``to_xml`` to write the output xml file. + + Parameters + ---------- + path_to_excel_file : Path + Path to the excel file. + """ + + def __init__(self, path_to_excel_file: Path): + # Read in all sheets from the excel file + self.sheets = pd.read_excel(path_to_excel_file, sheet_name=None) + # Set up the packet mapping from packetName to Apid + packet_sheet = self.sheets["Packets"] + if "apId" not in packet_sheet.columns: + # Create the apId column from the apIdHex (base=0 works with the 0x prefix) + packet_sheet["apId"] = packet_sheet["apIdHex"].apply(int, base=0) + self._packet_mapping = packet_sheet.set_index("packetName")["apId"].to_dict() + + # Create the XML containers that will be populated later + self._setup_xml_containers() + # Add the CCSDS Header information to the containers + self._setup_ccsds_header() + # Create the sequence containers (also adding parameters within) + self._create_container_sets() + + def _setup_xml_containers(self) -> None: + """Create an XML representation of telemetry data.""" + # Register the XML namespace + source_link = "http://www.omg.org/space/xtce" + Et.register_namespace("xtce", source_link) + + # Create the root element and add namespaces + root = Et.Element("xtce:SpaceSystem") + self._root = root + root.attrib["xmlns:xtce"] = source_link + # Subsystem sheet name is used as the base name for this XTCE definition + subsystem = self.sheets["Subsystem"] + root.attrib["name"] = str( + subsystem.loc[subsystem["infoField"] == "subsystem", "infoValue"].values[0] + ) + # Create the Header element with attributes 'date', 'version', and 'author' + # Versioning is used to keep track of changes to the XML file. + header = Et.SubElement(root, "xtce:Header") + header.attrib["date"] = str( + subsystem.loc[ + subsystem["infoField"] == "sheetReleaseDate", "infoValue" + ].values[0] + ) + header.attrib["version"] = str( + subsystem.loc[ + subsystem["infoField"] == "sheetReleaseRev", "infoValue" + ].values[0] + ) + header.attrib["author"] = "IMAP SDC" + + # Create the TelemetryMetaData element + self._telemetry_metadata = Et.SubElement(root, "xtce:TelemetryMetaData") + + # Create the ParameterTypeSet element + self._parameter_type_set = Et.SubElement( + self._telemetry_metadata, "xtce:ParameterTypeSet" + ) + + # Create the ParameterSet element + self._parameter_set = Et.SubElement( + self._telemetry_metadata, "xtce:ParameterSet" + ) + + # Create ContainerSet element + self._container_sets = Et.SubElement( + self._telemetry_metadata, "xtce:ContainerSet" + ) + + def _setup_ccsds_header(self) -> None: + """Fill in the default CCSDS header information.""" + # Create CCSDSPacket SequenceContainer + ccsds_container = Et.SubElement(self._container_sets, "xtce:SequenceContainer") + ccsds_container.attrib["name"] = "CCSDSPacket" + ccsds_container.attrib["abstract"] = "true" + ccsds_entry_list = Et.SubElement(ccsds_container, "xtce:EntryList") + + # Populate EntryList for CCSDSPacket SequenceContainer + for parameter_data in _CCSDS_PARAMETERS: + parameter_ref_entry = Et.SubElement( + ccsds_entry_list, "xtce:ParameterRefEntry" + ) + name = str(parameter_data["name"]) + + parameter_ref_entry.attrib["parameterRef"] = name + + # Add the parameter to the ParameterSet + parameter = Et.SubElement(self._parameter_set, "xtce:Parameter") + parameter.attrib["name"] = name + parameter.attrib["parameterTypeRef"] = name + + description = Et.SubElement(parameter, "xtce:LongDescription") + description.text = str(parameter_data["description"]) + + # Add the typeref to the parameter type set + parameter_type = Et.SubElement( + self._parameter_type_set, "xtce:IntegerParameterType" + ) + parameter_type.attrib["name"] = name + parameter_type.attrib["signed"] = "false" + + encoding = Et.SubElement(parameter_type, "xtce:IntegerDataEncoding") + encoding.attrib["sizeInBits"] = str(parameter_data["lengthInBits"]) + encoding.attrib["encoding"] = "unsigned" + + def _create_container_sets(self) -> None: + """Create a container set for each packet in the Excel file.""" + # Iterate over all packets and create Packet SequenceContainers + for packet_name, apid in self._packet_mapping.items(): + # Populate EntryList for packet SequenceContainers + # The sheets are sometimes prefixed with P_, so we need to try both options + try: + packet_df = self.sheets[packet_name] + except KeyError: + try: + packet_df = self.sheets[f"P_{packet_name}"] + except KeyError: + print( + f"Packet definition for {packet_name} " + "not found in the excel file." + ) + continue + + # Create Packet SequenceContainer that use the CCSDSPacket SequenceContainer + # as the base container + science_container = Et.SubElement( + self._container_sets, "xtce:SequenceContainer" + ) + science_container.attrib["name"] = packet_name + + # Every container should inherit from the base container, CCSDSPacket + base_container = Et.SubElement(science_container, "xtce:BaseContainer") + base_container.attrib["containerRef"] = "CCSDSPacket" + + # Add RestrictionCriteria element to use the given APID for comparison + restriction_criteria = Et.SubElement( + base_container, "xtce:RestrictionCriteria" + ) + comparison = Et.SubElement(restriction_criteria, "xtce:Comparison") + comparison.attrib["parameterRef"] = "PKT_APID" + comparison.attrib["value"] = str(apid) + comparison.attrib["useCalibratedValue"] = "false" + + packet_entry_list = Et.SubElement(science_container, "xtce:EntryList") + # Needed for dynamic binary packet length + total_packet_bits = int(packet_df["lengthInBits"].sum()) + for i, row in packet_df.iterrows(): + if i < 7: + # Skip first 7 rows as they are the CCSDS header elements + continue + if pd.isna(row.get("packetName")): + # This is a poorly formatted row, skip it + continue + name = f"{row['packetName']}_{row['mnemonic']}" + parameter_ref_entry = Et.SubElement( + packet_entry_list, "xtce:ParameterRefEntry" + ) + parameter_ref_entry.attrib["parameterRef"] = name + # Add this parameter to the ParameterSet too + self._add_parameter(row, total_packet_bits) + + def _add_parameter(self, row: pd.Series, total_packet_bits: int) -> None: + """ + Row from a packet definition to be added to the XTCE file. + + Parameters + ---------- + row : pandas.Row + Row to be added to the XTCE file, containing mnemonic, lengthInBits, ... + total_packet_bits : int + Total number of bits in the packet, as summed from the lengthInBits column. + """ + parameter = Et.SubElement(self._parameter_set, "xtce:Parameter") + # Combine the packet name and mnemonic to create a unique parameter name + name = f"{row['packetName']}_{row['mnemonic']}" + parameter.attrib["name"] = name + # UINT8, ... + parameter.attrib["parameterTypeRef"] = name + + # Add descriptions if they exist + if pd.notna(row.get("shortDescription")): + parameter.attrib["shortDescription"] = row.get("shortDescription") + if pd.notna(row.get("longDescription")): + description = Et.SubElement(parameter, "xtce:LongDescription") + description.text = row.get("longDescription") + + length_in_bits = int(row["lengthInBits"]) + + # Add the parameterTypeRef for this row + if "UINT" in row["dataType"]: + parameter_type = Et.SubElement( + self._parameter_type_set, "xtce:IntegerParameterType" + ) + parameter_type.attrib["name"] = name + parameter_type.attrib["signed"] = "false" + + encoding = Et.SubElement(parameter_type, "xtce:IntegerDataEncoding") + encoding.attrib["sizeInBits"] = str(length_in_bits) + encoding.attrib["encoding"] = "unsigned" + + elif any(x in row["dataType"] for x in ["SINT", "INT"]): + parameter_type = Et.SubElement( + self._parameter_type_set, "xtce:IntegerParameterType" + ) + parameter_type.attrib["name"] = name + parameter_type.attrib["signed"] = "true" + encoding = Et.SubElement(parameter_type, "xtce:IntegerDataEncoding") + encoding.attrib["sizeInBits"] = str(length_in_bits) + encoding.attrib["encoding"] = "signed" + + elif "BYTE" in row["dataType"]: + parameter_type = Et.SubElement( + self._parameter_type_set, "xtce:BinaryParameterType" + ) + parameter_type.attrib["name"] = name + + encoding = Et.SubElement(parameter_type, "xtce:BinaryDataEncoding") + encoding.attrib["bitOrder"] = "mostSignificantBitFirst" + + size_in_bits = Et.SubElement(encoding, "xtce:SizeInBits") + + # If it is a byte field consider it a dynamic value. + dynamic_value = Et.SubElement(size_in_bits, "xtce:DynamicValue") + param_ref = Et.SubElement(dynamic_value, "xtce:ParameterInstanceRef") + param_ref.attrib["parameterRef"] = "PKT_LEN" + linear_adjustment = Et.SubElement(dynamic_value, "xtce:LinearAdjustment") + linear_adjustment.attrib["slope"] = str(8) + # The length of all other variables (other than this specific one) + other_variable_bits = total_packet_bits - length_in_bits + # PKT_LEN == number of bytes in the packet data field - 1 + # So we need to subtract the header bytes plus 1 to get the offset + # The amount to subtract to get the intercept is then: + # number of other bits in the packet - (6 + 1) * 8 + linear_adjustment.attrib["intercept"] = str(-int(other_variable_bits - 56)) + + # TODO: Do we want to allow fixed length values? + # fixed_value = Et.SubElement(size_in_bits, "xtce:FixedValue") + # fixed_value.text = str(row["lengthInBits"]) + + if row["convertAs"] == "ANALOG": + # Go look up the conversion in the AnalogConversions tab + # and add it to the encoding + self._add_analog_conversion(row, encoding) + + def _add_analog_conversion(self, row: pd.Series, encoding: Et.Element) -> None: + """ + Add an analog conversion to the encoding element. + + Parameters + ---------- + row : pandas.Row + Row to be added to the XTCE file, containing mnemonic, packetName. + encoding : Element + The encoding element to add the conversion to. + """ + # Look up the conversion in the AnalogConversions tab + analog_conversion = self.sheets["AnalogConversions"] + # conversion is a row from the AnalogConversions sheet + conversion = analog_conversion.loc[ + (analog_conversion["mnemonic"] == row["mnemonic"]) + & (analog_conversion["packetName"] == row["packetName"]) + ].iloc[0] + + # Create the Conversion element + default_calibrator = Et.SubElement(encoding, "xtce:DefaultCalibrator") + polynomial_calibrator = Et.SubElement( + default_calibrator, "xtce:PolynomialCalibrator" + ) + # FIXME: Use lowValue / highValue from the conversion sheet + # FIXME: Handle segmented polynomials (only using first segment now) + for i in range(8): + col = f"c{i}" + if conversion[col] != 0: + term = Et.SubElement(polynomial_calibrator, "xtce:Term") + term.attrib["coefficient"] = str(conversion[col]) + term.attrib["exponent"] = str(i) + + def to_xml(self, output_xml_path: Path) -> None: + """ + Create and output an XTCE file from the Element Tree representation. + + Parameters + ---------- + output_xml_path : Path + Path to the output XML file. + """ + # Create the XML tree and save the document + tree = Et.ElementTree(self._root) + Et.indent(tree, space="\t", level=0) + + # Use the provided output_xml_path + tree.write(output_xml_path, encoding="utf-8", xml_declaration=True) + + +# Function to parse command line arguments +def _parse_args() -> argparse.Namespace: + """ + Parse the command line arguments. + + The expected input format is a required argument of "/path/to/excel_file.xlsx" + with an optional argument containing the output path for the XTCE file + "/path/to/output.xml". + + Returns + ------- + args : argparse.Namespace + An object containing the parsed arguments and their values. + """ + description = ( + "This command line program generates an instrument specific XTCE file. " + "Example usage: imap_xtce " + "path/to/excel_packet_file.xlsx --output path/to/output_packet_definition.xml" + ) + output_help = ( + "Where to save the output XTCE file. " + "If not provided, the input file name will be used with a " + ".xml extension." + ) + file_path_help = "Provide the full path to the input excel file." + + parser = argparse.ArgumentParser(prog="imap_xtce", description=description) + parser.add_argument("excel_file", type=Path, help=file_path_help) + parser.add_argument("--output", type=Path, required=False, help=output_help) + + if not find_spec("openpyxl"): + parser.error( + "The openpyxl package is required for this script. " + "Please install it using 'pip install openpyxl'." + ) + + args = parser.parse_args() + + if not args.excel_file.exists(): + parser.error(f"File not found: {args.excel_file}") + + if not args.output: + args.output = args.excel_file.with_suffix(".xml") + + return args + + +def main() -> None: + """ + Generate xtce file from CLI information given. + + The xtce file will be written in an instrument specific subfolder. + """ + # Parse arguments from the command line + args = _parse_args() + + xtce_generator = XTCEGenerator( + path_to_excel_file=args.excel_file, + ) + xtce_generator.to_xml(args.output) + + +if __name__ == "__main__": + main() diff --git a/imap_processing/tests/ccsds/test_excel_to_xtce.py b/imap_processing/tests/ccsds/test_excel_to_xtce.py new file mode 100644 index 000000000..efecee5ad --- /dev/null +++ b/imap_processing/tests/ccsds/test_excel_to_xtce.py @@ -0,0 +1,60 @@ +"""Testing for xtce generator template.""" + +import sys +from pathlib import Path +from unittest import mock + +import pytest + +from imap_processing.ccsds import excel_to_xtce + +pytest.importorskip("openpyxl") + + +@pytest.fixture() +def filepath(tmpdir): + p = Path(tmpdir / "test_file.xlsx").resolve() + p.touch() + return p + + +# General test +@mock.patch("imap_processing.ccsds.excel_to_xtce.XTCEGenerator") +def test_main_general(mock_input, filepath): + """Testing base main function.""" + test_args = [ + "test_script", + "--output", + "swe.xml", + f"{filepath}", + ] + with mock.patch.object(sys, "argv", test_args): + excel_to_xtce.main() + mock_input.assert_called_once() + + +# Testing without required arguments +def test_main_inval_arg(): + """Testing with invalid instrument.""" + test_args = [ + "test_script", + "--output", + "glows.xml", + ] + with mock.patch.object(sys, "argv", test_args): + with pytest.raises(SystemExit): + excel_to_xtce.main() + + +# File does not exist +def test_main_inval_file(): + """Testing with invalid file.""" + test_args = [ + "test_script", + "--instrument", + "glows", + "not-a-valid-file.txt", + ] + with mock.patch.object(sys, "argv", test_args): + with pytest.raises(SystemExit): + excel_to_xtce.main() diff --git a/poetry.lock b/poetry.lock index 88066b2e0..e24e2123e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1143,6 +1143,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1150,8 +1151,16 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1168,6 +1177,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1175,6 +1185,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, diff --git a/pyproject.toml b/pyproject.toml index e353ddaea..a84c2e0ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,14 +86,13 @@ lint.ignore = ["D104", "PLR2004", "RUF200", "S311"] # S603 unchecked input in subprocess call is fine in our tests # PT006 Wrong type passed to first argument of @pytest.mark.parametrize "*/tests/*" = ["D", "S101", "S603", "PT006"] -"tools/xtce*" = ["D"] [tool.ruff.lint.pydocstyle] convention = "numpy" [tool.poetry.scripts] imap_cli = 'imap_processing.cli:main' -imap_xtce = 'tools.xtce_generation.xtce_generator_template:main' +imap_xtce = 'imap_processing.ccsds.excel_to_xtce:main' [tool.codespell] ignore-words-list = "livetime" diff --git a/tools/tests/test_xtce_generator_template.py b/tools/tests/test_xtce_generator_template.py deleted file mode 100644 index 1b95d298c..000000000 --- a/tools/tests/test_xtce_generator_template.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Testing for xtce generator template.""" - -import sys -from pathlib import Path -from unittest import mock - -import pytest - -from tools.xtce_generation.xtce_generator_template import main - - -# Create filepath for testing -def filepath(file_name: str): - file_path = Path(__file__).parent.parent / "xtce_generation" / file_name - return str(file_path) - - -# General test -@mock.patch("tools.xtce_generation.xtce_generator_template.main") -def test_main_general(mock_input): - """Testing base main function.""" - test_args = [ - "test_script", - "--instrument", - "swe", - "--file-path", - f"{filepath('TLM_SWP_20231006-121021.xlsx')}", - "--packets", - '{"P_SWP_HK": 1184, "P_SWP_SCI": 1188, "P_SWP_AUT": 1192}', - ] - with mock.patch.object(sys, "argv", test_args): - main() - - -# Testing invalid instrument -@mock.patch("tools.xtce_generation.xtce_generator_template.main") -def test_main_inval_instr(mock_input): - """Testing with invalid instrument.""" - test_args = [ - "test_script", - "--instrument", - "ana", - "--file-path", - f"{filepath('TLM_SWP_20231006-121021.xlsx')}", - "--packets", - '{"P_SWP_HK": 1184, "P_SWP_SCI": 1188, "P_SWP_AUT": 1192}', - ] - with mock.patch.object(sys, "argv", test_args): - with pytest.raises( - ValueError, match="ana is not in the supported instrument list:" - ): - main() - - -# Testing without required arguments -@mock.patch("tools.xtce_generation.xtce_generator_template.main") -def test_main_inval_arg(mock_input): - """Testing with invalid instrument.""" - test_args = [ - "test_script", - "--instrument", - "glows", - "--packets", - '{"P_SWP_HK": 1184, "P_SWP_SCI": 1188, "P_SWP_AUT": 1192}', - ] - with mock.patch.object(sys, "argv", test_args): - with pytest.raises(SystemExit): - main() - - -# File does not exist -@mock.patch("tools.xtce_generation.xtce_generator_template.main") -def test_main_inval_file(mock_input): - """Testing with invalid file.""" - test_args = [ - "test_script", - "--instrument", - "glows", - "--file-path", - "This file is silly", - "--packets", - '{"P_SWP_HK": 1184, "P_SWP_SCI": 1188, "P_SWP_AUT": 1192}', - ] - with mock.patch.object(sys, "argv", test_args): - with pytest.raises(FileNotFoundError): - main() diff --git a/tools/xtce_generation/26850.02-TLMDEF-04.xlsx b/tools/xtce_generation/26850.02-TLMDEF-04.xlsx deleted file mode 100644 index 44df5af1a..000000000 Binary files a/tools/xtce_generation/26850.02-TLMDEF-04.xlsx and /dev/null differ diff --git a/tools/xtce_generation/TLM_SWP_20231006-121021.xlsx b/tools/xtce_generation/TLM_SWP_20231006-121021.xlsx deleted file mode 100644 index d7a9bb0f1..000000000 Binary files a/tools/xtce_generation/TLM_SWP_20231006-121021.xlsx and /dev/null differ diff --git a/tools/xtce_generation/ccsds_header_xtce_generator.py b/tools/xtce_generation/ccsds_header_xtce_generator.py deleted file mode 100644 index 4de32474e..000000000 --- a/tools/xtce_generation/ccsds_header_xtce_generator.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -A class representing CCSDS packet parameters. - -This class defines the parameters commonly found in CCSDS telemetry packets. -Each parameter consists of a name, parameter type reference, and description. -""" - - -class CCSDSParameters: - """The defined class for CCSDS parameters. Contains the object itself.""" - - def __init__(self) -> None: - self.parameters = [ - { - "name": "VERSION", - "parameterTypeRef": "UINT3", - "description": "CCSDS Packet Version Number (always 0)", - }, - { - "name": "TYPE", - "parameterTypeRef": "UINT1", - "description": "CCSDS Packet Type Indicator (0=telemetry)", - }, - { - "name": "SEC_HDR_FLG", - "parameterTypeRef": "UINT1", - "description": "CCSDS Packet Secondary Header Flag (always 1)", - }, - { - "name": "PKT_APID", - "parameterTypeRef": "UINT11", - "description": "CCSDS Packet Application Process ID", - }, - { - "name": "SEQ_FLGS", - "parameterTypeRef": "UINT2", - "description": "CCSDS Packet Grouping Flags (3=not part of group)", - }, - { - "name": "SRC_SEQ_CTR", - "parameterTypeRef": "UINT14", - "description": "CCSDS Packet Sequence Count " - "(increments with each new packet)", - }, - { - "name": "PKT_LEN", - "parameterTypeRef": "UINT16", - "description": "CCSDS Packet Length " - "(number of bytes after Packet length minus 1)", - }, - ] - - -# Other utility functions related to CCSDS parameters can also be added here diff --git a/tools/xtce_generation/telemetry_generator.py b/tools/xtce_generation/telemetry_generator.py deleted file mode 100644 index 2d7560598..000000000 --- a/tools/xtce_generation/telemetry_generator.py +++ /dev/null @@ -1,393 +0,0 @@ -"""Class Creation for TelemetryGenerator.""" - -import xml.etree.ElementTree as Et -from datetime import datetime -from typing import Optional - -import pandas as pd - -from tools.xtce_generation.ccsds_header_xtce_generator import CCSDSParameters - - -class TelemetryGenerator: - """ - This class will automatically generate XTCE files from excel definition files. - - The excel file should have the following columns: mnemonic, sequence, lengthInBits, - startBit, dataType, convertAs, units, source, and either shortDescription or - longDescription. - - This class will correctly generate a CCSDS header and the provided data types. - It doesn't handle the following cases: - - Enum generation - - Variable packet lengths - - Multiple APIDs or complex APID comparison - - It is intended for use as a first pass of XTCE generation, and most cases, the - packet definitions will require manual updates. - - Use generate_telemetry_xml to create XML files. - - Parameters - ---------- - packet_name : str - Name of packet. - path_to_excel_file : dict todo check - Path to the excel file. - apid : int todo check - The application ID associated with the telemetry packet. - pkt : str todo check - Default set to None. - """ - - def __init__( - self, - packet_name: str, - path_to_excel_file: str, - apid: int, - pkt: Optional[str] = None, - ) -> None: - """Initialize TelemetryGenerator.""" - self.packet_name = packet_name - self.apid = apid - self.source_link = "http://www.omg.org/space/xtce" - - if pkt is None: - # Read excel sheet - xls = pd.ExcelFile(path_to_excel_file) - self.pkt = xls.parse(packet_name) - else: - self.pkt = pkt - - def create_telemetry_xml(self) -> tuple: - """ - Create an XML representation of telemetry data based on input parameters. - - Returns - ------- - root - The root element of the generated XML tree. - parameter_type_set - The ParameterTypeSet element. - parameter_set - The ParameterSet element. - telemetry_metadata - The TelemetryMetaData element. - """ - # Register the XML namespace - Et.register_namespace("xtce", self.source_link) - - # Get the current date and time - current_date = datetime.now().strftime("%Y-%m") - - # Create the root element and add namespaces - root = Et.Element("xtce:SpaceSystem") - root.attrib["xmlns:xtce"] = f"{self.source_link}" - root.attrib["name"] = self.packet_name - # xmlns:xtce="http://www.omg.org/space/xtce" - - # Create the Header element with attributes 'date', 'version', and 'author' - # Versioning is used to keep track of changes to the XML file. - header = Et.SubElement(root, "xtce:Header") - header.attrib["date"] = current_date - header.attrib["version"] = "1.0" - header.attrib["author"] = "IMAP SDC" - - # Create the TelemetryMetaData element - telemetry_metadata = Et.SubElement(root, "xtce:TelemetryMetaData") - - # Create the ParameterTypeSet element - parameter_type_set = Et.SubElement(telemetry_metadata, "xtce:ParameterTypeSet") - - # Create the ParameterSet element - parameter_set = Et.SubElement(telemetry_metadata, "xtce:ParameterSet") - - return root, parameter_type_set, parameter_set, telemetry_metadata - - def get_unique_bits_length(self) -> dict: - """ - Create dictionary. - - Get unique values from the 'lengthInBits' column and create dictionary - with key and value using the dataType and lengthInBits. - - Returns - ------- - unique_lengths : dict - Dictionary containing all unique bits lengths. - - Examples - -------- - { - 'UINT16': 16, - 'UINT32': 32, - 'BYTE13000': 13000, - } - """ - # Extract unique values from the 'lengthInBits' column - length_in_bits = self.pkt["lengthInBits"] - data_types = self.pkt["dataType"] - unique_lengths = {} - for index in range(len(length_in_bits)): - # Here, we are creating a dictionary like this: - # { - # 'UINT16': 16, - # 'UINT32': 32, - # 'BYTE13000': 13000, - # } - unique_lengths[f"{data_types[index]}{length_in_bits[index]}"] = ( - length_in_bits[index] - ) - # Sort the dictionary by the value (lengthInBits) - unique_lengths = dict(sorted(unique_lengths.items(), key=lambda item: item[1])) - return unique_lengths - - def create_parameter_types( - self, - parameter_type_set: Et.Element, - unique_lengths: dict, - ) -> Et.Element: - """ - Create parameter types based on 'dataType' for the unique 'lengthInBits' values. - - This will loop through the unique lengths and create a ParameterType element - for each length representing a data type. - - Parameters - ---------- - parameter_type_set : Et.Element - The ParameterTypeSet element where parameter types are. - unique_lengths : dict - Unique values from the 'lengthInBits' column. - - Returns - ------- - parameter_type_set : Et.Element - The updated ParameterTypeSet element. - """ - for parameter_type_ref_name, size in unique_lengths.items(): - if "UINT" in parameter_type_ref_name: - parameter_type = Et.SubElement( - parameter_type_set, - "xtce:IntegerParameterType", - ) - parameter_type.attrib["name"] = parameter_type_ref_name - parameter_type.attrib["signed"] = "false" - - encoding = Et.SubElement(parameter_type, "xtce:IntegerDataEncoding") - encoding.attrib["sizeInBits"] = str(size) - encoding.attrib["encoding"] = "unsigned" - - elif any(x in parameter_type_ref_name for x in ["SINT", "INT"]): - parameter_type = Et.SubElement( - parameter_type_set, - "xtce:IntegerParameterType", - ) - parameter_type.attrib["name"] = parameter_type_ref_name - parameter_type.attrib["signed"] = "true" - encoding = Et.SubElement(parameter_type, "xtce:IntegerDataEncoding") - encoding.attrib["sizeInBits"] = str(size) - encoding.attrib["encoding"] = "signed" - - elif "BYTE" in parameter_type_ref_name: - binary_parameter_type = Et.SubElement( - parameter_type_set, - "xtce:BinaryParameterType", - ) - binary_parameter_type.attrib["name"] = parameter_type_ref_name - - Et.SubElement(binary_parameter_type, "xtce:UnitSet") - - binary_data_encoding = Et.SubElement( - binary_parameter_type, "xtce:BinaryDataEncoding" - ) - binary_data_encoding.attrib["bitOrder"] = "mostSignificantBitFirst" - - size_in_bits = Et.SubElement(binary_data_encoding, "xtce:SizeInBits") - fixed_value = Et.SubElement(size_in_bits, "xtce:FixedValue") - fixed_value.text = str(size) - - return parameter_type_set - - def create_ccsds_packet_parameters( - self, parameter_set: Et.Element, ccsds_parameters: list - ) -> Et.Element: - """ - Create XML elements to define CCSDS packet parameters based on the given data. - - Parameters - ---------- - parameter_set : Et.Element - The ParameterSet element where parameters will be added. - ccsds_parameters : list - A list of dictionaries containing CCSDS parameter data. - - Returns - ------- - parameter_set : Et.Element - The updated ParameterSet element. - """ - for parameter_data in ccsds_parameters: - parameter = Et.SubElement(parameter_set, "xtce:Parameter") - parameter.attrib["name"] = parameter_data["name"] - parameter.attrib["parameterTypeRef"] = parameter_data["parameterTypeRef"] - - description = Et.SubElement(parameter, "xtce:LongDescription") - description.text = parameter_data["description"] - - return parameter_set - - def create_container_set( - self, - telemetry_metadata: Et.Element, - ccsds_parameters: list, - container_name: str, - ) -> Et.Element: - """ - Create XML elements. - - These elements are for ContainerSet, CCSDSPacket SequenceContainer, - and Packet SequenceContainer. - - Parameters - ---------- - telemetry_metadata : Et.Element - The TelemetryMetaData element where containers are. - ccsds_parameters : list - A list of dictionaries containing CCSDS parameter data. - container_name : str - The name of sequence container. - - Returns - ------- - telemetry_metadata : Et.Element - The updated TelemetryMetaData element. - """ - # Create ContainerSet element - container_set = Et.SubElement(telemetry_metadata, "xtce:ContainerSet") - # Create CCSDSPacket SequenceContainer - ccsds_packet_container = Et.SubElement(container_set, "xtce:SequenceContainer") - ccsds_packet_container.attrib["name"] = "CCSDSPacket" - ccsds_packet_entry_list = Et.SubElement( - ccsds_packet_container, "xtce:EntryList" - ) - - # Populate EntryList for CCSDSPacket SequenceContainer - for parameter_data in ccsds_parameters: - parameter_ref_entry = Et.SubElement( - ccsds_packet_entry_list, "xtce:ParameterRefEntry" - ) - parameter_ref_entry.attrib["parameterRef"] = parameter_data["name"] - - # Create Packet SequenceContainer that use CCSDSPacket SequenceContainer - # as base container - science_container = Et.SubElement(container_set, "xtce:SequenceContainer") - science_container.attrib["name"] = container_name - - base_container = Et.SubElement(science_container, "xtce:BaseContainer") - base_container.attrib["containerRef"] = "CCSDSPacket" - - # Add RestrictionCriteria element to use the given APID for comparison - restriction_criteria = Et.SubElement(base_container, "xtce:RestrictionCriteria") - comparison = Et.SubElement(restriction_criteria, "xtce:Comparison") - comparison.attrib["parameterRef"] = "PKT_APID" - comparison.attrib["value"] = f"{self.apid}" - comparison.attrib["useCalibratedValue"] = "false" - - # Populate EntryList for packet SequenceContainer - packet_entry_list = Et.SubElement(science_container, "xtce:EntryList") - parameter_refs = self.pkt.loc[7:, "mnemonic"].tolist() - - for parameter_ref in parameter_refs: - parameter_ref_entry = Et.SubElement( - packet_entry_list, "xtce:ParameterRefEntry" - ) - parameter_ref_entry.attrib["parameterRef"] = parameter_ref - - return telemetry_metadata - - def create_remaining_parameters(self, parameter_set: Et.Element) -> Et.Element: - """ - Create XML elements for parameters. - - These are based on DataFrame rows starting from SHCOARSE (also known as MET). - - Parameters - ---------- - parameter_set : Et.Element - The ParameterSet element where parameters will be added. - - Returns - ------- - parameter_set : Et.Element - The updated ParameterSet element. - """ - # Process rows from SHCOARSE until the last available row in the DataFrame - for index, row in self.pkt.iterrows(): - if index < 7: - # Skip rows until SHCOARSE(also known as MET) which are - # not part of CCSDS header - continue - - parameter = Et.SubElement(parameter_set, "xtce:Parameter") - parameter.attrib["name"] = row["mnemonic"] - parameter_type_ref = f"{row['dataType']}{row['lengthInBits']}" - - parameter.attrib["parameterTypeRef"] = parameter_type_ref - - # Add descriptions if they exist - if pd.notna(row.get("shortDescription")): - parameter.attrib["shortDescription"] = row.get("shortDescription") - if pd.notna(row.get("longDescription")): - description = Et.SubElement(parameter, "xtce:LongDescription") - description.text = row.get("longDescription") - - return parameter_set - - def generate_telemetry_xml(self, output_xml_path: str, container_name: str) -> None: - """ - Create and output an XTCE file based on the data within the class. - - Parameters - ---------- - output_xml_path : str - The path for the final xml file. - container_name : str - The name of the sequence container in the output xml. - """ - - unique_bits_lengths_data = self.get_unique_bits_length() - - # Here, we create the XML components so that we can add data in following steps - ( - telemetry_xml_root, - parameter_type_set, - parameter_set, - telemetry_metadata, - ) = self.create_telemetry_xml() - - parameter_type_set = self.create_parameter_types( - parameter_type_set, unique_bits_lengths_data - ) - - # Create CCSDSPacket parameters and add them to the ParameterSet element - parameter_set = self.create_ccsds_packet_parameters( - parameter_set, CCSDSParameters().parameters - ) - # Add remaining parameters to the ParameterSet element - parameter_set = self.create_remaining_parameters(parameter_set) - - # Create ContainerSet with CCSDSPacket SequenceContainer and packet - # SequenceContainer - telemetry_metadata = self.create_container_set( - telemetry_metadata, - CCSDSParameters().parameters, - container_name=container_name, - ) - - # Create the XML tree and save the document - tree = Et.ElementTree(telemetry_xml_root) - Et.indent(tree, space="\t", level=0) - - # Use the provided output_xml_path - tree.write(output_xml_path, encoding="utf-8", xml_declaration=True) diff --git a/tools/xtce_generation/xtce_generator_codice.py b/tools/xtce_generation/xtce_generator_codice.py deleted file mode 100644 index 76d8fc6ad..000000000 --- a/tools/xtce_generation/xtce_generator_codice.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -Module for generating XTCE files for telemetry packets for codice. - -This module provides functionality to generate XTCE files for telemetry packets -for codice. It includes a `TelemetryGenerator` class for creating XTCE files -based on packet definitions stored in an Excel file. -""" - -from pathlib import Path - -from tools.xtce_generation.telemetry_generator import TelemetryGenerator - - -def main() -> None: - """Function used by instrument to generate XTCE.""" - instrument_name = "codice" - current_directory = Path(__file__).parent - module_path = f"{current_directory}/../../imap_processing" - packet_definition_path = f"{module_path}/{instrument_name}/packet_definitions" - path_to_excel_file = "TLM_COD.xlsx" - - # CoDICE packets - packets = { - "P_COD_HSKP": 1136, - "P_COD_LO_IAL": 1152, - "P_COD_LO_PHA": 1153, - "P_COD_LO_SW_PRIORITY": 1155, - "P_COD_LO_SW_SPECIES": 1156, - "P_COD_LO_NSW_SPECIES": 1157, - "P_COD_LO_SW_ANGULAR": 1158, - "P_COD_LO_NSW_ANGULAR": 1159, - "P_COD_LO_NSW_PRIORITY": 1160, - "P_COD_LO_INST_COUNTS_AGGREGATED": 1161, - "P_COD_LO_INST_COUNTS_SINGLES": 1162, - "P_COD_HI_IAL": 1168, - "P_COD_HI_PHA": 1169, - "P_COD_HI_INST_COUNTS_AGGREGATED": 1170, - "P_COD_HI_INST_COUNTS_SINGLES": 1171, - "P_COD_HI_OMNI_SPECIES_COUNTS": 1172, - "P_COD_HI_SECT_SPECIES_COUNTS": 1173, - } - - for packet_name, app_id in packets.items(): - print(packet_name) - telemetry_generator = TelemetryGenerator( - packet_name=packet_name, path_to_excel_file=path_to_excel_file, apid=app_id - ) - telemetry_generator.generate_telemetry_xml( - f"{packet_definition_path}/{packet_name}.xml", packet_name - ) - - -if __name__ == "__main__": - main() diff --git a/tools/xtce_generation/xtce_generator_glows.py b/tools/xtce_generation/xtce_generator_glows.py deleted file mode 100644 index 31a9a9669..000000000 --- a/tools/xtce_generation/xtce_generator_glows.py +++ /dev/null @@ -1,59 +0,0 @@ -""" -Module for generating XTCE files for telemetry packets for glows. - -This module provides functionality to generate XTCE files for telemetry packets -for glows. It includes a `TelemetryGenerator` class for creating XTCE files -based on packet definitions stored in an Excel file. -""" - -from pathlib import Path - -from tools.xtce_generation.telemetry_generator import TelemetryGenerator - - -def main() -> None: - """ - Function is used to generate the GLOWS XTCE files for packet processing. - - This will create two XML files, P_GLX_TMSCDE.xml and P_GLX_TMSCHIST.xml. For - processing, these need to be manually combined into one file: GLX_COMBINED.xml. - - To do this, first duplicate P_GLX_TMSCHIST into a new file GLX_COMBINED.xml. - - Copy the DE specific parameter entries out of P_GLX_TMSCDE.xml. ALl the non-CCSDS - entries need to be copied into the combined XML file. This should go under the - combined ParameterSet tag. Then, remove the "SEC" entry, as this is a duplicate. - - Finally, copy the SequenceContainer named "P_GLX_TMSCDE" out of P_GLX_TMSCDE.xml - into the ContainerSet in GLX_COMBINED. Then, delete the "SEC" entry. This is a - duplicate and will cause an error. - """ - - instrument_name = "glows" - current_directory = Path(__file__).parent - module_path = f"{current_directory}/../../imap_processing" - packet_definition_path = f"{module_path}/{instrument_name}/packet_definitions" - path_to_excel_file = f"{current_directory}/tlm_glx_2023_06_22-edited.xlsx" - - # Eg. - # packets = { - # "P_COD_HI_PHA": 1169, - # "P_COD_LO_PHA": 1153, - # "P_COD_LO_NSW_SPECIES_COUNTS": 1157, - # "P_COD_HI_OMNI_SPECIES_COUNTS": 1172, - # } - - packets = {"P_GLX_TMSCHIST": 1480, "P_GLX_TMSCDE": 1481} - - for packet_name, app_id in packets.items(): - print(packet_name) - telemetry_generator = TelemetryGenerator( - packet_name=packet_name, path_to_excel_file=path_to_excel_file, apid=app_id - ) - telemetry_generator.generate_telemetry_xml( - f"{packet_definition_path}/{packet_name}.xml", packet_name - ) - - -if __name__ == "__main__": - main() diff --git a/tools/xtce_generation/xtce_generator_hi.py b/tools/xtce_generation/xtce_generator_hi.py deleted file mode 100644 index 4a74000a4..000000000 --- a/tools/xtce_generation/xtce_generator_hi.py +++ /dev/null @@ -1,37 +0,0 @@ -""" -Module for generating XTCE files for telemetry packets for hi. - -This module provides functionality to generate XTCE files for telemetry packets -for hi. It includes a `TelemetryGenerator` class for creating XTCE files -based on packet definitions stored in an Excel file. -""" - -from pathlib import Path - -from tools.xtce_generation.telemetry_generator import TelemetryGenerator - - -def main() -> None: - """Function used by hi to generate XTCE.""" - - instrument_name = "hi" - current_directory = Path(__file__).parent - module_path = f"{current_directory}/../../imap_processing" - packet_definition_path = f"{module_path}/{instrument_name}/packet_definitions" - # NOTE: Copy packet definition to tools/xtce_generation/ folder - path_to_excel_file = f"{current_directory}/26850.02-TLMDEF-04.xlsx" - - packets = {"H45_APP_NHK": 754, "H45_SCI_CNT": 769, "H45_SCI_DE": 770} - - for packet_name, app_id in packets.items(): - print(packet_name) - telemetry_generator = TelemetryGenerator( - packet_name=packet_name, path_to_excel_file=path_to_excel_file, apid=app_id - ) - telemetry_generator.generate_telemetry_xml( - f"{packet_definition_path}/{packet_name}.xml", packet_name - ) - - -if __name__ == "__main__": - main() diff --git a/tools/xtce_generation/xtce_generator_hit.py b/tools/xtce_generation/xtce_generator_hit.py deleted file mode 100644 index a8428804f..000000000 --- a/tools/xtce_generation/xtce_generator_hit.py +++ /dev/null @@ -1,45 +0,0 @@ -""" -Module for generating XTCE files for telemetry packets for hit. - -This module provides functionality to generate XTCE files for telemetry packets -for hit. It includes a `TelemetryGenerator` class for creating XTCE files -based on packet definitions stored in an Excel file. -""" - -from pathlib import Path - -from telemetry_generator import TelemetryGenerator - -from imap_processing import imap_module_directory - - -def main() -> None: - """Function used by hit to generate XTCE.""" - instrument_name = "hit" - current_directory = Path(__file__).parent - packet_definition_path = ( - f"{imap_module_directory}/{instrument_name}/packet_definitions" - ) - path_to_excel_file = f"{current_directory}/TLM_HIT_modified.xlsx" - - # Lo packets - packets = { - "P_HIT_AUT": 1250, - "P_HIT_HSKP": 1251, - "P_HIT_SCIENCE": 1252, - "P_HIT_MSGLOG": 1254, - "P_HIT_MEMDUMP": 1255, - } - - for packet_name, app_id in packets.items(): - print(packet_name) - telemetry_generator = TelemetryGenerator( - packet_name=packet_name, path_to_excel_file=path_to_excel_file, apid=app_id - ) - telemetry_generator.generate_telemetry_xml( - f"{packet_definition_path}/{packet_name}.xml", packet_name - ) - - -if __name__ == "__main__": - main() diff --git a/tools/xtce_generation/xtce_generator_lo.py b/tools/xtce_generation/xtce_generator_lo.py deleted file mode 100644 index 536d14107..000000000 --- a/tools/xtce_generation/xtce_generator_lo.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -Module for generating XTCE files for telemetry packets for lo. - -This module provides functionality to generate XTCE files for telemetry packets -for lo. It includes a `TelemetryGenerator` class for creating XTCE files -based on packet definitions stored in an Excel file. -""" - -from pathlib import Path - -from telemetry_generator import TelemetryGenerator - -# Following the creation of the XTCE files, manual updates need to be -# made to the following packets to make their binary field a variable -# length value in XTCE: -# P_ILO_BOOT_MEMDMP -# P_ILO_MEMDMP -# P_ILO_SCI_DE - - -def main() -> None: - """Function used by lo to generate XTCE.""" - instrument_name = "lo" - current_directory = Path(__file__).parent - module_path = f"{current_directory}/../../imap_processing" - packet_definition_path = f"{module_path}/{instrument_name}/packet_definitions" - # In the telemetry definition sheets, modifications need to be made to the - # P_ILO_RAW_DE and P_ILO_SCI_CNT. - # - # P_ILO_RAW_DE: the rows for bits 112 to 20591 were collapsed into a - # single binary row called RAW_DE. The reason for this is the number - # of fields in the packet caused the XTCE file to exceed the repo - # file size limit. Because of this modification, the binary field - # will need to be parsed in the python code. - # - # P_ILO_SCI_CNT: the rows for bits 80 to 26959 were collapsed into a - # single binary row called SCI_CNT. The reason for this is the number - # of fields in the packet caused the XTCE file to exceed the repo - # file size limit. Because of this modification, the binary field - # will need to be parsed in the python code. - path_to_excel_file = f"{current_directory}/telem_def_lo_modified.xls" - - # Lo packets - packets = { - "P_ILO_APP_NHK": 677, - "P_ILO_APP_SHK": 676, - "P_ILO_AUTO": 672, - "P_ILO_BOOT_HK": 673, - "P_ILO_BOOT_MEMDMP": 674, - "P_ILO_DIAG_BULK_HVPS": 724, - "P_ILO_DIAG_CDH": 721, - "P_ILO_DIAG_IFB": 722, - "P_ILO_DIAG_PCC": 725, - "P_ILO_DIAG_TOF_BD": 723, - "P_ILO_EVTMSG": 678, - "P_ILO_MEMDMP": 679, - "P_ILO_RAW_CNT": 689, - "P_ILO_RAW_DE": 690, - "P_ILO_RAW_STAR": 691, - "P_ILO_SCI_CNT": 705, - "P_ILO_SCI_DE": 706, - "P_ILO_SPIN": 708, - "P_ILO_STAR": 707, - } - - for packet_name, app_id in packets.items(): - print(packet_name) - telemetry_generator = TelemetryGenerator( - packet_name=packet_name, path_to_excel_file=path_to_excel_file, apid=app_id - ) - telemetry_generator.generate_telemetry_xml( - f"{packet_definition_path}/{packet_name}.xml", packet_name - ) - - -if __name__ == "__main__": - main() diff --git a/tools/xtce_generation/xtce_generator_mag.py b/tools/xtce_generation/xtce_generator_mag.py deleted file mode 100644 index b70b04176..000000000 --- a/tools/xtce_generation/xtce_generator_mag.py +++ /dev/null @@ -1,49 +0,0 @@ -""" -Module for generating XTCE files for telemetry packets for mag. - -This module provides functionality to generate XTCE files for telemetry packets -for mag. It includes a `TelemetryGenerator` class for creating XTCE files -based on packet definitions stored in an Excel file. -""" - -from pathlib import Path - -from tools.xtce_generation.telemetry_generator import TelemetryGenerator - - -def main() -> None: - """Function used by mag to generate XTCE.""" - - instrument_name = "mag" - current_directory = Path(__file__).parent - module_path = f"{current_directory}/../../imap_processing" - packet_definition_path = f"{module_path}/{instrument_name}/packet_definitions" - path_to_excel_file = f"{current_directory}/TLM_MAG_SCI.xls" - - # Eg. - # packets = { - # "P_COD_HI_PHA": 1169, - # "P_COD_LO_PHA": 1153, - # "P_COD_LO_NSW_SPECIES_COUNTS": 1157, - # "P_COD_HI_OMNI_SPECIES_COUNTS": 1172, - # } - - packets = { - "P_MAG_SCI_BURST": "1068", - "P_MAG_SCI_NORM": "1052", - } - - for packet_name, app_id in packets.items(): - print(packet_name) - telemetry_generator = TelemetryGenerator( - packet_name=packet_name, - path_to_excel_file=path_to_excel_file, - apid=int(app_id), - ) - telemetry_generator.generate_telemetry_xml( - f"{packet_definition_path}/{packet_name}.xml", packet_name - ) - - -if __name__ == "__main__": - main() diff --git a/tools/xtce_generation/xtce_generator_swapi.py b/tools/xtce_generation/xtce_generator_swapi.py deleted file mode 100644 index 80fd26e62..000000000 --- a/tools/xtce_generation/xtce_generator_swapi.py +++ /dev/null @@ -1,41 +0,0 @@ -""" -Module for generating XTCE files for telemetry packets for swapi. - -This module provides functionality to generate XTCE files for telemetry packets -for swapi. It includes a `TelemetryGenerator` class for creating XTCE files -based on packet definitions stored in an Excel file. -""" - -from pathlib import Path - -from tools.xtce_generation.telemetry_generator import TelemetryGenerator - - -def main() -> None: - """Function used by swapi to generate XTCE.""" - - instrument_name = "swapi" - current_directory = Path(__file__).parent - module_path = f"{current_directory}/../../imap_processing" - packet_definition_path = f"{module_path}/{instrument_name}/packet_definitions" - # NOTE: Copy packet definition to tools/xtce_generation/ folder if hasn't already - path_to_excel_file = f"{current_directory}/TLM_SWP_20231006-121021.xlsx" - - packets = { - "P_SWP_HK": 1184, - "P_SWP_SCI": 1188, - "P_SWP_AUT": 1192, - } - - for packet_name, app_id in packets.items(): - print(packet_name) - telemetry_generator = TelemetryGenerator( - packet_name=packet_name, path_to_excel_file=path_to_excel_file, apid=app_id - ) - telemetry_generator.generate_telemetry_xml( - f"{packet_definition_path}/{packet_name}.xml", packet_name - ) - - -if __name__ == "__main__": - main() diff --git a/tools/xtce_generation/xtce_generator_swe.py b/tools/xtce_generation/xtce_generator_swe.py deleted file mode 100644 index dab0d89b0..000000000 --- a/tools/xtce_generation/xtce_generator_swe.py +++ /dev/null @@ -1,43 +0,0 @@ -""" -Module for generating XTCE files for telemetry packets for swe. - -This module provides functionality to generate XTCE files for telemetry packets -for swe. It includes a `TelemetryGenerator` class for creating XTCE files -based on packet definitions stored in an Excel file. -""" - -from pathlib import Path - -from tools.xtce_generation.telemetry_generator import TelemetryGenerator - - -def main() -> None: - """Function used by swe to generate XTCE.""" - - instrument_name = "swe" - current_directory = Path(__file__).parent - module_path = f"{current_directory}/../../imap_processing" - packet_definition_path = f"{module_path}/{instrument_name}/packet_definitions" - - path_to_excel_file = f"{current_directory}/TLM_SWE_20230904.xlsx" - - # SWE packets - packets = { - "P_SWE_APP_HK": 1330, - "P_SWE_EVTMSG": 1317, - "P_SWE_CEM_RAW": 1334, - "P_SWE_SCIENCE": 1344, - } - - for packet_name, app_id in packets.items(): - print(packet_name) - telemetry_generator = TelemetryGenerator( - packet_name=packet_name, path_to_excel_file=path_to_excel_file, apid=app_id - ) - telemetry_generator.generate_telemetry_xml( - f"{packet_definition_path}/{packet_name}.xml", packet_name - ) - - -if __name__ == "__main__": - main() diff --git a/tools/xtce_generation/xtce_generator_template.py b/tools/xtce_generation/xtce_generator_template.py deleted file mode 100644 index 352075bfb..000000000 --- a/tools/xtce_generation/xtce_generator_template.py +++ /dev/null @@ -1,112 +0,0 @@ -"""XTCE generator script that uses CLI for generalized format.""" - -import argparse -import json -from pathlib import Path - -import imap_data_access - -from tools.xtce_generation.telemetry_generator import TelemetryGenerator - - -# Function to parse command line arguments -def _parse_args() -> argparse.Namespace: - """ - Parse the command line arguments. - - The expected input format is: - --instrument "instrument_name" - --file-path "full_file_path" - --packets '{"packet_name_1": app_id_1, "packet_name_2": app_id_2}' - - Returns - ------- - args : argparse.Namespace - An object containing the parsed arguments and their values. - """ - description = ( - "This command line program generates an instrument specific XTCE file." - "Example usage: " - '--instrument "swapi"' - "--file-path " - '"/Users/anma6676/Desktop/Repositories/imap_processing/tools/xtce_generation/TLM_SWP_20231006-121021.xlsx"' - "--packets '" - '{"P_SWP_HK": 1184, ' - '"P_SWP_SCI": 1188, ' - '"P_SWP_AUT": 1192, ' - "}'" - ) - instrument_help = ( - "The instrument to process. Acceptable values are: " - f"{imap_data_access.VALID_INSTRUMENTS}" - ) - file_path_help = "Provide full file path to write packets to." - packets_help = ( - "Provide packet dictionary using packet_name, and app_id." - '{"": }' - ) - - parser = argparse.ArgumentParser(prog="imap_xtce", description=description) - parser.add_argument("--instrument", type=str, required=True, help=instrument_help) - parser.add_argument("--file-path", type=str, required=True, help=file_path_help) - parser.add_argument("--packets", type=str, required=True, help=packets_help) - - args = parser.parse_args() - - return args - - -def _validate_args(args: argparse.Namespace) -> None: - """ - Ensure that the instrument is valid. - - Parameters - ---------- - args : argparse.Namespace - An object containing the parsed arguments and their values. - """ - if args.instrument not in imap_data_access.VALID_INSTRUMENTS: - raise ValueError( - f"{args.instrument} is not in the supported instrument list: " - f"{imap_data_access.VALID_INSTRUMENTS}" - ) - - directory = Path(args.file_path) - if not directory.exists(): - raise FileNotFoundError(f"{args.file_path} not found, and may not exist.") - - -def main() -> None: - """ - Generate xtce file from CLI information given. - - The xtce file will be written in an instrument specific subfolder. - """ - - # Parse arguments, and validate instrument - args = _parse_args() - _validate_args(args) - - instrument_name = args.instrument - current_directory = Path(__file__).parent.parent.parent - module_path = current_directory / "imap_processing" - packet_definition_path = module_path / instrument_name / "packet_definitions" - path_to_excel_file = args.file_path - - # Update packets dictionary with given CLI information - packets = json.loads(args.packets) - - for packet_name, app_id in packets.items(): - print(packet_name) - telemetry_generator = TelemetryGenerator( - packet_name=packet_name, - path_to_excel_file=path_to_excel_file, - apid=int(app_id), - ) - telemetry_generator.generate_telemetry_xml( - packet_definition_path / f"{packet_name}.xml", packet_name - ) - - -if __name__ == "__main__": - main()