Skip to content

Commit

Permalink
MNT/ENH: Add FLOAT and FILL options for spreadsheet dataType (#701)
Browse files Browse the repository at this point in the history 
* MNT/ENH: Add FLOAT and FILL options for spreadsheet dataType

This adds two new options to the handled data types, and also raises
an error if we get a datatype that isn't handled. This is important
because otherwise there will be a parameter added without a corresponding
parameterType entry.

* TST: Add tests for Excel to XTCE XML

Add a simple excel file with variable types we want to test and
assert against a known XML file we have produced.
  • Loading branch information
@greglucas
greglucas authored Jul 30, 2024
1 parent ca85cee commit ba50882
Show file tree
Hide file tree
Showing 6 changed files with 195 additions and 9 deletions.
13 changes: 12 additions & 1 deletion imap_processing/ccsds/excel_to_xtce.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,7 +262,7 @@ def _add_parameter(self, row: pd.Series, total_packet_bits: int) -> None:
length_in_bits = int(row["lengthInBits"])

# Add the parameterTypeRef for this row
if "UINT" in row["dataType"]:
if "UINT" in row["dataType"] or "FILL" in row["dataType"]:
parameter_type = Et.SubElement(
self._parameter_type_set, "xtce:IntegerParameterType"
)
Expand All @@ -283,6 +283,15 @@ def _add_parameter(self, row: pd.Series, total_packet_bits: int) -> None:
encoding.attrib["sizeInBits"] = str(length_in_bits)
encoding.attrib["encoding"] = "signed"

elif "FLOAT" in row["dataType"]:
parameter_type = Et.SubElement(
self._parameter_type_set, "xtce:FloatParameterType"
)
parameter_type.attrib["name"] = name
encoding = Et.SubElement(parameter_type, "xtce:FloatDataEncoding")
encoding.attrib["sizeInBits"] = str(length_in_bits)
encoding.attrib["encoding"] = "IEEE-754"

elif "BYTE" in row["dataType"]:
parameter_type = Et.SubElement(
self._parameter_type_set, "xtce:BinaryParameterType"
Expand Down Expand Up @@ -311,6 +320,8 @@ def _add_parameter(self, row: pd.Series, total_packet_bits: int) -> None:
# TODO: Do we want to allow fixed length values?
# fixed_value = Et.SubElement(size_in_bits, "xtce:FixedValue")
# fixed_value.text = str(row["lengthInBits"])
else:
raise ValueError(f"Unknown data type for {name}: {row['dataType']}")

if row["convertAs"] == "ANALOG":
# Go look up the conversion in the AnalogConversions tab
Expand Down
Binary file added imap_processing/tests/ccsds/test_data/excel_to_xtce_test_file.xlsx
View file
Binary file not shown.
160 changes: 160 additions & 0 deletions imap_processing/tests/ccsds/test_data/expected_output.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
<?xml version='1.0' encoding='utf-8'?>
<xtce:SpaceSystem xmlns:xtce="http://www.omg.org/space/xtce" name="Test Instrument">
<xtce:Header date="2024-07-26 00:00:00" version="v1.2" author="IMAP SDC" />
<xtce:TelemetryMetaData>
<xtce:ParameterTypeSet>
<xtce:IntegerParameterType name="VERSION" signed="false">
<xtce:IntegerDataEncoding sizeInBits="3" encoding="unsigned" />
</xtce:IntegerParameterType>
<xtce:IntegerParameterType name="TYPE" signed="false">
<xtce:IntegerDataEncoding sizeInBits="1" encoding="unsigned" />
</xtce:IntegerParameterType>
<xtce:IntegerParameterType name="SEC_HDR_FLG" signed="false">
<xtce:IntegerDataEncoding sizeInBits="1" encoding="unsigned" />
</xtce:IntegerParameterType>
<xtce:IntegerParameterType name="PKT_APID" signed="false">
<xtce:IntegerDataEncoding sizeInBits="11" encoding="unsigned" />
</xtce:IntegerParameterType>
<xtce:IntegerParameterType name="SEQ_FLGS" signed="false">
<xtce:IntegerDataEncoding sizeInBits="2" encoding="unsigned" />
</xtce:IntegerParameterType>
<xtce:IntegerParameterType name="SRC_SEQ_CTR" signed="false">
<xtce:IntegerDataEncoding sizeInBits="14" encoding="unsigned" />
</xtce:IntegerParameterType>
<xtce:IntegerParameterType name="PKT_LEN" signed="false">
<xtce:IntegerDataEncoding sizeInBits="16" encoding="unsigned" />
</xtce:IntegerParameterType>
<xtce:IntegerParameterType name="TEST_PACKET_SHCOARSE" signed="false">
<xtce:IntegerDataEncoding sizeInBits="32" encoding="unsigned" />
</xtce:IntegerParameterType>
<xtce:IntegerParameterType name="TEST_PACKET_VAR_UINT" signed="false">
<xtce:IntegerDataEncoding sizeInBits="2" encoding="unsigned">
<xtce:DefaultCalibrator>
<xtce:PolynomialCalibrator>
<xtce:Term coefficient="1.5" exponent="0" />
<xtce:Term coefficient="2.5" exponent="1" />
</xtce:PolynomialCalibrator>
</xtce:DefaultCalibrator>
</xtce:IntegerDataEncoding>
</xtce:IntegerParameterType>
<xtce:IntegerParameterType name="TEST_PACKET_VAR_INT" signed="true">
<xtce:IntegerDataEncoding sizeInBits="4" encoding="signed" />
</xtce:IntegerParameterType>
<xtce:IntegerParameterType name="TEST_PACKET_VAR_SINT" signed="true">
<xtce:IntegerDataEncoding sizeInBits="5" encoding="signed" />
</xtce:IntegerParameterType>
<xtce:BinaryParameterType name="TEST_PACKET_VAR_BYTE">
<xtce:BinaryDataEncoding bitOrder="mostSignificantBitFirst">
<xtce:SizeInBits>
<xtce:DynamicValue>
<xtce:ParameterInstanceRef parameterRef="PKT_LEN" />
<xtce:LinearAdjustment slope="8" intercept="-70" />
</xtce:DynamicValue>
</xtce:SizeInBits>
</xtce:BinaryDataEncoding>
</xtce:BinaryParameterType>
<xtce:IntegerParameterType name="TEST_PACKET_VAR_FILL" signed="false">
<xtce:IntegerDataEncoding sizeInBits="3" encoding="unsigned" />
</xtce:IntegerParameterType>
<xtce:FloatParameterType name="TEST_PACKET_VAR_FLOAT">
<xtce:FloatDataEncoding sizeInBits="32" encoding="IEEE-754" />
</xtce:FloatParameterType>
<xtce:IntegerParameterType name="TEST_PACKET2_SHCOARSE" signed="false">
<xtce:IntegerDataEncoding sizeInBits="32" encoding="unsigned" />
</xtce:IntegerParameterType>
<xtce:IntegerParameterType name="TEST_PACKET2_VAR1" signed="false">
<xtce:IntegerDataEncoding sizeInBits="2" encoding="unsigned" />
</xtce:IntegerParameterType>
</xtce:ParameterTypeSet>
<xtce:ParameterSet>
<xtce:Parameter name="VERSION" parameterTypeRef="VERSION">
<xtce:LongDescription>CCSDS Packet Version Number (always 0)</xtce:LongDescription>
</xtce:Parameter>
<xtce:Parameter name="TYPE" parameterTypeRef="TYPE">
<xtce:LongDescription>CCSDS Packet Type Indicator (0=telemetry)</xtce:LongDescription>
</xtce:Parameter>
<xtce:Parameter name="SEC_HDR_FLG" parameterTypeRef="SEC_HDR_FLG">
<xtce:LongDescription>CCSDS Packet Secondary Header Flag (always 1)</xtce:LongDescription>
</xtce:Parameter>
<xtce:Parameter name="PKT_APID" parameterTypeRef="PKT_APID">
<xtce:LongDescription>CCSDS Packet Application Process ID</xtce:LongDescription>
</xtce:Parameter>
<xtce:Parameter name="SEQ_FLGS" parameterTypeRef="SEQ_FLGS">
<xtce:LongDescription>CCSDS Packet Grouping Flags (3=not part of group)</xtce:LongDescription>
</xtce:Parameter>
<xtce:Parameter name="SRC_SEQ_CTR" parameterTypeRef="SRC_SEQ_CTR">
<xtce:LongDescription>CCSDS Packet Sequence Count (increments with each new packet)</xtce:LongDescription>
</xtce:Parameter>
<xtce:Parameter name="PKT_LEN" parameterTypeRef="PKT_LEN">
<xtce:LongDescription>CCSDS Packet Length (number of bytes after Packet length minus 1)</xtce:LongDescription>
</xtce:Parameter>
<xtce:Parameter name="TEST_PACKET_SHCOARSE" parameterTypeRef="TEST_PACKET_SHCOARSE">
<xtce:LongDescription>Mission elapsed time</xtce:LongDescription>
</xtce:Parameter>
<xtce:Parameter name="TEST_PACKET_VAR_UINT" parameterTypeRef="TEST_PACKET_VAR_UINT">
<xtce:LongDescription>Unsgned integer data with conversion</xtce:LongDescription>
</xtce:Parameter>
<xtce:Parameter name="TEST_PACKET_VAR_INT" parameterTypeRef="TEST_PACKET_VAR_INT">
<xtce:LongDescription>Integer data</xtce:LongDescription>
</xtce:Parameter>
<xtce:Parameter name="TEST_PACKET_VAR_SINT" parameterTypeRef="TEST_PACKET_VAR_SINT">
<xtce:LongDescription>Signed integer data</xtce:LongDescription>
</xtce:Parameter>
<xtce:Parameter name="TEST_PACKET_VAR_BYTE" parameterTypeRef="TEST_PACKET_VAR_BYTE">
<xtce:LongDescription>Binary data - variable length</xtce:LongDescription>
</xtce:Parameter>
<xtce:Parameter name="TEST_PACKET_VAR_FILL" parameterTypeRef="TEST_PACKET_VAR_FILL">
<xtce:LongDescription>Fill data</xtce:LongDescription>
</xtce:Parameter>
<xtce:Parameter name="TEST_PACKET_VAR_FLOAT" parameterTypeRef="TEST_PACKET_VAR_FLOAT">
<xtce:LongDescription>Float data</xtce:LongDescription>
</xtce:Parameter>
<xtce:Parameter name="TEST_PACKET2_SHCOARSE" parameterTypeRef="TEST_PACKET2_SHCOARSE">
<xtce:LongDescription>Mission elapsed time</xtce:LongDescription>
</xtce:Parameter>
<xtce:Parameter name="TEST_PACKET2_VAR1" parameterTypeRef="TEST_PACKET2_VAR1" shortDescription="Variable 1 short desc">
<xtce:LongDescription>Variable 1 - long desc</xtce:LongDescription>
</xtce:Parameter>
</xtce:ParameterSet>
<xtce:ContainerSet>
<xtce:SequenceContainer name="CCSDSPacket" abstract="true">
<xtce:EntryList>
<xtce:ParameterRefEntry parameterRef="VERSION" />
<xtce:ParameterRefEntry parameterRef="TYPE" />
<xtce:ParameterRefEntry parameterRef="SEC_HDR_FLG" />
<xtce:ParameterRefEntry parameterRef="PKT_APID" />
<xtce:ParameterRefEntry parameterRef="SEQ_FLGS" />
<xtce:ParameterRefEntry parameterRef="SRC_SEQ_CTR" />
<xtce:ParameterRefEntry parameterRef="PKT_LEN" />
</xtce:EntryList>
</xtce:SequenceContainer>
<xtce:SequenceContainer name="TEST_PACKET">
<xtce:BaseContainer containerRef="CCSDSPacket">
<xtce:RestrictionCriteria>
<xtce:Comparison parameterRef="PKT_APID" value="1" useCalibratedValue="false" />
</xtce:RestrictionCriteria>
</xtce:BaseContainer>
<xtce:EntryList>
<xtce:ParameterRefEntry parameterRef="TEST_PACKET_SHCOARSE" />
<xtce:ParameterRefEntry parameterRef="TEST_PACKET_VAR_UINT" />
<xtce:ParameterRefEntry parameterRef="TEST_PACKET_VAR_INT" />
<xtce:ParameterRefEntry parameterRef="TEST_PACKET_VAR_SINT" />
<xtce:ParameterRefEntry parameterRef="TEST_PACKET_VAR_BYTE" />
<xtce:ParameterRefEntry parameterRef="TEST_PACKET_VAR_FILL" />
<xtce:ParameterRefEntry parameterRef="TEST_PACKET_VAR_FLOAT" />
</xtce:EntryList>
</xtce:SequenceContainer>
<xtce:SequenceContainer name="TEST_PACKET2">
<xtce:BaseContainer containerRef="CCSDSPacket">
<xtce:RestrictionCriteria>
<xtce:Comparison parameterRef="PKT_APID" value="15" useCalibratedValue="false" />
</xtce:RestrictionCriteria>
</xtce:BaseContainer>
<xtce:EntryList>
<xtce:ParameterRefEntry parameterRef="TEST_PACKET2_SHCOARSE" />
<xtce:ParameterRefEntry parameterRef="TEST_PACKET2_VAR1" />
</xtce:EntryList>
</xtce:SequenceContainer>
</xtce:ContainerSet>
</xtce:TelemetryMetaData>
</xtce:SpaceSystem>
25 changes: 20 additions & 5 deletions imap_processing/tests/ccsds/test_excel_to_xtce.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,21 +12,36 @@


@pytest.fixture()
def filepath(tmpdir):
p = Path(tmpdir / "test_file.xlsx").resolve()
p.touch()
def excel_file():
p = Path(__file__).parent / "test_data" / "excel_to_xtce_test_file.xlsx"
return p


def test_generated_xml(excel_file, tmp_path):
"""Make sure we are producing the expected contents within the XML file.
To produce a new expected output file the following command can be used.
imap_xtce imap_processing/tests/ccsds/test_data/excel_to_xtce_test_file.xlsx
--output imap_processing/tests/ccsds/test_data/expected_output.xml
"""
generator = excel_to_xtce.XTCEGenerator(excel_file)
output_file = tmp_path / "output.xml"
generator.to_xml(output_file)

expected_file = excel_file.parent / "expected_output.xml"
with open(output_file) as f, open(expected_file) as f_expected:
assert f.read() == f_expected.read()


# General test
@mock.patch("imap_processing.ccsds.excel_to_xtce.XTCEGenerator")
def test_main_general(mock_input, filepath):
def test_main_general(mock_input, excel_file):
"""Testing base main function."""
test_args = [
"test_script",
"--output",
"swe.xml",
f"{filepath}",
f"{excel_file}",
]
with mock.patch.object(sys, "argv", test_args):
excel_to_xtce.main()
Expand Down
4 changes: 2 additions & 2 deletions poetry.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ mypy = {version="^1.10.1", optional=true}
[tool.poetry.extras]
dev = ["pre-commit", "ruff", "mypy"]
doc = ["numpydoc", "pydata-sphinx-theme", "sphinx", "sphinxcontrib-openapi"]
test = ["pytest", "pytest-cov"]
test = ["openpyxl", "pytest", "pytest-cov"]
tools= ["openpyxl", "pandas"]

[project.urls]
Expand Down

0 comments on commit ba50882

Please sign in to comment.