diff --git a/src/python_testing/TestMatterTestingSupport.py b/src/python_testing/TestMatterTestingSupport.py index f111250481032d..45f7efb9926e95 100644 --- a/src/python_testing/TestMatterTestingSupport.py +++ b/src/python_testing/TestMatterTestingSupport.py @@ -22,9 +22,9 @@ import chip.clusters as Clusters from chip.clusters.Types import Nullable, NullValue -from chip.testing.matter_testing import (MatterBaseTest, async_test_body, compare_time, default_matter_test_main, - get_wait_seconds_from_set_time, parse_matter_test_args, type_matches, - utc_time_in_matter_epoch) +from chip.testing.matter_testing import (MatterBaseTest, PIXITDefinition, PIXITType, PIXITValidationError, PIXITValidator, + async_test_body, compare_time, default_matter_test_main, get_wait_seconds_from_set_time, + parse_matter_test_args, type_matches, utc_time_in_matter_epoch) from chip.testing.pics import parse_pics, parse_pics_xml from chip.testing.taglist_and_topology_test import (TagProblem, create_device_type_list_for_root, create_device_type_lists, find_tag_list_problems, find_tree_roots, flat_list_ok, get_all_children, @@ -659,6 +659,194 @@ def test_parse_matter_test_args(self): asserts.assert_equal(parsed.global_test_params.get("PIXIT.TEST.STR.MULTI.2"), "bar") asserts.assert_equal(parsed.global_test_params.get("PIXIT.TEST.JSON"), {"key": "value"}) + def test_pixit_validation(self): + # Test integer validation + PIXITValidator.validate_int_pixit_value("123") # Should pass + PIXITValidator.validate_int_pixit_value("-123") # Should pass + PIXITValidator.validate_int_pixit_value("0") # Should pass + + try: + PIXITValidator.validate_int_pixit_value("abc") + asserts.fail("Expected ValueError for invalid integer") + except ValueError: + pass + + try: + PIXITValidator.validate_int_pixit_value("12.34") + asserts.fail("Expected ValueError for float value") + except ValueError: + pass + + # Test boolean validation + PIXITValidator.validate_bool_pixit_value("true") # Should pass + PIXITValidator.validate_bool_pixit_value("false") # Should pass + PIXITValidator.validate_bool_pixit_value("True") # Should pass + PIXITValidator.validate_bool_pixit_value("False") # Should pass + + try: + PIXITValidator.validate_bool_pixit_value("yes") + asserts.fail("Expected ValueError for invalid boolean") + except ValueError: + pass + + try: + PIXITValidator.validate_bool_pixit_value("1") + asserts.fail("Expected ValueError for numeric boolean") + except ValueError: + pass + + # Test float validation + PIXITValidator.validate_float_pixit_value("123.45") # Should pass + PIXITValidator.validate_float_pixit_value("-123.45") # Should pass + PIXITValidator.validate_float_pixit_value("0.0") # Should pass + + try: + PIXITValidator.validate_float_pixit_value("abc") + asserts.fail("Expected ValueError for invalid float") + except ValueError: + pass + + # Test string validation - should accept most inputs + PIXITValidator.validate_string_pixit_value("abc") # Should pass + PIXITValidator.validate_string_pixit_value("123") # Should pass + + # Test JSON validation + PIXITValidator.validate_json_pixit_value('{"key": "value"}') # Should pass + PIXITValidator.validate_json_pixit_value('[]') # Should pass + PIXITValidator.validate_json_pixit_value('null') # Should pass + + try: + PIXITValidator.validate_json_pixit_value('{invalid json}') + asserts.fail("Expected ValueError for invalid JSON") + except ValueError: + pass + + try: + PIXITValidator.validate_json_pixit_value('{"unclosed": "object"') + asserts.fail("Expected ValueError for malformed JSON") + except ValueError: + pass + + # Test hex validation + PIXITValidator.validate_hex_pixit_value("0x1234") # Should pass + PIXITValidator.validate_hex_pixit_value("1234") # Should pass + PIXITValidator.validate_hex_pixit_value("ABCD") # Should pass + PIXITValidator.validate_hex_pixit_value("abcd") # Should pass + PIXITValidator.validate_hex_pixit_value("hex:12ab") # Should pass + + try: + PIXITValidator.validate_hex_pixit_value("0x123") # Odd number of digits + asserts.fail("Expected ValueError for odd number of hex digits") + except ValueError: + pass + + try: + PIXITValidator.validate_hex_pixit_value("123") # Odd number of digits + asserts.fail("Expected ValueError for odd number of hex digits") + except ValueError: + pass + + try: + PIXITValidator.validate_hex_pixit_value("WXYZ") # Invalid hex chars + asserts.fail("Expected ValueError for invalid hex characters") + except ValueError: + pass + + # Test full PIXIT validation + pixit_def = PIXITDefinition( + name="PIXIT.TEST.INT", + pixit_type=PIXITType.INT, + description="Test integer PIXIT", + required=True + ) + + PIXITValidator.validate_value("123", pixit_def) # Should pass + + try: + PIXITValidator.validate_value("abc", pixit_def) + asserts.fail("Expected PIXITValidationError for invalid type") + except PIXITValidationError: + pass + + try: + PIXITValidator.validate_value(None, pixit_def) # Required but None + asserts.fail("Expected PIXITValidationError for missing required value") + except PIXITValidationError: + pass + + # Test optional PIXIT + optional_pixit = PIXITDefinition( + name="PIXIT.TEST.INT", + pixit_type=PIXITType.INT, + description="Test integer PIXIT", + required=False + ) + + PIXITValidator.validate_value(None, optional_pixit) # Should pass as it's optional + + def test_pixits_validation(self): + """Test bulk PIXIT validation functionality""" + pixits = [ + PIXITDefinition( + name="PIXIT.TEST.INT", + pixit_type=PIXITType.INT, + description="Test integer PIXIT" + ), + PIXITDefinition( + name="PIXIT.TEST.STRING", + pixit_type=PIXITType.STRING, + description="Test string PIXIT", + required=False + ), + PIXITDefinition( + name="PIXIT.TEST.HEX", + pixit_type=PIXITType.HEX, + description="Test hex PIXIT" + ) + ] + + # Test valid values + valid_values = { + "PIXIT.TEST.INT": "42", + "PIXIT.TEST.STRING": "test string", + "PIXIT.TEST.HEX": "0x1234" + } + PIXITValidator.validate_pixits(pixits, valid_values) # Should pass + + # Test missing required PIXIT + missing_required = { + "PIXIT.TEST.STRING": "test string", + "PIXIT.TEST.HEX": "0x1234" + } + try: + PIXITValidator.validate_pixits(pixits, missing_required) + asserts.fail("Expected PIXITValidationError for missing required PIXIT") + except PIXITValidationError as e: + asserts.assert_true("PIXIT.TEST.INT" in str(e), + "Error message should indicate missing PIXIT") + + # Test missing optional PIXIT (should pass) + missing_optional = { + "PIXIT.TEST.INT": "42", + "PIXIT.TEST.HEX": "0x1234" + } + PIXITValidator.validate_pixits(pixits, missing_optional) # Should pass + + # Test multiple validation errors + multiple_errors = { + "PIXIT.TEST.INT": "not a number", + "PIXIT.TEST.HEX": "invalid hex" + } + try: + PIXITValidator.validate_pixits(pixits, multiple_errors) + asserts.fail("Expected PIXITValidationError for multiple validation errors") + except PIXITValidationError as e: + error_msg = str(e) + asserts.assert_true("Invalid value for PIXIT.TEST.INT" in error_msg, + "Error message should indicate invalid INT PIXIT value") + asserts.assert_true("Invalid value for PIXIT.TEST.HEX" in error_msg, + "Error message should indicate invalid HEX PIXIT value") + if __name__ == "__main__": default_matter_test_main() diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/matter_testing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/matter_testing.py index 46c32a2e178b79..5d2d64a12bdf71 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/matter_testing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/matter_testing.py @@ -612,6 +612,228 @@ def test_skipped(self, filename: str, name: str): logging.info(f"Skipping test from {filename}: {name}") +class PIXITType(Enum): + INT = "int" + BOOL = "bool" + FLOAT = "float" + STRING = "string" + JSON = "json" + HEX = "hex" + + @property + def arg_flag(self) -> str: + """Maps PIXIT type to command line flag""" + return f"--{self.value}-arg" + + +@dataclass +class PIXITDefinition: + """Describes a PIXIT requirement for a test""" + name: str + pixit_type: PIXITType + description: str + required: bool = True + default: Optional[Any] = None + + @staticmethod + def is_pixit(arg_name: str) -> bool: + """Only validate args starting with PIXIT.""" + return arg_name.startswith("PIXIT.") + + +class PIXITValidationError(Exception): + """Raised when PIXIT validation fails""" + pass + + +class PIXITValidator: + """Handles validation of PIXIT values against their definitions""" + + @staticmethod + def validate_int_pixit_value(value: str) -> None: + """Validates that a value can be converted to int. + + Args: + value: Value to validate + + Raises: + ValueError: If value cannot be converted to int + """ + try: + int(value) + except ValueError as e: + raise ValueError(f"Invalid integer value: {e}") + + @staticmethod + def validate_bool_pixit_value(value: str) -> None: + """Validates that a value represents a valid boolean. + + Args: + value: Value to validate + + Raises: + ValueError: If value is not a valid boolean representation + """ + try: + if isinstance(value, str): + value_lower = value.lower() + if value_lower not in ('true', 'false'): + raise ValueError(f"String value must be 'true' or 'false', got '{value}'") + else: + bool(value) + except ValueError as e: + raise ValueError(f"Invalid boolean value: {e}") + + @staticmethod + def validate_float_pixit_value(value: str) -> None: + """Validates that a value can be converted to float. + + Args: + value: Value to validate + + Raises: + ValueError: If value cannot be converted to float + """ + try: + float(value) + except ValueError as e: + raise ValueError(f"Invalid float value: {e}") + + @staticmethod + def validate_string_pixit_value(value: str) -> None: + """Validates that a value can be converted to string. + + Args: + value: Value to validate + + Raises: + ValueError: If value cannot be converted to string + """ + try: + str(value) + except ValueError as e: + raise ValueError(f"Invalid string value: {e}") + + @staticmethod + def validate_json_pixit_value(value: str) -> None: + """Validates that a value can be parsed as valid JSON. + + Args: + value: Value to validate + + Raises: + ValueError: If value cannot be parsed as valid JSON + """ + try: + json.loads(value) + except json.JSONDecodeError as e: + raise ValueError(f"Invalid JSON value: {e}") + + @staticmethod + def validate_hex_pixit_value(value: str) -> None: + """Validates that a value represents valid hexadecimal data. + + Args: + value: Value to validate. Can include optional "0x" prefix or "hex:" prefix + + Raises: + ValueError: If value is not valid hexadecimal or has odd number of digits + """ + # Remove optional "0x" or "hex:" prefix + if value.startswith("0x"): + hex_value = value[2:] + elif value.startswith("hex:"): + hex_value = value[4:] + else: + hex_value = value + + try: + int(hex_value, 16) # Validate hex format + if len(hex_value) % 2 != 0: + raise ValueError("Hex string must have even number of digits") + except (ValueError, TypeError) as e: + raise ValueError(f"Invalid hex value: {e}") + + @classmethod + def validate_value(cls, value: Any, pixit_def: PIXITDefinition) -> None: + """Validate PIXIT value matches its declared type. + + Args: + value: The value to validate + pixit_def: The PIXIT definition containing type and requirements + + Raises: + PIXITValidationError: If validation fails + """ + if not PIXITDefinition.is_pixit(pixit_def.name): + return # Skip validation for non-PIXIT args + + if value is None: + if pixit_def.required: + raise PIXITValidationError( + f"Required PIXIT {pixit_def.name} ({pixit_def.description}) is missing" + ) + return + + # Mapping of PIXITType to validation function + type_validators = { + PIXITType.INT: cls.validate_int_pixit_value, + PIXITType.BOOL: cls.validate_bool_pixit_value, + PIXITType.FLOAT: cls.validate_float_pixit_value, + PIXITType.STRING: cls.validate_string_pixit_value, + PIXITType.JSON: cls.validate_json_pixit_value, + PIXITType.HEX: cls.validate_hex_pixit_value, + } + + validator = type_validators.get(pixit_def.pixit_type) + if not validator: + raise PIXITValidationError(f"Unknown PIXIT type: {pixit_def.pixit_type}") + + try: + validator(value) + except (ValueError, TypeError, json.JSONDecodeError) as e: + raise PIXITValidationError( + f"Invalid value for {pixit_def.name}: {value} (expected {pixit_def.pixit_type.value})" + ) from e + + @classmethod + def validate_pixits(cls, pixits: list[PIXITDefinition], + provided_values: dict[str, Any]) -> None: + """Validate all PIXITs against provided values. + + Args: + pixits: List of PIXIT definitions to validate + provided_values: Dictionary of provided PIXIT values + + Raises: + PIXITValidationError: If any PIXIT validation fails + """ + missing = [] + invalid = [] + + for pixit in pixits: + value = provided_values.get(pixit.name) + if value is None and pixit.required: + missing.append(f"{pixit.name} ({pixit.description})") + continue + + # Validate non-missing values + if value is not None: + try: + cls.validate_value(value, pixit) + except PIXITValidationError as e: + invalid.append(str(e)) + + # Collect all validation errors + if missing or invalid: + error_msg = "" + if missing: + error_msg += "Missing required PIXITs: " + ", ".join(missing) + if invalid: + error_msg += "Invalid PIXIT values: " + ", ".join(invalid) + raise PIXITValidationError(error_msg) + + @dataclass class MatterTestConfig: storage_path: pathlib.Path = pathlib.Path(".") @@ -960,6 +1182,7 @@ class TestInfo: desc: str steps: list[TestStep] pics: list[str] + pixits: list[PIXITDefinition] class MatterBaseTest(base_test.BaseTestClass): @@ -984,6 +1207,15 @@ async def commission_devices(self) -> bool: return True + def get_test_pixits(self, test: str) -> list[PIXITDefinition]: + """Get PIXIT definitions for a specific test""" + pixits_name = f'pixits_{test.removeprefix("test_")}' + try: + fn = getattr(self, pixits_name) + return fn() + except AttributeError: + return [] + def get_test_steps(self, test: str) -> list[TestStep]: ''' Retrieves the test step list for the given test @@ -1155,8 +1387,17 @@ def setup_test(self): self.step_start_time = datetime.now(timezone.utc) self.step_skipped = False self.failed = False + test_name = self.current_test_info.name + + if not self.is_commissioning: + pixits = self.get_test_pixits(test_name) + validator = PIXITValidator() + try: + validator.validate_pixits(pixits, self.matter_test_config.global_test_params) + except PIXITValidationError as e: + raise signals.TestFailure(f"PIXIT validation failed for test {test_name}: {str(e)}") + if self.runner_hook and not self.is_commissioning: - test_name = self.current_test_info.name steps = self.get_defined_test_steps(test_name) num_steps = 1 if steps is None else len(steps) filename = inspect.getfile(self.__class__) @@ -2469,7 +2710,8 @@ def get_test_info(test_class: MatterBaseTest, matter_test_config: MatterTestConf info = [] for t in tests: - info.append(TestInfo(t, steps=base.get_test_steps(t), desc=base.get_test_desc(t), pics=base.get_test_pics(t))) + info.append(TestInfo(t, steps=base.get_test_steps(t), desc=base.get_test_desc(t), + pics=base.get_test_pics(t), pixits=base.get_test_pixits(t))) return info