From 38f47dee484c0bc9e678934ed0cdcf77a5963507 Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Thu, 21 Nov 2024 09:21:49 -0500 Subject: [PATCH 01/45] Added modified spec_parsing.py to now use the data model package --- .../chip/testing/spec_parsing.py | 108 ++++++++++-------- 1 file changed, 61 insertions(+), 47 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 97a13606eabc45..d547c5c5d88f78 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -18,12 +18,14 @@ import glob import logging import os +import importlib +import importlib.resources import typing import xml.etree.ElementTree as ElementTree from copy import deepcopy from dataclasses import dataclass from enum import Enum, auto -from typing import Callable, Optional +from typing import Callable, Optional, Union import chip.clusters as Clusters import chip.testing.conformance as conformance_support @@ -519,49 +521,73 @@ class DataModelLevel(str, Enum): def _get_data_model_root() -> str: - """Attempts to find ${CHIP_ROOT}/data_model or equivalent.""" - - # Since this class is generally in a module, we have to rely on being bootstrapped or - # we use CWD if we cannot - choices = [os.getcwd()] - - if 'PW_PROJECT_ROOT' in os.environ: - choices.insert(0, os.environ['PW_PROJECT_ROOT']) - - for c in choices: - data_model_path = os.path.join(c, 'data_model') - if os.path.exists(os.path.join(data_model_path, 'master', 'scraper_version')): - return data_model_path - raise FileNotFoundError('Cannot find a CHIP_ROOT/data_model path. Tried %r as prefixes.' % choices) + """Attempts to find the 'data_model' directory inside the Python package (whl).""" + try: + # Use importlib.resources to get the directory contents + package = importlib.import_module('chip_testing') + data_model_path = None + + # This lists all resources under the package, filtering for the 'data_model' directory + for resource in importlib.resources.contents(package): + if resource.endswith('data_model'): + data_model_path = resource + break + + if not data_model_path: + raise FileNotFoundError("Data model directory not found in the package.") + + return data_model_path + + except Exception as e: + raise FileNotFoundError(f"Failed to find the data model root: {e}") def get_data_model_directory(data_model_directory: typing.Union[PrebuiltDataModelDirectory, str], data_model_level: DataModelLevel) -> str: + """ + Get the directory of the data model for a specific version and level from the installed package. + """ + data_model_root = _get_data_model_root() + if data_model_directory == PrebuiltDataModelDirectory.k1_3: - return os.path.join(_get_data_model_root(), '1.3', data_model_level) + return os.path.join(data_model_root, '1.3', data_model_level) elif data_model_directory == PrebuiltDataModelDirectory.k1_4: - return os.path.join(_get_data_model_root(), '1.4', data_model_level) + return os.path.join(data_model_root, '1.4', data_model_level) elif data_model_directory == PrebuiltDataModelDirectory.kMaster: - return os.path.join(_get_data_model_root(), 'master', data_model_level) + return os.path.join(data_model_root, 'master', data_model_level) else: return data_model_directory -def build_xml_clusters(data_model_directory: typing.Union[PrebuiltDataModelDirectory, str] = PrebuiltDataModelDirectory.k1_4) -> tuple[dict[uint, XmlCluster], list[ProblemNotice]]: +def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, str] = PrebuiltDataModelDirectory.k1_4) -> tuple[dict[int, XmlCluster], list[ProblemNotice]]: + # Get the data model directory path inside the wheel package dir = get_data_model_directory(data_model_directory, DataModelLevel.kCluster) clusters: dict[int, XmlCluster] = {} pure_base_clusters: dict[str, XmlCluster] = {} ids_by_name: dict[str, int] = {} problems: list[ProblemNotice] = [] - files = glob.glob(f'{dir}/*.xml') - if not files: - raise SpecParsingException(f'No data model files found in specified directory {dir}') - for xml in files: - logging.info(f'Parsing file {xml}') - tree = ElementTree.parse(f'{xml}') - root = tree.getroot() - add_cluster_data_from_xml(root, clusters, pure_base_clusters, ids_by_name, problems) + package_name = "chip_testing" # The name of the wheel package + + try: + # Use importlib.resources to list all files in the data model directory within the package + package = importlib.import_module(package_name) + xml_files = [f for f in importlib.resources.contents(package) if f.endswith('.xml') and f.startswith(dir)] + + if not xml_files: + raise SpecParsingException(f'No XML files found in the specified package directory {dir}') + + # Parse each XML file found inside the package + for xml in xml_files: + logging.info(f'Parsing file {xml}') + # Open and parse each XML file from the wheel package + with importlib.resources.open_text(package, xml) as file: + tree = ElementTree.parse(file) + root = tree.getroot() + add_cluster_data_from_xml(root, clusters, pure_base_clusters, ids_by_name, problems) + + except Exception as e: + raise SpecParsingException(f"Error reading XML files from the package: {e}") # There are a few clusters where the conformance columns are listed as desc. These clusters need specific, targeted tests # to properly assess conformance. Here, we list them as Optional to allow these for the general test. Targeted tests are described below. @@ -569,6 +595,7 @@ def build_xml_clusters(data_model_directory: typing.Union[PrebuiltDataModelDirec # Actions cluster - all commands - these need to be listed in the ActionsList attribute to be supported. # We do not currently have a test for this. Please see https://github.com/CHIP-Specifications/chip-test-plans/issues/3646. + # Define remove_problem function to modify the problems list def remove_problem(location: typing.Union[CommandPathLocation, FeaturePathLocation]): nonlocal problems problems = [p for p in problems if p.location != location] @@ -578,6 +605,7 @@ def remove_problem(location: typing.Union[CommandPathLocation, FeaturePathLocati mask = clusters[descriptor_id].feature_map[code] clusters[descriptor_id].features[mask].conformance = optional() remove_problem(FeaturePathLocation(endpoint_id=0, cluster_id=descriptor_id, feature_code=code)) + action_id = Clusters.Actions.id for c in Clusters.ClusterObjects.ALL_ACCEPTED_COMMANDS[action_id]: clusters[action_id].accepted_commands[c].conformance = optional() @@ -592,25 +620,11 @@ def remove_problem(location: typing.Union[CommandPathLocation, FeaturePathLocati # see 3.2.8. Defined Primaries Information Attribute Set, affects Primary<#>X/Y/Intensity attributes. cc_id = Clusters.ColorControl.id cc_attr = Clusters.ColorControl.Attributes - affected_attributes = [cc_attr.Primary1X, - cc_attr.Primary1Y, - cc_attr.Primary1Intensity, - cc_attr.Primary2X, - cc_attr.Primary2Y, - cc_attr.Primary2Intensity, - cc_attr.Primary3X, - cc_attr.Primary3Y, - cc_attr.Primary3Intensity, - cc_attr.Primary4X, - cc_attr.Primary4Y, - cc_attr.Primary4Intensity, - cc_attr.Primary5X, - cc_attr.Primary5Y, - cc_attr.Primary5Intensity, - cc_attr.Primary6X, - cc_attr.Primary6Y, - cc_attr.Primary6Intensity, - ] + affected_attributes = [cc_attr.Primary1X, cc_attr.Primary1Y, cc_attr.Primary1Intensity, cc_attr.Primary2X, + cc_attr.Primary2Y, cc_attr.Primary2Intensity, cc_attr.Primary3X, cc_attr.Primary3Y, + cc_attr.Primary3Intensity, cc_attr.Primary4X, cc_attr.Primary4Y, cc_attr.Primary4Intensity, + cc_attr.Primary5X, cc_attr.Primary5Y, cc_attr.Primary5Intensity, cc_attr.Primary6X, + cc_attr.Primary6Y, cc_attr.Primary6Intensity] for a in affected_attributes: clusters[cc_id].attributes[a.attribute_id].conformance = optional() @@ -640,7 +654,7 @@ def remove_problem(location: typing.Union[CommandPathLocation, FeaturePathLocati presents_id = clusters[Clusters.Thermostat.id].attribute_map[presets_name] schedules_id = clusters[Clusters.Thermostat.id].attribute_map[schedules_name] conformance = or_operation([conformance_support.attribute(presents_id, presets_name), - conformance_support.attribute(schedules_id, schedules_name)]) + conformance_support.attribute(schedules_id, schedules_name)]) clusters[Clusters.Thermostat.id].accepted_commands[atomic_request_cmd_id] = XmlCommand( id=atomic_request_cmd_id, name=atomic_request_name, conformance=conformance) clusters[Clusters.Thermostat.id].generated_commands[atomic_response_cmd_id] = XmlCommand( From 946f316b90d1e6bc0eb4e6835501c57536a38a04 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 21 Nov 2024 14:26:32 +0000 Subject: [PATCH 02/45] Restyled by autopep8 --- .../chip/testing/spec_parsing.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index d547c5c5d88f78..80f853c65b64c6 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -526,18 +526,18 @@ def _get_data_model_root() -> str: # Use importlib.resources to get the directory contents package = importlib.import_module('chip_testing') data_model_path = None - + # This lists all resources under the package, filtering for the 'data_model' directory for resource in importlib.resources.contents(package): if resource.endswith('data_model'): data_model_path = resource break - + if not data_model_path: raise FileNotFoundError("Data model directory not found in the package.") - + return data_model_path - + except Exception as e: raise FileNotFoundError(f"Failed to find the data model root: {e}") @@ -573,7 +573,7 @@ def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, s # Use importlib.resources to list all files in the data model directory within the package package = importlib.import_module(package_name) xml_files = [f for f in importlib.resources.contents(package) if f.endswith('.xml') and f.startswith(dir)] - + if not xml_files: raise SpecParsingException(f'No XML files found in the specified package directory {dir}') @@ -605,7 +605,7 @@ def remove_problem(location: typing.Union[CommandPathLocation, FeaturePathLocati mask = clusters[descriptor_id].feature_map[code] clusters[descriptor_id].features[mask].conformance = optional() remove_problem(FeaturePathLocation(endpoint_id=0, cluster_id=descriptor_id, feature_code=code)) - + action_id = Clusters.Actions.id for c in Clusters.ClusterObjects.ALL_ACCEPTED_COMMANDS[action_id]: clusters[action_id].accepted_commands[c].conformance = optional() From a51f664e01dae63d90670a6eb668b16927e552a8 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 21 Nov 2024 14:26:34 +0000 Subject: [PATCH 03/45] Restyled by isort --- .../chip/testing/spec_parsing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 80f853c65b64c6..193a96ecefc49a 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -16,10 +16,10 @@ # import glob -import logging -import os import importlib import importlib.resources +import logging +import os import typing import xml.etree.ElementTree as ElementTree from copy import deepcopy From 923470fd730d76a06248b80b579763b784cd3e78 Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani <152916324+vatsalghelani-csa@users.noreply.github.com> Date: Thu, 21 Nov 2024 09:33:13 -0500 Subject: [PATCH 04/45] Fixing the cosmetic changes --- .../chip/testing/spec_parsing.py | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 193a96ecefc49a..e28d6fde4d153b 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -595,7 +595,6 @@ def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, s # Actions cluster - all commands - these need to be listed in the ActionsList attribute to be supported. # We do not currently have a test for this. Please see https://github.com/CHIP-Specifications/chip-test-plans/issues/3646. - # Define remove_problem function to modify the problems list def remove_problem(location: typing.Union[CommandPathLocation, FeaturePathLocation]): nonlocal problems problems = [p for p in problems if p.location != location] @@ -620,11 +619,25 @@ def remove_problem(location: typing.Union[CommandPathLocation, FeaturePathLocati # see 3.2.8. Defined Primaries Information Attribute Set, affects Primary<#>X/Y/Intensity attributes. cc_id = Clusters.ColorControl.id cc_attr = Clusters.ColorControl.Attributes - affected_attributes = [cc_attr.Primary1X, cc_attr.Primary1Y, cc_attr.Primary1Intensity, cc_attr.Primary2X, - cc_attr.Primary2Y, cc_attr.Primary2Intensity, cc_attr.Primary3X, cc_attr.Primary3Y, - cc_attr.Primary3Intensity, cc_attr.Primary4X, cc_attr.Primary4Y, cc_attr.Primary4Intensity, - cc_attr.Primary5X, cc_attr.Primary5Y, cc_attr.Primary5Intensity, cc_attr.Primary6X, - cc_attr.Primary6Y, cc_attr.Primary6Intensity] + affected_attributes = [cc_attr.Primary1X, + cc_attr.Primary1Y, + cc_attr.Primary1Intensity, + cc_attr.Primary2X, + cc_attr.Primary2Y, + cc_attr.Primary2Intensity, + cc_attr.Primary3X, + cc_attr.Primary3Y, + cc_attr.Primary3Intensity, + cc_attr.Primary4X, + cc_attr.Primary4Y, + cc_attr.Primary4Intensity, + cc_attr.Primary5X, + cc_attr.Primary5Y, + cc_attr.Primary5Intensity, + cc_attr.Primary6X, + cc_attr.Primary6Y, + cc_attr.Primary6Intensity, + ] for a in affected_attributes: clusters[cc_id].attributes[a.attribute_id].conformance = optional() @@ -654,7 +667,7 @@ def remove_problem(location: typing.Union[CommandPathLocation, FeaturePathLocati presents_id = clusters[Clusters.Thermostat.id].attribute_map[presets_name] schedules_id = clusters[Clusters.Thermostat.id].attribute_map[schedules_name] conformance = or_operation([conformance_support.attribute(presents_id, presets_name), - conformance_support.attribute(schedules_id, schedules_name)]) + conformance_support.attribute(schedules_id, schedules_name)]) clusters[Clusters.Thermostat.id].accepted_commands[atomic_request_cmd_id] = XmlCommand( id=atomic_request_cmd_id, name=atomic_request_name, conformance=conformance) clusters[Clusters.Thermostat.id].generated_commands[atomic_response_cmd_id] = XmlCommand( From 99fcef5bf031c77ebcfd19e93613b9d5564d32aa Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Mon, 25 Nov 2024 09:02:16 -0500 Subject: [PATCH 05/45] Use chip.testing as a module to extract, work via PosixPath directory --- .../chip/testing/spec_parsing.py | 76 +++++++++---------- 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index e28d6fde4d153b..e94847b450adec 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -16,10 +16,8 @@ # import glob -import importlib -import importlib.resources +import pathlib import logging -import os import typing import xml.etree.ElementTree as ElementTree from copy import deepcopy @@ -27,6 +25,7 @@ from enum import Enum, auto from typing import Callable, Optional, Union +import chip.testing import chip.clusters as Clusters import chip.testing.conformance as conformance_support from chip.testing.conformance import (OPTIONAL_CONFORM, TOP_LEVEL_CONFORMANCE_TAGS, ConformanceDecision, ConformanceException, @@ -520,59 +519,58 @@ class DataModelLevel(str, Enum): kDeviceType = 'device_types' -def _get_data_model_root() -> str: - """Attempts to find the 'data_model' directory inside the Python package (whl).""" +def _get_data_model_root() -> pathlib.PosixPath: + """Attempts to find the 'data_model' directory inside the 'chip.testing' package.""" try: - # Use importlib.resources to get the directory contents - package = importlib.import_module('chip_testing') - data_model_path = None - - # This lists all resources under the package, filtering for the 'data_model' directory - for resource in importlib.resources.contents(package): - if resource.endswith('data_model'): - data_model_path = resource - break - - if not data_model_path: - raise FileNotFoundError("Data model directory not found in the package.") - - return data_model_path + # Locate the directory where the 'chip.testing' module is located + package_dir = pathlib.Path(chip.testing.__file__).parent + + # Construct the path to the 'data_model' directory (assuming this structure) + data_model_root = package_dir / 'data_model' + + if not data_model_root.exists(): + raise FileNotFoundError(f"Data model directory not found in the package at {data_model_root}.") + + return data_model_root except Exception as e: raise FileNotFoundError(f"Failed to find the data model root: {e}") - -def get_data_model_directory(data_model_directory: typing.Union[PrebuiltDataModelDirectory, str], data_model_level: DataModelLevel) -> str: +def get_data_model_directory(data_model_directory: Union[PrebuiltDataModelDirectory, str], data_model_level: str) -> pathlib.PosixPath: """ Get the directory of the data model for a specific version and level from the installed package. """ data_model_root = _get_data_model_root() - - if data_model_directory == PrebuiltDataModelDirectory.k1_3: - return os.path.join(data_model_root, '1.3', data_model_level) - elif data_model_directory == PrebuiltDataModelDirectory.k1_4: - return os.path.join(data_model_root, '1.4', data_model_level) - elif data_model_directory == PrebuiltDataModelDirectory.kMaster: - return os.path.join(data_model_root, 'master', data_model_level) + + # Build path based on the version and data model level + if isinstance(data_model_directory, PrebuiltDataModelDirectory): + version_map = { + PrebuiltDataModelDirectory.k1_3: '1.3', + PrebuiltDataModelDirectory.k1_4: '1.4', + PrebuiltDataModelDirectory.kMaster: 'master', + } + + version = version_map.get(data_model_directory) + if not version: + raise ValueError(f"Unsupported data model directory: {data_model_directory}") + + return data_model_root / version / data_model_level else: - return data_model_directory - + # If it's a custom directory, return it directly + return pathlib.Path(data_model_directory) def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, str] = PrebuiltDataModelDirectory.k1_4) -> tuple[dict[int, XmlCluster], list[ProblemNotice]]: - # Get the data model directory path inside the wheel package - dir = get_data_model_directory(data_model_directory, DataModelLevel.kCluster) + # Get the data model directory path inside the package + dir = get_data_model_directory(data_model_directory, 'kCluster') clusters: dict[int, XmlCluster] = {} pure_base_clusters: dict[str, XmlCluster] = {} ids_by_name: dict[str, int] = {} problems: list[ProblemNotice] = [] - package_name = "chip_testing" # The name of the wheel package - try: - # Use importlib.resources to list all files in the data model directory within the package - package = importlib.import_module(package_name) - xml_files = [f for f in importlib.resources.contents(package) if f.endswith('.xml') and f.startswith(dir)] + # Use pathlib.Path to list all XML files in the data model directory + xml_files = [f for f in dir.glob('**/*.xml')] # Recursively find all .xml files if not xml_files: raise SpecParsingException(f'No XML files found in the specified package directory {dir}') @@ -580,8 +578,8 @@ def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, s # Parse each XML file found inside the package for xml in xml_files: logging.info(f'Parsing file {xml}') - # Open and parse each XML file from the wheel package - with importlib.resources.open_text(package, xml) as file: + # Open and parse each XML file from the directory + with xml.open('r') as file: tree = ElementTree.parse(file) root = tree.getroot() add_cluster_data_from_xml(root, clusters, pure_base_clusters, ids_by_name, problems) From d14f4e0b56e3d350aea86c9f4ded96136da6cff3 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Mon, 25 Nov 2024 14:03:03 +0000 Subject: [PATCH 06/45] Restyled by autopep8 --- .../chip/testing/spec_parsing.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index e94847b450adec..6f771ce75150bb 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -524,24 +524,25 @@ def _get_data_model_root() -> pathlib.PosixPath: try: # Locate the directory where the 'chip.testing' module is located package_dir = pathlib.Path(chip.testing.__file__).parent - + # Construct the path to the 'data_model' directory (assuming this structure) data_model_root = package_dir / 'data_model' - + if not data_model_root.exists(): raise FileNotFoundError(f"Data model directory not found in the package at {data_model_root}.") - + return data_model_root except Exception as e: raise FileNotFoundError(f"Failed to find the data model root: {e}") + def get_data_model_directory(data_model_directory: Union[PrebuiltDataModelDirectory, str], data_model_level: str) -> pathlib.PosixPath: """ Get the directory of the data model for a specific version and level from the installed package. """ data_model_root = _get_data_model_root() - + # Build path based on the version and data model level if isinstance(data_model_directory, PrebuiltDataModelDirectory): version_map = { @@ -549,16 +550,17 @@ def get_data_model_directory(data_model_directory: Union[PrebuiltDataModelDirect PrebuiltDataModelDirectory.k1_4: '1.4', PrebuiltDataModelDirectory.kMaster: 'master', } - + version = version_map.get(data_model_directory) if not version: raise ValueError(f"Unsupported data model directory: {data_model_directory}") - + return data_model_root / version / data_model_level else: # If it's a custom directory, return it directly return pathlib.Path(data_model_directory) + def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, str] = PrebuiltDataModelDirectory.k1_4) -> tuple[dict[int, XmlCluster], list[ProblemNotice]]: # Get the data model directory path inside the package dir = get_data_model_directory(data_model_directory, 'kCluster') From c57a3c1a1d86bcfa2cb6e0880f8b35ad3eacc800 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Mon, 25 Nov 2024 14:03:06 +0000 Subject: [PATCH 07/45] Restyled by isort --- .../chip/testing/spec_parsing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 6f771ce75150bb..4be0d2adcab709 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -16,8 +16,8 @@ # import glob -import pathlib import logging +import pathlib import typing import xml.etree.ElementTree as ElementTree from copy import deepcopy @@ -25,8 +25,8 @@ from enum import Enum, auto from typing import Callable, Optional, Union -import chip.testing import chip.clusters as Clusters +import chip.testing import chip.testing.conformance as conformance_support from chip.testing.conformance import (OPTIONAL_CONFORM, TOP_LEVEL_CONFORMANCE_TAGS, ConformanceDecision, ConformanceException, ConformanceParseParameters, feature, is_disallowed, mandatory, optional, or_operation, From 0d9eb8726d6001037d96ffa0fcd442a93214f546 Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Mon, 25 Nov 2024 12:06:16 -0500 Subject: [PATCH 08/45] Fixed correct directory search --- .../matter_testing_infrastructure/chip/testing/spec_parsing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 4be0d2adcab709..ff344bb8d049cb 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -563,7 +563,7 @@ def get_data_model_directory(data_model_directory: Union[PrebuiltDataModelDirect def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, str] = PrebuiltDataModelDirectory.k1_4) -> tuple[dict[int, XmlCluster], list[ProblemNotice]]: # Get the data model directory path inside the package - dir = get_data_model_directory(data_model_directory, 'kCluster') + dir = get_data_model_directory(data_model_directory, 'clusters') clusters: dict[int, XmlCluster] = {} pure_base_clusters: dict[str, XmlCluster] = {} From 296c91fd0dee37980dd96ff982d2bee6347997f6 Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Tue, 26 Nov 2024 10:45:44 -0500 Subject: [PATCH 09/45] Solving the wrapping for try-catch according to comments --- .../chip/testing/spec_parsing.py | 42 +++++++++---------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index ff344bb8d049cb..223b6522c7753c 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -521,20 +521,16 @@ class DataModelLevel(str, Enum): def _get_data_model_root() -> pathlib.PosixPath: """Attempts to find the 'data_model' directory inside the 'chip.testing' package.""" - try: - # Locate the directory where the 'chip.testing' module is located - package_dir = pathlib.Path(chip.testing.__file__).parent + # Locate the directory where the 'chip.testing' module is located + package_dir = pathlib.Path(chip.testing.__file__).parent - # Construct the path to the 'data_model' directory (assuming this structure) - data_model_root = package_dir / 'data_model' + # Construct the path to the 'data_model' directory (assuming this structure) + data_model_root = package_dir / 'data_model' - if not data_model_root.exists(): - raise FileNotFoundError(f"Data model directory not found in the package at {data_model_root}.") + if not data_model_root.exists(): + raise FileNotFoundError(f"Data model directory not found in the package at {data_model_root}.") - return data_model_root - - except Exception as e: - raise FileNotFoundError(f"Failed to find the data model root: {e}") + return data_model_root def get_data_model_directory(data_model_directory: Union[PrebuiltDataModelDirectory, str], data_model_level: str) -> pathlib.PosixPath: @@ -570,24 +566,24 @@ def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, s ids_by_name: dict[str, int] = {} problems: list[ProblemNotice] = [] - try: - # Use pathlib.Path to list all XML files in the data model directory - xml_files = [f for f in dir.glob('**/*.xml')] # Recursively find all .xml files + # Use pathlib.Path to list all XML files in the data model directory + xml_files = [f for f in dir.glob('**/*.xml')] # Recursively find all .xml files - if not xml_files: - raise SpecParsingException(f'No XML files found in the specified package directory {dir}') + if not xml_files: + raise SpecParsingException(f'No XML files found in the specified package directory {dir}') - # Parse each XML file found inside the package - for xml in xml_files: - logging.info(f'Parsing file {xml}') - # Open and parse each XML file from the directory - with xml.open('r') as file: + # Parse each XML file found inside the package + for xml in xml_files: + logging.info(f'Parsing file {xml}') + # Open and parse each XML file from the directory + with xml.open('r') as file: + try: tree = ElementTree.parse(file) root = tree.getroot() add_cluster_data_from_xml(root, clusters, pure_base_clusters, ids_by_name, problems) + except Exception as e: + logging.error(f"Error parsing XML file {xml}: {e}") - except Exception as e: - raise SpecParsingException(f"Error reading XML files from the package: {e}") # There are a few clusters where the conformance columns are listed as desc. These clusters need specific, targeted tests # to properly assess conformance. Here, we list them as Optional to allow these for the general test. Targeted tests are described below. From d07b52f3884b03d764bf196eaa4aa467b2f129e0 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Tue, 26 Nov 2024 15:46:30 +0000 Subject: [PATCH 10/45] Restyled by autopep8 --- .../matter_testing_infrastructure/chip/testing/spec_parsing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 223b6522c7753c..65a5629fb9c3db 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -584,7 +584,6 @@ def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, s except Exception as e: logging.error(f"Error parsing XML file {xml}: {e}") - # There are a few clusters where the conformance columns are listed as desc. These clusters need specific, targeted tests # to properly assess conformance. Here, we list them as Optional to allow these for the general test. Targeted tests are described below. # Descriptor - TagList feature - this feature is mandated when the duplicate condition holds for the endpoint. It is tested in DESC-2.2 From c849ea9f59886f01f30aad352c7579831576d573 Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Tue, 3 Dec 2024 12:56:55 -0500 Subject: [PATCH 11/45] Added fixes for both a pre-built location or a full path --- .../chip/testing/spec_parsing.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 65a5629fb9c3db..e04e5e7d6c1ede 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -539,7 +539,7 @@ def get_data_model_directory(data_model_directory: Union[PrebuiltDataModelDirect """ data_model_root = _get_data_model_root() - # Build path based on the version and data model level + # If it's a prebuilt directory, build the path based on the version and data model level if isinstance(data_model_directory, PrebuiltDataModelDirectory): version_map = { PrebuiltDataModelDirectory.k1_3: '1.3', @@ -558,8 +558,15 @@ def get_data_model_directory(data_model_directory: Union[PrebuiltDataModelDirect def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, str] = PrebuiltDataModelDirectory.k1_4) -> tuple[dict[int, XmlCluster], list[ProblemNotice]]: - # Get the data model directory path inside the package - dir = get_data_model_directory(data_model_directory, 'clusters') + """ + Build XML clusters from the specified data model directory. + This function supports both pre-built locations (via `PrebuiltDataModelDirectory`) and full paths (strings). + """ + # If a pre-built directory is provided, resolve the full path + if isinstance(data_model_directory, PrebuiltDataModelDirectory): + data_model_directory = get_data_model_directory(data_model_directory, 'clusters') + + dir = pathlib.Path(data_model_directory) clusters: dict[int, XmlCluster] = {} pure_base_clusters: dict[str, XmlCluster] = {} @@ -584,6 +591,7 @@ def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, s except Exception as e: logging.error(f"Error parsing XML file {xml}: {e}") + # There are a few clusters where the conformance columns are listed as desc. These clusters need specific, targeted tests # to properly assess conformance. Here, we list them as Optional to allow these for the general test. Targeted tests are described below. # Descriptor - TagList feature - this feature is mandated when the duplicate condition holds for the endpoint. It is tested in DESC-2.2 From e3addddd780a5120f22d7e0d6e11e8f0a862b4a0 Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Tue, 3 Dec 2024 13:32:41 -0500 Subject: [PATCH 12/45] Fix comment --- .../matter_testing_infrastructure/chip/testing/spec_parsing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index e04e5e7d6c1ede..1d570a5376a443 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -553,7 +553,7 @@ def get_data_model_directory(data_model_directory: Union[PrebuiltDataModelDirect return data_model_root / version / data_model_level else: - # If it's a custom directory, return it directly + # If it's a custom directory, returns directly return pathlib.Path(data_model_directory) From e9ba66858a6a6dc3047bf2011824b551dfda1685 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Tue, 3 Dec 2024 18:33:22 +0000 Subject: [PATCH 13/45] Restyled by autopep8 --- .../matter_testing_infrastructure/chip/testing/spec_parsing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 1d570a5376a443..ef5f396d0f2996 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -591,7 +591,6 @@ def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, s except Exception as e: logging.error(f"Error parsing XML file {xml}: {e}") - # There are a few clusters where the conformance columns are listed as desc. These clusters need specific, targeted tests # to properly assess conformance. Here, we list them as Optional to allow these for the general test. Targeted tests are described below. # Descriptor - TagList feature - this feature is mandated when the duplicate condition holds for the endpoint. It is tested in DESC-2.2 From f08316e41b31216ebad3fe63686cace30d4d7988 Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Wed, 4 Dec 2024 11:36:44 -0500 Subject: [PATCH 14/45] Adding importlib.resources capability --- .../chip/testing/spec_parsing.py | 45 +++++++++++-------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index ef5f396d0f2996..93946da8a502ae 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -17,6 +17,7 @@ import glob import logging +import importlib.resources as pkg_resources import pathlib import typing import xml.etree.ElementTree as ElementTree @@ -519,16 +520,17 @@ class DataModelLevel(str, Enum): kDeviceType = 'device_types' -def _get_data_model_root() -> pathlib.PosixPath: +def _get_data_model_root() -> pathlib.Path: """Attempts to find the 'data_model' directory inside the 'chip.testing' package.""" - # Locate the directory where the 'chip.testing' module is located - package_dir = pathlib.Path(chip.testing.__file__).parent + # Access the 'chip.testing' package using importlib.resources + package_dir = pkg_resources.files(chip.testing) - # Construct the path to the 'data_model' directory (assuming this structure) + # Construct the path to the 'data_model' directory inside the package data_model_root = package_dir / 'data_model' + # Check if the 'data_model' directory exists if not data_model_root.exists(): - raise FileNotFoundError(f"Data model directory not found in the package at {data_model_root}.") + raise FileNotFoundError(f"Data model directory not found in the package at {data_model_root}") return data_model_root @@ -552,44 +554,51 @@ def get_data_model_directory(data_model_directory: Union[PrebuiltDataModelDirect raise ValueError(f"Unsupported data model directory: {data_model_directory}") return data_model_root / version / data_model_level + else: - # If it's a custom directory, returns directly - return pathlib.Path(data_model_directory) + # If it's a custom directory, return it directly + return pathlib.PosixPath(data_model_directory) def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, str] = PrebuiltDataModelDirectory.k1_4) -> tuple[dict[int, XmlCluster], list[ProblemNotice]]: """ Build XML clusters from the specified data model directory. - This function supports both pre-built locations (via `PrebuiltDataModelDirectory`) and full paths (strings). + This function supports both pre-built locations (via PrebuiltDataModelDirectory) and full paths (strings). """ # If a pre-built directory is provided, resolve the full path if isinstance(data_model_directory, PrebuiltDataModelDirectory): data_model_directory = get_data_model_directory(data_model_directory, 'clusters') - dir = pathlib.Path(data_model_directory) + # Use importlib.resources to access the data model root directory + data_model_root = pkg_resources.files(chip.testing) / 'data_model' + + # We can now use pkg_resources to list XML files inside the data_model directory + xml_files = [] + + # Recursively find all XML files in the 'data_model' directory + for xml in pkg_resources.contents(data_model_root): + if xml.endswith('.xml'): + xml_files.append(pathlib.Path(xml)) # Create Path objects for XML files + + if not xml_files: + raise SpecParsingException(f'No XML files found in the specified package directory {data_model_root}') + # Now we can parse each XML file found inside the package clusters: dict[int, XmlCluster] = {} pure_base_clusters: dict[str, XmlCluster] = {} ids_by_name: dict[str, int] = {} problems: list[ProblemNotice] = [] - # Use pathlib.Path to list all XML files in the data model directory - xml_files = [f for f in dir.glob('**/*.xml')] # Recursively find all .xml files - - if not xml_files: - raise SpecParsingException(f'No XML files found in the specified package directory {dir}') - - # Parse each XML file found inside the package for xml in xml_files: logging.info(f'Parsing file {xml}') - # Open and parse each XML file from the directory + # Open and parse each XML file with xml.open('r') as file: try: tree = ElementTree.parse(file) root = tree.getroot() add_cluster_data_from_xml(root, clusters, pure_base_clusters, ids_by_name, problems) except Exception as e: - logging.error(f"Error parsing XML file {xml}: {e}") + logging.error(f"Error parsing XML file {xml}: {e}") # There are a few clusters where the conformance columns are listed as desc. These clusters need specific, targeted tests # to properly assess conformance. Here, we list them as Optional to allow these for the general test. Targeted tests are described below. From 70e36226c660b3b72bc2726263bd4cc959cbb92c Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Wed, 4 Dec 2024 16:37:32 +0000 Subject: [PATCH 15/45] Restyled by autopep8 --- .../chip/testing/spec_parsing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 93946da8a502ae..2000f15b905728 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -557,7 +557,7 @@ def get_data_model_directory(data_model_directory: Union[PrebuiltDataModelDirect else: # If it's a custom directory, return it directly - return pathlib.PosixPath(data_model_directory) + return pathlib.PosixPath(data_model_directory) def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, str] = PrebuiltDataModelDirectory.k1_4) -> tuple[dict[int, XmlCluster], list[ProblemNotice]]: @@ -598,7 +598,7 @@ def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, s root = tree.getroot() add_cluster_data_from_xml(root, clusters, pure_base_clusters, ids_by_name, problems) except Exception as e: - logging.error(f"Error parsing XML file {xml}: {e}") + logging.error(f"Error parsing XML file {xml}: {e}") # There are a few clusters where the conformance columns are listed as desc. These clusters need specific, targeted tests # to properly assess conformance. Here, we list them as Optional to allow these for the general test. Targeted tests are described below. From 1b3494e6953c03365573599d30451fcdab262aed Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Wed, 4 Dec 2024 16:37:34 +0000 Subject: [PATCH 16/45] Restyled by isort --- .../matter_testing_infrastructure/chip/testing/spec_parsing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 2000f15b905728..a415cbd816d7a0 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -16,8 +16,8 @@ # import glob -import logging import importlib.resources as pkg_resources +import logging import pathlib import typing import xml.etree.ElementTree as ElementTree From d28c40eb8461991b4bf7cbb2f29888e984d69592 Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Wed, 4 Dec 2024 13:23:37 -0500 Subject: [PATCH 17/45] Fixed _spec_ path error --- .../matter_testing_infrastructure/chip/testing/spec_parsing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index a415cbd816d7a0..5faeab1542cd00 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -570,7 +570,7 @@ def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, s data_model_directory = get_data_model_directory(data_model_directory, 'clusters') # Use importlib.resources to access the data model root directory - data_model_root = pkg_resources.files(chip.testing) / 'data_model' + data_model_root = _get_data_model_root() # Get the correct path # We can now use pkg_resources to list XML files inside the data_model directory xml_files = [] From 891c2789f45ecc258cebb30b4a2d23a4e76b1d12 Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Wed, 4 Dec 2024 15:06:09 -0500 Subject: [PATCH 18/45] Fixed to use module as string --- .../matter_testing_infrastructure/chip/testing/spec_parsing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 5faeab1542cd00..b186d8e8b82d71 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -523,7 +523,7 @@ class DataModelLevel(str, Enum): def _get_data_model_root() -> pathlib.Path: """Attempts to find the 'data_model' directory inside the 'chip.testing' package.""" # Access the 'chip.testing' package using importlib.resources - package_dir = pkg_resources.files(chip.testing) + package_dir = pkg_resources.files('chip.testing') # Construct the path to the 'data_model' directory inside the package data_model_root = package_dir / 'data_model' From fd5ab8375d29cc3206d5d8eca57eac4301f62737 Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Wed, 4 Dec 2024 15:50:29 -0500 Subject: [PATCH 19/45] Removed unused import --- .../matter_testing_infrastructure/chip/testing/spec_parsing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index b186d8e8b82d71..a81fc0efcc68c4 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -27,7 +27,6 @@ from typing import Callable, Optional, Union import chip.clusters as Clusters -import chip.testing import chip.testing.conformance as conformance_support from chip.testing.conformance import (OPTIONAL_CONFORM, TOP_LEVEL_CONFORMANCE_TAGS, ConformanceDecision, ConformanceException, ConformanceParseParameters, feature, is_disallowed, mandatory, optional, or_operation, From 68ab20e6290afa220e5268511fe70549d4952702 Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Thu, 5 Dec 2024 10:18:07 -0500 Subject: [PATCH 20/45] Added fixes for path issues --- .../chip/testing/spec_parsing.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index a81fc0efcc68c4..ebe46f97ff94f0 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -525,7 +525,7 @@ def _get_data_model_root() -> pathlib.Path: package_dir = pkg_resources.files('chip.testing') # Construct the path to the 'data_model' directory inside the package - data_model_root = package_dir / 'data_model' + data_model_root = pathlib.Path(package_dir) / 'data_model' # Check if the 'data_model' directory exists if not data_model_root.exists(): @@ -569,15 +569,10 @@ def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, s data_model_directory = get_data_model_directory(data_model_directory, 'clusters') # Use importlib.resources to access the data model root directory - data_model_root = _get_data_model_root() # Get the correct path - - # We can now use pkg_resources to list XML files inside the data_model directory - xml_files = [] + data_model_root = _get_data_model_root() - # Recursively find all XML files in the 'data_model' directory - for xml in pkg_resources.contents(data_model_root): - if xml.endswith('.xml'): - xml_files.append(pathlib.Path(xml)) # Create Path objects for XML files + # List all .xml files in the directory using pathlib + xml_files = [file for file in data_model_root.iterdir() if file.suffix == '.xml'] if not xml_files: raise SpecParsingException(f'No XML files found in the specified package directory {data_model_root}') From 0a3d947ad1872b12c21801c7b5bd14ed502a7ec1 Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Thu, 5 Dec 2024 15:57:54 -0500 Subject: [PATCH 21/45] Fixing the xml not found error --- .../chip/testing/spec_parsing.py | 76 +++++++++---------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index ebe46f97ff94f0..e806bd779a6928 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -519,33 +519,31 @@ class DataModelLevel(str, Enum): kDeviceType = 'device_types' -def _get_data_model_root() -> pathlib.Path: +def _get_data_model_root() -> pathlib.PosixPath: """Attempts to find the 'data_model' directory inside the 'chip.testing' package.""" - # Access the 'chip.testing' package using importlib.resources - package_dir = pkg_resources.files('chip.testing') - - # Construct the path to the 'data_model' directory inside the package - data_model_root = pathlib.Path(package_dir) / 'data_model' - - # Check if the 'data_model' directory exists - if not data_model_root.exists(): + # Access the 'chip.testing' package directly via pkg_resources + package = pkg_resources.files(pkg_resources.import_module('chip.testing')) # Corrected: Removed importlib + try: + # We assume the 'data_model' directory is inside the package itself + data_model_root = package / 'data_model' + except FileNotFoundError: raise FileNotFoundError(f"Data model directory not found in the package at {data_model_root}") - + return data_model_root -def get_data_model_directory(data_model_directory: Union[PrebuiltDataModelDirectory, str], data_model_level: str) -> pathlib.PosixPath: +def get_data_model_directory(data_model_directory: Union[str], data_model_level: str) -> pathlib.PosixPath: """ Get the directory of the data model for a specific version and level from the installed package. """ data_model_root = _get_data_model_root() # If it's a prebuilt directory, build the path based on the version and data model level - if isinstance(data_model_directory, PrebuiltDataModelDirectory): + if isinstance(data_model_directory, str): version_map = { - PrebuiltDataModelDirectory.k1_3: '1.3', - PrebuiltDataModelDirectory.k1_4: '1.4', - PrebuiltDataModelDirectory.kMaster: 'master', + 'k1_3': '1.3', + 'k1_4': '1.4', + 'kMaster': 'master', } version = version_map.get(data_model_directory) @@ -553,46 +551,48 @@ def get_data_model_directory(data_model_directory: Union[PrebuiltDataModelDirect raise ValueError(f"Unsupported data model directory: {data_model_directory}") return data_model_root / version / data_model_level - else: - # If it's a custom directory, return it directly - return pathlib.PosixPath(data_model_directory) + # If it's a custom directory, returns directly + return pathlib.Path(data_model_directory) -def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, str] = PrebuiltDataModelDirectory.k1_4) -> tuple[dict[int, XmlCluster], list[ProblemNotice]]: +def build_xml_clusters(data_model_directory: Union[str] = 'k1_4') -> tuple[dict[int, dict], list]: """ Build XML clusters from the specified data model directory. - This function supports both pre-built locations (via PrebuiltDataModelDirectory) and full paths (strings). + This function supports both pre-built locations (via `str` directory names) and full paths (strings). """ # If a pre-built directory is provided, resolve the full path - if isinstance(data_model_directory, PrebuiltDataModelDirectory): + if isinstance(data_model_directory, str): data_model_directory = get_data_model_directory(data_model_directory, 'clusters') - # Use importlib.resources to access the data model root directory - data_model_root = _get_data_model_root() + # Convert the resolved directory path into a resource path using pkg_resources + dir = pathlib.Path(data_model_directory) - # List all .xml files in the directory using pathlib - xml_files = [file for file in data_model_root.iterdir() if file.suffix == '.xml'] + clusters = {} + pure_base_clusters = {} + ids_by_name = {} + problems = [] + # Use pkg_resources to list all XML files in the data model directory + try: + xml_files = [f for f in pkg_resources.contents(pkg_resources.files(pkg_resources.import_module('chip.testing')).joinpath('data_model').joinpath(data_model_directory)).splitlines() if f.endswith('.xml')] + except Exception as e: + logging.error(f"Error accessing XML files in {data_model_directory}: {e}") + return clusters, problems + if not xml_files: - raise SpecParsingException(f'No XML files found in the specified package directory {data_model_root}') - - # Now we can parse each XML file found inside the package - clusters: dict[int, XmlCluster] = {} - pure_base_clusters: dict[str, XmlCluster] = {} - ids_by_name: dict[str, int] = {} - problems: list[ProblemNotice] = [] + raise FileNotFoundError(f'No XML files found in the specified package directory {dir}') + # Parse each XML file found inside the package for xml in xml_files: logging.info(f'Parsing file {xml}') - # Open and parse each XML file - with xml.open('r') as file: - try: + try: + # Open the resource file using pkg_resources and parse it + with pkg_resources.open_text(pkg_resources.files(pkg_resources.import_module('chip.testing')).joinpath('data_model').joinpath(data_model_directory), xml) as file: tree = ElementTree.parse(file) root = tree.getroot() - add_cluster_data_from_xml(root, clusters, pure_base_clusters, ids_by_name, problems) - except Exception as e: - logging.error(f"Error parsing XML file {xml}: {e}") + except Exception as e: + logging.error(f"Error parsing XML file {xml}: {e}") # There are a few clusters where the conformance columns are listed as desc. These clusters need specific, targeted tests # to properly assess conformance. Here, we list them as Optional to allow these for the general test. Targeted tests are described below. From bd4172f9a1af8a915a5083cd60d5f7bdc88ac48f Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Thu, 5 Dec 2024 20:58:42 +0000 Subject: [PATCH 22/45] Restyled by autopep8 --- .../chip/testing/spec_parsing.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index e806bd779a6928..633d71b7e1964b 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -528,7 +528,7 @@ def _get_data_model_root() -> pathlib.PosixPath: data_model_root = package / 'data_model' except FileNotFoundError: raise FileNotFoundError(f"Data model directory not found in the package at {data_model_root}") - + return data_model_root @@ -568,18 +568,19 @@ def build_xml_clusters(data_model_directory: Union[str] = 'k1_4') -> tuple[dict[ # Convert the resolved directory path into a resource path using pkg_resources dir = pathlib.Path(data_model_directory) - clusters = {} + clusters = {} pure_base_clusters = {} ids_by_name = {} problems = [] # Use pkg_resources to list all XML files in the data model directory try: - xml_files = [f for f in pkg_resources.contents(pkg_resources.files(pkg_resources.import_module('chip.testing')).joinpath('data_model').joinpath(data_model_directory)).splitlines() if f.endswith('.xml')] + xml_files = [f for f in pkg_resources.contents(pkg_resources.files(pkg_resources.import_module( + 'chip.testing')).joinpath('data_model').joinpath(data_model_directory)).splitlines() if f.endswith('.xml')] except Exception as e: logging.error(f"Error accessing XML files in {data_model_directory}: {e}") return clusters, problems - + if not xml_files: raise FileNotFoundError(f'No XML files found in the specified package directory {dir}') From 5b758e6fe06a24a5c6707a5809859a2cbd65e78e Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Thu, 5 Dec 2024 16:16:02 -0500 Subject: [PATCH 23/45] Fixed code lint error --- .../matter_testing_infrastructure/chip/testing/spec_parsing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 633d71b7e1964b..8846f169dce215 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -591,7 +591,6 @@ def build_xml_clusters(data_model_directory: Union[str] = 'k1_4') -> tuple[dict[ # Open the resource file using pkg_resources and parse it with pkg_resources.open_text(pkg_resources.files(pkg_resources.import_module('chip.testing')).joinpath('data_model').joinpath(data_model_directory), xml) as file: tree = ElementTree.parse(file) - root = tree.getroot() except Exception as e: logging.error(f"Error parsing XML file {xml}: {e}") From aa5564e28faf629aa7fc41221ebdfcc0b4dd46a5 Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Thu, 5 Dec 2024 16:56:45 -0500 Subject: [PATCH 24/45] Fixed code lint error tree --- .../chip/testing/spec_parsing.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 8846f169dce215..65b133980023be 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -584,13 +584,17 @@ def build_xml_clusters(data_model_directory: Union[str] = 'k1_4') -> tuple[dict[ if not xml_files: raise FileNotFoundError(f'No XML files found in the specified package directory {dir}') - # Parse each XML file found inside the package for xml in xml_files: logging.info(f'Parsing file {xml}') try: # Open the resource file using pkg_resources and parse it - with pkg_resources.open_text(pkg_resources.files(pkg_resources.import_module('chip.testing')).joinpath('data_model').joinpath(data_model_directory), xml) as file: - tree = ElementTree.parse(file) + with pkg_resources.open_text( + pkg_resources.files(pkg_resources.import_module('chip.testing')) + .joinpath('data_model') + .joinpath(data_model_directory), xml + ) as file: + # Directly parse the XML file without assigning it to 'tree' + ElementTree.parse(file) # Parse the file to process it except Exception as e: logging.error(f"Error parsing XML file {xml}: {e}") From 79a7db4473da827fe644daab337289fcca3393bd Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Fri, 6 Dec 2024 09:28:07 -0500 Subject: [PATCH 25/45] Fixed importlib errors --- .../chip/testing/spec_parsing.py | 22 +++++-------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 65b133980023be..4368f5a1f21c5b 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -16,6 +16,7 @@ # import glob +import importlib import importlib.resources as pkg_resources import logging import pathlib @@ -519,19 +520,6 @@ class DataModelLevel(str, Enum): kDeviceType = 'device_types' -def _get_data_model_root() -> pathlib.PosixPath: - """Attempts to find the 'data_model' directory inside the 'chip.testing' package.""" - # Access the 'chip.testing' package directly via pkg_resources - package = pkg_resources.files(pkg_resources.import_module('chip.testing')) # Corrected: Removed importlib - try: - # We assume the 'data_model' directory is inside the package itself - data_model_root = package / 'data_model' - except FileNotFoundError: - raise FileNotFoundError(f"Data model directory not found in the package at {data_model_root}") - - return data_model_root - - def get_data_model_directory(data_model_directory: Union[str], data_model_level: str) -> pathlib.PosixPath: """ Get the directory of the data model for a specific version and level from the installed package. @@ -552,11 +540,11 @@ def get_data_model_directory(data_model_directory: Union[str], data_model_level: return data_model_root / version / data_model_level else: - # If it's a custom directory, returns directly + # If it's a custom directory, return directly return pathlib.Path(data_model_directory) -def build_xml_clusters(data_model_directory: Union[str] = 'k1_4') -> tuple[dict[int, dict], list]: +def build_xml_clusters(data_model_directory: Union[str] = 'k1_4') -> typing.Tuple[dict[int, dict], list]: """ Build XML clusters from the specified data model directory. This function supports both pre-built locations (via `str` directory names) and full paths (strings). @@ -575,7 +563,7 @@ def build_xml_clusters(data_model_directory: Union[str] = 'k1_4') -> tuple[dict[ # Use pkg_resources to list all XML files in the data model directory try: - xml_files = [f for f in pkg_resources.contents(pkg_resources.files(pkg_resources.import_module( + xml_files = [f for f in pkg_resources.contents(pkg_resources.files(importlib.import_module( 'chip.testing')).joinpath('data_model').joinpath(data_model_directory)).splitlines() if f.endswith('.xml')] except Exception as e: logging.error(f"Error accessing XML files in {data_model_directory}: {e}") @@ -589,7 +577,7 @@ def build_xml_clusters(data_model_directory: Union[str] = 'k1_4') -> tuple[dict[ try: # Open the resource file using pkg_resources and parse it with pkg_resources.open_text( - pkg_resources.files(pkg_resources.import_module('chip.testing')) + pkg_resources.files(importlib.import_module('chip.testing')) .joinpath('data_model') .joinpath(data_model_directory), xml ) as file: From fb9f6809b54bf1aa666a0cbaf29776c782432a4d Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Fri, 6 Dec 2024 10:18:07 -0500 Subject: [PATCH 26/45] Fixed code lint --- .../chip/testing/spec_parsing.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 4368f5a1f21c5b..ca3e9b51a31062 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -520,6 +520,19 @@ class DataModelLevel(str, Enum): kDeviceType = 'device_types' +def _get_data_model_root() -> pathlib.PosixPath: + """Attempts to find the 'data_model' directory inside the 'chip.testing' package.""" + # Access the 'chip.testing' package directly via importlib and pkg_resources + package = pkg_resources.files(importlib.import_module('chip.testing')) # Corrected: using importlib + try: + # We assume the 'data_model' directory is inside the package itself + data_model_root = package / 'data_model' + except FileNotFoundError: + raise FileNotFoundError(f"Data model directory not found in the package at {data_model_root}") + + return data_model_root + + def get_data_model_directory(data_model_directory: Union[str], data_model_level: str) -> pathlib.PosixPath: """ Get the directory of the data model for a specific version and level from the installed package. From 4120e4225b5fa76e9652116b6d1cb660ea1c9f67 Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Fri, 6 Dec 2024 12:36:21 -0500 Subject: [PATCH 27/45] Fixed errors --- .../chip/testing/spec_parsing.py | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index ca3e9b51a31062..e2126d3ee5c3f7 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -532,8 +532,7 @@ def _get_data_model_root() -> pathlib.PosixPath: return data_model_root - -def get_data_model_directory(data_model_directory: Union[str], data_model_level: str) -> pathlib.PosixPath: +def get_data_model_directory(data_model_directory: Union[str, pathlib.Path], data_model_level: str) -> pathlib.PosixPath: """ Get the directory of the data model for a specific version and level from the installed package. """ @@ -553,46 +552,50 @@ def get_data_model_directory(data_model_directory: Union[str], data_model_level: return data_model_root / version / data_model_level else: - # If it's a custom directory, return directly + # If it's a custom directory (path-like object), return directly return pathlib.Path(data_model_directory) -def build_xml_clusters(data_model_directory: Union[str] = 'k1_4') -> typing.Tuple[dict[int, dict], list]: +def build_xml_clusters(data_model_directory: Union[str, pathlib.Path] = 'k1_4') -> typing.Tuple[dict[int, dict], list]: """ Build XML clusters from the specified data model directory. - This function supports both pre-built locations (via `str` directory names) and full paths (strings). + This function supports both pre-built locations (via `str` directory names) and full paths (strings or pathlib.Path). """ # If a pre-built directory is provided, resolve the full path if isinstance(data_model_directory, str): data_model_directory = get_data_model_directory(data_model_directory, 'clusters') - # Convert the resolved directory path into a resource path using pkg_resources - dir = pathlib.Path(data_model_directory) + # Ensure that data_model_directory is a pathlib.Path object + dir_path = pathlib.Path(data_model_directory) clusters = {} pure_base_clusters = {} ids_by_name = {} problems = [] - # Use pkg_resources to list all XML files in the data model directory + # Use importlib.resources to list all XML files in the data model directory try: - xml_files = [f for f in pkg_resources.contents(pkg_resources.files(importlib.import_module( - 'chip.testing')).joinpath('data_model').joinpath(data_model_directory)).splitlines() if f.endswith('.xml')] + # List the contents of the data_model directory using importlib.resources + xml_files = [ + f for f in pkg_resources.contents(pkg_resources.files(importlib.import_module( + 'chip.testing')).joinpath('data_model').joinpath(dir_path.name)) + if f.endswith('.xml') + ] except Exception as e: logging.error(f"Error accessing XML files in {data_model_directory}: {e}") return clusters, problems if not xml_files: - raise FileNotFoundError(f'No XML files found in the specified package directory {dir}') + raise FileNotFoundError(f'No XML files found in the specified package directory {dir_path}') for xml in xml_files: logging.info(f'Parsing file {xml}') try: - # Open the resource file using pkg_resources and parse it + # Open the resource file using importlib.resources and parse it with pkg_resources.open_text( pkg_resources.files(importlib.import_module('chip.testing')) .joinpath('data_model') - .joinpath(data_model_directory), xml + .joinpath(dir_path.name), xml ) as file: # Directly parse the XML file without assigning it to 'tree' ElementTree.parse(file) # Parse the file to process it From 4a986d1fba1e310fe055630d2b817d21842954b6 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Fri, 6 Dec 2024 17:37:02 +0000 Subject: [PATCH 28/45] Restyled by autopep8 --- .../matter_testing_infrastructure/chip/testing/spec_parsing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index e2126d3ee5c3f7..55292ece1c3745 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -532,6 +532,7 @@ def _get_data_model_root() -> pathlib.PosixPath: return data_model_root + def get_data_model_directory(data_model_directory: Union[str, pathlib.Path], data_model_level: str) -> pathlib.PosixPath: """ Get the directory of the data model for a specific version and level from the installed package. From 3c5442e74e0f68f8e91956f6248d5a523075bd69 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 9 Dec 2024 09:11:23 -0500 Subject: [PATCH 29/45] Some type updates and iteration logic updates to be consistent --- .../chip/testing/spec_parsing.py | 114 +++++++++--------- 1 file changed, 54 insertions(+), 60 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 55292ece1c3745..1682bb2d504c9b 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -17,6 +17,7 @@ import glob import importlib +from importlib.abc import Traversable import importlib.resources as pkg_resources import logging import pathlib @@ -514,10 +515,28 @@ class PrebuiltDataModelDirectory(Enum): k1_4 = auto() kMaster = auto() + @property + def dirname(self): + if self == PrebuiltDataModelDirectory.k1_3: + return "1.3" + if self == PrebuiltDataModelDirectory.k1_4: + return "1.4" + if self == PrebuiltDataModelDirectory.kMaster: + return "master" + raise KeyError("Invalid enum: %r" % self) + class DataModelLevel(str, Enum): - kCluster = 'clusters' - kDeviceType = 'device_types' + kCluster = auto() + kDeviceType = auto() + + @property + def dirname(self): + if self == DataModelLevel.kCluster: + return "clusters" + if self == DataModelLevel.kDeviceType: + return "device_types" + raise KeyError("Invalid enum: %r" % self) def _get_data_model_root() -> pathlib.PosixPath: @@ -533,75 +552,51 @@ def _get_data_model_root() -> pathlib.PosixPath: return data_model_root -def get_data_model_directory(data_model_directory: Union[str, pathlib.Path], data_model_level: str) -> pathlib.PosixPath: +def get_data_model_directory(data_model_directory: Union[PrebuiltDataModelDirectory, Traversable], data_model_level: DataModelLevel) -> Traversable: """ Get the directory of the data model for a specific version and level from the installed package. """ - data_model_root = _get_data_model_root() - # If it's a prebuilt directory, build the path based on the version and data model level - if isinstance(data_model_directory, str): - version_map = { - 'k1_3': '1.3', - 'k1_4': '1.4', - 'kMaster': 'master', - } - - version = version_map.get(data_model_directory) - if not version: - raise ValueError(f"Unsupported data model directory: {data_model_directory}") - - return data_model_root / version / data_model_level + if isinstance(data_model_directory, PrebuiltDataModelDirectory): + top = pkg_resources.files(importlib.import_module('chip.testing')).joinpath('data_model').joinpath(data_model_directory.dirname) else: - # If it's a custom directory (path-like object), return directly - return pathlib.Path(data_model_directory) + top = data_model_directory + + return top.joinpath(data_model_level.dirname) -def build_xml_clusters(data_model_directory: Union[str, pathlib.Path] = 'k1_4') -> typing.Tuple[dict[int, dict], list]: +def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, pathlib.Path] = PrebuiltDataModelDirectory.k1_4) -> typing.Tuple[dict[int, dict], list]: """ Build XML clusters from the specified data model directory. - This function supports both pre-built locations (via `str` directory names) and full paths (strings or pathlib.Path). + This function supports both pre-built locations and full paths """ - # If a pre-built directory is provided, resolve the full path - if isinstance(data_model_directory, str): - data_model_directory = get_data_model_directory(data_model_directory, 'clusters') - # Ensure that data_model_directory is a pathlib.Path object - dir_path = pathlib.Path(data_model_directory) + if isinstance(data_model_directory, PrebuiltDataModelDirectory): + top = pkg_resources.files(importlib.import_module("chip.testing")).joinpath('data_model').joinpath(data_model_directory.dirname) + else: + top = data_model_directory - clusters = {} - pure_base_clusters = {} - ids_by_name = {} - problems = [] + top = top.joinpath(DataModelLevel.kCluster.dirname) - # Use importlib.resources to list all XML files in the data model directory - try: - # List the contents of the data_model directory using importlib.resources - xml_files = [ - f for f in pkg_resources.contents(pkg_resources.files(importlib.import_module( - 'chip.testing')).joinpath('data_model').joinpath(dir_path.name)) - if f.endswith('.xml') - ] - except Exception as e: - logging.error(f"Error accessing XML files in {data_model_directory}: {e}") - return clusters, problems - - if not xml_files: - raise FileNotFoundError(f'No XML files found in the specified package directory {dir_path}') - - for xml in xml_files: - logging.info(f'Parsing file {xml}') - try: - # Open the resource file using importlib.resources and parse it - with pkg_resources.open_text( - pkg_resources.files(importlib.import_module('chip.testing')) - .joinpath('data_model') - .joinpath(dir_path.name), xml - ) as file: - # Directly parse the XML file without assigning it to 'tree' - ElementTree.parse(file) # Parse the file to process it - except Exception as e: - logging.error(f"Error parsing XML file {xml}: {e}") + clusters: dict[int, XmlCluster] = {} + pure_base_clusters: dict[str, XmlCluster] = {} + ids_by_name: dict[str, int] = {} + problems: list[ProblemNotice] = [] + + logging.info("Reading XML clusters from %r", top) + for f in top.iterdir(): + if not f.name.endswith('.xml'): + logging.info("Ignoring non-XML file %s", f.name) + continue + + logging.info('Parsing file %s', f.name) + + # Open the resource file using importlib.resources and parse it + with f.open("r", encoding="utf8") as file: + # Directly parse the XML file without assigning it to 'tree' + tree = ElementTree.parse(file) # Parse the file to process it + root = tree.getroot() + add_cluster_data_from_xml(root, clusters, pure_base_clusters, ids_by_name, problems) # There are a few clusters where the conformance columns are listed as desc. These clusters need specific, targeted tests # to properly assess conformance. Here, we list them as Optional to allow these for the general test. Targeted tests are described below. @@ -618,7 +613,6 @@ def remove_problem(location: typing.Union[CommandPathLocation, FeaturePathLocati mask = clusters[descriptor_id].feature_map[code] clusters[descriptor_id].features[mask].conformance = optional() remove_problem(FeaturePathLocation(endpoint_id=0, cluster_id=descriptor_id, feature_code=code)) - action_id = Clusters.Actions.id for c in Clusters.ClusterObjects.ALL_ACCEPTED_COMMANDS[action_id]: clusters[action_id].accepted_commands[c].conformance = optional() @@ -834,7 +828,7 @@ def parse_single_device_type(root: ElementTree.Element) -> tuple[list[ProblemNot return device_types, problems -def build_xml_device_types(data_model_directory: typing.Union[PrebuiltDataModelDirectory, str] = PrebuiltDataModelDirectory.k1_4) -> tuple[dict[int, XmlDeviceType], list[ProblemNotice]]: +def build_xml_device_types(data_model_directory: typing.Union[PrebuiltDataModelDirectory, Traversable] = PrebuiltDataModelDirectory.k1_4) -> tuple[dict[int, XmlDeviceType], list[ProblemNotice]]: dir = get_data_model_directory(data_model_directory, DataModelLevel.kDeviceType) device_types: dict[int, XmlDeviceType] = {} problems = [] From 0793ba0dd9e9358471da3b4a0e791fbe9626f9dd Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 9 Dec 2024 09:13:15 -0500 Subject: [PATCH 30/45] Remove unused method --- .../chip/testing/spec_parsing.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 1682bb2d504c9b..059ff85c4e30bd 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -539,19 +539,6 @@ def dirname(self): raise KeyError("Invalid enum: %r" % self) -def _get_data_model_root() -> pathlib.PosixPath: - """Attempts to find the 'data_model' directory inside the 'chip.testing' package.""" - # Access the 'chip.testing' package directly via importlib and pkg_resources - package = pkg_resources.files(importlib.import_module('chip.testing')) # Corrected: using importlib - try: - # We assume the 'data_model' directory is inside the package itself - data_model_root = package / 'data_model' - except FileNotFoundError: - raise FileNotFoundError(f"Data model directory not found in the package at {data_model_root}") - - return data_model_root - - def get_data_model_directory(data_model_directory: Union[PrebuiltDataModelDirectory, Traversable], data_model_level: DataModelLevel) -> Traversable: """ Get the directory of the data model for a specific version and level from the installed package. From 35edf22b852bf6432cbd232a182c8804ada5e6dd Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 9 Dec 2024 09:16:47 -0500 Subject: [PATCH 31/45] Fix logic to match existing usage: we need clusters to be part of the passed in path if applicable --- .../chip/testing/spec_parsing.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 059ff85c4e30bd..d65d56c93a112a 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -552,19 +552,20 @@ def get_data_model_directory(data_model_directory: Union[PrebuiltDataModelDirect return top.joinpath(data_model_level.dirname) -def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, pathlib.Path] = PrebuiltDataModelDirectory.k1_4) -> typing.Tuple[dict[int, dict], list]: +def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, Traversable] = PrebuiltDataModelDirectory.k1_4) -> typing.Tuple[dict[int, dict], list]: """ Build XML clusters from the specified data model directory. This function supports both pre-built locations and full paths + + if data_model_directory is a Travesable, it is assumed to already contain `clusters` (i.e. be a directory + with all XML files in it) """ if isinstance(data_model_directory, PrebuiltDataModelDirectory): - top = pkg_resources.files(importlib.import_module("chip.testing")).joinpath('data_model').joinpath(data_model_directory.dirname) + top = pkg_resources.files(importlib.import_module("chip.testing")).joinpath('data_model').joinpath(data_model_directory.dirname).joinpath(DataModelLevel.kCluster.dirname) else: top = data_model_directory - top = top.joinpath(DataModelLevel.kCluster.dirname) - clusters: dict[int, XmlCluster] = {} pure_base_clusters: dict[str, XmlCluster] = {} ids_by_name: dict[str, int] = {} From ec01ebb53f8266251e52dc70a222eec5462da245 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Mon, 9 Dec 2024 14:17:33 +0000 Subject: [PATCH 32/45] Restyled by autopep8 --- .../chip/testing/spec_parsing.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index d65d56c93a112a..b6e07d874a3d10 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -545,7 +545,8 @@ def get_data_model_directory(data_model_directory: Union[PrebuiltDataModelDirect """ # If it's a prebuilt directory, build the path based on the version and data model level if isinstance(data_model_directory, PrebuiltDataModelDirectory): - top = pkg_resources.files(importlib.import_module('chip.testing')).joinpath('data_model').joinpath(data_model_directory.dirname) + top = pkg_resources.files(importlib.import_module('chip.testing')).joinpath( + 'data_model').joinpath(data_model_directory.dirname) else: top = data_model_directory @@ -562,7 +563,8 @@ def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, T """ if isinstance(data_model_directory, PrebuiltDataModelDirectory): - top = pkg_resources.files(importlib.import_module("chip.testing")).joinpath('data_model').joinpath(data_model_directory.dirname).joinpath(DataModelLevel.kCluster.dirname) + top = pkg_resources.files(importlib.import_module("chip.testing")).joinpath( + 'data_model').joinpath(data_model_directory.dirname).joinpath(DataModelLevel.kCluster.dirname) else: top = data_model_directory From 1e7a1e2e2876dcf4e2655c663452dd551f424032 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Mon, 9 Dec 2024 14:17:35 +0000 Subject: [PATCH 33/45] Restyled by isort --- .../matter_testing_infrastructure/chip/testing/spec_parsing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index b6e07d874a3d10..84a336c59b0511 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -17,7 +17,6 @@ import glob import importlib -from importlib.abc import Traversable import importlib.resources as pkg_resources import logging import pathlib @@ -26,6 +25,7 @@ from copy import deepcopy from dataclasses import dataclass from enum import Enum, auto +from importlib.abc import Traversable from typing import Callable, Optional, Union import chip.clusters as Clusters From 931a746608350e6de82cf42ba3d80f86d90f9a35 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 9 Dec 2024 09:30:37 -0500 Subject: [PATCH 34/45] remove unused import --- .../matter_testing_infrastructure/chip/testing/spec_parsing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 84a336c59b0511..6d78a7a91df407 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -19,7 +19,6 @@ import importlib import importlib.resources as pkg_resources import logging -import pathlib import typing import xml.etree.ElementTree as ElementTree from copy import deepcopy From 222c7425dda5770dd0872958efd69bf11e168fad Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 9 Dec 2024 09:37:14 -0500 Subject: [PATCH 35/45] Cleanup some odd comments --- .../chip/testing/spec_parsing.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 6d78a7a91df407..66b7c7e086e6a0 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -580,11 +580,8 @@ def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, T logging.info('Parsing file %s', f.name) - # Open the resource file using importlib.resources and parse it with f.open("r", encoding="utf8") as file: - # Directly parse the XML file without assigning it to 'tree' - tree = ElementTree.parse(file) # Parse the file to process it - root = tree.getroot() + root = ElementTree.parse(file).getroot() add_cluster_data_from_xml(root, clusters, pure_base_clusters, ids_by_name, problems) # There are a few clusters where the conformance columns are listed as desc. These clusters need specific, targeted tests From 100ac230939d35fc4a5eb46877d7b4ef98d44fcb Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 9 Dec 2024 09:40:43 -0500 Subject: [PATCH 36/45] Another update to avoid using globs --- .../chip/testing/spec_parsing.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 66b7c7e086e6a0..baccd09310136c 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -815,16 +815,18 @@ def parse_single_device_type(root: ElementTree.Element) -> tuple[list[ProblemNot def build_xml_device_types(data_model_directory: typing.Union[PrebuiltDataModelDirectory, Traversable] = PrebuiltDataModelDirectory.k1_4) -> tuple[dict[int, XmlDeviceType], list[ProblemNotice]]: - dir = get_data_model_directory(data_model_directory, DataModelLevel.kDeviceType) + top = get_data_model_directory(data_model_directory, DataModelLevel.kDeviceType) device_types: dict[int, XmlDeviceType] = {} problems = [] - for xml in glob.glob(f"{dir}/*.xml"): - logging.info(f'Parsing file {xml}') - tree = ElementTree.parse(f'{xml}') - root = tree.getroot() - tmp_device_types, tmp_problems = parse_single_device_type(root) - problems = problems + tmp_problems - device_types.update(tmp_device_types) + for file in top.iterdir(): + if not file.name.endswith('.xml'): + continue + logging.info('Parsing file %r / %s', top, file.name) + with file.open('r', encoding="utf8") as xml: + root = ElementTree.parse(xml) + tmp_device_types, tmp_problems = parse_single_device_type(root) + problems = problems + tmp_problems + device_types.update(tmp_device_types) if -1 not in device_types.keys(): raise ConformanceException("Base device type not found in device type xml data") From 2f2352afda069f897de86936a0e36431e989efbc Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 9 Dec 2024 09:42:47 -0500 Subject: [PATCH 37/45] Fix up types and return --- .../chip/testing/spec_parsing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index baccd09310136c..fe8cfb963dc8b4 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -742,7 +742,7 @@ def combine_attributes(base: dict[uint, XmlAttribute], derived: dict[uint, XmlAt xml_clusters[id] = new -def parse_single_device_type(root: ElementTree.Element) -> tuple[list[ProblemNotice], dict[int, XmlDeviceType]]: +def parse_single_device_type(root: ElementTree.Element) -> tuple[dict[int, XmlDeviceType], list[ProblemNotice]]: problems: list[ProblemNotice] = [] device_types: dict[int, XmlDeviceType] = {} device = root.iter('deviceType') @@ -823,7 +823,7 @@ def build_xml_device_types(data_model_directory: typing.Union[PrebuiltDataModelD continue logging.info('Parsing file %r / %s', top, file.name) with file.open('r', encoding="utf8") as xml: - root = ElementTree.parse(xml) + root = ElementTree.parse(xml).getroot() tmp_device_types, tmp_problems = parse_single_device_type(root) problems = problems + tmp_problems device_types.update(tmp_device_types) From cd7a9de59bfa52ad001a83ab94bdaea3812443c8 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 9 Dec 2024 09:43:03 -0500 Subject: [PATCH 38/45] Remove unused import --- .../matter_testing_infrastructure/chip/testing/spec_parsing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index fe8cfb963dc8b4..eea855b9f20c3b 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -15,7 +15,6 @@ # limitations under the License. # -import glob import importlib import importlib.resources as pkg_resources import logging From c1611d2e9f5542ce40d13fa26344ea80d11f63a1 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 9 Dec 2024 09:45:43 -0500 Subject: [PATCH 39/45] Another dep cleanup --- .../matter_testing_infrastructure/chip/testing/spec_parsing.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index eea855b9f20c3b..df4f648456f768 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -524,7 +524,7 @@ def dirname(self): raise KeyError("Invalid enum: %r" % self) -class DataModelLevel(str, Enum): +class DataModelLevel(Enum): kCluster = auto() kDeviceType = auto() @@ -578,7 +578,6 @@ def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, T continue logging.info('Parsing file %s', f.name) - with f.open("r", encoding="utf8") as file: root = ElementTree.parse(file).getroot() add_cluster_data_from_xml(root, clusters, pure_base_clusters, ids_by_name, problems) From b8b50fa269253375fba7002d2da6519fd2d21b66 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 9 Dec 2024 11:35:58 -0500 Subject: [PATCH 40/45] Remove one test step: unclear about the value of throwing a specparse exception on invalid input type --- src/python_testing/TestSpecParsingSupport.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/python_testing/TestSpecParsingSupport.py b/src/python_testing/TestSpecParsingSupport.py index b4c908c232fa94..959550325fa5b4 100644 --- a/src/python_testing/TestSpecParsingSupport.py +++ b/src/python_testing/TestSpecParsingSupport.py @@ -276,9 +276,6 @@ def test_build_xml_override(self): asserts.assert_count_equal(string_override_check.keys(), self.spec_xml_clusters.keys(), "Mismatched cluster generation") - with asserts.assert_raises(SpecParsingException): - build_xml_clusters("baddir") - def test_spec_parsing_access(self): strs = [None, 'view', 'operate', 'manage', 'admin'] for read in strs: From 4bcc9a153a4e00a6c8f83a44a7be3b10096b003a Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 9 Dec 2024 11:53:31 -0500 Subject: [PATCH 41/45] Remove unused import --- src/python_testing/TestSpecParsingSupport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python_testing/TestSpecParsingSupport.py b/src/python_testing/TestSpecParsingSupport.py index 959550325fa5b4..7ab5847a276883 100644 --- a/src/python_testing/TestSpecParsingSupport.py +++ b/src/python_testing/TestSpecParsingSupport.py @@ -21,7 +21,7 @@ import jinja2 from chip.testing.global_attribute_ids import GlobalAttributeIds from chip.testing.matter_testing import MatterBaseTest, ProblemNotice, default_matter_test_main -from chip.testing.spec_parsing import (ClusterParser, DataModelLevel, PrebuiltDataModelDirectory, SpecParsingException, XmlCluster, +from chip.testing.spec_parsing import (ClusterParser, DataModelLevel, PrebuiltDataModelDirectory, XmlCluster, add_cluster_data_from_xml, build_xml_clusters, check_clusters_for_unknown_commands, combine_derived_clusters_with_base, get_data_model_directory) from mobly import asserts From bb0ace93bb8e626041fe99c6fee69961441cd884 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Tue, 10 Dec 2024 11:00:39 -0500 Subject: [PATCH 42/45] update logic to throw specparsing when no XMLs found ... this preserves previous logic somewhat --- .../chip/testing/spec_parsing.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index df4f648456f768..53b59c50b1438a 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -572,16 +572,27 @@ def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, T problems: list[ProblemNotice] = [] logging.info("Reading XML clusters from %r", top) + + found_xmls = 0 for f in top.iterdir(): if not f.name.endswith('.xml'): logging.info("Ignoring non-XML file %s", f.name) continue logging.info('Parsing file %s', f.name) + found_xmls += 1 with f.open("r", encoding="utf8") as file: root = ElementTree.parse(file).getroot() add_cluster_data_from_xml(root, clusters, pure_base_clusters, ids_by_name, problems) + # For now we assume even a single XML means the directory was probaly OK + # we may increase this later as most our data model directories are larger + # + # Intent here is to make user aware of typos in paths instead of silently having + # empty parsing + if found_xmls < 1: + raise SpecParsingException(f'No data model files found in specified directory {top:r}') + # There are a few clusters where the conformance columns are listed as desc. These clusters need specific, targeted tests # to properly assess conformance. Here, we list them as Optional to allow these for the general test. Targeted tests are described below. # Descriptor - TagList feature - this feature is mandated when the duplicate condition holds for the endpoint. It is tested in DESC-2.2 From 985375252a0f84b0b9bd8737b8107a24b4a9b0df Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Tue, 10 Dec 2024 11:11:41 -0500 Subject: [PATCH 43/45] Make data model directory consistent with cluster logic --- .../chip/testing/spec_parsing.py | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index 53b59c50b1438a..ff831d04ed6764 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -537,18 +537,18 @@ def dirname(self): raise KeyError("Invalid enum: %r" % self) -def get_data_model_directory(data_model_directory: Union[PrebuiltDataModelDirectory, Traversable], data_model_level: DataModelLevel) -> Traversable: +def get_data_model_directory(data_model_directory: Union[PrebuiltDataModelDirectory, Traversable], data_model_level: DataModelLevel = DataModelLevel.kCluster) -> Traversable: """ Get the directory of the data model for a specific version and level from the installed package. + + If `data_model_directory` is given as a PATH, it is returned directly WITHOUT using the data_model level at all. """ # If it's a prebuilt directory, build the path based on the version and data model level if isinstance(data_model_directory, PrebuiltDataModelDirectory): - top = pkg_resources.files(importlib.import_module('chip.testing')).joinpath( - 'data_model').joinpath(data_model_directory.dirname) + return pkg_resources.files(importlib.import_module('chip.testing')).joinpath( + 'data_model').joinpath(data_model_directory.dirname).joinpath(data_model_level.dirname) else: - top = data_model_directory - - return top.joinpath(data_model_level.dirname) + return data_model_directory def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, Traversable] = PrebuiltDataModelDirectory.k1_4) -> typing.Tuple[dict[int, dict], list]: @@ -560,17 +560,12 @@ def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, T with all XML files in it) """ - if isinstance(data_model_directory, PrebuiltDataModelDirectory): - top = pkg_resources.files(importlib.import_module("chip.testing")).joinpath( - 'data_model').joinpath(data_model_directory.dirname).joinpath(DataModelLevel.kCluster.dirname) - else: - top = data_model_directory - clusters: dict[int, XmlCluster] = {} pure_base_clusters: dict[str, XmlCluster] = {} ids_by_name: dict[str, int] = {} problems: list[ProblemNotice] = [] + top = get_data_model_directory(data_model_directory, DataModelLevel.kCluster) logging.info("Reading XML clusters from %r", top) found_xmls = 0 @@ -591,7 +586,7 @@ def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, T # Intent here is to make user aware of typos in paths instead of silently having # empty parsing if found_xmls < 1: - raise SpecParsingException(f'No data model files found in specified directory {top:r}') + raise SpecParsingException(f'No data model files found in specified directory {top:!r}') # There are a few clusters where the conformance columns are listed as desc. These clusters need specific, targeted tests # to properly assess conformance. Here, we list them as Optional to allow these for the general test. Targeted tests are described below. From 37f230e7cf626f5847d215d60c465fd99cb51eed Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Tue, 10 Dec 2024 11:15:47 -0500 Subject: [PATCH 44/45] Comments update --- .../chip/testing/spec_parsing.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index ff831d04ed6764..a438ab14b92f59 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -541,7 +541,9 @@ def get_data_model_directory(data_model_directory: Union[PrebuiltDataModelDirect """ Get the directory of the data model for a specific version and level from the installed package. - If `data_model_directory` is given as a PATH, it is returned directly WITHOUT using the data_model level at all. + + `data_model_directory` given as a path MUST be of type Traversable (often `pathlib.Path(somepathstring)`). + If `data_model_directory` is given as a Traversable, it is returned directly WITHOUT using the data_model_level at all. """ # If it's a prebuilt directory, build the path based on the version and data model level if isinstance(data_model_directory, PrebuiltDataModelDirectory): @@ -554,9 +556,10 @@ def get_data_model_directory(data_model_directory: Union[PrebuiltDataModelDirect def build_xml_clusters(data_model_directory: Union[PrebuiltDataModelDirectory, Traversable] = PrebuiltDataModelDirectory.k1_4) -> typing.Tuple[dict[int, dict], list]: """ Build XML clusters from the specified data model directory. - This function supports both pre-built locations and full paths + This function supports both pre-built locations and full paths. - if data_model_directory is a Travesable, it is assumed to already contain `clusters` (i.e. be a directory + `data_model_directory`` given as a path MUST be of type Traversable (often `pathlib.Path(somepathstring)`). + If data_model_directory is a Travesable, it is assumed to already contain `clusters` (i.e. be a directory with all XML files in it) """ From ddde2d958ac9b8a8e20675f6ad101ab96e83c081 Mon Sep 17 00:00:00 2001 From: Vatsal Ghelani Date: Fri, 13 Dec 2024 10:13:09 -0500 Subject: [PATCH 45/45] Added warning levels for checking xml --- .../chip/testing/spec_parsing.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py index a438ab14b92f59..3607f515d3a9ec 100644 --- a/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py +++ b/src/python_testing/matter_testing_infrastructure/chip/testing/spec_parsing.py @@ -825,16 +825,23 @@ def build_xml_device_types(data_model_directory: typing.Union[PrebuiltDataModelD top = get_data_model_directory(data_model_directory, DataModelLevel.kDeviceType) device_types: dict[int, XmlDeviceType] = {} problems = [] + + found_xmls = 0 + for file in top.iterdir(): if not file.name.endswith('.xml'): continue logging.info('Parsing file %r / %s', top, file.name) + found_xmls += 1 with file.open('r', encoding="utf8") as xml: root = ElementTree.parse(xml).getroot() tmp_device_types, tmp_problems = parse_single_device_type(root) problems = problems + tmp_problems device_types.update(tmp_device_types) + if found_xmls < 1: + logging.warning("No XML files found in the specified device type directory: %r", top) + if -1 not in device_types.keys(): raise ConformanceException("Base device type not found in device type xml data")