diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..d939d93 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "test/public_regulated_data_types"] + path = test/public_regulated_data_types + url = https://github.com/OpenCyphal/public_regulated_data_types.git diff --git a/pydsdl/__init__.py b/pydsdl/__init__.py index d07f1f2..1982145 100644 --- a/pydsdl/__init__.py +++ b/pydsdl/__init__.py @@ -25,8 +25,8 @@ _sys.path = [str(_Path(__file__).parent / "third_party")] + _sys.path # Never import anything that is not available here - API stability guarantees are only provided for the exposed items. +from .visitors import PrintOutputHandler as PrintOutputHandler # for backwards compatibility from ._namespace import read_namespace as read_namespace -from ._namespace import PrintOutputHandler as PrintOutputHandler # Error model. from ._error import FrontendError as FrontendError diff --git a/pydsdl/_data_type_builder.py b/pydsdl/_data_type_builder.py index 4572da3..9c8ecbc 100644 --- a/pydsdl/_data_type_builder.py +++ b/pydsdl/_data_type_builder.py @@ -2,16 +2,44 @@ # This software is distributed under the terms of the MIT License. # Author: Pavel Kirienko -from typing import Optional, Callable, Iterable +import abc import logging from pathlib import Path -from . import _serializable -from . import _expression -from . import _error -from . import _dsdl_definition -from . import _parser -from . import _data_schema_builder -from . import _port_id_ranges +from typing import Callable, Iterable, Optional, Set + +from . import _data_schema_builder, _error, _expression, _parser, _port_id_ranges, _serializable +from .visitors import DsdlFile, NamespaceVisitor + + +class DsdlFileBuildable(DsdlFile): + """ + A DSDL file that can construct a composite type from its contents. + """ + + @abc.abstractmethod + def read( + self, + lookup_definitions: Iterable["DsdlFileBuildable"], + namespace_visitors: Iterable[NamespaceVisitor], + print_output_handler: Callable[[int, str], None], + allow_unregulated_fixed_port_id: bool, + ) -> _serializable.CompositeType: + """ + Reads the data type definition and returns its high-level data type representation. + The output should be cached; all following invocations should read from this cache. + Caching is very important, because it is expected that the same definition may be referred to multiple + times (e.g., for composition or when accessing external constants). Re-processing a definition every time + it is accessed would be a huge waste of time. + Note, however, that this may lead to unexpected complications if one is attempting to re-read a definition + with different inputs (e.g., different lookup paths) expecting to get a different result: caching would + get in the way. That issue is easy to avoid by creating a new instance of the object. + :param lookup_definitions: List of definitions available for referring to. + :param namespace_visitors: Namespace visitors to notify about discovered dependencies. + :param print_output_handler: Used for @print and for diagnostics: (line_number, text) -> None. + :param allow_unregulated_fixed_port_id: Do not complain about fixed unregulated port IDs. + :return: The data type representation. + """ + raise NotImplementedError() class AssertionCheckFailureError(_error.InvalidDefinitionError): @@ -44,19 +72,21 @@ class MissingSerializationModeError(_error.InvalidDefinitionError): class DataTypeBuilder(_parser.StatementStreamProcessor): def __init__( self, - definition: _dsdl_definition.DSDLDefinition, - lookup_definitions: Iterable[_dsdl_definition.DSDLDefinition], + definition: DsdlFileBuildable, + lookup_definitions: Iterable[DsdlFileBuildable], + namespace_visitors: Iterable[NamespaceVisitor], print_output_handler: Callable[[int, str], None], allow_unregulated_fixed_port_id: bool, ): self._definition = definition self._lookup_definitions = list(lookup_definitions) + self._namespace_visitors = namespace_visitors self._print_output_handler = print_output_handler self._allow_unregulated_fixed_port_id = allow_unregulated_fixed_port_id self._element_callback = None # type: Optional[Callable[[str], None]] - assert isinstance(self._definition, _dsdl_definition.DSDLDefinition) - assert all(map(lambda x: isinstance(x, _dsdl_definition.DSDLDefinition), lookup_definitions)) + assert isinstance(self._definition, DsdlFileBuildable) + assert all(map(lambda x: isinstance(x, DsdlFileBuildable), lookup_definitions)) assert callable(self._print_output_handler) assert isinstance(self._allow_unregulated_fixed_port_id, bool) @@ -198,6 +228,9 @@ def resolve_versioned_data_type(self, name: str, version: _serializable.Version) del name found = list(filter(lambda d: d.full_name == full_name and d.version == version, self._lookup_definitions)) if not found: + for visitor in self._namespace_visitors: + visitor.on_discover_lookup_dependent_type(self._definition, full_name, version) + # Play Sherlock to help the user with mistakes like https://forum.opencyphal.org/t/904/2 requested_ns = full_name.split(_serializable.CompositeType.NAME_COMPONENT_SEPARATOR)[0] lookup_nss = set(x.root_namespace for x in self._lookup_definitions) @@ -221,15 +254,20 @@ def resolve_versioned_data_type(self, name: str, version: _serializable.Version) raise _error.InternalError("Conflicting definitions: %r" % found) target_definition = found[0] - assert isinstance(target_definition, _dsdl_definition.DSDLDefinition) + for visitor in self._namespace_visitors: + visitor.on_discover_lookup_dependent_file(self._definition, target_definition) + + assert isinstance(target_definition, DsdlFileBuildable) assert target_definition.full_name == full_name assert target_definition.version == version # Recursion is cool. - return target_definition.read( + dt = target_definition.read( lookup_definitions=self._lookup_definitions, + namespace_visitors=self._namespace_visitors, print_output_handler=self._print_output_handler, allow_unregulated_fixed_port_id=self._allow_unregulated_fixed_port_id, ) + return dt def _queue_attribute(self, element_callback: Callable[[str], None]) -> None: self._flush_attribute("") diff --git a/pydsdl/_dsdl_definition.py b/pydsdl/_dsdl_definition.py index a8114da..5b8a129 100644 --- a/pydsdl/_dsdl_definition.py +++ b/pydsdl/_dsdl_definition.py @@ -2,14 +2,16 @@ # This software is distributed under the terms of the MIT License. # Author: Pavel Kirienko -import time -from typing import Iterable, Callable, Optional, List import logging +import time from pathlib import Path -from ._error import FrontendError, InvalidDefinitionError, InternalError -from ._serializable import CompositeType, Version -from . import _parser +from typing import Callable, Iterable, List, Optional +from . import _parser +from ._data_type_builder import DataTypeBuilder, DsdlFileBuildable +from ._error import FrontendError, InternalError, InvalidDefinitionError +from ._serializable import CompositeType, Version +from .visitors import NamespaceVisitor _logger = logging.getLogger(__name__) @@ -23,7 +25,7 @@ def __init__(self, text: str, path: Path): super().__init__(text=text, path=Path(path)) -class DSDLDefinition: +class DSDLDefinition(DsdlFileBuildable): """ A DSDL type definition source abstracts the filesystem level details away, presenting a higher-level interface that operates solely on the level of type names, namespaces, fixed identifiers, and so on. @@ -86,26 +88,16 @@ def __init__(self, file_path: Path, root_namespace_path: Path): self._cached_type: Optional[CompositeType] = None + # +-----------------------------------------------------------------------+ + # | DsdlFileBuildable :: INTERFACE | + # +-----------------------------------------------------------------------+ def read( self, - lookup_definitions: Iterable["DSDLDefinition"], + lookup_definitions: Iterable[DsdlFileBuildable], + namespace_visitors: Iterable[NamespaceVisitor], print_output_handler: Callable[[int, str], None], allow_unregulated_fixed_port_id: bool, ) -> CompositeType: - """ - Reads the data type definition and returns its high-level data type representation. - The output is cached; all following invocations will read from the cache. - Caching is very important, because it is expected that the same definition may be referred to multiple - times (e.g., for composition or when accessing external constants). Re-processing a definition every time - it is accessed would be a huge waste of time. - Note, however, that this may lead to unexpected complications if one is attempting to re-read a definition - with different inputs (e.g., different lookup paths) expecting to get a different result: caching would - get in the way. That issue is easy to avoid by creating a new instance of the object. - :param lookup_definitions: List of definitions available for referring to. - :param print_output_handler: Used for @print and for diagnostics: (line_number, text) -> None. - :param allow_unregulated_fixed_port_id: Do not complain about fixed unregulated port IDs. - :return: The data type representation. - """ log_prefix = "%s.%d.%d" % (self.full_name, self.version.major, self.version.minor) if self._cached_type is not None: _logger.debug("%s: Cache hit", log_prefix) @@ -124,17 +116,17 @@ def read( ", ".join(set(sorted(map(lambda x: x.root_namespace, lookup_definitions)))), ) try: - builder = _data_type_builder.DataTypeBuilder( + builder = DataTypeBuilder( definition=self, lookup_definitions=lookup_definitions, + namespace_visitors=namespace_visitors, print_output_handler=print_output_handler, allow_unregulated_fixed_port_id=allow_unregulated_fixed_port_id, ) - with open(self.file_path) as f: - _parser.parse(f.read(), builder) - self._cached_type = builder.finalize() + _parser.parse(self._text, builder) + self._cached_type = builder.finalize() _logger.info( "%s: Processed in %.0f ms; category: %s, fixed port ID: %s", log_prefix, @@ -151,34 +143,35 @@ def read( except Exception as ex: # pragma: no cover raise InternalError(culprit=ex, path=self.file_path) from ex + # +-----------------------------------------------------------------------+ + # | DsdlFile :: INTERFACE | + # +-----------------------------------------------------------------------+ + @property + def composite_type(self) -> Optional[CompositeType]: + return self._cached_type + @property def full_name(self) -> str: - """The full name, e.g., uavcan.node.Heartbeat""" return self._name @property def name_components(self) -> List[str]: - """Components of the full name as a list, e.g., ['uavcan', 'node', 'Heartbeat']""" return self._name.split(CompositeType.NAME_COMPONENT_SEPARATOR) @property def short_name(self) -> str: - """The last component of the full name, e.g., Heartbeat of uavcan.node.Heartbeat""" return self.name_components[-1] @property def full_namespace(self) -> str: - """The full name without the short name, e.g., uavcan.node for uavcan.node.Heartbeat""" return str(CompositeType.NAME_COMPONENT_SEPARATOR.join(self.name_components[:-1])) @property def root_namespace(self) -> str: - """The first component of the full name, e.g., uavcan of uavcan.node.Heartbeat""" return self.name_components[0] @property def text(self) -> str: - """The source text in its raw unprocessed form (with comments, formatting intact, and everything)""" return self._text @property @@ -187,7 +180,6 @@ def version(self) -> Version: @property def fixed_port_id(self) -> Optional[int]: - """Either the fixed port ID as integer, or None if not defined for this type.""" return self._fixed_port_id @property @@ -202,6 +194,10 @@ def file_path(self) -> Path: def root_namespace_path(self) -> Path: return self._root_namespace_path + # +-----------------------------------------------------------------------+ + # | Python :: SPECIAL FUNCTIONS | + # +-----------------------------------------------------------------------+ + def __eq__(self, other: object) -> bool: """ Two definitions will compare equal if they share the same name AND version number. @@ -220,8 +216,3 @@ def __str__(self) -> str: ) __repr__ = __str__ - - -# Moved this import here to break recursive dependency. -# Maybe I have messed up the architecture? Should think about it later. -from . import _data_type_builder # pylint: disable=wrong-import-position diff --git a/pydsdl/_namespace.py b/pydsdl/_namespace.py index 8e9501e..7f7dc85 100644 --- a/pydsdl/_namespace.py +++ b/pydsdl/_namespace.py @@ -4,13 +4,15 @@ # pylint: disable=logging-not-lazy -from typing import Iterable, Callable, DefaultDict, List, Optional, Union, Set, Dict -import logging import collections +import logging from pathlib import Path -from . import _serializable -from . import _dsdl_definition -from . import _error +from typing import Callable, DefaultDict, Dict, Iterable, List, Optional, Set, Union + +from . import _dsdl_definition, _error, _serializable +from ._data_type_builder import DsdlFileBuildable +from .visitors._common import DsdlFile, PrintOutputHandler, Visitor +from .visitors._definition_visitors import NamespaceVisitor, DependentFileError class RootNamespaceNameCollisionError(_error.InvalidDefinitionError): @@ -69,16 +71,14 @@ class SealingConsistencyError(_error.InvalidDefinitionError): """ -PrintOutputHandler = Callable[[Path, int, str], None] -"""Invoked when the frontend encounters a print directive or needs to output a generic diagnostic.""" - - +# pylint: disable=too-many-arguments, dangerous-default-value def read_namespace( root_namespace_directory: Union[Path, str], lookup_directories: Union[None, Path, str, Iterable[Union[Path, str]]] = None, print_output_handler: Optional[PrintOutputHandler] = None, allow_unregulated_fixed_port_id: bool = False, allow_root_namespace_name_collision: bool = True, + visitors: List[Visitor] = [], ) -> List[_serializable.CompositeType]: """ This function is the main entry point of the library. @@ -108,6 +108,11 @@ def read_namespace( the same root namespace name multiple times in the lookup dirs. This will enable defining a namespace partially and let other entities define new messages or new sub-namespaces in the same root namespace. + :param visitors: A list of visitor objects. Each visitor object will be handled differently based on their concrete + type. See the documentation for each visitor in the pydsdl.visitors module for more information. + This is an optional argument for backwards compatibility with an older version of this method. There is no + difference between None and an empty list. + :return: A list of :class:`pydsdl.CompositeType` sorted lexicographically by full data type name, then by major version (newest version first), then by minor version (newest version first). The ordering guarantee allows the caller to always find the newest version simply by picking @@ -139,6 +144,16 @@ def read_namespace( for a in lookup_directories_path_list: _logger.debug(_LOG_LIST_ITEM_PREFIX + str(a)) + namespace_visitors = [visitor for visitor in visitors if isinstance(visitor, NamespaceVisitor)] + for visitor in visitors: + visitor.on_read_namespace_start( + root_namespace_directory, + lookup_directories_path_list, + print_output_handler, + allow_unregulated_fixed_port_id, + allow_root_namespace_name_collision, + ) + # Check for common usage errors and warn the user if anything looks suspicious. _ensure_no_common_usage_errors(root_namespace_directory, lookup_directories_path_list, _logger.warning) @@ -157,7 +172,7 @@ def read_namespace( for x in target_dsdl_definitions: _logger.debug(_LOG_LIST_ITEM_PREFIX + str(x)) - lookup_dsdl_definitions = [] # type: List[_dsdl_definition.DSDLDefinition] + lookup_dsdl_definitions = [] # type: List[DsdlFileBuildable] for ld in lookup_directories_path_list: lookup_dsdl_definitions += _construct_dsdl_definitions_from_namespace(ld) @@ -179,7 +194,11 @@ def read_namespace( # Read the constructed definitions. types = _read_namespace_definitions( - target_dsdl_definitions, lookup_dsdl_definitions, print_output_handler, allow_unregulated_fixed_port_id + target_dsdl_definitions, + lookup_dsdl_definitions, + namespace_visitors, + print_output_handler, + allow_unregulated_fixed_port_id, ) # Note that we check for collisions in the read namespace only. @@ -191,6 +210,8 @@ def read_namespace( _ensure_no_fixed_port_id_collisions(types) _ensure_minor_version_compatibility(types) + for visitor in visitors: + visitor.on_read_namespace_end(types) return types @@ -202,8 +223,9 @@ def read_namespace( def _read_namespace_definitions( - target_definitions: List[_dsdl_definition.DSDLDefinition], - lookup_definitions: List[_dsdl_definition.DSDLDefinition], + target_definitions: List[DsdlFileBuildable], + lookup_definitions: List[DsdlFileBuildable], + namespace_visitors: Iterable[NamespaceVisitor], print_output_handler: Optional[PrintOutputHandler] = None, allow_unregulated_fixed_port_id: bool = False, ) -> List[_serializable.CompositeType]: @@ -215,7 +237,7 @@ def _read_namespace_definitions( :return: A list of types. """ - def make_print_handler(definition: _dsdl_definition.DSDLDefinition) -> Callable[[int, str], None]: + def make_print_handler(definition: DsdlFile) -> Callable[[int, str], None]: def handler(line_number: int, text: str) -> None: if print_output_handler: # pragma: no branch assert isinstance(line_number, int) and isinstance(text, str) @@ -226,8 +248,15 @@ def handler(line_number: int, text: str) -> None: types = [] # type: List[_serializable.CompositeType] for tdd in target_definitions: + if not all(visitor.on_discover_target_file(tdd) for visitor in namespace_visitors): + _logger.debug("Skipping target file %s due to visitor veto", tdd.file_path) + continue try: - dt = tdd.read(lookup_definitions, make_print_handler(tdd), allow_unregulated_fixed_port_id) + dt = tdd.read( + lookup_definitions, namespace_visitors, make_print_handler(tdd), allow_unregulated_fixed_port_id + ) + except DependentFileError: + _logger.debug("Skipping target file %s due to dependent file error", tdd.file_path) except _error.FrontendError as ex: # pragma: no cover ex.set_error_location_if_unknown(path=tdd.file_path) raise ex @@ -242,8 +271,8 @@ def handler(line_number: int, text: str) -> None: def _ensure_no_collisions( - target_definitions: List[_dsdl_definition.DSDLDefinition], - lookup_definitions: List[_dsdl_definition.DSDLDefinition], + target_definitions: List[DsdlFileBuildable], + lookup_definitions: List[DsdlFileBuildable], ) -> None: for tg in target_definitions: tg_full_namespace_period = tg.full_namespace.lower() + "." @@ -435,7 +464,7 @@ def _ensure_no_namespace_name_collisions(directories: Iterable[Path]) -> None: raise RootNamespaceNameCollisionError("The name of this namespace conflicts with %s" % b, path=a) -def _construct_dsdl_definitions_from_namespace(root_namespace_path: Path) -> List[_dsdl_definition.DSDLDefinition]: +def _construct_dsdl_definitions_from_namespace(root_namespace_path: Path) -> List[DsdlFileBuildable]: """ Accepts a directory path, returns a sorted list of abstract DSDL file representations. Those can be read later. The definitions are sorted by name lexicographically, then by major version (greatest version first), @@ -450,7 +479,7 @@ def _construct_dsdl_definitions_from_namespace(root_namespace_path: Path) -> Lis "File uses deprecated extension %r, please rename to use %r: %s", DSDL_FILE_GLOB_LEGACY, DSDL_FILE_GLOB, p ) - output = [] # type: List[_dsdl_definition.DSDLDefinition] + output = [] # type: List[DsdlFileBuildable] for fp in sorted(source_file_paths): dsdl_def = _dsdl_definition.DSDLDefinition(fp, root_namespace_path) output.append(dsdl_def) @@ -461,6 +490,7 @@ def _construct_dsdl_definitions_from_namespace(root_namespace_path: Path) -> Lis def _unittest_dsdl_definition_constructor() -> None: import tempfile + from ._dsdl_definition import FileNameFormatError with tempfile.TemporaryDirectory() as directory: @@ -474,7 +504,7 @@ def _unittest_dsdl_definition_constructor() -> None: dsdl_defs = _construct_dsdl_definitions_from_namespace(root) print(dsdl_defs) - lut = {x.full_name: x for x in dsdl_defs} # type: Dict[str, _dsdl_definition.DSDLDefinition] + lut = {x.full_name: x for x in dsdl_defs} # type: Dict[str, DsdlFileBuildable] assert len(lut) == 3 assert str(lut["foo.Qwerty"]) == repr(lut["foo.Qwerty"]) @@ -584,7 +614,7 @@ def _unittest_dsdl_definition_constructor_legacy() -> None: (root / "123.Qwerty.123.234.uavcan").write_text("# TEST A") dsdl_defs = _construct_dsdl_definitions_from_namespace(root) print(dsdl_defs) - lut = {x.full_name: x for x in dsdl_defs} # type: Dict[str, _dsdl_definition.DSDLDefinition] + lut = {x.full_name: x for x in dsdl_defs} # type: Dict[str, DsdlFileBuildable] assert len(lut) == 1 t = lut["foo.Qwerty"] assert t.file_path == root / "123.Qwerty.123.234.uavcan" @@ -633,9 +663,10 @@ def _unittest_common_usage_errors() -> None: def _unittest_nested_roots() -> None: - from pytest import raises import tempfile + from pytest import raises + with tempfile.TemporaryDirectory() as directory: di = Path(directory) (di / "a").mkdir() diff --git a/pydsdl/_test.py b/pydsdl/_test.py index 3e4048f..39301e8 100644 --- a/pydsdl/_test.py +++ b/pydsdl/_test.py @@ -62,6 +62,7 @@ def parse_definition( ) -> _serializable.CompositeType: return definition.read( lookup_definitions, + [], print_output_handler=lambda line, text: print("Output from line %d:" % line, text), allow_unregulated_fixed_port_id=False, ) @@ -422,7 +423,7 @@ def _unittest_error(wrkspc: Workspace) -> None: def standalone(rel_path: str, definition: str, allow_unregulated: bool = False) -> _serializable.CompositeType: return wrkspc.parse_new(rel_path, definition + "\n").read( - [], lambda *_: None, allow_unregulated + [], [], lambda *_: None, allow_unregulated ) # pragma: no branch with raises(_error.InvalidDefinitionError, match="(?i).*port ID.*"): @@ -754,20 +755,20 @@ def print_handler(line_number: int, text: str) -> None: wrkspc.parse_new( "ns/A.1.0.dsdl", "# line number 1\n" "# line number 2\n" "@print 2 + 2 == 4 # line number 3\n" "# line number 4\n" "@sealed\n", - ).read([], print_handler, False) + ).read([], [], print_handler, False) assert printed_items assert printed_items[0] == 3 assert printed_items[1] == "true" - wrkspc.parse_new("ns/B.1.0.dsdl", "@print false\n@sealed").read([], print_handler, False) + wrkspc.parse_new("ns/B.1.0.dsdl", "@print false\n@sealed").read([], [], print_handler, False) assert printed_items assert printed_items[0] == 1 assert printed_items[1] == "false" wrkspc.parse_new( "ns/Offset.1.0.dsdl", "@print _offset_ # Not recorded\n" "uint8 a\n" "@print _offset_\n" "@extent 800\n" - ).read([], print_handler, False) + ).read([], [], print_handler, False) assert printed_items assert printed_items[0] == 3 assert printed_items[1] == "{8}" diff --git a/pydsdl/visitors/__init__.py b/pydsdl/visitors/__init__.py new file mode 100644 index 0000000..f73e3f2 --- /dev/null +++ b/pydsdl/visitors/__init__.py @@ -0,0 +1,12 @@ +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT +""" +This module contains public visitors. There are other internal visitors used by the core parser but they are not +exposed to the user. While the core parser drives these visitors, when provided, it does not extend them. +""" + +from ._common import DsdlFile as DsdlFile +from ._common import PrintOutputHandler as PrintOutputHandler +from ._common import Visitor as Visitor +from ._definition_visitors import NamespaceVisitor as NamespaceVisitor diff --git a/pydsdl/visitors/_common.py b/pydsdl/visitors/_common.py new file mode 100644 index 0000000..bd1bb7a --- /dev/null +++ b/pydsdl/visitors/_common.py @@ -0,0 +1,133 @@ +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT + +import abc +from pathlib import Path +from typing import Callable, Iterable, List, Optional, Union + +from .._serializable import CompositeType, Version + +PrintOutputHandler = Callable[[Path, int, str], None] +"""Invoked when the frontend encounters a print directive or needs to output a generic diagnostic.""" + + +class Visitor(abc.ABC): + """ + Common interface for all visitors. Visitors are used to participate in the parsing process where + different visitors are called at different stages of the parsing process. See the pydsdl.visitors module + for available visitors. + """ + + # pylint: disable=too-many-arguments + @abc.abstractmethod + def on_read_namespace_start( + self, + root_namespace_directory: Union[Path, str], + lookup_directories: Union[None, Path, str, Iterable[Union[Path, str]]], + print_output_handler: Optional[PrintOutputHandler], + allow_unregulated_fixed_port_id: bool, + allow_root_namespace_name_collision: bool, + ) -> None: + """ + Called before the parser starts reading a namespace. + :param Path root_namespace_directory: The root namespace directory the parser is about to read. + :param Union[None, Path, str, Iterable[Union[Path, str]]] lookup_directories: The lookup directories the parser + will use to resolve the namespace dependencies. + :param Optional[PrintOutputHandler] print_output_handler: The print output handler the parser will use to + output DSDL print statements. + :param bool allow_unregulated_fixed_port_id: Whether the parser will allow unregulated fixed port IDs. + :param bool allow_root_namespace_name_collision: Whether the parser will allow root namespace name collisions. + """ + raise NotImplementedError() + + def on_read_namespace_end(self, result: List[CompositeType]) -> None: + """ + Called when the parser finishes reading a namespace. + :param List[_serializable.CompositeType] composite_types: The composite types that were read. + """ + raise NotImplementedError() + + +class DsdlFile(abc.ABC): + """ + Interface for DSDL files. This interface is implemented by the parser to provide a common interface for + DSDL files that are read. + """ + + @property + @abc.abstractmethod + def composite_type(self) -> Optional[CompositeType]: + """The composite type that was read from the DSDL file or None if the type has not been parsed yet.""" + raise NotImplementedError() + + @property + @abc.abstractmethod + def full_name(self) -> str: + """The full name, e.g., uavcan.node.Heartbeat""" + raise NotImplementedError() + + @property + def name_components(self) -> List[str]: + """Components of the full name as a list, e.g., ['uavcan', 'node', 'Heartbeat']""" + raise NotImplementedError() + + @property + @abc.abstractmethod + def short_name(self) -> str: + """The last component of the full name, e.g., Heartbeat of uavcan.node.Heartbeat""" + raise NotImplementedError() + + @property + @abc.abstractmethod + def full_namespace(self) -> str: + """The full name without the short name, e.g., uavcan.node for uavcan.node.Heartbeat""" + raise NotImplementedError() + + @property + @abc.abstractmethod + def root_namespace(self) -> str: + """The first component of the full name, e.g., uavcan of uavcan.node.Heartbeat""" + raise NotImplementedError() + + @property + @abc.abstractmethod + def text(self) -> str: + """The source text in its raw unprocessed form (with comments, formatting intact, and everything)""" + raise NotImplementedError() + + @property + @abc.abstractmethod + def version(self) -> Version: + """ + The version of the DSDL definition. + """ + raise NotImplementedError() + + @property + @abc.abstractmethod + def fixed_port_id(self) -> Optional[int]: + """Either the fixed port ID as integer, or None if not defined for this type.""" + raise NotImplementedError() + + @property + @abc.abstractmethod + def has_fixed_port_id(self) -> bool: + """ + If the type has a fixed port ID defined, this method returns True. Equivalent to ``fixed_port_id is not None``. + """ + raise NotImplementedError() + + @property + @abc.abstractmethod + def file_path(self) -> Path: + """The path to the DSDL file on the filesystem.""" + raise NotImplementedError() + + @property + @abc.abstractmethod + def root_namespace_path(self) -> Path: + """ + The path to the root namespace directory on the filesystem. + """ + raise NotImplementedError() diff --git a/pydsdl/visitors/_definition_visitors.py b/pydsdl/visitors/_definition_visitors.py new file mode 100644 index 0000000..fe0377e --- /dev/null +++ b/pydsdl/visitors/_definition_visitors.py @@ -0,0 +1,76 @@ +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT + +from pathlib import Path +from typing import Iterable, List, Optional, Union + +from .._serializable import CompositeType, Version +from ._common import PrintOutputHandler, Visitor, DsdlFile + +class DependentFileError(RuntimeError): + """ + Raised by the NamespaceVisitor when it encounters a dependent dsdl file that is not allowed by the user. + """ + +class NamespaceVisitor(Visitor): + """ + A visitor interface that allows the user to vote on whether the parser should parse DSDL files it finds on the + filesystem. This allows the user to exclude certain DSDL files from the parsing process based on their own criteria. + For example, by excluding target DSDL files the user can select a few target DSDL files to parse from a large + collection of DSDL files within a given namespace. + """ + + # +-----------------------------------------------------------------------+ + # | Visitor :: TEMPLATE METHODS | + # | We convert the Visitor interface methods to template methods to | + # | allow the user to override only the methods they are interested | + # | in. | + # +-----------------------------------------------------------------------+ + # pylint: disable=too-many-arguments + def on_read_namespace_start( + self, + root_namespace_directory: Union[Path, str], + lookup_directories: Union[None, Path, str, Iterable[Union[Path, str]]], + print_output_handler: Optional[PrintOutputHandler], + allow_unregulated_fixed_port_id: bool, + allow_root_namespace_name_collision: bool, + ) -> None: + pass + + def on_read_namespace_end(self, result: List[CompositeType]) -> None: + pass + + # +-----------------------------------------------------------------------+ + # | NamespaceVisitor :: TEMPLATE METHODS | + # +-----------------------------------------------------------------------+ + def on_discover_target_file(self, target_dsdl_file: DsdlFile) -> bool: + """ + Voting template method called by the parser when it finds a target DSDL file on the filesystem but + before it starts parsing it (although, the parser may have already read the contents of the file). + Override this method to vote on whether the parser should parse the target DSDL file or exclude it. + :param DsdlFile target_dsdl_file: The target DSDL file the parser found. + :return: True if the parser should parse the target DSDL file, False otherwise. + """ + return True + + def on_discover_lookup_dependent_type( + self, target_dsdl_file: DsdlFile, full_name: str, version: Version + ) -> None: + """ + Notification template method called by the parser when it finds a dependent type of a target DSDL file that it + hasn't parsed yet. This method is called before on_discover_lookup_dependent_file. + Override this method to receive notifications about dependent types the parser finds. + :param DsdlFile target_dsdl_file: The target DSDL file that has dependencies the parser is searching for. + :param str full_name: The full name of the dependent type the parser found. + :param Version version: The version of the dependent type the parser found. + """ + + def on_discover_lookup_dependent_file(self, target_dsdl_file: DsdlFile, lookup_file: DsdlFile) -> None: + """ + Called by the parser after if finds a dependent type but before it parses a file in a lookup namespace. + Override this method to vote on whether the parser should parse the lookup file. + :param DsdlFile target_dsdl_file: The target DSDL file that has dependencies the parser is searching for. + :param DsdlFile lookup_file: The lookup file the parser is about to parse. + :raises DependentFileError: If the dependent file is not allowed by the user. + """ diff --git a/setup.cfg b/setup.cfg index 8d934ba..c58e783 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,7 +35,7 @@ include = pydsdl* # -------------------------------------------------- PYTEST -------------------------------------------------- [tool:pytest] -testpaths = pydsdl +testpaths = pydsdl test norecursedirs = third_party python_files = *.py python_classes = _UnitTest diff --git a/test/public_regulated_data_types b/test/public_regulated_data_types new file mode 160000 index 0000000..f9f6790 --- /dev/null +++ b/test/public_regulated_data_types @@ -0,0 +1 @@ +Subproject commit f9f67906cc0ca5d7c1b429924852f6b28f313cbf diff --git a/test/test_public_types.py b/test/test_public_types.py new file mode 100644 index 0000000..018b79c --- /dev/null +++ b/test/test_public_types.py @@ -0,0 +1,46 @@ +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT + +# pylint: disable=redefined-outer-name +# pylint: disable=logging-fstring-interpolation +import cProfile +import io +import logging +import pstats +from pathlib import Path +from pstats import SortKey + +import pytest + +import pydsdl + + +@pytest.fixture +def public_types() -> Path: + return Path("test") / "public_regulated_data_types" / "uavcan" + + +def dsdl_printer(dsdl_file: Path, line: int, message: str) -> None: + """ + Prints the DSDL file. + """ + logging.info(f"{dsdl_file}:{line}: {message}") + + +def _unittest_public_types(public_types: Path) -> None: + """ + Sanity check to ensure that the public types can be read. This also allows us to debug + against a real dataset. + """ + pr = cProfile.Profile() + pr.enable() + _ = pydsdl.read_namespace(public_types) + pr.disable() + s = io.StringIO() + sortby = SortKey.TIME + ps = pstats.Stats(pr, stream=s).sort_stats(sortby) + ps.print_stats() + print(s.getvalue()) + +