Merge pull request #124 from klauer/makefile_helper

ENH: add Makefile information helper

MANIFEST.in

@@ -10,3 +10,4 @@ recursive-include whatrecord/grammar *.lark
 recursive-include whatrecord/tests/epicsarch/ *.txt
 recursive-include whatrecord/tests *.db *.cmd *.dbd *.proto *.pvlist *.acf t1-*.txt *.st LICENSE
 recursive-include whatrecord/tests *.substitutions *.substitutions.expanded
+recursive-include whatrecord/tests Makefile

README.rst

@@ -63,11 +63,18 @@ Plugins
 * LCLS-specific epicsArch / logbook DAQ PVs
 * TwinCAT PLC source code (pytmc)
 
+Makefile / build system information
+
+* Determine build dependencies from a ``Makefile``
+* Recursively inspect sub-dependencies
+* Graph IOC dependency information or output it as JSON
+
 Command-line tools
 
 * ``whatrecord lint`` - lint a database
 * ``whatrecord parse`` - parse supported formats
 * ``whatrecord server`` - start the API server
+* ``whatrecord graph`` - graph PV relationships, SNL diagrams, IOC dependencies
 
 Record?
 -------

docs/docs/index.md

@@ -56,11 +56,18 @@ It contains interfaces for:
 * LCLS-specific epicsArch / logbook DAQ PVs
 * TwinCAT PLC source code (pytmc)
 
+### Makefile / build system information
+
+* Determine build dependencies from a ``Makefile``
+* Recursively inspect sub-dependencies
+* Graph IOC dependency information or output it as JSON
+
 ### Command-line tools
 
 * ``whatrecord lint`` - lint a database
 * ``whatrecord parse`` - parse supported formats to JSON
 * ``whatrecord server`` - start the API server
+* ``whatrecord graph`` - graph PV relationships, SNL diagrams, IOC dependencies
 * Plugins can similarly be executed to provide parsed information in JSON
 
 ## Installing

whatrecord/__init__.py

@@ -3,19 +3,22 @@
 __version__ = _version.get_versions()['version']
 
 from .access_security import AccessSecurityConfig
+from .common import FileFormat
 from .db import Database
 from .dbtemplate import TemplateSubstitution
 from .gateway import GatewayConfig
 from .gateway import PVList as GatewayPVList
 from .iocsh import parse_iocsh_line
 from .macro import MacroContext
+from .parse import parse
 from .plugins.epicsarch import LclsEpicsArchFile
 from .snl import SequencerProgram
 from .streamdevice import StreamProtocol
 
 __all__ = [
     "AccessSecurityConfig",
     "Database",
+    "FileFormat",
     "GatewayConfig",
     "GatewayPVList",
     "LclsEpicsArchFile",
@@ -24,4 +27,5 @@
     "StreamProtocol",
     "TemplateSubstitution",
     "parse_iocsh_line",
+    "parse",
 ]

whatrecord/bin/deps.py

@@ -0,0 +1,102 @@
+"""
+"whatrecord deps" is used to get dependency information from EPICS IOC or
+module Makefiles.
+
+Under the hood, this uses GNU make, which is an external dependency required
+for correct functionality.
+"""
+
+import argparse
+import json
+import logging
+import sys
+from typing import Optional
+
+import apischema
+
+from ..common import AnyPath
+from ..format import FormatContext
+from ..makefile import DependencyGroup, DependencyGroupGraph, Makefile
+from .graph import render_graph_to_file
+
+logger = logging.getLogger(__name__)
+DESCRIPTION = __doc__
+
+
+def build_arg_parser(parser=None):
+    if parser is None:
+        parser = argparse.ArgumentParser()
+
+    parser.description = DESCRIPTION
+    parser.formatter_class = argparse.RawTextHelpFormatter
+
+    parser.add_argument(
+        "path", type=str, help="Path to IOC or Makefile itself"
+    )
+
+    parser.add_argument(
+        "--keep-os-env",
+        action="store_true",
+        help=(
+            "Keep environment variables present outside of ``make`` in the "
+            ".env dictionaries"
+        )
+    )
+
+    parser.add_argument(
+        "--no-recurse",
+        action="store_true",
+        help="Do not recurse into dependencies",
+    )
+
+    parser.add_argument(
+        "--friendly",
+        dest="friendly",
+        action="store_true",
+        help="Output user-friendly text instead of JSON",
+    )
+
+    parser.add_argument(
+        "--graph",
+        action="store_true",
+        help="Output a graph of dependencies",
+    )
+
+    parser.add_argument(
+        "-o", "--graph-output",
+        type=str,
+        required=False,
+        help="Output file to write to.  Defaults to standard output.",
+    )
+
+    return parser
+
+
+def main(
+    path: AnyPath,
+    friendly: bool = False,
+    no_recurse: bool = False,
+    keep_os_env: bool = False,
+    graph: bool = False,
+    graph_output: Optional[str] = None,
+    file=sys.stdout,
+):
+    makefile_path = Makefile.find_makefile(path)
+    makefile = Makefile.from_file(makefile_path, keep_os_env=keep_os_env)
+    info = DependencyGroup.from_makefile(
+        makefile, recurse=not no_recurse, keep_os_env=keep_os_env
+    )
+
+    if graph:
+        group_graph = DependencyGroupGraph(info)
+        render_graph_to_file(group_graph.to_digraph(), filename=graph_output)
+        # An alternative to 'whatrecord graph'; both should have the same
+        # result in the end.
+        return
+
+    if not friendly:
+        json_info = apischema.serialize(info)
+        print(json.dumps(json_info, indent=4))
+    else:
+        fmt = FormatContext()
+        print(fmt.render_object(info, "console"), file=file)

whatrecord/bin/graph.py

@@ -18,6 +18,7 @@
 from ..common import AnyPath
 from ..db import Database, LinterResults
 from ..graph import RecordLinkGraph, build_database_relations, graph_links
+from ..makefile import DependencyGroup, DependencyGroupGraph, Makefile
 from ..shell import LoadedIoc
 from ..snl import SequencerProgram
 from .parse import parse_from_cli_args
@@ -217,6 +218,8 @@ def main(
 
         if isinstance(item, SequencerProgram):
             graph = item.as_graph(include_code=code)
+        elif isinstance(item, Makefile):
+            graph = DependencyGroupGraph(DependencyGroup.from_makefile(item))
         else:
             raise RuntimeError(
                 f"Sorry, graph isn't supported yet for {item.__class__.__name__}"

whatrecord/bin/main.py

@@ -16,7 +16,18 @@
 DESCRIPTION = __doc__
 RETURN_VALUE = None
 
-MODULES = ("server", "iocmanager_loader", "info", "lint", "parse", "graph")
+# The whatrecord.bin modules are listed here and imported dynamically such
+# that any missing dependencies for that command will not stop other
+# entrypoints from working:
+MODULES = (
+    "deps",
+    "graph",
+    "info",
+    "iocmanager_loader",
+    "lint",
+    "parse",
+    "server",
+)
 
 
 def _try_import(module):

whatrecord/bin/parse.py

@@ -5,24 +5,16 @@
 """
 
 import argparse
-import asyncio
 import json
 import logging
-import pathlib
-from typing import Dict, List, Optional, Union
+from typing import List, Optional
 
 import apischema
 
-from ..access_security import AccessSecurityConfig
 from ..common import AnyPath, FileFormat, IocMetadata
-from ..db import Database, LinterResults
-from ..dbtemplate import TemplateSubstitution
 from ..format import FormatContext
-from ..gateway import PVList as GatewayPVList
-from ..macro import MacroContext
+from ..parse import ParseResult, parse
 from ..shell import LoadedIoc
-from ..snl import SequencerProgram
-from ..streamdevice import StreamProtocol
 
 logger = logging.getLogger(__name__)
 DESCRIPTION = __doc__
@@ -104,132 +96,6 @@ def build_arg_parser(parser=None):
     return parser
 
 
-ParseResult = Union[
-    AccessSecurityConfig,
-    Database,
-    GatewayPVList,
-    LinterResults,
-    LoadedIoc,
-    SequencerProgram,
-    StreamProtocol,
-    TemplateSubstitution,
-]
-
-
-def parse(
-    filename: AnyPath,
-    dbd: Optional[str] = None,
-    standin_directories: Optional[Dict[str, str]] = None,
-    macros: Optional[str] = None,
-    use_gdb: bool = False,
-    format: Optional[FileFormat] = None,
-    expand: bool = False,
-    v3: bool = False,
-) -> ParseResult:
-    """
-    Generically parse either a startup script or a database file.
-
-    Hopefully does the right thing based on file extension.  If not, use
-    the ``format`` kwarg to specify it directly.
-
-    Parameters
-    ----------
-    filename : str or pathlib.Path
-        The filename to parse.
-
-    dbd : str or pathlib.Path, optional
-        The associated database definition file, if parsing a database or
-        substitutions file.
-
-    standin_directories : dict, optional
-        Rewrite hard-coded directory prefixes by setting::
-
-            standin_directories = {"/replace_this/": "/with/this"}
-
-    macros : str, optional
-        Macro string to use when parsing the file.
-
-    expand : bool, optional
-        Expand a substitutions file.
-
-    v3 : bool, optional
-        Use V3 database grammar where applicable.
-    """
-    standin_directories = standin_directories or {}
-
-    filename = pathlib.Path(filename)
-
-    # The shared macro context - used in different ways below:
-    macro_context = MacroContext(macro_string=macros or "")
-
-    if format is None:
-        format = FileFormat.from_filename(filename)
-
-    if format in (FileFormat.database, FileFormat.database_definition):
-        if format == FileFormat.database_definition or not dbd:
-            return Database.from_file(
-                filename, macro_context=macro_context, version=3 if v3 else 4
-            )
-        return LinterResults.from_database_file(
-            db_filename=filename,
-            dbd=Database.from_file(dbd, version=3 if v3 else 4),
-            macro_context=macro_context
-        )
-
-    if format == FileFormat.iocsh:
-        md = IocMetadata.from_filename(
-            filename,
-            standin_directories=standin_directories,
-            macros=dict(macro_context),
-        )
-        if use_gdb:
-            try:
-                asyncio.run(md.get_binary_information())
-            except KeyboardInterrupt:
-                logger.info("Skipping gdb information...")
-
-        return LoadedIoc.from_metadata(md)
-
-    if format == FileFormat.substitution:
-        template = TemplateSubstitution.from_file(filename)
-        if not expand:
-            return template
-
-        database_text = template.expand_files()
-        # It's technically possible that this *isn't* a database file; so
-        # perhaps a `whatrecord msi` could be implemented in the future.
-        return Database.from_string(
-            database_text,
-            macro_context=macro_context,
-            dbd=Database.from_file(dbd) if dbd is not None else None,
-            filename=filename,
-            version=3 if v3 else 4,
-        )
-
-    if format == FileFormat.state_notation:
-        return SequencerProgram.from_file(filename)
-
-    with open(filename, "rt") as fp:
-        contents = fp.read()
-
-    if macros:
-        contents = macro_context.expand_file(contents)
-
-    if format == FileFormat.gateway_pvlist:
-        return GatewayPVList.from_string(contents, filename=filename)
-
-    if format == FileFormat.access_security:
-        return AccessSecurityConfig.from_string(contents, filename=filename)
-
-    if format == FileFormat.stream_protocol:
-        return StreamProtocol.from_string(contents, filename=filename)
-
-    raise RuntimeError(
-        f"Sorry, whatrecord doesn't support the {format!r} format just yet in the "
-        f"CLI parsing tool.  Please open an issue."
-    )
-
-
 def parse_from_cli_args(
     filename: AnyPath,
     dbd: Optional[str] = None,