Ensure that the parse tool produces trees with the same structure as …

…the generator (#232)

The parse trees generated by ANTLR consist only of rule nodes and
do not contain information about the decisions made on alternations
or quantifiers. However, tests generated by Grammarinator store
such information in the form of alternative and quantifier nodes.
To make these trees compatible, the current patch recreates the
same structure that the generator produces in the trees obtained
from parsed input.
The approach reuses the grammar graph implementation of processor,
therefore it requires to define a lib_dir if included grammars
are stored in a separate directory. To define such directories,
similarily to grammarinator-process, grammarinator-parse also supports
the `--lib_dir` CLI argument.

Co-authored-by: Akos Kiss <[email protected]>

.pylintrc

@@ -33,6 +33,7 @@ disable=
     redefined-builtin,
     too-few-public-methods,
     too-many-arguments,
+    too-many-boolean-expressions,
     too-many-branches,
     too-many-instance-attributes,
     too-many-lines,

grammarinator/parse.py

@@ -56,6 +56,8 @@ def execute():
                         help='directory to save the trees (default: %(default)s).')
     parser.add_argument('--parser-dir', metavar='DIR',
                         help='directory to save the parser grammars (default: <OUTDIR>/grammars).')
+    parser.add_argument('--lib', metavar='DIR',
+                        help='alternative location of import grammars.')
     add_tree_format_argument(parser)
     add_encoding_argument(parser, help='input file encoding (default: %(default)s).')
     add_encoding_errors_argument(parser)
@@ -80,7 +82,7 @@ def execute():
         parser.error(e)
 
     with ParserTool(grammars=args.grammar, hidden=args.hidden, transformers=args.transformer, parser_dir=args.parser_dir, antlr=args.antlr, rule=args.rule,
-                    population=DefaultPopulation(args.out, args.tree_extension, codec=args.tree_codec), max_depth=args.max_depth, cleanup=args.cleanup, encoding=args.encoding, errors=args.encoding_errors) as parser:
+                    population=DefaultPopulation(args.out, args.tree_extension, codec=args.tree_codec), max_depth=args.max_depth, lib_dir=args.lib, cleanup=args.cleanup, encoding=args.encoding, errors=args.encoding_errors) as parser:
         if args.jobs > 1:
             with Pool(args.jobs) as pool:
                 for _ in pool.imap_unordered(parser.parse, iter_files(args)):

tests/parser/Parse.g4

@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2024 Renata Hodovan, Akos Kiss.
+ *
+ * Licensed under the BSD 3-Clause License
+ * <LICENSE.rst or https://opensource.org/licenses/BSD-3-Clause>.
+ * This file may not be copied, modified, or distributed except
+ * according to those terms.
+ */
+
+/*
+ * This test checks whether the parser utility of Grammarinator creates
+ * the same tree structures as generator would do.
+ */
+
+grammar Parse;
+
+start
+  : element (' ' element)*    # Quantifiers_test
+  | element (' | ' element)+  # Quantifiers_test
+  | list_with_recursion       # Recursion_test
+  ;
+
+element
+  : 'pass' ('?' | '!')?
+  ;
+
+list_with_recursion
+  : list_with_recursion (', ' element)
+  | element
+  ;

tests/parser/exp1.grtj

@@ -0,0 +1 @@
+{"t": "p", "n": "start", "c": [{"t": "a", "ai": 0, "i": 0, "c": [{"t": "p", "n": "start_Quantifiers_test", "c": [{"t": "p", "n": "element", "c": [{"t": "l", "n": "<INVALID>", "s": "pass", "z": [1, 1]}, {"t": "q", "i": 0, "b": 0, "e": 1, "c": []}]}, {"t": "q", "i": 0, "b": 0, "e": Infinity, "c": []}]}]}]}

tests/parser/exp2.grtj

@@ -0,0 +1 @@
+{"t": "p", "n": "start", "c": [{"t": "a", "ai": 0, "i": 0, "c": [{"t": "p", "n": "start_Quantifiers_test", "c": [{"t": "p", "n": "element", "c": [{"t": "l", "n": "<INVALID>", "s": "pass", "z": [1, 1]}, {"t": "q", "i": 0, "b": 0, "e": 1, "c": [{"t": "qd", "c": [{"t": "a", "ai": 0, "i": 0, "c": [{"t": "l", "n": "<INVALID>", "s": "?", "z": [1, 1]}]}]}]}]}, {"t": "q", "i": 0, "b": 0, "e": Infinity, "c": []}]}]}]}

tests/parser/exp3.grtj

@@ -0,0 +1 @@
+{"t": "p", "n": "start", "c": [{"t": "a", "ai": 0, "i": 0, "c": [{"t": "p", "n": "start_Quantifiers_test", "c": [{"t": "p", "n": "element", "c": [{"t": "l", "n": "<INVALID>", "s": "pass", "z": [1, 1]}, {"t": "q", "i": 0, "b": 0, "e": 1, "c": [{"t": "qd", "c": [{"t": "a", "ai": 0, "i": 1, "c": [{"t": "l", "n": "<INVALID>", "s": "!", "z": [1, 1]}]}]}]}]}, {"t": "q", "i": 0, "b": 0, "e": Infinity, "c": [{"t": "qd", "c": [{"t": "l", "n": "<INVALID>", "s": " ", "z": [1, 1]}, {"t": "p", "n": "element", "c": [{"t": "l", "n": "<INVALID>", "s": "pass", "z": [1, 1]}, {"t": "q", "i": 0, "b": 0, "e": 1, "c": [{"t": "qd", "c": [{"t": "a", "ai": 0, "i": 0, "c": [{"t": "l", "n": "<INVALID>", "s": "?", "z": [1, 1]}]}]}]}]}]}]}]}]}]}

tests/parser/exp4.grtj

@@ -0,0 +1 @@
+{"t": "p", "n": "start", "c": [{"t": "a", "ai": 0, "i": 1, "c": [{"t": "p", "n": "start_Quantifiers_test", "c": [{"t": "p", "n": "element", "c": [{"t": "l", "n": "<INVALID>", "s": "pass", "z": [1, 1]}, {"t": "q", "i": 0, "b": 0, "e": 1, "c": [{"t": "qd", "c": [{"t": "a", "ai": 0, "i": 0, "c": [{"t": "l", "n": "<INVALID>", "s": "?", "z": [1, 1]}]}]}]}]}, {"t": "q", "i": 0, "b": 1, "e": Infinity, "c": [{"t": "qd", "c": [{"t": "l", "n": "<INVALID>", "s": " | ", "z": [1, 1]}, {"t": "p", "n": "element", "c": [{"t": "l", "n": "<INVALID>", "s": "pass", "z": [1, 1]}, {"t": "q", "i": 0, "b": 0, "e": 1, "c": []}]}]}]}]}]}]}

tests/parser/exp5.grtj

@@ -0,0 +1 @@
+{"t": "p", "n": "start", "c": [{"t": "a", "ai": 0, "i": 2, "c": [{"t": "p", "n": "start_Recursion_test", "c": [{"t": "p", "n": "list_with_recursion", "c": [{"t": "a", "ai": 0, "i": 0, "c": [{"t": "p", "n": "list_with_recursion", "c": [{"t": "a", "ai": 0, "i": 0, "c": [{"t": "p", "n": "list_with_recursion", "c": [{"t": "a", "ai": 0, "i": 1, "c": [{"t": "p", "n": "element", "c": [{"t": "l", "n": "<INVALID>", "s": "pass", "z": [1, 1]}, {"t": "q", "i": 0, "b": 0, "e": 1, "c": [{"t": "qd", "c": [{"t": "a", "ai": 0, "i": 0, "c": [{"t": "l", "n": "<INVALID>", "s": "?", "z": [1, 1]}]}]}]}]}]}]}, {"t": "l", "n": "<INVALID>", "s": ", ", "z": [1, 1]}, {"t": "p", "n": "element", "c": [{"t": "l", "n": "<INVALID>", "s": "pass", "z": [1, 1]}, {"t": "q", "i": 0, "b": 0, "e": 1, "c": [{"t": "qd", "c": [{"t": "a", "ai": 0, "i": 1, "c": [{"t": "l", "n": "<INVALID>", "s": "!", "z": [1, 1]}]}]}]}]}]}]}, {"t": "l", "n": "<INVALID>", "s": ", ", "z": [1, 1]}, {"t": "p", "n": "element", "c": [{"t": "l", "n": "<INVALID>", "s": "pass", "z": [1, 1]}, {"t": "q", "i": 0, "b": 0, "e": 1, "c": []}]}]}]}]}]}]}

tests/parser/inp1.txt

@@ -0,0 +1 @@
+pass

tests/parser/inp2.txt

@@ -0,0 +1 @@
+pass?

tests/parser/inp3.txt

@@ -0,0 +1 @@
+pass! pass?

tests/parser/inp4.txt

@@ -0,0 +1 @@
+pass? | pass

tests/parser/inp5.txt

@@ -0,0 +1 @@
+pass?, pass!, pass

tests/test_parser.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2024 Renata Hodovan, Akos Kiss.
+#
+# Licensed under the BSD 3-Clause License
+# <LICENSE.rst or https://opensource.org/licenses/BSD-3-Clause>.
+# This file may not be copied, modified, or distributed except
+# according to those terms.
+
+import os
+
+import pytest
+
+from antlerinator import default_antlr_jar_path
+from antlr4 import InputStream
+
+from grammarinator.parse import ParserTool
+from grammarinator.tool import JsonTreeCodec
+
+parser_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'parser')
+
+
+@pytest.mark.parametrize('inp, expected', [
+    (os.path.join(parser_dir, 'inp1.txt'), os.path.join(parser_dir, 'exp1.grtj')),
+    (os.path.join(parser_dir, 'inp2.txt'), os.path.join(parser_dir, 'exp2.grtj')),
+    (os.path.join(parser_dir, 'inp3.txt'), os.path.join(parser_dir, 'exp3.grtj')),
+    (os.path.join(parser_dir, 'inp4.txt'), os.path.join(parser_dir, 'exp4.grtj')),
+    (os.path.join(parser_dir, 'inp5.txt'), os.path.join(parser_dir, 'exp5.grtj')),
+])
+def test_parser(inp, expected, tmpdir):
+    with open(inp, 'r') as f:
+        src = f.read()
+
+    tool = ParserTool(grammars=[os.path.join(parser_dir, 'Parse.g4')], rule='start', parser_dir=str(tmpdir), antlr=default_antlr_jar_path(), population=None)
+    root = tool._create_tree(InputStream(src), None)
+
+    with open(expected, 'rb') as f:
+        expected_root = JsonTreeCodec().decode(f.read())
+
+    assert root.equals(expected_root), f'{root:|} != {expected_root:|}'