Merge pull request #38 from jdpage/pgen

Replace ANTLR4 with homegrown LALR parser generator

Makefile

@@ -1,6 +1,7 @@
 .PHONY: all antlr check demo develop install
 
-all: antlr
+all:
+
 
 install: all
 	pip install .
@@ -9,11 +10,6 @@ develop: all
 	pip install -Ur requirements.txt
 	pip install -e .
 
-antlr: jeff65/gold/grammar/Gold.py
-
-jeff65/gold/grammar/Gold.py: jeff65/gold/grammar/Gold.g4
-	antlr4 -Dlanguage=Python3 $^
-
 check: all
 	flake8 jeff65 tests
 	python setup.py nosetests

jeff65/gold/__init__.py

@@ -14,11 +14,9 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-from .ast import ParseError
 from .compiler import parse, translate
 
 __all__ = [
-    'ParseError',
     'parse',
     'translate',
 ]

jeff65/gold/ast.py

@@ -14,13 +14,6 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-from .grammar import ParseListener
-
-
-class ParseError(Exception):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
 
 class AstNode:
     def __init__(self, t, position, attrs=None, children=None):
@@ -48,7 +41,7 @@ def __eq__(self, other):
             and self.attrs == other.attrs
             and self.children == other.children)
 
-    def transform(self, transformer):
+    def transform(self, transformer, always_list=False):
         node = transformer.transform_enter(self.t, self)
 
         if transformer.transform_attrs and type(node) is AstNode:
@@ -70,7 +63,7 @@ def transform(self, transformer):
             children = []
             for child in node.children:
                 if type(child) is AstNode:
-                    children.extend(child.transform(transformer))
+                    children.extend(child.transform(transformer, always_list))
                 else:
                     children.append(child)
             if children != node.children:
@@ -85,7 +78,7 @@ def transform(self, transformer):
         elif type(nodes) is not list:
             nodes = [nodes]
 
-        if self.t == 'unit':
+        if not always_list and self.t == 'unit':
             assert len(nodes) == 1
             return nodes[0]
         return nodes
@@ -140,162 +133,3 @@ def __generic_enter(self, node):
 
     def __generic_exit(self, node):
         return [node]
-
-
-class AstBuilder(ParseListener):
-    def __init__(self):
-        self.stack = []
-
-    @property
-    def ast(self):
-        return self.stack[0]
-
-    def _push(self, node):
-        self.stack.append(node)
-
-    def _pop(self):
-        c = self.stack.pop()
-        self.stack[-1].children.append(c)
-        return c
-
-    def _pop_attr(self, attr):
-        a = self.stack.pop()
-        self.stack[-1].attrs[attr] = a
-
-    def _pos(self, ctx):
-        return (ctx.start.line, ctx.start.column)
-
-    def enterUnit(self, ctx):
-        self._push(AstNode("unit", self._pos(ctx)))
-
-    def enterStmtUse(self, ctx):
-        self._push(AstNode("use", self._pos(ctx), {
-            "name": ctx.unitId.text
-        }))
-
-    def exitStmtUse(self, ctx):
-        self._pop()
-
-    def enterStmtConstant(self, ctx):
-        self._push(AstNode("constant", self._pos(ctx), {
-            "name": ctx.declaration().name.text
-        }))
-
-    def exitStmtConstant(self, ctx):
-        self._pop()
-
-    def enterStmtLet(self, ctx):
-        node = AstNode("let", self._pos(ctx))
-        node.attrs['name'] = ctx.declaration().name.text
-        if ctx.storage():
-            node.attrs['storage'] = ctx.storage().storage_class.text
-        self._push(node)
-
-    def exitStmtLet(self, ctx):
-        self._pop()
-
-    def enterStmtFun(self, ctx):
-        self._push(AstNode("fun", self._pos(ctx), {
-            "name": ctx.name.text,
-            'return': None,
-            'args': [],
-        }))
-
-    def exitStmtFun(self, ctx):
-        self._pop()
-
-    def enterStmtAssignVal(self, ctx):
-        self._push(AstNode("set", self._pos(ctx)))
-
-    def exitStmtAssignVal(self, ctx):
-        self._pop()
-
-    def enterTypePrimitive(self, ctx):
-        self.stack[-1].attrs["type"] = ctx.name.text
-
-    def enterTypePointer(self, ctx):
-        self._push(AstNode("type_ref", self._pos(ctx)))
-
-    def exitTypePointer(self, ctx):
-        self._pop_attr("type")
-
-    def enterTypeArray(self, ctx):
-        self._push(AstNode('type_array', self._pos(ctx)))
-
-    def exitTypeArray(self, ctx):
-        self._pop_attr('type')
-
-    def enterExprMember(self, ctx):
-        self._push(AstNode("member_access", self._pos(ctx), {
-            "member": ctx.member.text
-        }))
-
-    def exitExprMember(self, ctx):
-        self._pop()
-
-    def enterExprId(self, ctx):
-        self.stack[-1].children.append(AstNode("identifier", self._pos(ctx), {
-            'name': ctx.name.text,
-        }))
-
-    def enterExprNumber(self, ctx):
-        text = ctx.value.text.lower()
-        if text.startswith('0x'):
-            value = int(text[2:], 16)
-        elif text.startswith('0o'):
-            value = int(text[2:], 8)
-        elif text.startswith('0b'):
-            value = int(text[2:], 2)
-        else:
-            value = int(text)
-        self.stack[-1].children.append(AstNode("numeric", self._pos(ctx), {
-            'value': value,
-        }))
-
-    def enterExprFunCall(self, ctx):
-        self._push(AstNode("call", self._pos(ctx)))
-
-    def exitExprFunCall(self, ctx):
-        call = self.stack[-1]
-        call.attrs['target'] = call.children[0]
-        call.children = call.children[1:]
-        self._pop()
-
-    def enterExprDeref(self, ctx):
-        self._push(AstNode("deref", self._pos(ctx)))
-
-    def exitExprDeref(self, ctx):
-        self._pop()
-
-    def enterExprSum(self, ctx):
-        if ctx.op.text == '+':
-            self._push(AstNode("add", self._pos(ctx)))
-        elif ctx.op.text == '-':
-            self._push(AstNode("sub", self._pos(ctx)))
-        else:
-            assert False
-
-    def exitExprSum(self, ctx):
-        self._pop()
-
-    def enterExprProduct(self, ctx):
-        if ctx.op.text == '*':
-            self._push(AstNode("mul", self._pos(ctx)))
-        elif ctx.op.text == '/':
-            self._push(AstNode("div", self._pos(ctx)))
-        else:
-            assert False
-
-    def exitExprProduct(self, ctx):
-        self._pop()
-
-    def enterExprNegation(self, ctx):
-        self._push(AstNode("negate", self._pos(ctx)))
-
-    def exitExprNegation(self, ctx):
-        self._pop()
-
-    def enterString(self, ctx):
-        self.stack[-1].children.append(AstNode('string', self._pos(ctx), {
-            'value': "".join(s.text for s in ctx.s),
-        }))

jeff65/gold/compiler.py

@@ -15,12 +15,9 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 import sys
-import antlr4
-from . import ast
-from .. import blum
-from .grammar import Parser
-from .lexer import Lexer
-from .passes import asm, binding, lower, resolve, typepasses
+from . import ast, grammar
+from .. import blum, parsing
+from .passes import asm, binding, lower, resolve, simplify, typepasses
 
 
 passes = [
@@ -48,15 +45,15 @@ def open_unit(unit):
 
 
 def parse(fileobj, name):
-    lexer = Lexer(fileobj, name=name)
-    tokens = antlr4.CommonTokenStream(lexer)
-    parser = Parser(tokens)
-    tree = parser.unit()
-    if parser._syntaxErrors > 0:
-        raise ast.ParseError("Unit {} had errors; terminating".format(name))
-    builder = ast.AstBuilder()
-    antlr4.ParseTreeWalker.DEFAULT.walk(builder, tree)
-    return builder.ast
+    stream = parsing.ReStream(fileobj)
+    tree = grammar.parse(
+        stream, grammar.lex,
+        lambda t, s, c, m: ast.AstNode(t, s.start, children=c))
+    # if parser._syntaxErrors > 0:
+    #     raise ast.ParseError("Unit {} had errors; terminating".format(name))
+    unit = tree.transform(simplify.Simplify(), always_list=True)
+    assert len(unit) == 1
+    return unit[0]
 
 
 def translate(unit, verbose=False):