-
Notifications
You must be signed in to change notification settings - Fork 6
/
arlington-fn-lex.py
205 lines (180 loc) · 6.68 KB
/
arlington-fn-lex.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
#!/usr/bin/python3
# Copyright 2021 PDF Association, Inc. https://www.pdfa.org
#
# This material is based upon work supported by the Defense Advanced
# Research Projects Agency (DARPA) under Contract No. HR001119C0079.
# Any opinions, findings and conclusions or recommendations expressed
# in this material are those of the author(s) and do not necessarily
# reflect the views of the Defense Advanced Research Projects Agency
# (DARPA). Approved for public release.
#
# SPDX-License-Identifier: Apache-2.0
# Contributors: Peter Wyatt, PDF Association
#
# Syntactically validate all declarative functions in the Arlington PDF model.
#
# Usage: arlington-fn-lex.py [- | files ]
#
# Lexical errors (such as typos) will result in a Python exception and premature failure.
#
# Normal usage with Arlington PDF model (Linux CLI):
# $ echo "fn:Eval(@key,-27,1.23,'string',path::key1,((expr1==expr2) && (a!=b)" || (4>3))) | python3 arlington-fn-lex.py
#
# $ grep --color=never -Pho "fn:[a-zA-Z0-9]+\([^\t\]\;]*\)" ../tsv/latest/*.tsv | sort | uniq | python3 arlington-fn-lex.py
# --> expected outcome: no errors or crashes means that the predicates in Arlington are likely correct and valid
#
#
# Requires Python 3 and sly:
# - pip3 install sly
# - See https://sly.readthedocs.io/en/latest/sly.html
#
# Python QA:
# - flake8 --ignore E501,E221,E226 arlington-fn-lex.py
# - pyflakes arlington-fn-lex.py
# - mypy arlington-fn-lex.py
#
import fileinput
import pprint
from typing import Any, Dict, List, Optional, Tuple, Union
import sly # type: ignore
##############################################################################################
class ArlingtonFnLexer(sly.Lexer):
"""
Sly parser
"""
# debugfile = 'parser.out'
# Set of token names. This is always required.
tokens = {
FUNC_NAME, KEY_VALUE, KEY_NAME, KEY_PATH, MOD, PDF_PATH, EQ, NE, GE, LE, GT, LT,
LOGICAL_AND, LOGICAL_OR, REAL, INTEGER, PLUS, MINUS, TIMES, DIVIDE, LPAREN,
RPAREN, COMMA, ARRAY_START, ARRAY_END, ELLIPSIS, PDF_TRUE, PDF_FALSE, PDF_STRING
}
# Precedence rules
precedence = (
('nonassoc', EQ, NE, LE, GE, LT, GT), # Non-associative operators
('left', PLUS, MINUS),
('left', TIMES, DIVIDE, MOD),
('left', LOGICAL_AND, LOGICAL_OR)
)
# String containing ignored characters between tokens (just SPACE)
# Because we are reading from stdin and/or text files, also ignore EOLs
ignore = ' '
# Regular expression rules for tokens
FUNC_NAME = r'fn\:[A-Z][a-zA-Z0-9]+\('
PDF_TRUE = r'(true)|(TRUE)'
PDF_FALSE = r'(false)|(FALSE)'
PDF_STRING = r'\'[^\']+\''
MOD = r'mod'
ELLIPSIS = r'\.\.\.'
KEY_VALUE = r'@(\*|[0-9]+|[0-9]+\*|[a-zA-Z0-9_\.\-]+)'
# Key name is both a PDF name or a TSV filename
# Key name of just '*' is ambiguous TIMES (multiply) operator.
# Key name which is numeric array index (0-9*) is ambiguous with integers.
# Array indices are integers, or integer + ASTERISK (wildcard) - need to use SPACEs to disambiguate
KEY_PATH = r'(parent::)?(([a-zA-Z]|[a-zA-Z][0-9]*|[0-9]*\*|[0-9]*[a-zA-Z])[a-zA-Z0-9_\.\-]*::)+'
KEY_NAME = r'([_a-zA-Z]|[_a-zA-Z][0-9]*|[0-9]*\*|[0-9]*[_a-zA-Z])[a-zA-Z0-9_:\.\-]*'
PDF_PATH = r'::'
ARRAY_START = r'\['
ARRAY_END = r'\]'
EQ = r'=='
NE = r'!='
GE = r'>='
LE = r'<='
LOGICAL_AND = r'\&\&'
LOGICAL_OR = r'\|\|'
GT = r'>'
LT = r'<'
REAL = r'\-?\d+\.\d+'
INTEGER = r'\-?\d+'
PLUS = r'\+'
MINUS = r'-'
TIMES = r'\*'
DIVIDE = r'/'
LPAREN = r'\('
RPAREN = r'\)'
COMMA = r'\,'
@_(r'\-?\d+\.\d+')
def REAL(self, t):
t.value = float(t.value)
return t
@_(r'\-?\d+')
def INTEGER(self, t):
t.value = int(t.value)
return t
@_(r'(false)|(FALSE)')
def PDF_FALSE(self, t):
t.value = False
return t
@_(r'(true)|(TRUE)')
def PDF_TRUE(self, t):
t.value = True
return t
# Terse version of sly.lex.Token.__str__/__repr__ dunder methods
def MyTokenStr(self) -> str:
return "TOKEN(type='%s', value='%s')" % (self.type, self.value)
# Function to JSON-ify sly.lex.Token objects
def sly_lex_Token_to_json(self) -> Dict[str, Union[str, sly.lex.Token]]:
if isinstance(self, sly.lex.Token):
return {'object': 'sly.lex.Token', 'type': self.type, 'value': self.value}
return {'error': '!not a sly.lex.Token!'}
def ToNestedAST(stk: List[sly.lex.Token], idx: int = 0) -> Tuple[int, List[sly.lex.Token]]:
"""
Assumes a fully valid parse tree with fully bracketed "( .. )" expressions.
Recursive.
@param[in] stk: stack of sly.lex.Tokens
@param[in] idx: token index into stk
@returns: index to next token in token stack, AST
"""
ast: List[sly.lex.Token] = []
i: int = idx
while (i < len(stk)):
if (stk[i].type == 'FUNC_NAME'):
ast.append(stk[i])
j, k = ToNestedAST(stk, i+1)
ast.append(k)
i = j
elif (stk[i].type == 'LPAREN'):
j, k = ToNestedAST(stk, i+1)
ast.append(k)
i = j
elif (stk[i].type == 'RPAREN'):
# go up recursion 1 level
return i+1, ast
elif (stk[i].type == 'COMMA'):
# skip COMMA
i = i + 1
else:
ast.append(stk[i])
i = i + 1
return i, ast
if __name__ == '__main__':
# "Monkey patch" sly.lex.Token __str__ and __repr__ dunder methods to make JSON nicer
# Don't do this if we want to read the JSON back in!
old_str = sly.lex.Token.__str__
sly.lex.Token.__str__ = MyTokenStr
old_repr = sly.lex.Token.__repr__
sly.lex.Token.__repr__ = MyTokenStr
lexer = ArlingtonFnLexer()
i: int
j: int
line: str
tok: sly.lex.Token
ast: List[sly.lex.Token]
for line in fileinput.input():
# Skip blank lines and those starting with '#' (comments)
line = line.strip()
if (line != '') and (line[0] != '#'):
stk: List[sly.lex.Token] = []
print(line, end='')
for tok in lexer.tokenize(line):
# print('type=%20r, value=%26r' % (tok.type, tok.value))
stk.append(tok)
print()
i, ast = ToNestedAST(stk)
#pprint.pprint(ast)
for j, a in enumerate(ast):
# De-tokenize only the top level PDF keynames
if (not isinstance(a, list)) and (a.type == 'KEY_NAME'):
ast[j] = a.value
pprint.pprint(ast)
print()