From ed1dbdc9e1b64847cc02de1539c2af99e41b541d Mon Sep 17 00:00:00 2001 From: hbernard Date: Sun, 17 Oct 2021 21:53:31 -0400 Subject: [PATCH 1/4] proposal to trace the includes files --- mappyfile/cli.py | 20 +++++++++---- mappyfile/parser.py | 46 ++++++++++++++++++++++-------- mappyfile/transformer.py | 20 +++++++++---- mappyfile/utils.py | 29 ++++++++++++++----- mappyfile/validator.py | 61 ++++++++++++++++++++++++++++------------ 5 files changed, 130 insertions(+), 46 deletions(-) diff --git a/mappyfile/cli.py b/mappyfile/cli.py index b3864abc..6cf95561 100644 --- a/mappyfile/cli.py +++ b/mappyfile/cli.py @@ -36,6 +36,7 @@ import click import mappyfile from mappyfile.validator import Validator +from sys import argv def get_mapfiles(mapfiles): @@ -57,7 +58,11 @@ def configure_logging(verbosity): None """ log_level = max(10, 30 - 10 * verbosity) - logging.basicConfig(stream=sys.stderr, level=log_level) + # use stdout for output librairie Warning (ex: ... >> output.txt) + if log_level < 40: + logging.basicConfig(stream=sys.stdout, level=log_level) + else: + logging.basicConfig(stream=sys.stderr, level=log_level) logger = logging.getLogger(__name__) @@ -111,7 +116,7 @@ def format(ctx, input_mapfile, output_mapfile, indent, spacer, quote, newlinecha spacer = codecs.decode(spacer, 'unicode_escape') # ensure \t is handled as a tab newlinechar = codecs.decode(newlinechar, 'unicode_escape') # ensure \n is handled as a newline - d = mappyfile.open(input_mapfile, expand_includes=expand, include_comments=comments, include_position=True) + d, trace_o_incl = mappyfile.open(input_mapfile, expand_includes=expand, include_comments=comments, include_position=True) mappyfile.save(d, output_mapfile, indent=indent, spacer=spacer, quote=quote, newlinechar=newlinechar) sys.exit(0) @@ -151,17 +156,17 @@ def validate(ctx, mapfiles, expand, version): for fn in all_mapfiles: fn = click.format_filename(fn) try: - d = mappyfile.open(fn, expand_includes=expand, include_position=True) + d, trace_o_incl = mappyfile.open(fn, expand_includes=expand, include_position=True) except Exception as ex: logger.exception(ex) click.echo("{} failed to parse successfully".format(fn)) continue - validation_messages = mappyfile.validate(d, version) + validation_messages = mappyfile.validate(d, trace_o_incl, version) if validation_messages: for v in validation_messages: v["fn"] = fn - msg = "{fn} (Line: {line} Column: {column}) {message} - {error}".format(**v) + msg = "{fn} (File: {file} Line: {line} Column: {column}) {message} - {error}".format(**v) click.echo(msg) errors += 1 else: @@ -171,6 +176,11 @@ def validate(ctx, mapfiles, expand, version): click.echo("{} file(s) validated ({} successfully)".format(len(all_mapfiles), validation_count)) sys.exit(errors) +if __name__ == '__main__': + if getattr(sys, 'frozen', False): + main(argv[1:]) + else: + main() @main.command(short_help="Export a Mapfile Schema") @click.argument('output_file', nargs=1, type=click.Path()) diff --git a/mappyfile/parser.py b/mappyfile/parser.py index ded53589..82fdc640 100644 --- a/mappyfile/parser.py +++ b/mappyfile/parser.py @@ -52,7 +52,9 @@ def __init__(self, expand_includes=True, include_comments=False, **kwargs): def load_grammar(self, grammar_file): gf = os.path.join(os.path.dirname(__file__), grammar_file) - return open(gf).read() + # fix to ensure Mapfiles are closed after reading + with open(gf) as f: + return f.read() def _create_lalr_parser(self): grammar_text = self.load_grammar("mapfile.lalr.g") @@ -77,11 +79,15 @@ def _get_include_filename(self, line): return inc_file_path.strip("'").strip('"') - def load_includes(self, text, fn=None, _nested_includes=0): + def load_includes(self, text, fn=None, trace_o_incl=None, _nested_includes=0): # Per default use working directory of the process if fn is None: fn = os.getcwd() + os.sep + if not trace_o_incl and _nested_includes == 0: + trace_o_incl = [] + trace_o_incl.append([fn]) + lines = text.split('\n') includes = {} for idx, l in enumerate(lines): @@ -98,13 +104,15 @@ def load_includes(self, text, fn=None, _nested_includes=0): except IOError as ex: log.warning("Include file '%s' not found in '%s'", inc_file_path, fn) raise ex - # recursively load any further includes - includes[idx] = self.load_includes(include_text, fn=fn, _nested_includes=_nested_includes+1) + trace_o_incl[0].append(inc_file_path) + includes[idx], trace_o_incl = self.load_includes(include_text, fn=fn, trace_o_incl=trace_o_incl, _nested_includes=_nested_includes+1) + else: + trace_o_incl.append(len(trace_o_incl[0])-1) for idx, txt in includes.items(): lines.pop(idx) # remove the original include lines.insert(idx, txt) - return '\n'.join(lines) + return '\n'.join(lines), trace_o_incl def assign_comments(self, tree, comments): """ @@ -221,18 +229,34 @@ def parse(self, text, fn=None): text = unicode(text, 'utf-8') if self.expand_includes: - text = self.load_includes(text, fn=fn) + trace_o_incl = [] + trace_o_incl.append([fn]) + text, trace_o_incl = self.load_includes(text, fn=fn) + else: + trace_o_incl=None try: self._comments[:] = [] # clear any comments from a previous parse tree = self.lalr.parse(text) if self.include_comments: self.assign_comments(tree, self._comments) - return tree + return tree, trace_o_incl except (ParseError, UnexpectedInput) as ex: - if fn: - log.error("Parsing of {} unsuccessful".format(fn)) - else: - log.error("Parsing of Mapfile unsuccessful") + error_message = "Parsing of Mapfile unsuccessful" + + # check the case where the lark parser does not stop on the error. + # In this case ex.line is on the end of the mapfile. + # ex on tag STYLE: OUTLINECOLOR 128 128 128x + if trace_o_incl and ex.line < len(trace_o_incl) - 3: + # position of error in file origin + trace = trace_o_incl[ex.line] + originFile = trace_o_incl[0][trace] + lineOrigin = trace_o_incl[1:ex.line].count(trace) + 1 + error_message = "Parsing of {file} unsuccessful (Line: {line} Column: {column})".format(file = originFile, line = lineOrigin, column = ex.column) + elif fn: + error_message = "Parsing of {} unsuccessful".format(fn) + + log.error(error_message) log.info(ex) + raise diff --git a/mappyfile/transformer.py b/mappyfile/transformer.py index 9af2de1b..50b08fef 100644 --- a/mappyfile/transformer.py +++ b/mappyfile/transformer.py @@ -56,10 +56,11 @@ class MapfileTransformer(Transformer, object): - def __init__(self, include_position=False, include_comments=False): + def __init__(self, include_position=False, include_comments=False, trace_o_incl=None): self.quoter = Quoter() self.include_position = include_position self.include_comments = include_comments + self.trace_o_incl = trace_o_incl def key_name(self, token): return token.value.lower() @@ -360,8 +361,15 @@ def process_value_pairs(self, tokens, type_): v = self.clean_string(t[1].value) if k in d.keys(): - log.warning("A duplicate key ({}) was found in {}. Only the last value ({}) will be used. ".format( - k, type_, v)) + if self.trace_o_incl: + trace = self.trace_o_incl[key.line] + originFile = self.trace_o_incl[0][trace] + lineOrigin = self.trace_o_incl[1:key.line].count(trace) + 1 + log.warning("A duplicate key ({}) was found in {}. Only the last value ({}) will be used. File: {}, parent key {}, approximate parent key line {}".format( + k, type_, v, originFile, key_name.upper(), lineOrigin)) + else: + log.warning("A duplicate key ({}) was found in {}. Only the last value ({}) will be used. Line of parent key of parent in file {} ".format( + k, type_, v, key.line)) d[k] = v @@ -705,11 +713,12 @@ def _save_composite_comments(self, tree): class MapfileToDict(object): - def __init__(self, include_position=False, include_comments=False, + def __init__(self, include_position=False, include_comments=False, trace_o_incl=None, transformerClass=MapfileTransformer, **kwargs): self.include_position = include_position self.include_comments = include_comments + self.trace_o_incl = trace_o_incl self.transformerClass = transformerClass self.kwargs = kwargs @@ -717,7 +726,8 @@ def transform(self, tree): tree = Canonize().transform(tree) self.mapfile_transformer = self.transformerClass(include_position=self.include_position, - include_comments=self.include_comments, **self.kwargs) + include_comments=self.include_comments, + trace_o_incl=self.trace_o_incl , **self.kwargs) if self.include_comments: comments_transformer = CommentsTransformer(self.mapfile_transformer) diff --git a/mappyfile/utils.py b/mappyfile/utils.py index b80dbda8..5dbbadc1 100644 --- a/mappyfile/utils.py +++ b/mappyfile/utils.py @@ -60,7 +60,7 @@ def new_func(*args, **kwargs): return new_func -def open(fn, expand_includes=True, include_comments=False, include_position=False, **kwargs): +def open(fn, output_o_trace=False, expand_includes=True, include_comments=False, include_position=False, **kwargs): """ Load a Mapfile from the supplied filename into a Python dictionary. @@ -69,6 +69,8 @@ def open(fn, expand_includes=True, include_comments=False, include_position=Fals fn: string The path to the Mapfile, or partial Mapfile + output_o_trace: boolean + To include a trace of origin include files in the output: tuple (dict, list) expand_includes: boolean Load any ``INCLUDE`` files in the MapFile include_comments: boolean @@ -82,6 +84,9 @@ def open(fn, expand_includes=True, include_comments=False, include_position=Fals dict A Python dictionary representing the Mapfile in the mappyfile format + trace_o_incl* + A trace of the origin of lines for include files, use to find the original location of an error + Example ------- @@ -97,11 +102,17 @@ def open(fn, expand_includes=True, include_comments=False, include_position=Fals """ p = Parser(expand_includes=expand_includes, include_comments=include_comments, **kwargs) - ast = p.parse_file(fn) + ast, trace_o_incl = p.parse_file(fn) m = MapfileToDict(include_position=include_position, - include_comments=include_comments, **kwargs) + include_comments=include_comments, + trace_o_incl=trace_o_incl, + **kwargs) + d = m.transform(ast) - return d + if output_o_trace == True: + return d, trace_o_incl + else: + return d def load(fp, expand_includes=True, include_position=False, include_comments=False, **kwargs): @@ -590,7 +601,7 @@ def update(d1, d2): return d1 -def validate(d, version=None): +def validate(d, *trace_o_incl, version=None): """ Validate a mappyfile dictionary by using the Mapfile schema. An optional version number can be used to specify a specific @@ -601,7 +612,9 @@ def validate(d, version=None): d: dict A Python dictionary based on the the mappyfile schema - version: float + trace_o_incl: list (optional) + A trace of the origin of lines for include files, use to find the original location of an error + version: float The MapServer version number used to validate the Mapfile Returns @@ -612,7 +625,9 @@ def validate(d, version=None): """ v = Validator() - return v.validate(d, version=version) + if not trace_o_incl: + trace_o_incl = None + return v.validate(d, trace_o_incl, version=version) def _save(output_file, string): diff --git a/mappyfile/validator.py b/mappyfile/validator.py index 8bb87aa4..9a18fb60 100644 --- a/mappyfile/validator.py +++ b/mappyfile/validator.py @@ -178,7 +178,7 @@ def convert_lowercase(self, x): return x - def create_message(self, rootdict, path, error, add_comments): + def create_message(self, rootdict, trace_o_incl, path, error, add_comments): """ Add a validation comment to the dictionary path is the path to the error object, it can be empty if the error is in the root object @@ -190,19 +190,36 @@ def create_message(self, rootdict, path, error, add_comments): See https://github.com/Julian/jsonschema/issues/119 """ + keyPathParent = None if not path: # error applies to the root type d = rootdict key = d["__type__"] - elif isinstance(path[-1], int): - # the error is on an object in a list - d = utils.findkey(rootdict, *path) - key = d["__type__"] else: - key = path[-1] - d = utils.findkey(rootdict, *path[:-1]) - - error_message = "ERROR: Invalid value in {}".format(key.upper()) + if isinstance(path[-1], int): + # the error is on an object in a list + d = utils.findkey(rootdict, *path) + key = d["__type__"] + else: + key = path[-1] + d = utils.findkey(rootdict, *path[:-1]) + + # get the site of the parent key + l = len(path) + n = int(l / 2) + keyPathParent = [] + for i in range(1, n + 1): + c = i * 2 + keySource = utils.findkey(rootdict, *path[:c]) + if (keySource.get('name', None) != None): + keyPathParent.append(str(keySource["__type__"] + " name: " + keySource["name"])) + elif (keySource.get('text', None) != None): + keyPathParent.append(str(keySource["__type__"] + " text: " + keySource["text"])) + else: + keyPathParent.append(str(keySource["__type__"])) + + # add the site of the parent key + error_message = "ERROR: Invalid value in {}, parent path: {}".format(key.upper(), keyPathParent) # add a comment to the dict structure @@ -225,33 +242,41 @@ def create_message(self, rootdict, path, error, add_comments): else: pd = d["__position__"][key] - error_message["line"] = pd.get("line") - error_message["column"] = pd.get("column") + if trace_o_incl: + trace = trace_o_incl[pd.get("line")] + originFile = trace_o_incl[0][trace] + lineOrigin = trace_o_incl[1:pd.get("line")].count(trace) + 1 + error_message["line"] = lineOrigin + error_message["column"] = pd.get("column") + error_message["file"] = originFile + else: + error_message["line"] = pd.get("line") + error_message["column"] = pd.get("column") return error_message - def get_error_messages(self, d, errors, add_comments): + def get_error_messages(self, d, trace_o_incl, errors, add_comments): error_messages = [] for error in errors: pth = error.absolute_path pth = list(pth) # convert deque to list - em = self.create_message(d, pth, error, add_comments) + em = self.create_message(d, trace_o_incl, pth, error, add_comments) error_messages.append(em) return error_messages - def _validate(self, d, validator, add_comments, schema_name): + def _validate(self, d, trace_o_incl, validator, add_comments, schema_name): lowercase_dict = self.convert_lowercase(d) jsn = json.loads(json.dumps(lowercase_dict), object_pairs_hook=OrderedDict) errors = list(validator.iter_errors(jsn)) - error_messages = self.get_error_messages(d, errors, add_comments) + error_messages = self.get_error_messages(d, trace_o_incl, errors, add_comments) return error_messages - def validate(self, value, add_comments=False, schema_name="map", version=None): + def validate(self, value, trace_o_incl=None, add_comments=False, schema_name="map", version=None): """ verbose - also return the jsonschema error details """ @@ -265,9 +290,9 @@ def validate(self, value, add_comments=False, schema_name="map", version=None): if isinstance(value, list): for d in value: - error_messages += self._validate(d, validator, add_comments, schema_name) + error_messages += self._validate(d, trace_o_incl, validator, add_comments, schema_name) else: - error_messages = self._validate(value, validator, add_comments, schema_name) + error_messages = self._validate(value, trace_o_incl, validator, add_comments, schema_name) return error_messages From 1d6f1cff30a0bb15328a92337736178e236b2f2d Mon Sep 17 00:00:00 2001 From: hbernard Date: Mon, 18 Oct 2021 08:26:53 -0400 Subject: [PATCH 2/4] command call correction --- mappyfile/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mappyfile/cli.py b/mappyfile/cli.py index 6cf95561..77931c64 100644 --- a/mappyfile/cli.py +++ b/mappyfile/cli.py @@ -156,7 +156,7 @@ def validate(ctx, mapfiles, expand, version): for fn in all_mapfiles: fn = click.format_filename(fn) try: - d, trace_o_incl = mappyfile.open(fn, expand_includes=expand, include_position=True) + d, trace_o_incl = mappyfile.open(fn, True, expand_includes=expand, include_position=True) except Exception as ex: logger.exception(ex) click.echo("{} failed to parse successfully".format(fn)) From 640df45b446e3ca0770a043239477ebcc1ba478b Mon Sep 17 00:00:00 2001 From: hbernard Date: Mon, 18 Oct 2021 09:05:33 -0400 Subject: [PATCH 3/4] optional argument correction --- mappyfile/utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mappyfile/utils.py b/mappyfile/utils.py index 5dbbadc1..f038bf4d 100644 --- a/mappyfile/utils.py +++ b/mappyfile/utils.py @@ -601,7 +601,7 @@ def update(d1, d2): return d1 -def validate(d, *trace_o_incl, version=None): +def validate(d, trace_o_incl=None, version=None): """ Validate a mappyfile dictionary by using the Mapfile schema. An optional version number can be used to specify a specific @@ -612,7 +612,7 @@ def validate(d, *trace_o_incl, version=None): d: dict A Python dictionary based on the the mappyfile schema - trace_o_incl: list (optional) + trace_o_incl: list A trace of the origin of lines for include files, use to find the original location of an error version: float The MapServer version number used to validate the Mapfile @@ -625,8 +625,6 @@ def validate(d, *trace_o_incl, version=None): """ v = Validator() - if not trace_o_incl: - trace_o_incl = None return v.validate(d, trace_o_incl, version=version) From 1e2c6f6bc1987ab470f18ee3d01ab8f9fbe736cc Mon Sep 17 00:00:00 2001 From: hbernard Date: Mon, 18 Oct 2021 10:57:44 -0400 Subject: [PATCH 4/4] command call adjust --- mappyfile/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mappyfile/cli.py b/mappyfile/cli.py index 77931c64..60de0b9c 100644 --- a/mappyfile/cli.py +++ b/mappyfile/cli.py @@ -156,7 +156,7 @@ def validate(ctx, mapfiles, expand, version): for fn in all_mapfiles: fn = click.format_filename(fn) try: - d, trace_o_incl = mappyfile.open(fn, True, expand_includes=expand, include_position=True) + d, trace_o_incl = mappyfile.open(fn, output_trace=True, expand_includes=expand, include_position=True) except Exception as ex: logger.exception(ex) click.echo("{} failed to parse successfully".format(fn))