proposal to trace the includes files

mappyfile/cli.py

@@ -36,6 +36,7 @@
 import click
 import mappyfile
 from mappyfile.validator import Validator
+from sys import argv
 
 
 def get_mapfiles(mapfiles):
@@ -57,7 +58,11 @@ def configure_logging(verbosity):
     None
     """
     log_level = max(10, 30 - 10 * verbosity)
-    logging.basicConfig(stream=sys.stderr, level=log_level)
+    # use stdout for output librairie Warning (ex: ... >> output.txt)
+    if log_level < 40:
+        logging.basicConfig(stream=sys.stdout, level=log_level)
+    else:
+        logging.basicConfig(stream=sys.stderr, level=log_level)
 
 
 logger = logging.getLogger(__name__)
@@ -111,7 +116,7 @@ def format(ctx, input_mapfile, output_mapfile, indent, spacer, quote, newlinecha
     spacer = codecs.decode(spacer, 'unicode_escape')  # ensure \t is handled as a tab
     newlinechar = codecs.decode(newlinechar, 'unicode_escape')  # ensure \n is handled as a newline
 
-    d = mappyfile.open(input_mapfile, expand_includes=expand, include_comments=comments, include_position=True)
+    d, trace_o_incl = mappyfile.open(input_mapfile, expand_includes=expand, include_comments=comments, include_position=True)
     mappyfile.save(d, output_mapfile, indent=indent, spacer=spacer, quote=quote, newlinechar=newlinechar)
     sys.exit(0)
 
@@ -151,17 +156,17 @@ def validate(ctx, mapfiles, expand, version):
     for fn in all_mapfiles:
         fn = click.format_filename(fn)
         try:
-            d = mappyfile.open(fn, expand_includes=expand, include_position=True)
+            d, trace_o_incl = mappyfile.open(fn, output_trace=True, expand_includes=expand, include_position=True)
         except Exception as ex:
             logger.exception(ex)
             click.echo("{} failed to parse successfully".format(fn))
             continue
 
-        validation_messages = mappyfile.validate(d, version)
+        validation_messages = mappyfile.validate(d, trace_o_incl, version)
         if validation_messages:
             for v in validation_messages:
                 v["fn"] = fn
-                msg = "{fn} (Line: {line} Column: {column}) {message} - {error}".format(**v)
+                msg = "{fn} (File: {file} Line: {line} Column: {column}) {message} - {error}".format(**v)
                 click.echo(msg)
                 errors += 1
         else:
@@ -171,6 +176,11 @@ def validate(ctx, mapfiles, expand, version):
     click.echo("{} file(s) validated ({} successfully)".format(len(all_mapfiles), validation_count))
     sys.exit(errors)
 
+if __name__ == '__main__':
+    if getattr(sys, 'frozen', False):
+        main(argv[1:])
+    else:
+        main()
 
 @main.command(short_help="Export a Mapfile Schema")
 @click.argument('output_file', nargs=1, type=click.Path())

mappyfile/parser.py

@@ -52,7 +52,9 @@ def __init__(self, expand_includes=True, include_comments=False, **kwargs):
 
     def load_grammar(self, grammar_file):
         gf = os.path.join(os.path.dirname(__file__), grammar_file)
-        return open(gf).read()
+        # fix to ensure Mapfiles are closed after reading
+        with open(gf) as f:
+            return f.read()
 
     def _create_lalr_parser(self):
         grammar_text = self.load_grammar("mapfile.lalr.g")
@@ -77,11 +79,15 @@ def _get_include_filename(self, line):
 
         return inc_file_path.strip("'").strip('"')
 
-    def load_includes(self, text, fn=None, _nested_includes=0):
+    def load_includes(self, text, fn=None, trace_o_incl=None, _nested_includes=0):
         # Per default use working directory of the process
         if fn is None:
             fn = os.getcwd() + os.sep
 
+        if not trace_o_incl and _nested_includes == 0:
+            trace_o_incl = []
+            trace_o_incl.append([fn])
+
         lines = text.split('\n')
         includes = {}
         for idx, l in enumerate(lines):
@@ -98,13 +104,15 @@ def load_includes(self, text, fn=None, _nested_includes=0):
                 except IOError as ex:
                     log.warning("Include file '%s' not found in '%s'", inc_file_path, fn)
                     raise ex
-                # recursively load any further includes
-                includes[idx] = self.load_includes(include_text, fn=fn, _nested_includes=_nested_includes+1)
+                trace_o_incl[0].append(inc_file_path)
+                includes[idx], trace_o_incl = self.load_includes(include_text, fn=fn, trace_o_incl=trace_o_incl, _nested_includes=_nested_includes+1)
+            else:
+                trace_o_incl.append(len(trace_o_incl[0])-1)
 
         for idx, txt in includes.items():
             lines.pop(idx)  # remove the original include
             lines.insert(idx, txt)
-        return '\n'.join(lines)
+        return '\n'.join(lines), trace_o_incl
 
     def assign_comments(self, tree, comments):
         """
@@ -221,18 +229,34 @@ def parse(self, text, fn=None):
             text = unicode(text, 'utf-8')
 
         if self.expand_includes:
-            text = self.load_includes(text, fn=fn)
+            trace_o_incl = []
+            trace_o_incl.append([fn])
+            text, trace_o_incl = self.load_includes(text, fn=fn)
+        else:
+            trace_o_incl=None
 
         try:
             self._comments[:] = []  # clear any comments from a previous parse
             tree = self.lalr.parse(text)
             if self.include_comments:
                 self.assign_comments(tree, self._comments)
-            return tree
+            return tree, trace_o_incl
         except (ParseError, UnexpectedInput) as ex:
-            if fn:
-                log.error("Parsing of {} unsuccessful".format(fn))
-            else:
-                log.error("Parsing of Mapfile unsuccessful")
+            error_message = "Parsing of Mapfile unsuccessful"
+
+            # check the case where the lark parser does not stop on the error.
+            # In this case ex.line is on the end of the mapfile.
+            #   ex on tag STYLE: OUTLINECOLOR 128 128 128x
+            if trace_o_incl and ex.line < len(trace_o_incl) - 3:
+                #  position of error in file origin
+                trace = trace_o_incl[ex.line]
+                originFile = trace_o_incl[0][trace]
+                lineOrigin = trace_o_incl[1:ex.line].count(trace) + 1
+                error_message = "Parsing of {file} unsuccessful (Line: {line} Column: {column})".format(file = originFile, line = lineOrigin, column = ex.column)
+            elif fn:
+                error_message = "Parsing of {} unsuccessful".format(fn)
+
+            log.error(error_message)
             log.info(ex)
+
             raise

mappyfile/transformer.py

@@ -56,10 +56,11 @@
 
 class MapfileTransformer(Transformer, object):
 
-    def __init__(self, include_position=False, include_comments=False):
+    def __init__(self, include_position=False, include_comments=False, trace_o_incl=None):
         self.quoter = Quoter()
         self.include_position = include_position
         self.include_comments = include_comments
+        self.trace_o_incl = trace_o_incl
 
     def key_name(self, token):
         return token.value.lower()
@@ -360,8 +361,15 @@ def process_value_pairs(self, tokens, type_):
             v = self.clean_string(t[1].value)
 
             if k in d.keys():
-                log.warning("A duplicate key ({}) was found in {}. Only the last value ({}) will be used. ".format(
-                            k, type_, v))
+                if self.trace_o_incl:
+                    trace = self.trace_o_incl[key.line]
+                    originFile = self.trace_o_incl[0][trace]
+                    lineOrigin = self.trace_o_incl[1:key.line].count(trace) + 1
+                    log.warning("A duplicate key ({}) was found in {}. Only the last value ({}) will be used. File: {}, parent key {}, approximate parent key line {}".format(
+                                k, type_, v, originFile, key_name.upper(), lineOrigin))
+                else:
+                    log.warning("A duplicate key ({}) was found in {}. Only the last value ({}) will be used. Line of parent key of parent in file {} ".format(
+                        k, type_, v, key.line))
 
             d[k] = v
 
@@ -705,19 +713,21 @@ def _save_composite_comments(self, tree):
 
 class MapfileToDict(object):
 
-    def __init__(self, include_position=False, include_comments=False,
+    def __init__(self, include_position=False, include_comments=False, trace_o_incl=None,
                  transformerClass=MapfileTransformer, **kwargs):
 
         self.include_position = include_position
         self.include_comments = include_comments
+        self.trace_o_incl = trace_o_incl
         self.transformerClass = transformerClass
         self.kwargs = kwargs
 
     def transform(self, tree):
         tree = Canonize().transform(tree)
 
         self.mapfile_transformer = self.transformerClass(include_position=self.include_position,
-                                                         include_comments=self.include_comments, **self.kwargs)
+                                                         include_comments=self.include_comments,
+                                                         trace_o_incl=self.trace_o_incl , **self.kwargs)
 
         if self.include_comments:
             comments_transformer = CommentsTransformer(self.mapfile_transformer)

mappyfile/utils.py

@@ -60,7 +60,7 @@ def new_func(*args, **kwargs):
     return new_func
 
 
-def open(fn, expand_includes=True, include_comments=False, include_position=False, **kwargs):
+def open(fn, output_o_trace=False, expand_includes=True, include_comments=False, include_position=False, **kwargs):
     """
     Load a Mapfile from the supplied filename into a Python dictionary.
 
@@ -69,6 +69,8 @@ def open(fn, expand_includes=True, include_comments=False, include_position=Fals
 
     fn: string
         The path to the Mapfile, or partial Mapfile
+    output_o_trace: boolean
+        To include a trace of origin include files in the output: tuple (dict, list)
     expand_includes: boolean
         Load any ``INCLUDE`` files in the MapFile
     include_comments: boolean
@@ -82,6 +84,9 @@ def open(fn, expand_includes=True, include_comments=False, include_position=Fals
     dict
         A Python dictionary representing the Mapfile in the mappyfile format
 
+    trace_o_incl*
+        A trace of the origin of lines for include files, use to find the original location of an error
+
     Example
     -------
 
@@ -97,11 +102,17 @@ def open(fn, expand_includes=True, include_comments=False, include_position=Fals
     """
     p = Parser(expand_includes=expand_includes,
                include_comments=include_comments, **kwargs)
-    ast = p.parse_file(fn)
+    ast, trace_o_incl = p.parse_file(fn)
     m = MapfileToDict(include_position=include_position,
-                      include_comments=include_comments, **kwargs)
+                      include_comments=include_comments,
+                      trace_o_incl=trace_o_incl,
+                       **kwargs)
+
     d = m.transform(ast)
-    return d
+    if output_o_trace == True:
+        return d, trace_o_incl
+    else:
+        return d
 
 
 def load(fp, expand_includes=True, include_position=False, include_comments=False, **kwargs):
@@ -590,7 +601,7 @@ def update(d1, d2):
     return d1
 
 
-def validate(d, version=None):
+def validate(d, trace_o_incl=None, version=None):
     """
     Validate a mappyfile dictionary by using the Mapfile schema.
     An optional version number can be used to specify a specific
@@ -601,7 +612,9 @@ def validate(d, version=None):
 
     d: dict
         A Python dictionary based on the the mappyfile schema
-   version: float
+    trace_o_incl: list
+        A trace of the origin of lines for include files, use to find the original location of an error
+    version: float
         The MapServer version number used to validate the Mapfile
 
     Returns
@@ -612,7 +625,7 @@ def validate(d, version=None):
 
     """
     v = Validator()
-    return v.validate(d, version=version)
+    return v.validate(d, trace_o_incl, version=version)
 
 
 def _save(output_file, string):

mappyfile/validator.py

@@ -178,7 +178,7 @@ def convert_lowercase(self, x):
 
         return x
 
-    def create_message(self, rootdict, path, error, add_comments):
+    def create_message(self, rootdict, trace_o_incl, path, error, add_comments):
         """
         Add a validation comment to the dictionary
         path is the path to the error object, it can be empty if the error is in the root object
@@ -190,19 +190,36 @@ def create_message(self, rootdict, path, error, add_comments):
         See https://github.com/Julian/jsonschema/issues/119
         """
 
+        keyPathParent = None
         if not path:
             # error applies to the root type
             d = rootdict
             key = d["__type__"]
-        elif isinstance(path[-1], int):
-            # the error is on an object in a list
-            d = utils.findkey(rootdict, *path)
-            key = d["__type__"]
         else:
-            key = path[-1]
-            d = utils.findkey(rootdict, *path[:-1])
-
-        error_message = "ERROR: Invalid value in {}".format(key.upper())
+            if isinstance(path[-1], int):
+                # the error is on an object in a list
+                d = utils.findkey(rootdict, *path)
+                key = d["__type__"]
+            else:
+                key = path[-1]
+                d = utils.findkey(rootdict, *path[:-1])
+
+            # get the site of the parent key
+            l = len(path)
+            n = int(l / 2)
+            keyPathParent = []
+            for i in range(1, n + 1):
+                c = i * 2
+                keySource = utils.findkey(rootdict, *path[:c])
+                if (keySource.get('name', None) != None):
+                    keyPathParent.append(str(keySource["__type__"] + " name: " + keySource["name"]))
+                elif (keySource.get('text', None) != None):
+                    keyPathParent.append(str(keySource["__type__"] + " text: " + keySource["text"]))
+                else:
+                    keyPathParent.append(str(keySource["__type__"]))
+
+        # add the site of the parent key
+        error_message = "ERROR: Invalid value in {}, parent path: {}".format(key.upper(), keyPathParent)
 
         # add a comment to the dict structure
 
@@ -225,33 +242,41 @@ def create_message(self, rootdict, path, error, add_comments):
             else:
                 pd = d["__position__"][key]
 
-            error_message["line"] = pd.get("line")
-            error_message["column"] = pd.get("column")
+            if trace_o_incl:
+                trace = trace_o_incl[pd.get("line")]
+                originFile = trace_o_incl[0][trace]
+                lineOrigin = trace_o_incl[1:pd.get("line")].count(trace) + 1
+                error_message["line"] = lineOrigin
+                error_message["column"] = pd.get("column")
+                error_message["file"] = originFile
+            else:
+                error_message["line"] = pd.get("line")
+                error_message["column"] = pd.get("column")
 
         return error_message
 
-    def get_error_messages(self, d, errors, add_comments):
+    def get_error_messages(self, d, trace_o_incl, errors, add_comments):
 
         error_messages = []
 
         for error in errors:
             pth = error.absolute_path
             pth = list(pth)  # convert deque to list
-            em = self.create_message(d, pth, error, add_comments)
+            em = self.create_message(d, trace_o_incl, pth, error, add_comments)
             error_messages.append(em)
 
         return error_messages
 
-    def _validate(self, d, validator, add_comments, schema_name):
+    def _validate(self, d, trace_o_incl, validator, add_comments, schema_name):
         lowercase_dict = self.convert_lowercase(d)
         jsn = json.loads(json.dumps(lowercase_dict), object_pairs_hook=OrderedDict)
 
         errors = list(validator.iter_errors(jsn))
-        error_messages = self.get_error_messages(d, errors, add_comments)
+        error_messages = self.get_error_messages(d, trace_o_incl, errors, add_comments)
 
         return error_messages
 
-    def validate(self, value, add_comments=False, schema_name="map", version=None):
+    def validate(self, value, trace_o_incl=None, add_comments=False, schema_name="map", version=None):
         """
         verbose - also return the jsonschema error details
         """
@@ -265,9 +290,9 @@ def validate(self, value, add_comments=False, schema_name="map", version=None):
 
         if isinstance(value, list):
             for d in value:
-                error_messages += self._validate(d, validator, add_comments, schema_name)
+                error_messages += self._validate(d, trace_o_incl, validator, add_comments, schema_name)
         else:
-            error_messages = self._validate(value, validator, add_comments, schema_name)
+            error_messages = self._validate(value, trace_o_incl, validator, add_comments, schema_name)
 
         return error_messages