fix percent escape not working when not at the beginning of the line

Fixed parsing issue where attempting to render a single percent sign ``%``
using an escaped percent ``%%`` would not function correctly if the escaped
percent were not the first character on a line.  Pull request courtesy Hai
Zhu.

Fixes: #323
Closes: #383
Pull-request: sqlalchemy/mako#383
Pull-request-sha: ab8e74756d1ddfae6584854b2765e3706ba87ad5

Change-Id: I3c494222443320e3681758e3892f44cf7748fe5f

doc/build/unreleased/323.rst

@@ -0,0 +1,8 @@
+.. change::
+    :tags: bug, lexer
+    :tickets: 323
+
+    Fixed parsing issue where attempting to render a single percent sign ``%``
+    using an escaped percent ``%%`` would not function correctly if the escaped
+    percent were not the first character on a line.  Pull request courtesy Hai
+    Zhu.

mako/lexer.py

@@ -357,9 +357,12 @@ def match_text(self):
             r"""
                 (.*?)         # anything, followed by:
                 (
-                 (?<=\n)(?=[ \t]*(?=%|\#\#)) # an eval or line-based
-                                             # comment preceded by a
-                                             # consumed newline and whitespace
+                 (?<=\n)(?=[ \t]*(?=%(?!%)|\#\#))  # an eval or line-based
+                                            # comment, preceded by a
+                                            # consumed newline and whitespace
+                 |
+                 (?<!%)(?=%%+) # consume the first percent sign
+                               # out of a group of percent signs
                  |
                  (?=\${)      # an expression
                  |

mako/testing/helpers.py

@@ -19,6 +19,10 @@ def result_lines(result):
     ]
 
 
+def result_raw_lines(result):
+    return [x for x in re.split(r"\r?\n", result) if x.strip() != ""]
+
+
 def make_path(
     filespec: Union[Path, str],
     make_absolute: bool = True,

test/test_lexer.py

@@ -201,15 +201,59 @@ def test_percent_escape(self):
                 {},
                 [
                     Text("""\n\n""", (1, 1)),
-                    Text("""% some whatever.\n\n""", (3, 2)),
-                    Text("   %% more some whatever\n", (5, 2)),
+                    Text("""% some whatever.\n\n    """, (3, 2)),
+                    Text("% more some whatever\n", (5, 6)),
                     ControlLine("if", "if foo:", False, (6, 1)),
                     ControlLine("if", "endif", True, (7, 1)),
                     Text("        ", (8, 1)),
                 ],
             ),
         )
 
+    def test_percent_escape2(self):
+        template = """%% do something
+%%% do something
+if <some condition>:
+    %%%% do something
+        """
+        node = Lexer(template).parse()
+        self._compare(
+            node,
+            TemplateNode(
+                {},
+                [
+                    Text("% do something\n", (1, 2)),
+                    Text(
+                        "%% do something\nif <some condition>:\n    ", (2, 2)
+                    ),
+                    Text("%%% do something\n        ", (4, 6)),
+                ],
+            ),
+        )
+
+    def test_percent_escape_with_control_block(self):
+        template = """
+% for i in [1, 2, 3]:
+    %% do something ${i}
+% endfor
+"""
+        node = Lexer(template).parse()
+        self._compare(
+            node,
+            TemplateNode(
+                {},
+                [
+                    Text("\n", (1, 1)),
+                    ControlLine("for", "for i in [1, 2, 3]:", False, (2, 1)),
+                    Text("    ", (3, 1)),
+                    Text("% do something ", (3, 6)),
+                    Expression("i", [], (3, 21)),
+                    Text("\n", (3, 25)),
+                    ControlLine("for", "endfor", True, (4, 1)),
+                ],
+            ),
+        )
+
     def test_old_multiline_comment(self):
         template = """#*"""
         node = Lexer(template).parse()

test/test_template.py

@@ -15,6 +15,7 @@
 from mako.testing.fixtures import TemplateTest
 from mako.testing.helpers import flatten_result
 from mako.testing.helpers import result_lines
+from mako.testing.helpers import result_raw_lines
 
 
 class ctx:
@@ -1667,3 +1668,34 @@ class FuturesTest(TemplateTest):
     def test_future_import(self):
         t = Template("${ x / y }", future_imports=["division"])
         assert result_lines(t.render(x=12, y=5)) == ["2.4"]
+
+
+class EscapeTest(TemplateTest):
+    def test_percent_escape(self):
+        t = Template(
+            """%% do something
+%%% do something
+if <some condition>:
+    %%%% do something
+"""
+        )
+        assert result_raw_lines(t.render()) == [
+            "% do something",
+            "%% do something",
+            "if <some condition>:",
+            "    %%% do something",
+        ]
+
+    def test_percent_escape2(self):
+        t = Template(
+            """
+% for i in [1, 2, 3]:
+    %% do something ${i}
+% endfor
+"""
+        )
+        assert result_raw_lines(t.render()) == [
+            "    % do something 1",
+            "    % do something 2",
+            "    % do something 3",
+        ]