fix percent escape not working when not at the beginning of the line

Fixed parsing issue where attempting to render a single percent sign %
using an escaped percent %% would not function correctly if the escaped
percent were not the first character on a line.  Note that this is a revised
version of a similar change made in Mako 1.3.1 which caused unexpected
parsing regressions, resulting in the release being yanked.
Pull request courtesy Hai Zhu.

Fixes: #323
Closes: #383
Pull-request: sqlalchemy/mako#383
Pull-request-sha: db9309737277d46976413d90c13811a9c61c46df

Change-Id: Ia13b652ccdb3cc51bb8a28aed329b4677d49e3ba

doc/build/changelog.rst

@@ -23,6 +23,9 @@ Changelog
         percent were not the first character on a line.  Pull request courtesy Hai
         Zhu.
 
+        .. note::  Mako 1.3.1 was yanked from pypi and this change was reverted,
+           replaced with a modified version for Mako 1.3.2.
+
 .. changelog::
     :version: 1.3.0
     :released: Wed Nov 8 2023

doc/build/unreleased/323.rst

@@ -0,0 +1,10 @@
+.. change::
+    :tags: bug, lexer
+    :tickets: 323
+
+    Fixed parsing issue where attempting to render a single percent sign %
+    using an escaped percent %% would not function correctly if the escaped
+    percent were not the first character on a line.  Note that this is a revised
+    version of a similar change made in Mako 1.3.1 which caused unexpected
+    parsing regressions, resulting in the release being yanked.
+    Pull request courtesy Hai Zhu.

mako/lexer.py

@@ -247,6 +247,8 @@ def parse(self):
                 continue
             if self.match_python_block():
                 continue
+            if self.match_percent():
+                continue
             if self.match_text():
                 continue
 
@@ -352,14 +354,24 @@ def match_end(self):
         else:
             return True
 
+    def match_percent(self):
+        match = self.match(r"(?<=^)(\s*)%%(%*)", re.M)
+        if match:
+            self.append_node(
+                parsetree.Text, match.group(1) + "%" + match.group(2)
+            )
+            return True
+        else:
+            return False
+
     def match_text(self):
         match = self.match(
             r"""
                 (.*?)         # anything, followed by:
                 (
-                 (?<=\n)(?=[ \t]*(?=%|\#\#)) # an eval or line-based
-                                             # comment preceded by a
-                                             # consumed newline and whitespace
+                 (?<=\n)(?=[ \t]*(?=%|\#\#))  # an eval or line-based
+                                            # comment, preceded by a
+                                            # consumed newline and whitespace
                  |
                  (?=\${)      # an expression
                  |

mako/testing/helpers.py

@@ -19,6 +19,10 @@ def result_lines(result):
     ]
 
 
+def result_raw_lines(result):
+    return [x for x in re.split(r"\r?\n", result) if x.strip() != ""]
+
+
 def make_path(
     filespec: Union[Path, str],
     make_absolute: bool = True,

test/test_lexer.py

@@ -200,16 +200,103 @@ def test_percent_escape(self):
             TemplateNode(
                 {},
                 [
-                    Text("""\n\n""", (1, 1)),
-                    Text("""% some whatever.\n\n""", (3, 2)),
-                    Text("   %% more some whatever\n", (5, 2)),
+                    Text("\n\n%", (1, 1)),
+                    Text(" some whatever.\n\n", (3, 3)),
+                    Text("    %", (5, 1)),
+                    Text(" more some whatever\n", (5, 7)),
                     ControlLine("if", "if foo:", False, (6, 1)),
                     ControlLine("if", "endif", True, (7, 1)),
                     Text("        ", (8, 1)),
                 ],
             ),
         )
 
+    def test_percent_escape2(self):
+        template = """%% do something
+%%% do something
+if <some condition>:
+    %%%% do something
+        """
+        node = Lexer(template).parse()
+        self._compare(
+            node,
+            TemplateNode(
+                {},
+                [
+                    Text("%", (1, 1)),
+                    Text(" do something\n", (1, 3)),
+                    Text("%%", (2, 1)),
+                    Text(" do something\nif <some condition>:\n", (2, 4)),
+                    Text("    %%%", (4, 1)),
+                    Text(" do something\n        ", (4, 9)),
+                ],
+            ),
+        )
+
+    def test_percent_escape_with_control_block(self):
+        template = """
+% for i in [1, 2, 3]:
+    %% do something ${i}
+% endfor
+"""
+        node = Lexer(template).parse()
+        self._compare(
+            node,
+            TemplateNode(
+                {},
+                [
+                    Text("\n", (1, 1)),
+                    ControlLine("for", "for i in [1, 2, 3]:", False, (2, 1)),
+                    Text("    %", (3, 1)),
+                    Text(" do something ", (3, 7)),
+                    Expression("i", [], (3, 21)),
+                    Text("\n", (3, 25)),
+                    ControlLine("for", "endfor", True, (4, 1)),
+                ],
+            ),
+        )
+
+    def test_inline_percent(self):
+        template = """
+%% foo
+bar %% baz
+"""
+        node = Lexer(template).parse()
+        self._compare(
+            node,
+            TemplateNode(
+                {},
+                [Text("\n%", (1, 1)), Text(" foo\nbar %% baz\n", (2, 3))],
+            ),
+        )
+
+        def test_inline_percent_with_control_block(self):
+            template = """
+% for i in [1, 2, 3]:
+%% foo
+bar %% baz
+% endfor
+"""
+            node = Lexer(template).parse()
+            self._compare(
+                node,
+                TemplateNode(
+                    {},
+                    [
+                        Text("\n", (1, 1)),
+                        ControlLine(
+                            "for", "for i in [1, 2, 3]:", False, (2, 1)
+                        ),
+                        Text("%", (3, 1)),
+                        Text(" foo\nbar ", (3, 3)),
+                        Text("%", (3, 3)),
+                        Text("%", (3, 3)),
+                        Text(" baz\n", (4, 7)),
+                        ControlLine("for", "endfor", True, (5, 1)),
+                    ],
+                ),
+            )
+
     def test_old_multiline_comment(self):
         template = """#*"""
         node = Lexer(template).parse()

test/test_template.py

@@ -15,6 +15,7 @@
 from mako.testing.fixtures import TemplateTest
 from mako.testing.helpers import flatten_result
 from mako.testing.helpers import result_lines
+from mako.testing.helpers import result_raw_lines
 
 
 class ctx:
@@ -1667,3 +1668,52 @@ class FuturesTest(TemplateTest):
     def test_future_import(self):
         t = Template("${ x / y }", future_imports=["division"])
         assert result_lines(t.render(x=12, y=5)) == ["2.4"]
+
+
+class EscapeTest(TemplateTest):
+    def test_percent_escape(self):
+        t = Template(
+            """%% do something
+%%% do something
+if <some condition>:
+    %%%% do something
+"""
+        )
+        assert result_raw_lines(t.render()) == [
+            "% do something",
+            "%% do something",
+            "if <some condition>:",
+            "    %%% do something",
+        ]
+
+    def test_percent_escape2(self):
+        t = Template(
+            """
+% for i in [1, 2, 3]:
+    %% do something ${i}
+% endfor
+"""
+        )
+        assert result_raw_lines(t.render()) == [
+            "    % do something 1",
+            "    % do something 2",
+            "    % do something 3",
+        ]
+
+    def test_inline_percent(self):
+        t = Template(
+            """
+% for i in [1, 2, 3]:
+%% foo
+bar %% baz
+% endfor
+"""
+        )
+        assert result_raw_lines(t.render()) == [
+            "% foo",
+            "bar %% baz",
+            "% foo",
+            "bar %% baz",
+            "% foo",
+            "bar %% baz",
+        ]