From 2031330db34b00c8cb4b95bb964e0b0ba582e5cd Mon Sep 17 00:00:00 2001
From: Hai Zhu <35182391+cocolato@users.noreply.github.com>
Date: Fri, 19 Jan 2024 11:11:37 -0500
Subject: [PATCH] fix percent escape not working when not at the beginning of
 the line

Fixed parsing issue where attempting to render a single percent sign ``%``
using an escaped percent ``%%`` would not function correctly if the escaped
percent were not the first character on a line.  Pull request courtesy Hai
Zhu.

Fixes: #323
Closes: #383
Pull-request: https://github.com/sqlalchemy/mako/pull/383
Pull-request-sha: ab8e74756d1ddfae6584854b2765e3706ba87ad5

Change-Id: I3c494222443320e3681758e3892f44cf7748fe5f
---
 doc/build/unreleased/323.rst |  8 ++++++
 mako/lexer.py                |  9 ++++---
 mako/testing/helpers.py      |  4 +++
 test/test_lexer.py           | 48 ++++++++++++++++++++++++++++++++++--
 test/test_template.py        | 32 ++++++++++++++++++++++++
 5 files changed, 96 insertions(+), 5 deletions(-)
 create mode 100644 doc/build/unreleased/323.rst

diff --git a/doc/build/unreleased/323.rst b/doc/build/unreleased/323.rst
new file mode 100644
index 00000000..8f23d8be
--- /dev/null
+++ b/doc/build/unreleased/323.rst
@@ -0,0 +1,8 @@
+.. change::
+    :tags: bug, lexer
+    :tickets: 323
+
+    Fixed parsing issue where attempting to render a single percent sign ``%``
+    using an escaped percent ``%%`` would not function correctly if the escaped
+    percent were not the first character on a line.  Pull request courtesy Hai
+    Zhu.
diff --git a/mako/lexer.py b/mako/lexer.py
index 34f17dc9..8ce3fd6e 100644
--- a/mako/lexer.py
+++ b/mako/lexer.py
@@ -357,9 +357,12 @@ def match_text(self):
             r"""
                 (.*?)         # anything, followed by:
                 (
-                 (?<=\n)(?=[ \t]*(?=%|\#\#)) # an eval or line-based
-                                             # comment preceded by a
-                                             # consumed newline and whitespace
+                 (?<=\n)(?=[ \t]*(?=%(?!%)|\#\#))  # an eval or line-based
+                                            # comment, preceded by a
+                                            # consumed newline and whitespace
+                 |
+                 (?<!%)(?=%%+) # consume the first percent sign
+                               # out of a group of percent signs
                  |
                  (?=\${)      # an expression
                  |
diff --git a/mako/testing/helpers.py b/mako/testing/helpers.py
index 77cca367..5ae9d38d 100644
--- a/mako/testing/helpers.py
+++ b/mako/testing/helpers.py
@@ -19,6 +19,10 @@ def result_lines(result):
     ]
 
 
+def result_raw_lines(result):
+    return [x for x in re.split(r"\r?\n", result) if x.strip() != ""]
+
+
 def make_path(
     filespec: Union[Path, str],
     make_absolute: bool = True,
diff --git a/test/test_lexer.py b/test/test_lexer.py
index f4983a3a..f168f80c 100644
--- a/test/test_lexer.py
+++ b/test/test_lexer.py
@@ -201,8 +201,8 @@ def test_percent_escape(self):
                 {},
                 [
                     Text("""\n\n""", (1, 1)),
-                    Text("""% some whatever.\n\n""", (3, 2)),
-                    Text("   %% more some whatever\n", (5, 2)),
+                    Text("""% some whatever.\n\n    """, (3, 2)),
+                    Text("% more some whatever\n", (5, 6)),
                     ControlLine("if", "if foo:", False, (6, 1)),
                     ControlLine("if", "endif", True, (7, 1)),
                     Text("        ", (8, 1)),
@@ -210,6 +210,50 @@ def test_percent_escape(self):
             ),
         )
 
+    def test_percent_escape2(self):
+        template = """%% do something
+%%% do something
+if <some condition>:
+    %%%% do something
+        """
+        node = Lexer(template).parse()
+        self._compare(
+            node,
+            TemplateNode(
+                {},
+                [
+                    Text("% do something\n", (1, 2)),
+                    Text(
+                        "%% do something\nif <some condition>:\n    ", (2, 2)
+                    ),
+                    Text("%%% do something\n        ", (4, 6)),
+                ],
+            ),
+        )
+
+    def test_percent_escape_with_control_block(self):
+        template = """
+% for i in [1, 2, 3]:
+    %% do something ${i}
+% endfor
+"""
+        node = Lexer(template).parse()
+        self._compare(
+            node,
+            TemplateNode(
+                {},
+                [
+                    Text("\n", (1, 1)),
+                    ControlLine("for", "for i in [1, 2, 3]:", False, (2, 1)),
+                    Text("    ", (3, 1)),
+                    Text("% do something ", (3, 6)),
+                    Expression("i", [], (3, 21)),
+                    Text("\n", (3, 25)),
+                    ControlLine("for", "endfor", True, (4, 1)),
+                ],
+            ),
+        )
+
     def test_old_multiline_comment(self):
         template = """#*"""
         node = Lexer(template).parse()
diff --git a/test/test_template.py b/test/test_template.py
index 62fd21d6..79017f7b 100644
--- a/test/test_template.py
+++ b/test/test_template.py
@@ -15,6 +15,7 @@
 from mako.testing.fixtures import TemplateTest
 from mako.testing.helpers import flatten_result
 from mako.testing.helpers import result_lines
+from mako.testing.helpers import result_raw_lines
 
 
 class ctx:
@@ -1667,3 +1668,34 @@ class FuturesTest(TemplateTest):
     def test_future_import(self):
         t = Template("${ x / y }", future_imports=["division"])
         assert result_lines(t.render(x=12, y=5)) == ["2.4"]
+
+
+class EscapeTest(TemplateTest):
+    def test_percent_escape(self):
+        t = Template(
+            """%% do something
+%%% do something
+if <some condition>:
+    %%%% do something
+"""
+        )
+        assert result_raw_lines(t.render()) == [
+            "% do something",
+            "%% do something",
+            "if <some condition>:",
+            "    %%% do something",
+        ]
+
+    def test_percent_escape2(self):
+        t = Template(
+            """
+% for i in [1, 2, 3]:
+    %% do something ${i}
+% endfor
+"""
+        )
+        assert result_raw_lines(t.render()) == [
+            "    % do something 1",
+            "    % do something 2",
+            "    % do something 3",
+        ]