From 00e690ead040dcbd2ee1ef8a64cb62b925dd8a2a Mon Sep 17 00:00:00 2001 From: Carter Dodd Date: Fri, 23 Aug 2024 14:53:27 -0500 Subject: [PATCH 1/8] Initial attempt at fixing issue-17706 --- mypy/fastparse.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index abcce74c6064..1f7ee1d956f0 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -414,7 +414,36 @@ def visit(self, node: AST | None) -> Any: method = "visit_" + node.__class__.__name__ visitor = getattr(self, method) self.visitor_cache[typeobj] = visitor - return visitor(node) + + try: + + return visitor(node) + + except RecursionError as e: + # For very complex expressions it is possible to hit recursion limit + # before reaching a leaf node. + # E.G. x1+x2+x3+...+xn -> BinOp(left=BinOp(left=BinOp(left=... + try: + # But to prove that is the cause of this particular recursion error, + # try to unparse the node + ast3.unparse(node) + except RecursionError: + self.errors.report( + node.lineno, + node.col_offset, + "Expression too complex to parse", + blocker=True, + code=codes.MISC, + ) + + expr = TempNode(AnyType(TypeOfAny.from_error), no_rhs=True) + self.set_line(expr, node) + return expr + + else: + # re-raise original recursion error if it *can* be unparsed, + # maybe this is some other issue that shouldn't be silenced/misdirected + raise e def set_line(self, node: N, n: AstNode) -> N: node.line = n.lineno From 5484503a2bdb7e4dcae2e265e33744a623793c17 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 23 Aug 2024 20:06:52 +0000 Subject: [PATCH 2/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mypy/fastparse.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 1f7ee1d956f0..d8c78a6f7dd7 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -441,9 +441,9 @@ def visit(self, node: AST | None) -> Any: return expr else: - # re-raise original recursion error if it *can* be unparsed, - # maybe this is some other issue that shouldn't be silenced/misdirected - raise e + # re-raise original recursion error if it *can* be unparsed, + # maybe this is some other issue that shouldn't be silenced/misdirected + raise e def set_line(self, node: N, n: AstNode) -> N: node.line = n.lineno From cc875de497b98e377281d6da6940202b9d455a38 Mon Sep 17 00:00:00 2001 From: Carter Dodd Date: Sat, 24 Aug 2024 14:11:07 -0500 Subject: [PATCH 3/8] change unparse to ast.NodeVisitor --- mypy/fastparse.py | 58 +++++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index d8c78a6f7dd7..d317581a3b0b 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -240,6 +240,34 @@ def parse( strip_function_bodies=strip_function_bodies, path=fnam, ).visit(ast) + + + except RecursionError as e: + # For very complex expressions it is possible to hit recursion limit + # before reaching a leaf node. + # Should reject at top level instead at bottom, since bottom would already + # be at the threshold of the recursion limit, and may fail again later. + # E.G. x1+x2+x3+...+xn -> BinOp(left=BinOp(left=BinOp(left=... + try: + # But to prove that is the cause of this particular recursion error, + # try to walk the tree using builtin visitor + ast3.NodeVisitor().visit(ast) + except RecursionError: + errors.report( + -1, + -1, + "Source expression too complex to parse", + blocker=False, + code=codes.MISC, + ) + + tree = MypyFile([], [], False, {}) + + else: + # re-raise original recursion error if it *can* be unparsed, + # maybe this is some other issue that shouldn't be silenced/misdirected + raise e + except SyntaxError as e: # alias to please mypyc is_py38_or_earlier = sys.version_info < (3, 9) @@ -415,35 +443,7 @@ def visit(self, node: AST | None) -> Any: visitor = getattr(self, method) self.visitor_cache[typeobj] = visitor - try: - - return visitor(node) - - except RecursionError as e: - # For very complex expressions it is possible to hit recursion limit - # before reaching a leaf node. - # E.G. x1+x2+x3+...+xn -> BinOp(left=BinOp(left=BinOp(left=... - try: - # But to prove that is the cause of this particular recursion error, - # try to unparse the node - ast3.unparse(node) - except RecursionError: - self.errors.report( - node.lineno, - node.col_offset, - "Expression too complex to parse", - blocker=True, - code=codes.MISC, - ) - - expr = TempNode(AnyType(TypeOfAny.from_error), no_rhs=True) - self.set_line(expr, node) - return expr - - else: - # re-raise original recursion error if it *can* be unparsed, - # maybe this is some other issue that shouldn't be silenced/misdirected - raise e + return visitor(node) def set_line(self, node: N, n: AstNode) -> N: node.line = n.lineno From 23c233a535f6d9a023d64d9b65e6de208e933f8a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 24 Aug 2024 19:11:34 +0000 Subject: [PATCH 4/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mypy/fastparse.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index d317581a3b0b..5ad505f2b7e1 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -241,7 +241,6 @@ def parse( path=fnam, ).visit(ast) - except RecursionError as e: # For very complex expressions it is possible to hit recursion limit # before reaching a leaf node. @@ -254,11 +253,7 @@ def parse( ast3.NodeVisitor().visit(ast) except RecursionError: errors.report( - -1, - -1, - "Source expression too complex to parse", - blocker=False, - code=codes.MISC, + -1, -1, "Source expression too complex to parse", blocker=False, code=codes.MISC ) tree = MypyFile([], [], False, {}) From 56e95e4259f1599f1d15d570405cf300bd86bd94 Mon Sep 17 00:00:00 2001 From: Carter Dodd Date: Sun, 25 Aug 2024 09:16:27 -0500 Subject: [PATCH 5/8] Change recursion limit of dmypy to be same as mypy --- mypy/dmypy/client.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mypy/dmypy/client.py b/mypy/dmypy/client.py index 9f0751e93609..0399dfb9d617 100644 --- a/mypy/dmypy/client.py +++ b/mypy/dmypy/client.py @@ -267,6 +267,10 @@ class BadStatus(Exception): def main(argv: list[str]) -> None: """The code is top-down.""" check_python_version("dmypy") + + # set recursion limit consistent with mypy/main.py + sys.setrecursionlimit(2**14) + args = parser.parse_args(argv) if not args.action: parser.print_usage() From 4cf6183c777cf1c770203d125970ff0c784c6d5c Mon Sep 17 00:00:00 2001 From: Carter Dodd Date: Sun, 25 Aug 2024 09:34:59 -0500 Subject: [PATCH 6/8] Add recursion limit variable --- mypy/dmypy/client.py | 3 ++- mypy/main.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/mypy/dmypy/client.py b/mypy/dmypy/client.py index 0399dfb9d617..36c07218321f 100644 --- a/mypy/dmypy/client.py +++ b/mypy/dmypy/client.py @@ -16,6 +16,7 @@ import traceback from typing import Any, Callable, Mapping, NoReturn +from mypy.main import RECURSION_LIMIT from mypy.dmypy_os import alive, kill from mypy.dmypy_util import DEFAULT_STATUS_FILE, receive, send from mypy.ipc import IPCClient, IPCException @@ -269,7 +270,7 @@ def main(argv: list[str]) -> None: check_python_version("dmypy") # set recursion limit consistent with mypy/main.py - sys.setrecursionlimit(2**14) + sys.setrecursionlimit(RECURSION_LIMIT) args = parser.parse_args(argv) if not args.action: diff --git a/mypy/main.py b/mypy/main.py index f177bb1c2062..2764b8b9d044 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -29,7 +29,7 @@ orig_stat: Final = os.stat MEM_PROFILE: Final = False # If True, dump memory profile - +RECURSION_LIMIT: Final = 2**14 def stat_proxy(path: str) -> os.stat_result: try: @@ -63,7 +63,7 @@ def main( util.check_python_version("mypy") t0 = time.time() # To log stat() calls: os.stat = stat_proxy - sys.setrecursionlimit(2**14) + sys.setrecursionlimit(RECURSION_LIMIT) if args is None: args = sys.argv[1:] From 18562ef57a06e131c9d7d7edd9ed998dccd7f16a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 25 Aug 2024 14:36:40 +0000 Subject: [PATCH 7/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mypy/dmypy/client.py | 2 +- mypy/main.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/dmypy/client.py b/mypy/dmypy/client.py index 36c07218321f..b50fee83ec9d 100644 --- a/mypy/dmypy/client.py +++ b/mypy/dmypy/client.py @@ -16,10 +16,10 @@ import traceback from typing import Any, Callable, Mapping, NoReturn -from mypy.main import RECURSION_LIMIT from mypy.dmypy_os import alive, kill from mypy.dmypy_util import DEFAULT_STATUS_FILE, receive, send from mypy.ipc import IPCClient, IPCException +from mypy.main import RECURSION_LIMIT from mypy.util import check_python_version, get_terminal_width, should_force_color from mypy.version import __version__ diff --git a/mypy/main.py b/mypy/main.py index 2764b8b9d044..bf34edefe25d 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -31,6 +31,7 @@ MEM_PROFILE: Final = False # If True, dump memory profile RECURSION_LIMIT: Final = 2**14 + def stat_proxy(path: str) -> os.stat_result: try: st = orig_stat(path) From 3f18caa31a86124c186197ad39412050b88923d9 Mon Sep 17 00:00:00 2001 From: Carter Dodd Date: Sun, 25 Aug 2024 09:44:14 -0500 Subject: [PATCH 8/8] fixup black --- mypy/dmypy/client.py | 2 +- mypy/main.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/dmypy/client.py b/mypy/dmypy/client.py index 36c07218321f..b50fee83ec9d 100644 --- a/mypy/dmypy/client.py +++ b/mypy/dmypy/client.py @@ -16,10 +16,10 @@ import traceback from typing import Any, Callable, Mapping, NoReturn -from mypy.main import RECURSION_LIMIT from mypy.dmypy_os import alive, kill from mypy.dmypy_util import DEFAULT_STATUS_FILE, receive, send from mypy.ipc import IPCClient, IPCException +from mypy.main import RECURSION_LIMIT from mypy.util import check_python_version, get_terminal_width, should_force_color from mypy.version import __version__ diff --git a/mypy/main.py b/mypy/main.py index 2764b8b9d044..bf34edefe25d 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -31,6 +31,7 @@ MEM_PROFILE: Final = False # If True, dump memory profile RECURSION_LIMIT: Final = 2**14 + def stat_proxy(path: str) -> os.stat_result: try: st = orig_stat(path)