From bc39f1714e2e5f39eb90f05f74575a7c397f068b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 4 Aug 2024 12:50:05 +0100 Subject: [PATCH] Revert "Fix Literal strings containing pipe characters" (#17638) Reverts python/mypy#17148 --- mypy/fastparse.py | 11 ++- mypy/semanal.py | 31 ++++---- mypy/server/astmerge.py | 3 +- mypy/stubutil.py | 16 +--- mypy/type_visitor.py | 4 - mypy/typeanal.py | 21 ++++-- mypy/types.py | 75 +++++++++++-------- mypy/typetraverser.py | 3 +- mypyc/irbuild/classdef.py | 9 ++- test-data/unit/check-final.test | 2 - test-data/unit/check-literal.test | 4 - test-data/unit/check-namedtuple.test | 8 +- .../unit/check-parameter-specification.test | 23 +----- test-data/unit/check-typeguard.test | 11 --- test-data/unit/check-typeis.test | 11 --- 15 files changed, 90 insertions(+), 142 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 363fc8375259..ab7eef924bd1 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -331,7 +331,14 @@ def parse_type_string( """ try: _, node = parse_type_comment(f"({expr_string})", line=line, column=column, errors=None) - return RawExpressionType(expr_string, expr_fallback_name, line, column, node=node) + if isinstance(node, UnboundType) and node.original_str_expr is None: + node.original_str_expr = expr_string + node.original_str_fallback = expr_fallback_name + return node + elif isinstance(node, UnionType): + return node + else: + return RawExpressionType(expr_string, expr_fallback_name, line, column) except (SyntaxError, ValueError): # Note: the parser will raise a `ValueError` instead of a SyntaxError if # the string happens to contain things like \x00. @@ -1057,8 +1064,6 @@ def set_type_optional(self, type: Type | None, initializer: Expression | None) - return # Indicate that type should be wrapped in an Optional if arg is initialized to None. optional = isinstance(initializer, NameExpr) and initializer.name == "None" - if isinstance(type, RawExpressionType) and type.node is not None: - type = type.node if isinstance(type, UnboundType): type.optional = optional diff --git a/mypy/semanal.py b/mypy/semanal.py index f36149076fe6..782985e3fbab 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -3437,10 +3437,10 @@ def analyze_typeddict_assign(self, s: AssignmentStmt) -> bool: def analyze_lvalues(self, s: AssignmentStmt) -> None: # We cannot use s.type, because analyze_simple_literal_type() will set it. explicit = s.unanalyzed_type is not None - final_type = self.unwrap_final_type(s.unanalyzed_type) - if final_type is not None: + if self.is_final_type(s.unanalyzed_type): # We need to exclude bare Final. - if not final_type.args: + assert isinstance(s.unanalyzed_type, UnboundType) + if not s.unanalyzed_type.args: explicit = False if s.rvalue: @@ -3506,19 +3506,19 @@ def unwrap_final(self, s: AssignmentStmt) -> bool: Returns True if Final[...] was present. """ - final_type = self.unwrap_final_type(s.unanalyzed_type) - if final_type is None: + if not s.unanalyzed_type or not self.is_final_type(s.unanalyzed_type): return False - if len(final_type.args) > 1: - self.fail("Final[...] takes at most one type argument", final_type) + assert isinstance(s.unanalyzed_type, UnboundType) + if len(s.unanalyzed_type.args) > 1: + self.fail("Final[...] takes at most one type argument", s.unanalyzed_type) invalid_bare_final = False - if not final_type.args: + if not s.unanalyzed_type.args: s.type = None if isinstance(s.rvalue, TempNode) and s.rvalue.no_rhs: invalid_bare_final = True self.fail("Type in Final[...] can only be omitted if there is an initializer", s) else: - s.type = final_type.args[0] + s.type = s.unanalyzed_type.args[0] if s.type is not None and self.is_classvar(s.type): self.fail("Variable should not be annotated with both ClassVar and Final", s) @@ -4937,18 +4937,13 @@ def is_classvar(self, typ: Type) -> bool: return False return sym.node.fullname == "typing.ClassVar" - def unwrap_final_type(self, typ: Type | None) -> UnboundType | None: - if typ is None: - return None - typ = typ.resolve_string_annotation() + def is_final_type(self, typ: Type | None) -> bool: if not isinstance(typ, UnboundType): - return None + return False sym = self.lookup_qualified(typ.name, typ) if not sym or not sym.node: - return None - if sym.node.fullname in FINAL_TYPE_NAMES: - return typ - return None + return False + return sym.node.fullname in FINAL_TYPE_NAMES def fail_invalid_classvar(self, context: Context) -> None: self.fail(message_registry.CLASS_VAR_OUTSIDE_OF_CLASS, context) diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index e6648fbb4be7..174c2922c767 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -507,8 +507,7 @@ def visit_typeddict_type(self, typ: TypedDictType) -> None: typ.fallback.accept(self) def visit_raw_expression_type(self, t: RawExpressionType) -> None: - if t.node is not None: - t.node.accept(self) + pass def visit_literal_type(self, typ: LiteralType) -> None: typ.fallback.accept(self) diff --git a/mypy/stubutil.py b/mypy/stubutil.py index 2f2db0dbbe53..04b36e149957 100644 --- a/mypy/stubutil.py +++ b/mypy/stubutil.py @@ -17,16 +17,7 @@ from mypy.modulefinder import ModuleNotFoundReason from mypy.moduleinspect import InspectError, ModuleInspect from mypy.stubdoc import ArgSig, FunctionSig -from mypy.types import ( - AnyType, - NoneType, - RawExpressionType, - Type, - TypeList, - TypeStrVisitor, - UnboundType, - UnionType, -) +from mypy.types import AnyType, NoneType, Type, TypeList, TypeStrVisitor, UnboundType, UnionType # Modules that may fail when imported, or that may have side effects (fully qualified). NOT_IMPORTABLE_MODULES = () @@ -302,11 +293,12 @@ def args_str(self, args: Iterable[Type]) -> str: The main difference from list_str is the preservation of quotes for string arguments """ + types = ["builtins.bytes", "builtins.str"] res = [] for arg in args: arg_str = arg.accept(self) - if isinstance(arg, RawExpressionType): - res.append(repr(arg.literal_value)) + if isinstance(arg, UnboundType) and arg.original_str_fallback in types: + res.append(f"'{arg_str}'") else: res.append(arg_str) return ", ".join(res) diff --git a/mypy/type_visitor.py b/mypy/type_visitor.py index e685c49904bc..59e13d12485c 100644 --- a/mypy/type_visitor.py +++ b/mypy/type_visitor.py @@ -382,8 +382,6 @@ def visit_typeddict_type(self, t: TypedDictType) -> T: return self.query_types(t.items.values()) def visit_raw_expression_type(self, t: RawExpressionType) -> T: - if t.node is not None: - return t.node.accept(self) return self.strategy([]) def visit_literal_type(self, t: LiteralType) -> T: @@ -524,8 +522,6 @@ def visit_typeddict_type(self, t: TypedDictType) -> bool: return self.query_types(list(t.items.values())) def visit_raw_expression_type(self, t: RawExpressionType) -> bool: - if t.node is not None: - return t.node.accept(self) return self.default def visit_literal_type(self, t: LiteralType) -> bool: diff --git a/mypy/typeanal.py b/mypy/typeanal.py index f63aef30a09a..f88c3f91d1c6 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1112,7 +1112,6 @@ def visit_callable_type( return ret def anal_type_guard(self, t: Type) -> Type | None: - t = t.resolve_string_annotation() if isinstance(t, UnboundType): sym = self.lookup_qualified(t.name, t) if sym is not None and sym.node is not None: @@ -1131,7 +1130,6 @@ def anal_type_guard_arg(self, t: UnboundType, fullname: str) -> Type | None: return None def anal_type_is(self, t: Type) -> Type | None: - t = t.resolve_string_annotation() if isinstance(t, UnboundType): sym = self.lookup_qualified(t.name, t) if sym is not None and sym.node is not None: @@ -1149,7 +1147,6 @@ def anal_type_is_arg(self, t: UnboundType, fullname: str) -> Type | None: def anal_star_arg_type(self, t: Type, kind: ArgKind, nested: bool) -> Type: """Analyze signature argument type for *args and **kwargs argument.""" - t = t.resolve_string_annotation() if isinstance(t, UnboundType) and t.name and "." in t.name and not t.args: components = t.name.split(".") tvar_name = ".".join(components[:-1]) @@ -1275,8 +1272,6 @@ def visit_raw_expression_type(self, t: RawExpressionType) -> Type: # make signatures like "foo(x: 20) -> None" legal, we can change # this method so it generates and returns an actual LiteralType # instead. - if t.node is not None: - return t.node.accept(self) if self.report_invalid_types: if t.base_type_name in ("builtins.int", "builtins.bool"): @@ -1539,7 +1534,6 @@ def analyze_callable_args( invalid_unpacks: list[Type] = [] second_unpack_last = False for i, arg in enumerate(arglist.items): - arg = arg.resolve_string_annotation() if isinstance(arg, CallableArgument): args.append(arg.typ) names.append(arg.name) @@ -1620,6 +1614,18 @@ def analyze_literal_type(self, t: UnboundType) -> Type: return UnionType.make_union(output, line=t.line) def analyze_literal_param(self, idx: int, arg: Type, ctx: Context) -> list[Type] | None: + # This UnboundType was originally defined as a string. + if isinstance(arg, UnboundType) and arg.original_str_expr is not None: + assert arg.original_str_fallback is not None + return [ + LiteralType( + value=arg.original_str_expr, + fallback=self.named_type(arg.original_str_fallback), + line=arg.line, + column=arg.column, + ) + ] + # If arg is an UnboundType that was *not* originally defined as # a string, try expanding it in case it's a type alias or something. if isinstance(arg, UnboundType): @@ -2605,8 +2611,7 @@ def visit_typeddict_type(self, t: TypedDictType) -> None: self.process_types(list(t.items.values())) def visit_raw_expression_type(self, t: RawExpressionType) -> None: - if t.node is not None: - t.node.accept(self) + pass def visit_literal_type(self, t: LiteralType) -> None: pass diff --git a/mypy/types.py b/mypy/types.py index 52b3121f9fb3..72374c1555c9 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -271,9 +271,6 @@ def can_be_true_default(self) -> bool: def can_be_false_default(self) -> bool: return True - def resolve_string_annotation(self) -> Type: - return self - def accept(self, visitor: TypeVisitor[T]) -> T: raise RuntimeError("Not implemented", type(self)) @@ -906,7 +903,14 @@ def copy_modified( class UnboundType(ProperType): """Instance type that has not been bound during semantic analysis.""" - __slots__ = ("name", "args", "optional", "empty_tuple_index") + __slots__ = ( + "name", + "args", + "optional", + "empty_tuple_index", + "original_str_expr", + "original_str_fallback", + ) def __init__( self, @@ -916,6 +920,8 @@ def __init__( column: int = -1, optional: bool = False, empty_tuple_index: bool = False, + original_str_expr: str | None = None, + original_str_fallback: str | None = None, ) -> None: super().__init__(line, column) if not args: @@ -927,6 +933,21 @@ def __init__( self.optional = optional # Special case for X[()] self.empty_tuple_index = empty_tuple_index + # If this UnboundType was originally defined as a str or bytes, keep track of + # the original contents of that string-like thing. This way, if this UnboundExpr + # ever shows up inside of a LiteralType, we can determine whether that + # Literal[...] is valid or not. E.g. Literal[foo] is most likely invalid + # (unless 'foo' is an alias for another literal or something) and + # Literal["foo"] most likely is. + # + # We keep track of the entire string instead of just using a boolean flag + # so we can distinguish between things like Literal["foo"] vs + # Literal[" foo "]. + # + # We also keep track of what the original base fallback type was supposed to be + # so we don't have to try and recompute it later + self.original_str_expr = original_str_expr + self.original_str_fallback = original_str_fallback def copy_modified(self, args: Bogus[Sequence[Type] | None] = _dummy) -> UnboundType: if args is _dummy: @@ -938,19 +959,25 @@ def copy_modified(self, args: Bogus[Sequence[Type] | None] = _dummy) -> UnboundT column=self.column, optional=self.optional, empty_tuple_index=self.empty_tuple_index, + original_str_expr=self.original_str_expr, + original_str_fallback=self.original_str_fallback, ) def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_unbound_type(self) def __hash__(self) -> int: - return hash((self.name, self.optional, tuple(self.args))) + return hash((self.name, self.optional, tuple(self.args), self.original_str_expr)) def __eq__(self, other: object) -> bool: if not isinstance(other, UnboundType): return NotImplemented return ( - self.name == other.name and self.optional == other.optional and self.args == other.args + self.name == other.name + and self.optional == other.optional + and self.args == other.args + and self.original_str_expr == other.original_str_expr + and self.original_str_fallback == other.original_str_fallback ) def serialize(self) -> JsonDict: @@ -958,12 +985,19 @@ def serialize(self) -> JsonDict: ".class": "UnboundType", "name": self.name, "args": [a.serialize() for a in self.args], + "expr": self.original_str_expr, + "expr_fallback": self.original_str_fallback, } @classmethod def deserialize(cls, data: JsonDict) -> UnboundType: assert data[".class"] == "UnboundType" - return UnboundType(data["name"], [deserialize_type(a) for a in data["args"]]) + return UnboundType( + data["name"], + [deserialize_type(a) for a in data["args"]], + original_str_expr=data["expr"], + original_str_fallback=data["expr_fallback"], + ) class CallableArgument(ProperType): @@ -2646,7 +2680,7 @@ class RawExpressionType(ProperType): This synthetic type is only used at the beginning stages of semantic analysis and should be completely removing during the process for mapping UnboundTypes to - actual types: we turn it into its "node" argument, a LiteralType, or an AnyType. + actual types: we either turn it into a LiteralType or an AnyType. For example, suppose `Foo[1]` is initially represented as the following: @@ -2684,7 +2718,7 @@ class RawExpressionType(ProperType): ) """ - __slots__ = ("literal_value", "base_type_name", "note", "node") + __slots__ = ("literal_value", "base_type_name", "note") def __init__( self, @@ -2693,13 +2727,11 @@ def __init__( line: int = -1, column: int = -1, note: str | None = None, - node: Type | None = None, ) -> None: super().__init__(line, column) self.literal_value = literal_value self.base_type_name = base_type_name self.note = note - self.node = node def simple_name(self) -> str: return self.base_type_name.replace("builtins.", "") @@ -2709,21 +2741,6 @@ def accept(self, visitor: TypeVisitor[T]) -> T: ret: T = visitor.visit_raw_expression_type(self) return ret - def copy_modified(self, node: Type | None) -> RawExpressionType: - return RawExpressionType( - literal_value=self.literal_value, - base_type_name=self.base_type_name, - line=self.line, - column=self.column, - note=self.note, - node=node, - ) - - def resolve_string_annotation(self) -> Type: - if self.node is not None: - return self.node.resolve_string_annotation() - return self - def serialize(self) -> JsonDict: assert False, "Synthetic types don't serialize" @@ -2735,7 +2752,6 @@ def __eq__(self, other: object) -> bool: return ( self.base_type_name == other.base_type_name and self.literal_value == other.literal_value - and self.node == other.node ) else: return NotImplemented @@ -3411,8 +3427,6 @@ def item_str(name: str, typ: str) -> str: return f"TypedDict({prefix}{s})" def visit_raw_expression_type(self, t: RawExpressionType) -> str: - if t.node is not None: - return t.node.accept(self) return repr(t.literal_value) def visit_literal_type(self, t: LiteralType) -> str: @@ -3476,9 +3490,6 @@ def visit_ellipsis_type(self, t: EllipsisType) -> Type: return t def visit_raw_expression_type(self, t: RawExpressionType) -> Type: - if t.node is not None: - node = t.node.accept(self) - return t.copy_modified(node=node) return t def visit_type_list(self, t: TypeList) -> Type: diff --git a/mypy/typetraverser.py b/mypy/typetraverser.py index 4d740a802b55..a28bbf422b61 100644 --- a/mypy/typetraverser.py +++ b/mypy/typetraverser.py @@ -130,8 +130,7 @@ def visit_partial_type(self, t: PartialType) -> None: pass def visit_raw_expression_type(self, t: RawExpressionType) -> None: - if t.node is not None: - t.node.accept(self) + pass def visit_type_alias_type(self, t: TypeAliasType) -> None: # TODO: sometimes we want to traverse target as well diff --git a/mypyc/irbuild/classdef.py b/mypyc/irbuild/classdef.py index 2152da099e81..7e0a842b1b41 100644 --- a/mypyc/irbuild/classdef.py +++ b/mypyc/irbuild/classdef.py @@ -26,7 +26,7 @@ TypeParam, is_class_var, ) -from mypy.types import ENUM_REMOVED_PROPS, Instance, RawExpressionType, get_proper_type +from mypy.types import ENUM_REMOVED_PROPS, Instance, UnboundType, get_proper_type from mypyc.common import PROPSET_PREFIX from mypyc.ir.class_ir import ClassIR, NonExtClassInfo from mypyc.ir.func_ir import FuncDecl, FuncSignature @@ -640,15 +640,16 @@ def add_non_ext_class_attr_ann( if typ is None: # FIXME: if get_type_info is not provided, don't fall back to stmt.type? ann_type = get_proper_type(stmt.type) - if isinstance(stmt.unanalyzed_type, RawExpressionType) and isinstance( - stmt.unanalyzed_type.literal_value, str + if ( + isinstance(stmt.unanalyzed_type, UnboundType) + and stmt.unanalyzed_type.original_str_expr is not None ): # Annotation is a forward reference, so don't attempt to load the actual # type and load the string instead. # # TODO: is it possible to determine whether a non-string annotation is # actually a forward reference due to the __annotations__ future? - typ = builder.load_str(stmt.unanalyzed_type.literal_value) + typ = builder.load_str(stmt.unanalyzed_type.original_str_expr) elif isinstance(ann_type, Instance): typ = load_type(builder, ann_type.type, stmt.line) else: diff --git a/test-data/unit/check-final.test b/test-data/unit/check-final.test index dadf76a283b0..763183159e94 100644 --- a/test-data/unit/check-final.test +++ b/test-data/unit/check-final.test @@ -6,13 +6,11 @@ [case testFinalDefiningModuleVar] from typing import Final -w: 'Final' = int() x: Final = int() y: Final[float] = int() z: Final[int] = int() bad: Final[str] = int() # E: Incompatible types in assignment (expression has type "int", variable has type "str") -reveal_type(w) # N: Revealed type is "builtins.int" reveal_type(x) # N: Revealed type is "builtins.int" reveal_type(y) # N: Revealed type is "builtins.float" reveal_type(z) # N: Revealed type is "builtins.int" diff --git a/test-data/unit/check-literal.test b/test-data/unit/check-literal.test index 6d76ce176aaf..e8d942f5d74d 100644 --- a/test-data/unit/check-literal.test +++ b/test-data/unit/check-literal.test @@ -12,12 +12,8 @@ reveal_type(g1) # N: Revealed type is "def (x: Literal['A['])" def f2(x: 'A B') -> None: pass # E: Invalid type comment or annotation def g2(x: Literal['A B']) -> None: pass -def h2(x: 'A|int') -> None: pass # E: Name "A" is not defined -def i2(x: Literal['A|B']) -> None: pass reveal_type(f2) # N: Revealed type is "def (x: Any)" reveal_type(g2) # N: Revealed type is "def (x: Literal['A B'])" -reveal_type(h2) # N: Revealed type is "def (x: Union[Any, builtins.int])" -reveal_type(i2) # N: Revealed type is "def (x: Literal['A|B'])" [builtins fixtures/tuple.pyi] [out] diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 147270dff72e..df2c7ffc8067 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -824,20 +824,14 @@ class Fraction(Real): [builtins fixtures/tuple.pyi] [case testForwardReferenceInNamedTuple] -from typing import List, NamedTuple +from typing import NamedTuple class A(NamedTuple): b: 'B' x: int - y: List['B'] class B: pass - -def f(a: A): - reveal_type(a.b) # N: Revealed type is "__main__.B" - reveal_type(a.x) # N: Revealed type is "builtins.int" - reveal_type(a.y) # N: Revealed type is "builtins.list[__main__.B]" [builtins fixtures/tuple.pyi] [case testTypeNamedTupleClassmethod] diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index e6d8cec3f0b0..c2afb61586a8 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -1193,28 +1193,7 @@ def func(callback: Callable[P, str]) -> Callable[P, str]: return inner [builtins fixtures/paramspec.pyi] -[case testParamSpecArgsAndKwargsStringified] -from typing import Callable -from typing_extensions import ParamSpec - -P1 = ParamSpec("P1") - -def func(callback: Callable[P1, str]) -> Callable[P1, str]: - def inner(*args: "P1.args", **kwargs: "P1.kwargs") -> str: - return "foo" - return inner - -@func -def outer(a: int) -> str: - return "" - -outer(1) # OK -outer("x") # E: Argument 1 to "outer" has incompatible type "str"; expected "int" -outer(a=1) # OK -outer(b=1) # E: Unexpected keyword argument "b" for "outer" -[builtins fixtures/paramspec.pyi] - -[case testParamSpecArgsAndKwargsMismatch] +[case testParamSpecArgsAndKwargsMissmatch] from typing import Callable from typing_extensions import ParamSpec diff --git a/test-data/unit/check-typeguard.test b/test-data/unit/check-typeguard.test index e1b7a86aba63..27b88553fb43 100644 --- a/test-data/unit/check-typeguard.test +++ b/test-data/unit/check-typeguard.test @@ -9,17 +9,6 @@ def main(a: object) -> None: reveal_type(a) # N: Revealed type is "builtins.object" [builtins fixtures/tuple.pyi] -[case testTypeGuardStringified] -from typing_extensions import TypeGuard -class Point: pass -def is_point(a: object) -> "TypeGuard[Point]": pass -def main(a: object) -> None: - if is_point(a): - reveal_type(a) # N: Revealed type is "__main__.Point" - else: - reveal_type(a) # N: Revealed type is "builtins.object" -[builtins fixtures/tuple.pyi] - [case testTypeGuardTypeArgsNone] from typing_extensions import TypeGuard def foo(a: object) -> TypeGuard: # E: TypeGuard must have exactly one type argument diff --git a/test-data/unit/check-typeis.test b/test-data/unit/check-typeis.test index 83467d5e3683..6b96845504ab 100644 --- a/test-data/unit/check-typeis.test +++ b/test-data/unit/check-typeis.test @@ -9,17 +9,6 @@ def main(a: object) -> None: reveal_type(a) # N: Revealed type is "builtins.object" [builtins fixtures/tuple.pyi] -[case testTypeIsStringified] -from typing_extensions import TypeIs -class Point: pass -def is_point(a: object) -> "TypeIs[Point]": pass -def main(a: object) -> None: - if is_point(a): - reveal_type(a) # N: Revealed type is "__main__.Point" - else: - reveal_type(a) # N: Revealed type is "builtins.object" -[builtins fixtures/tuple.pyi] - [case testTypeIsElif] from typing_extensions import TypeIs from typing import Union