From abdaf6a571a4a539d755db1d6dcfdc45b69d97c5 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 22 Jun 2024 23:19:22 +0100 Subject: [PATCH] Use (simplified) unions instead of joins for tuple fallbacks (#17408) Ref https://github.com/python/mypy/issues/12056 If `mypy_primer` will look good, I will add some logic to shorted unions in error messages. cc @JukkaL --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Alex Waygood --- mypy/checker.py | 3 ++ mypy/messages.py | 68 ++++++++++++++++++++++--- mypy/semanal_shared.py | 6 +-- mypy/typeops.py | 7 ++- test-data/unit/check-enum.test | 6 +-- test-data/unit/check-expressions.test | 2 +- test-data/unit/check-namedtuple.test | 6 +-- test-data/unit/check-newsemanal.test | 12 ++--- test-data/unit/check-statements.test | 2 +- test-data/unit/check-tuples.test | 4 +- test-data/unit/check-typevar-tuple.test | 6 +-- test-data/unit/check-unions.test | 57 +++++++++++++++++++++ test-data/unit/semanal-classes.test | 2 +- 13 files changed, 146 insertions(+), 35 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index d2562d5dd722..792e751691fd 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -49,6 +49,7 @@ SUGGESTED_TEST_FIXTURES, MessageBuilder, append_invariance_notes, + append_union_note, format_type, format_type_bare, format_type_distinctly, @@ -6814,6 +6815,8 @@ def check_subtype( ) if isinstance(subtype, Instance) and isinstance(supertype, Instance): notes = append_invariance_notes(notes, subtype, supertype) + if isinstance(subtype, UnionType) and isinstance(supertype, UnionType): + notes = append_union_note(notes, subtype, supertype, self.options) if extra_info: msg = msg.with_additional_msg(" (" + ", ".join(extra_info) + ")") diff --git a/mypy/messages.py b/mypy/messages.py index 27f152413151..62846c536f3d 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -90,6 +90,7 @@ UninhabitedType, UnionType, UnpackType, + flatten_nested_unions, get_proper_type, get_proper_types, ) @@ -145,6 +146,9 @@ "numbers.Integral", } +MAX_TUPLE_ITEMS = 10 +MAX_UNION_ITEMS = 10 + class MessageBuilder: """Helper class for reporting type checker error messages with parameters. @@ -2338,7 +2342,7 @@ def try_report_long_tuple_assignment_error( """ if isinstance(subtype, TupleType): if ( - len(subtype.items) > 10 + len(subtype.items) > MAX_TUPLE_ITEMS and isinstance(supertype, Instance) and supertype.type.fullname == "builtins.tuple" ): @@ -2347,7 +2351,7 @@ def try_report_long_tuple_assignment_error( self.generate_incompatible_tuple_error(lhs_types, subtype.items, context, msg) return True elif isinstance(supertype, TupleType) and ( - len(subtype.items) > 10 or len(supertype.items) > 10 + len(subtype.items) > MAX_TUPLE_ITEMS or len(supertype.items) > MAX_TUPLE_ITEMS ): if len(subtype.items) != len(supertype.items): if supertype_label is not None and subtype_label is not None: @@ -2370,7 +2374,7 @@ def try_report_long_tuple_assignment_error( def format_long_tuple_type(self, typ: TupleType) -> str: """Format very long tuple type using an ellipsis notation""" item_cnt = len(typ.items) - if item_cnt > 10: + if item_cnt > MAX_TUPLE_ITEMS: return "{}[{}, {}, ... <{} more items>]".format( "tuple" if self.options.use_lowercase_names() else "Tuple", format_type_bare(typ.items[0], self.options), @@ -2497,11 +2501,21 @@ def format(typ: Type) -> str: def format_list(types: Sequence[Type]) -> str: return ", ".join(format(typ) for typ in types) - def format_union(types: Sequence[Type]) -> str: + def format_union_items(types: Sequence[Type]) -> list[str]: formatted = [format(typ) for typ in types if format(typ) != "None"] + if len(formatted) > MAX_UNION_ITEMS and verbosity == 0: + more = len(formatted) - MAX_UNION_ITEMS // 2 + formatted = formatted[: MAX_UNION_ITEMS // 2] + else: + more = 0 + if more: + formatted.append(f"<{more} more items>") if any(format(typ) == "None" for typ in types): formatted.append("None") - return " | ".join(formatted) + return formatted + + def format_union(types: Sequence[Type]) -> str: + return " | ".join(format_union_items(types)) def format_literal_value(typ: LiteralType) -> str: if typ.is_enum_literal(): @@ -2605,6 +2619,9 @@ def format_literal_value(typ: LiteralType) -> str: elif isinstance(typ, LiteralType): return f"Literal[{format_literal_value(typ)}]" elif isinstance(typ, UnionType): + typ = get_proper_type(ignore_last_known_values(typ)) + if not isinstance(typ, UnionType): + return format(typ) literal_items, union_items = separate_union_literals(typ) # Coalesce multiple Literal[] members. This also changes output order. @@ -2624,7 +2641,7 @@ def format_literal_value(typ: LiteralType) -> str: return ( f"{literal_str} | {format_union(union_items)}" if options.use_or_syntax() - else f"Union[{format_list(union_items)}, {literal_str}]" + else f"Union[{', '.join(format_union_items(union_items))}, {literal_str}]" ) else: return literal_str @@ -2645,7 +2662,7 @@ def format_literal_value(typ: LiteralType) -> str: s = ( format_union(typ.items) if options.use_or_syntax() - else f"Union[{format_list(typ.items)}]" + else f"Union[{', '.join(format_union_items(typ.items))}]" ) return s elif isinstance(typ, NoneType): @@ -3182,6 +3199,23 @@ def append_invariance_notes( return notes +def append_union_note( + notes: list[str], arg_type: UnionType, expected_type: UnionType, options: Options +) -> list[str]: + """Point to specific union item(s) that may cause failure in subtype check.""" + non_matching = [] + items = flatten_nested_unions(arg_type.items) + if len(items) < MAX_UNION_ITEMS: + return notes + for item in items: + if not is_subtype(item, expected_type): + non_matching.append(item) + if non_matching: + types = ", ".join([format_type(typ, options) for typ in non_matching]) + notes.append(f"Item{plural_s(non_matching)} in the first union not in the second: {types}") + return notes + + def append_numbers_notes( notes: list[str], arg_type: Instance, expected_type: Instance ) -> list[str]: @@ -3235,3 +3269,23 @@ def format_key_list(keys: list[str], *, short: bool = False) -> str: return f"{td}key {formatted_keys[0]}" else: return f"{td}keys ({', '.join(formatted_keys)})" + + +def ignore_last_known_values(t: UnionType) -> Type: + """This will avoid types like str | str in error messages. + + last_known_values are kept during union simplification, but may cause + weird formatting for e.g. tuples of literals. + """ + union_items: list[Type] = [] + seen_instances = set() + for item in t.items: + if isinstance(item, ProperType) and isinstance(item, Instance): + erased = item.copy_modified(last_known_value=None) + if erased in seen_instances: + continue + seen_instances.add(erased) + union_items.append(erased) + else: + union_items.append(item) + return UnionType.make_union(union_items, t.line, t.column) diff --git a/mypy/semanal_shared.py b/mypy/semanal_shared.py index 01d8e9aafffb..db19f074911f 100644 --- a/mypy/semanal_shared.py +++ b/mypy/semanal_shared.py @@ -8,7 +8,6 @@ from mypy_extensions import trait -from mypy import join from mypy.errorcodes import LITERAL_REQ, ErrorCode from mypy.nodes import ( CallExpr, @@ -30,6 +29,7 @@ from mypy.plugin import SemanticAnalyzerPluginInterface from mypy.tvar_scope import TypeVarLikeScope from mypy.type_visitor import ANY_STRATEGY, BoolTypeQuery +from mypy.typeops import make_simplified_union from mypy.types import ( TPDICT_FB_NAMES, AnyType, @@ -58,7 +58,7 @@ # Priorities for ordering of patches within the "patch" phase of semantic analysis # (after the main pass): -# Fix fallbacks (does joins) +# Fix fallbacks (does subtype checks). PRIORITY_FALLBACKS: Final = 1 @@ -304,7 +304,7 @@ def calculate_tuple_fallback(typ: TupleType) -> None: raise NotImplementedError else: items.append(item) - fallback.args = (join.join_type_list(items),) + fallback.args = (make_simplified_union(items),) class _NamedTypeCallback(Protocol): diff --git a/mypy/typeops.py b/mypy/typeops.py index 62c850452516..4fe187f811ca 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -95,8 +95,6 @@ def is_recursive_pair(s: Type, t: Type) -> bool: def tuple_fallback(typ: TupleType) -> Instance: """Return fallback type for a tuple.""" - from mypy.join import join_type_list - info = typ.partial_fallback.type if info.fullname != "builtins.tuple": return typ.partial_fallback @@ -115,8 +113,9 @@ def tuple_fallback(typ: TupleType) -> Instance: raise NotImplementedError else: items.append(item) - # TODO: we should really use a union here, tuple types are special. - return Instance(info, [join_type_list(items)], extra_attrs=typ.partial_fallback.extra_attrs) + return Instance( + info, [make_simplified_union(items)], extra_attrs=typ.partial_fallback.extra_attrs + ) def get_self_type(func: CallableType, default_self: Instance | TupleType) -> Type | None: diff --git a/test-data/unit/check-enum.test b/test-data/unit/check-enum.test index d53935085325..78a114eda764 100644 --- a/test-data/unit/check-enum.test +++ b/test-data/unit/check-enum.test @@ -1010,7 +1010,7 @@ _empty: Final = Empty.token def func(x: Union[int, None, Empty] = _empty) -> int: boom = x + 42 # E: Unsupported left operand type for + ("None") \ # E: Unsupported left operand type for + ("Empty") \ - # N: Left operand is of type "Union[int, None, Empty]" + # N: Left operand is of type "Union[int, Empty, None]" if x is _empty: reveal_type(x) # N: Revealed type is "Literal[__main__.Empty.token]" return 0 @@ -1056,7 +1056,7 @@ _empty = Empty.token def func(x: Union[int, None, Empty] = _empty) -> int: boom = x + 42 # E: Unsupported left operand type for + ("None") \ # E: Unsupported left operand type for + ("Empty") \ - # N: Left operand is of type "Union[int, None, Empty]" + # N: Left operand is of type "Union[int, Empty, None]" if x is _empty: reveal_type(x) # N: Revealed type is "Literal[__main__.Empty.token]" return 0 @@ -1084,7 +1084,7 @@ _empty = Empty.token def func(x: Union[int, None, Empty] = _empty) -> int: boom = x + 42 # E: Unsupported left operand type for + ("None") \ # E: Unsupported left operand type for + ("Empty") \ - # N: Left operand is of type "Union[int, None, Empty]" + # N: Left operand is of type "Union[int, Empty, None]" if x is _empty: reveal_type(x) # N: Revealed type is "Literal[__main__.Empty.token]" return 0 diff --git a/test-data/unit/check-expressions.test b/test-data/unit/check-expressions.test index 4fc6e9a75c83..f9bd60f4dcc8 100644 --- a/test-data/unit/check-expressions.test +++ b/test-data/unit/check-expressions.test @@ -1640,7 +1640,7 @@ from typing import Generator def g() -> Generator[int, None, None]: x = yield from () # E: Function does not return a value (it only ever returns None) x = yield from (0, 1, 2) # E: Function does not return a value (it only ever returns None) - x = yield from (0, "ERROR") # E: Incompatible types in "yield from" (actual type "object", expected type "int") \ + x = yield from (0, "ERROR") # E: Incompatible types in "yield from" (actual type "Union[int, str]", expected type "int") \ # E: Function does not return a value (it only ever returns None) x = yield from ("ERROR",) # E: Incompatible types in "yield from" (actual type "str", expected type "int") \ # E: Function does not return a value (it only ever returns None) diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 2007d574f922..e9d156754d9c 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1249,7 +1249,7 @@ nti: NT[int] reveal_type(nti * x) # N: Revealed type is "builtins.tuple[builtins.int, ...]" nts: NT[str] -reveal_type(nts * x) # N: Revealed type is "builtins.tuple[builtins.object, ...]" +reveal_type(nts * x) # N: Revealed type is "builtins.tuple[Union[builtins.int, builtins.str], ...]" [builtins fixtures/tuple.pyi] [typing fixtures/typing-namedtuple.pyi] @@ -1310,9 +1310,9 @@ reveal_type(foo(nti, nts)) # N: Revealed type is "Tuple[builtins.int, builtins. reveal_type(foo(nts, nti)) # N: Revealed type is "Tuple[builtins.int, builtins.object, fallback=__main__.NT[builtins.object]]" reveal_type(foo(nti, x)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" -reveal_type(foo(nts, x)) # N: Revealed type is "builtins.tuple[builtins.object, ...]" +reveal_type(foo(nts, x)) # N: Revealed type is "builtins.tuple[Union[builtins.int, builtins.str], ...]" reveal_type(foo(x, nti)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" -reveal_type(foo(x, nts)) # N: Revealed type is "builtins.tuple[builtins.object, ...]" +reveal_type(foo(x, nts)) # N: Revealed type is "builtins.tuple[Union[builtins.int, builtins.str], ...]" [builtins fixtures/tuple.pyi] [typing fixtures/typing-namedtuple.pyi] diff --git a/test-data/unit/check-newsemanal.test b/test-data/unit/check-newsemanal.test index 47e508ee1a6b..511c7b003015 100644 --- a/test-data/unit/check-newsemanal.test +++ b/test-data/unit/check-newsemanal.test @@ -1947,7 +1947,7 @@ class NTStr(NamedTuple): y: str t1: T -reveal_type(t1.__iter__) # N: Revealed type is "def () -> typing.Iterator[__main__.A]" +reveal_type(t1.__iter__) # N: Revealed type is "def () -> typing.Iterator[Union[__main__.B, __main__.C]]" t2: NTInt reveal_type(t2.__iter__) # N: Revealed type is "def () -> typing.Iterator[builtins.int]" @@ -1960,7 +1960,6 @@ t: Union[Tuple[int, int], Tuple[str, str]] for x in t: reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" [builtins fixtures/for.pyi] -[out] [case testNewAnalyzerFallbackUpperBoundCheckAndFallbacks] from typing import TypeVar, Generic, Tuple @@ -1973,10 +1972,9 @@ S = TypeVar('S', bound='Tuple[G[A], ...]') class GG(Generic[S]): pass -g: GG[Tuple[G[B], G[C]]] \ - # E: Type argument "Tuple[G[B], G[C]]" of "GG" must be a subtype of "Tuple[G[A], ...]" \ - # E: Type argument "B" of "G" must be a subtype of "A" \ - # E: Type argument "C" of "G" must be a subtype of "A" +g: GG[Tuple[G[B], G[C]]] # E: Type argument "Tuple[G[B], G[C]]" of "GG" must be a subtype of "Tuple[G[A], ...]" \ + # E: Type argument "B" of "G" must be a subtype of "A" \ + # E: Type argument "C" of "G" must be a subtype of "A" T = TypeVar('T', bound=A, covariant=True) @@ -1984,7 +1982,7 @@ class G(Generic[T]): pass t: Tuple[G[B], G[C]] # E: Type argument "B" of "G" must be a subtype of "A" \ # E: Type argument "C" of "G" must be a subtype of "A" -reveal_type(t.__iter__) # N: Revealed type is "def () -> typing.Iterator[builtins.object]" +reveal_type(t.__iter__) # N: Revealed type is "def () -> typing.Iterator[__main__.G[__main__.B]]" [builtins fixtures/tuple.pyi] [case testNewAnalyzerClassKeywordsForward] diff --git a/test-data/unit/check-statements.test b/test-data/unit/check-statements.test index 34df5a8ab336..d1464423e90f 100644 --- a/test-data/unit/check-statements.test +++ b/test-data/unit/check-statements.test @@ -1339,7 +1339,7 @@ from typing import Generator def g() -> Generator[int, None, None]: yield from () yield from (0, 1, 2) - yield from (0, "ERROR") # E: Incompatible types in "yield from" (actual type "object", expected type "int") + yield from (0, "ERROR") # E: Incompatible types in "yield from" (actual type "Union[int, str]", expected type "int") yield from ("ERROR",) # E: Incompatible types in "yield from" (actual type "str", expected type "int") [builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-tuples.test b/test-data/unit/check-tuples.test index ad4893c2890a..bf36977b56e3 100644 --- a/test-data/unit/check-tuples.test +++ b/test-data/unit/check-tuples.test @@ -1408,8 +1408,8 @@ y = "" reveal_type(t[x]) # N: Revealed type is "Union[builtins.int, builtins.str]" t[y] # E: No overload variant of "__getitem__" of "tuple" matches argument type "str" \ # N: Possible overload variants: \ - # N: def __getitem__(self, int, /) -> object \ - # N: def __getitem__(self, slice, /) -> Tuple[object, ...] + # N: def __getitem__(self, int, /) -> Union[int, str] \ + # N: def __getitem__(self, slice, /) -> Tuple[Union[int, str], ...] [builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index 8f7dd12d9cd4..49298114e069 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -24,7 +24,7 @@ def g(a: Tuple[Unpack[Ts]], b: Tuple[Unpack[Ts]]) -> Tuple[Unpack[Ts]]: reveal_type(g(args, args)) # N: Revealed type is "Tuple[builtins.int, builtins.str]" reveal_type(g(args, args2)) # N: Revealed type is "Tuple[builtins.int, builtins.str]" -reveal_type(g(args, args3)) # N: Revealed type is "builtins.tuple[builtins.object, ...]" +reveal_type(g(args, args3)) # N: Revealed type is "builtins.tuple[Union[builtins.int, builtins.str], ...]" reveal_type(g(any, any)) # N: Revealed type is "builtins.tuple[Any, ...]" [builtins fixtures/tuple.pyi] @@ -989,7 +989,7 @@ from typing_extensions import Unpack def pipeline(*xs: Unpack[Tuple[int, Unpack[Tuple[float, ...]], bool]]) -> None: for x in xs: - reveal_type(x) # N: Revealed type is "builtins.float" + reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.float]" [builtins fixtures/tuple.pyi] [case testFixedUnpackItemInInstanceArguments] @@ -1715,7 +1715,7 @@ vt: Tuple[int, Unpack[Tuple[float, ...]], int] reveal_type(vt + (1, 2)) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.float, ...]], builtins.int, Literal[1]?, Literal[2]?]" reveal_type((1, 2) + vt) # N: Revealed type is "Tuple[Literal[1]?, Literal[2]?, builtins.int, Unpack[builtins.tuple[builtins.float, ...]], builtins.int]" -reveal_type(vt + vt) # N: Revealed type is "builtins.tuple[builtins.float, ...]" +reveal_type(vt + vt) # N: Revealed type is "builtins.tuple[Union[builtins.int, builtins.float], ...]" reveal_type(vtf + (1, 2)) # N: Revealed type is "Tuple[Unpack[builtins.tuple[builtins.float, ...]], Literal[1]?, Literal[2]?]" reveal_type((1, 2) + vtf) # N: Revealed type is "Tuple[Literal[1]?, Literal[2]?, Unpack[builtins.tuple[builtins.float, ...]]]" diff --git a/test-data/unit/check-unions.test b/test-data/unit/check-unions.test index 2ca2f1ba9eb3..329896f7a1a7 100644 --- a/test-data/unit/check-unions.test +++ b/test-data/unit/check-unions.test @@ -1289,3 +1289,60 @@ x: str = a_class_or_none.field a_or_none: Optional[A] y: int = a_or_none.field [builtins fixtures/list.pyi] + +[case testLargeUnionsShort] +from typing import Union + +class C1: ... +class C2: ... +class C3: ... +class C4: ... +class C5: ... +class C6: ... +class C7: ... +class C8: ... +class C9: ... +class C10: ... +class C11: ... + +u: Union[C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, C11] +x: int = u # E: Incompatible types in assignment (expression has type "Union[C1, C2, C3, C4, C5, <6 more items>]", variable has type "int") + +[case testLargeUnionsLongIfNeeded] +from typing import Union + +class C1: ... +class C2: ... +class C3: ... +class C4: ... +class C5: ... +class C6: ... +class C7: ... +class C8: ... +class C9: ... +class C10: ... + +x: Union[C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, int] +y: Union[C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, str] +x = y # E: Incompatible types in assignment (expression has type "Union[C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, str]", variable has type "Union[C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, int]") \ + # N: Item in the first union not in the second: "str" + +[case testLargeUnionsNoneShown] +from typing import Union + +class C1: ... +class C2: ... +class C3: ... +class C4: ... +class C5: ... +class C6: ... +class C7: ... +class C8: ... +class C9: ... +class C10: ... +class C11: ... + +x: Union[C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, C11] +y: Union[C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, C11, None] +x = y # E: Incompatible types in assignment (expression has type "Union[C1, C2, C3, C4, C5, <6 more items>, None]", variable has type "Union[C1, C2, C3, C4, C5, <6 more items>]") \ + # N: Item in the first union not in the second: "None" diff --git a/test-data/unit/semanal-classes.test b/test-data/unit/semanal-classes.test index 951791e23490..b14358509f85 100644 --- a/test-data/unit/semanal-classes.test +++ b/test-data/unit/semanal-classes.test @@ -585,7 +585,7 @@ MypyFile:1( TupleType( Tuple[builtins.int, builtins.str]) BaseType( - builtins.tuple[builtins.object, ...]) + builtins.tuple[Union[builtins.int, builtins.str], ...]) PassStmt:2())) [case testBaseClassFromIgnoredModule]