From 51fb484ed329de1cd10c11d8b4d906f0473fa2c6 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Thu, 27 Jun 2024 15:11:51 +0900 Subject: [PATCH 01/11] Start treating erased TypeVarTuple like erased TypeVar or erased ParamSpec --- mypy/erasetype.py | 3 ++- mypy/subtypes.py | 6 ++++-- mypy/types.py | 17 +++++++++++++---- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/mypy/erasetype.py b/mypy/erasetype.py index b41eefcd4821..25bf5038c5bd 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -212,7 +212,8 @@ def visit_tuple_type(self, t: TupleType) -> Type: def visit_type_var_tuple(self, t: TypeVarTupleType) -> Type: if self.erase_id(t.id): - return t.tuple_fallback.copy_modified(args=[self.replacement]) + # TODO: should t.tuple_fallback become a TupleType? + return TupleType([], t.tuple_fallback, erased_typevartuple=True) return t def visit_param_spec(self, t: ParamSpecType) -> Type: diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 649cbae4c831..4aa28254f36d 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -1602,6 +1602,8 @@ def are_parameters_compatible( if are_trivial_parameters(right) and not is_proper_subtype: return True trivial_suffix = is_trivial_suffix(right) and not is_proper_subtype + # erased typevartuples, like erased paramspecs or erased typevars are trivial + trivial_varargs = right_star and isinstance(right_star.typ, UnpackType) and isinstance(right_star.typ.type, TupleType) and right_star.typ.type.erased_typevartuple if ( right.arg_kinds == [ARG_STAR] @@ -1644,7 +1646,7 @@ def _incompatible(left_arg: FormalArgument | None, right_arg: FormalArgument | N if right_arg is None: return False if left_arg is None: - return not allow_partial_overlap and not trivial_suffix + return not allow_partial_overlap and not trivial_suffix and not trivial_varargs return not is_compat(right_arg.typ, left_arg.typ) if _incompatible(left_star, right_star) or _incompatible(left_star2, right_star2): @@ -1673,7 +1675,7 @@ def _incompatible(left_arg: FormalArgument | None, right_arg: FormalArgument | N # arguments. Get all further positional args of left, and make sure # they're more general than the corresponding member in right. # TODO: are we handling UnpackType correctly here? - if right_star is not None and not trivial_suffix: + if right_star is not None and not trivial_suffix and not trivial_varargs: # Synthesize an anonymous formal argument for the right right_by_position = right.try_synthesizing_arg_from_vararg(None) assert right_by_position is not None diff --git a/mypy/types.py b/mypy/types.py index 52f8a8d63f09..fda6d1de0f78 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -2126,6 +2126,8 @@ def with_normalized_var_args(self) -> Self: # this should be done once in semanal_typeargs.py for user-defined types, # and we ourselves rarely construct such type. return self + if unpacked.erased_typevartuple: + return self unpack_index = find_unpack_in_list(unpacked.items) if unpack_index == 0 and len(unpacked.items) > 1: # Already normalized. @@ -2352,13 +2354,16 @@ class TupleType(ProperType): a tuple base class. Use mypy.typeops.tuple_fallback to calculate the precise fallback type derived from item types. implicit: If True, derived from a tuple expression (t,....) instead of Tuple[t, ...] + erased_typevartuple: If True, this came from a (now-erased) TypeVarTuple. This + indicates that this tuple should act more like an Any. """ - __slots__ = ("items", "partial_fallback", "implicit") + __slots__ = ("items", "partial_fallback", "implicit", "erased_typevartuple") items: list[Type] partial_fallback: Instance implicit: bool + erased_typevartuple: bool def __init__( self, @@ -2367,11 +2372,13 @@ def __init__( line: int = -1, column: int = -1, implicit: bool = False, + erased_typevartuple: bool = False, ) -> None: super().__init__(line, column) self.partial_fallback = fallback self.items = items self.implicit = implicit + self.erased_typevartuple = erased_typevartuple def can_be_true_default(self) -> bool: if self.can_be_any_bool(): @@ -2412,12 +2419,12 @@ def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_tuple_type(self) def __hash__(self) -> int: - return hash((tuple(self.items), self.partial_fallback)) + return hash((tuple(self.items), self.partial_fallback, self.erased_typevartuple)) def __eq__(self, other: object) -> bool: if not isinstance(other, TupleType): return NotImplemented - return self.items == other.items and self.partial_fallback == other.partial_fallback + return self.items == other.items and self.partial_fallback == other.partial_fallback and self.erased_typevartuple == other.erased_typevartuple def serialize(self) -> JsonDict: return { @@ -2425,6 +2432,7 @@ def serialize(self) -> JsonDict: "items": [t.serialize() for t in self.items], "partial_fallback": self.partial_fallback.serialize(), "implicit": self.implicit, + "erased_typevartuple": self.erased_typevartuple, } @classmethod @@ -2434,6 +2442,7 @@ def deserialize(cls, data: JsonDict) -> TupleType: [deserialize_type(t) for t in data["items"]], Instance.deserialize(data["partial_fallback"]), implicit=data["implicit"], + erased_typevartuple=data["erased_typevartuple"], ) def copy_modified( @@ -2496,7 +2505,7 @@ def slice( return None else: slice_items = self.items[begin:end:stride] - return TupleType(slice_items, fallback, self.line, self.column, self.implicit) + return TupleType(slice_items, fallback, self.line, self.column, self.implicit, self.erased_typevartuple) class TypedDictType(ProperType): From 5e554ef4f6017cfdf40e7856f9b41bd66d09e3b3 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Thu, 27 Jun 2024 15:47:21 +0900 Subject: [PATCH 02/11] Add simple case handling of kwargs with typevartuples --- mypy/constraints.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index 49a2aea8fa05..98dff97ba2e3 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -1076,6 +1076,32 @@ def visit_callable_type(self, template: CallableType) -> list[Constraint]: # but not vice versa. # TODO: infer more from prefixes when possible. if unpack_present is not None and not cactual.param_spec(): + # if there's anything that would get ignored later, handle them now. + # (assumes that if there's a kwarg on template, it should get matched. + # ... which isn't always a right assumption) + for arg in template.formal_arguments(): + if arg.pos: + continue + + # this arg will get dropped in `repack_callable_args` later; + # handle it instead! ... this isn't very thorough though + other = cactual.argument_by_name(arg.name) + assert not other or arg.required + if not other: + continue + + # for now, simplify the problem: if `other` isn't at the end, + # or kw-only, give up + if other.pos is not None and other.pos + 1 != cactual.max_possible_positional_args(): + continue + + cactual = cactual.copy_modified( + cactual.arg_types, + [k if i != other.pos else ArgKind.ARG_NAMED for (i,k) in enumerate(cactual.arg_kinds)], + cactual.arg_names + ) + res.extend(infer_constraints(arg.typ, other.typ, self.direction)) + # We need to re-normalize args to the form they appear in tuples, # for callables we always pack the suffix inside another tuple. unpack = template.arg_types[unpack_present] @@ -1426,7 +1452,7 @@ def repack_callable_args(callable: CallableType, tuple_type: TypeInfo) -> list[T in e.g. a TupleType). """ if ARG_STAR not in callable.arg_kinds: - return callable.arg_types + return [t for (t,k) in zip(callable.arg_types, callable.arg_kinds) if k != ArgKind.ARG_NAMED] star_index = callable.arg_kinds.index(ARG_STAR) arg_types = callable.arg_types[:star_index] star_type = callable.arg_types[star_index] From 2ebe619df9ed84c410458e3a63f83745758c88c9 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Thu, 27 Jun 2024 15:58:42 +0900 Subject: [PATCH 03/11] Update for lints --- mypy/constraints.py | 16 ++++++++++++---- mypy/erasetype.py | 2 +- mypy/subtypes.py | 9 ++++++++- mypy/types.py | 10 ++++++++-- 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index 98dff97ba2e3..74480a1764f6 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -1092,13 +1092,19 @@ def visit_callable_type(self, template: CallableType) -> list[Constraint]: # for now, simplify the problem: if `other` isn't at the end, # or kw-only, give up - if other.pos is not None and other.pos + 1 != cactual.max_possible_positional_args(): + if ( + other.pos is not None + and other.pos + 1 != cactual.max_possible_positional_args() + ): continue cactual = cactual.copy_modified( cactual.arg_types, - [k if i != other.pos else ArgKind.ARG_NAMED for (i,k) in enumerate(cactual.arg_kinds)], - cactual.arg_names + [ + k if i != other.pos else ArgKind.ARG_NAMED + for (i, k) in enumerate(cactual.arg_kinds) + ], + cactual.arg_names, ) res.extend(infer_constraints(arg.typ, other.typ, self.direction)) @@ -1452,7 +1458,9 @@ def repack_callable_args(callable: CallableType, tuple_type: TypeInfo) -> list[T in e.g. a TupleType). """ if ARG_STAR not in callable.arg_kinds: - return [t for (t,k) in zip(callable.arg_types, callable.arg_kinds) if k != ArgKind.ARG_NAMED] + return [ + t for (t, k) in zip(callable.arg_types, callable.arg_kinds) if k != ArgKind.ARG_NAMED + ] star_index = callable.arg_kinds.index(ARG_STAR) arg_types = callable.arg_types[:star_index] star_type = callable.arg_types[star_index] diff --git a/mypy/erasetype.py b/mypy/erasetype.py index 25bf5038c5bd..9b4276019d6d 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -213,7 +213,7 @@ def visit_tuple_type(self, t: TupleType) -> Type: def visit_type_var_tuple(self, t: TypeVarTupleType) -> Type: if self.erase_id(t.id): # TODO: should t.tuple_fallback become a TupleType? - return TupleType([], t.tuple_fallback, erased_typevartuple=True) + return TupleType([], t.tuple_fallback, erased_typevartuple=True) return t def visit_param_spec(self, t: ParamSpecType) -> Type: diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 4aa28254f36d..ed3ec9e299e8 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -1603,7 +1603,14 @@ def are_parameters_compatible( return True trivial_suffix = is_trivial_suffix(right) and not is_proper_subtype # erased typevartuples, like erased paramspecs or erased typevars are trivial - trivial_varargs = right_star and isinstance(right_star.typ, UnpackType) and isinstance(right_star.typ.type, TupleType) and right_star.typ.type.erased_typevartuple + if right_star and isinstance(right_star.typ, UnpackType): + right_star_inner_type = get_proper_type(right_star.typ.type) + trivial_varargs = ( + isinstance(right_star_inner_type, TupleType) + and right_star_inner_type.erased_typevartuple + ) + else: + trivial_varargs = False if ( right.arg_kinds == [ARG_STAR] diff --git a/mypy/types.py b/mypy/types.py index fda6d1de0f78..b70afcf8a5b6 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -2424,7 +2424,11 @@ def __hash__(self) -> int: def __eq__(self, other: object) -> bool: if not isinstance(other, TupleType): return NotImplemented - return self.items == other.items and self.partial_fallback == other.partial_fallback and self.erased_typevartuple == other.erased_typevartuple + return ( + self.items == other.items + and self.partial_fallback == other.partial_fallback + and self.erased_typevartuple == other.erased_typevartuple + ) def serialize(self) -> JsonDict: return { @@ -2505,7 +2509,9 @@ def slice( return None else: slice_items = self.items[begin:end:stride] - return TupleType(slice_items, fallback, self.line, self.column, self.implicit, self.erased_typevartuple) + return TupleType( + slice_items, fallback, self.line, self.column, self.implicit, self.erased_typevartuple + ) class TypedDictType(ProperType): From 61c62cf5ee6d59952292f549ef0cbfc0b20753bd Mon Sep 17 00:00:00 2001 From: A5rocks Date: Mon, 8 Jul 2024 13:47:09 +0900 Subject: [PATCH 04/11] Handle the new Any-tuple somewhere else --- mypy/subtypes.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index ed3ec9e299e8..538f639accce 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -462,6 +462,9 @@ def visit_instance(self, left: Instance) -> bool: if isinstance(right, TupleType) and right.partial_fallback.type.is_enum: return self._is_subtype(left, mypy.typeops.tuple_fallback(right)) if isinstance(right, TupleType): + if right.erased_typevartuple: + return True # treat it like Any + if len(right.items) == 1: # Non-normalized Tuple type (may be left after semantic analysis # because semanal_typearg visitor is not a type translator). From c66fad26b705dce9c44ab9f7e2a12171f71f1b3d Mon Sep 17 00:00:00 2001 From: A5rocks Date: Wed, 10 Jul 2024 12:49:22 +0900 Subject: [PATCH 05/11] Handle another subtyping case for Any-tuple --- mypy/erasetype.py | 1 + mypy/subtypes.py | 2 ++ mypy/types.py | 2 ++ mypy/typevars.py | 1 + 4 files changed, 6 insertions(+) diff --git a/mypy/erasetype.py b/mypy/erasetype.py index 9b4276019d6d..7c0ea488e512 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -80,6 +80,7 @@ def visit_instance(self, t: Instance) -> ProperType: args: list[Type] = [] for tv in t.type.defn.type_vars: # Valid erasure for *Ts is *tuple[Any, ...], not just Any. + # TODO: try updating this to use TupleType if isinstance(tv, TypeVarTupleType): args.append( UnpackType( diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 538f639accce..1aab499a818c 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -827,6 +827,8 @@ def variadic_tuple_subtype(self, left: TupleType, right: TupleType) -> bool: right_unpack = right.items[right_unpack_index] assert isinstance(right_unpack, UnpackType) right_unpacked = get_proper_type(right_unpack.type) + if isinstance(right_unpacked, TupleType) and right_unpacked.erased_typevartuple: + return True # treat it as Any if not isinstance(right_unpacked, Instance): # This case should be handled by the caller. return False diff --git a/mypy/types.py b/mypy/types.py index b70afcf8a5b6..cf9295546e93 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -3388,6 +3388,8 @@ def visit_overloaded(self, t: Overloaded) -> str: return f"Overload({', '.join(a)})" def visit_tuple_type(self, t: TupleType) -> str: + if t.erased_typevartuple: + return "tuple[...]" s = self.list_str(t.items) or "()" tuple_name = "tuple" if self.options.use_lowercase_names() else "Tuple" if t.partial_fallback and t.partial_fallback.type: diff --git a/mypy/typevars.py b/mypy/typevars.py index 3d74a40c303f..738b38114fa1 100644 --- a/mypy/typevars.py +++ b/mypy/typevars.py @@ -67,6 +67,7 @@ def fill_typevars_with_any(typ: TypeInfo) -> Instance | TupleType: args: list[Type] = [] for tv in typ.defn.type_vars: # Valid erasure for *Ts is *tuple[Any, ...], not just Any. + # TODO: use TupleType if isinstance(tv, TypeVarTupleType): args.append( UnpackType(tv.tuple_fallback.copy_modified(args=[AnyType(TypeOfAny.special_form)])) From 7fa032da590fce377920124dca4b607934af0425 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Wed, 10 Jul 2024 14:23:35 +0900 Subject: [PATCH 06/11] Final work on Any-tuple --- mypy/expandtype.py | 45 ++++++++++++++++--------- mypy/subtypes.py | 2 ++ mypy/test/testtypes.py | 2 +- mypy/types.py | 12 +++++-- test-data/unit/check-typevar-tuple.test | 1 + 5 files changed, 43 insertions(+), 19 deletions(-) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 5c4d6af9458e..6c9750353f56 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -100,7 +100,15 @@ def expand_type_by_instance(typ: Type, instance: Instance) -> Type: ) tvar = tvars_middle[0] assert isinstance(tvar, TypeVarTupleType) - variables = {tvar.id: TupleType(list(args_middle), tvar.tuple_fallback)} + + tvar_value: Type = TupleType(list(args_middle), tvar.tuple_fallback) + if len(args_middle) == 1: + # prevent nested Unpacks + middle_arg = get_proper_type(args_middle[0]) + if isinstance(middle_arg, UnpackType): + tvar_value = middle_arg.type + + variables = {tvar.id: tvar_value} instance_args = args_prefix + args_suffix tvars = tvars_prefix + tvars_suffix else: @@ -207,7 +215,7 @@ def visit_erased_type(self, t: ErasedType) -> Type: return t def visit_instance(self, t: Instance) -> Type: - args = self.expand_types_with_unpack(list(t.args)) + args = self.expand_types_with_unpack(list(t.args))[0] if t.type.fullname == "builtins.tuple": # Normalize Tuple[*Tuple[X, ...], ...] -> Tuple[X, ...] arg = args[0] @@ -291,23 +299,24 @@ def visit_unpack_type(self, t: UnpackType) -> Type: # example is non-normalized types when called from semanal.py. return UnpackType(t.type.accept(self)) - def expand_unpack(self, t: UnpackType) -> list[Type]: + # TODO: there must be a cleaner way to not discard `erased_typevartuple` + def expand_unpack(self, t: UnpackType) -> tuple[list[Type], bool]: assert isinstance(t.type, TypeVarTupleType) repl = get_proper_type(self.variables.get(t.type.id, t.type)) if isinstance(repl, UnpackType): repl = get_proper_type(repl.type) if isinstance(repl, TupleType): - return repl.items + return repl.items, repl.erased_typevartuple elif ( isinstance(repl, Instance) and repl.type.fullname == "builtins.tuple" or isinstance(repl, TypeVarTupleType) ): - return [UnpackType(typ=repl)] + return [UnpackType(typ=repl)], False elif isinstance(repl, (AnyType, UninhabitedType)): # Replace *Ts = Any with *Ts = *tuple[Any, ...] and same for Never. # These types may appear here as a result of user error or failed inference. - return [UnpackType(t.type.tuple_fallback.copy_modified(args=[repl]))] + return [UnpackType(t.type.tuple_fallback.copy_modified(args=[repl]))], False else: raise RuntimeError(f"Invalid type replacement to expand: {repl}") @@ -329,13 +338,12 @@ def interpolate_args_for_unpack(self, t: CallableType, var_arg: UnpackType) -> l expanded_tuple = var_arg_type.accept(self) assert isinstance(expanded_tuple, ProperType) and isinstance(expanded_tuple, TupleType) expanded_items = expanded_tuple.items - fallback = var_arg_type.partial_fallback - new_unpack = UnpackType(TupleType(expanded_items, fallback)) + new_unpack = UnpackType(var_arg_type.copy_modified(items=expanded_items)) elif isinstance(var_arg_type, TypeVarTupleType): # We have plain Unpack[Ts] fallback = var_arg_type.tuple_fallback - expanded_items = self.expand_unpack(var_arg) - new_unpack = UnpackType(TupleType(expanded_items, fallback)) + expanded_items, etv = self.expand_unpack(var_arg) + new_unpack = UnpackType(TupleType(expanded_items, fallback, erased_typevartuple=etv)) else: # We have invalid type in Unpack. This can happen when expanding aliases # to Callable[[*Invalid], Ret] @@ -415,18 +423,23 @@ def visit_overloaded(self, t: Overloaded) -> Type: items.append(new_item) return Overloaded(items) - def expand_types_with_unpack(self, typs: Sequence[Type]) -> list[Type]: + def expand_types_with_unpack(self, typs: Sequence[Type]) -> tuple[list[Type], bool]: """Expands a list of types that has an unpack.""" items: list[Type] = [] + met_erased_typevartuple = False # not sure this is the right behavior. for item in typs: if isinstance(item, UnpackType) and isinstance(item.type, TypeVarTupleType): - items.extend(self.expand_unpack(item)) + its, etv = self.expand_unpack(item) + met_erased_typevartuple = met_erased_typevartuple or etv + items.extend(its) else: items.append(item.accept(self)) - return items + + assert not met_erased_typevartuple or len(typs) == 1 + return items, met_erased_typevartuple def visit_tuple_type(self, t: TupleType) -> Type: - items = self.expand_types_with_unpack(t.items) + items, etv = self.expand_types_with_unpack(t.items) if len(items) == 1: # Normalize Tuple[*Tuple[X, ...]] -> Tuple[X, ...] item = items[0] @@ -441,7 +454,7 @@ def visit_tuple_type(self, t: TupleType) -> Type: return unpacked fallback = t.partial_fallback.accept(self) assert isinstance(fallback, ProperType) and isinstance(fallback, Instance) - return t.copy_modified(items=items, fallback=fallback) + return t.copy_modified(items=items, fallback=fallback, erased_typevartuple=etv) def visit_typeddict_type(self, t: TypedDictType) -> Type: fallback = t.fallback.accept(self) @@ -480,7 +493,7 @@ def visit_type_type(self, t: TypeType) -> Type: def visit_type_alias_type(self, t: TypeAliasType) -> Type: # Target of the type alias cannot contain type variables (not bound by the type # alias itself), so we just expand the arguments. - args = self.expand_types_with_unpack(t.args) + args = self.expand_types_with_unpack(t.args)[0] # TODO: normalize if target is Tuple, and args are [*tuple[X, ...]]? return t.copy_modified(args=args) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 1aab499a818c..3295e7f7743e 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -787,6 +787,8 @@ def visit_tuple_type(self, left: TupleType) -> bool: return True return False elif isinstance(right, TupleType): + if right.erased_typevartuple: + return True # treat it like Any # If right has a variadic unpack this needs special handling. If there is a TypeVarTuple # unpack, item count must coincide. If the left has variadic unpack but right # doesn't have one, we will fall through to False down the line. diff --git a/mypy/test/testtypes.py b/mypy/test/testtypes.py index 0218d33cc124..d69350d59e46 100644 --- a/mypy/test/testtypes.py +++ b/mypy/test/testtypes.py @@ -1560,7 +1560,7 @@ def make_call(*items: tuple[str, str | None]) -> CallExpr: class TestExpandTypeLimitGetProperType(TestCase): # WARNING: do not increase this number unless absolutely necessary, # and you understand what you are doing. - ALLOWED_GET_PROPER_TYPES = 9 + ALLOWED_GET_PROPER_TYPES = 10 @skipUnless(mypy.expandtype.__file__.endswith(".py"), "Skip for compiled mypy") def test_count_get_proper_type(self) -> None: diff --git a/mypy/types.py b/mypy/types.py index cf9295546e93..c9564bfb1ec4 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -2450,13 +2450,21 @@ def deserialize(cls, data: JsonDict) -> TupleType: ) def copy_modified( - self, *, fallback: Instance | None = None, items: list[Type] | None = None + self, + *, + fallback: Instance | None = None, + items: list[Type] | None = None, + erased_typevartuple: bool | None = None, ) -> TupleType: if fallback is None: fallback = self.partial_fallback if items is None: items = self.items - return TupleType(items, fallback, self.line, self.column) + if erased_typevartuple is None: + erased_typevartuple = self.erased_typevartuple + return TupleType( + items, fallback, self.line, self.column, erased_typevartuple=erased_typevartuple + ) def slice( self, begin: int | None, end: int | None, stride: int | None, *, fallback: Instance | None diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index ea692244597c..e3cace17de83 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -2136,6 +2136,7 @@ get_items(b) # E: Argument 1 to "get_items" has incompatible type "Bad"; expect # N: def items(self) -> Tuple[Never, ...] \ # N: Got: \ # N: def items(self) -> List[int] +# TODO: this *should* work. match(b) # E: Argument 1 to "match" has incompatible type "Bad"; expected "PC[Unpack[Tuple[Never, ...]]]" \ # N: Following member(s) of "Bad" have conflicts: \ # N: Expected: \ From a1048e20a731967219a5fb542ce7cd3e388e9ad3 Mon Sep 17 00:00:00 2001 From: EXPLOSION Date: Wed, 10 Jul 2024 22:19:51 +0900 Subject: [PATCH 07/11] It is ok if arg is optional --- mypy/constraints.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index 74480a1764f6..8ff643d2824c 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -1086,7 +1086,6 @@ def visit_callable_type(self, template: CallableType) -> list[Constraint]: # this arg will get dropped in `repack_callable_args` later; # handle it instead! ... this isn't very thorough though other = cactual.argument_by_name(arg.name) - assert not other or arg.required if not other: continue From 06abecd6aabf9ce57861bfb163dd0203b717453b Mon Sep 17 00:00:00 2001 From: A5rocks Date: Wed, 28 Aug 2024 07:30:25 +0900 Subject: [PATCH 08/11] Get rid of Any-tuple --- mypy/erasetype.py | 3 +-- mypy/expandtype.py | 45 +++++++++++++++--------------------------- mypy/subtypes.py | 16 ++++++--------- mypy/test/testtypes.py | 2 +- mypy/types.py | 37 ++++++---------------------------- 5 files changed, 30 insertions(+), 73 deletions(-) diff --git a/mypy/erasetype.py b/mypy/erasetype.py index f7830c7e781a..5d95b221af15 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -203,8 +203,7 @@ def visit_tuple_type(self, t: TupleType) -> Type: def visit_type_var_tuple(self, t: TypeVarTupleType) -> Type: if self.erase_id(t.id): - # TODO: should t.tuple_fallback become a TupleType? - return TupleType([], t.tuple_fallback, erased_typevartuple=True) + return t.tuple_fallback.copy_modified(args=[self.replacement]) return t def visit_param_spec(self, t: ParamSpecType) -> Type: diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 76fd42316cbf..9336be54437b 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -100,15 +100,7 @@ def expand_type_by_instance(typ: Type, instance: Instance) -> Type: ) tvar = tvars_middle[0] assert isinstance(tvar, TypeVarTupleType) - - tvar_value: Type = TupleType(list(args_middle), tvar.tuple_fallback) - if len(args_middle) == 1: - # prevent nested Unpacks - middle_arg = get_proper_type(args_middle[0]) - if isinstance(middle_arg, UnpackType): - tvar_value = middle_arg.type - - variables = {tvar.id: tvar_value} + variables = {tvar.id: TupleType(list(args_middle), tvar.tuple_fallback)} instance_args = args_prefix + args_suffix tvars = tvars_prefix + tvars_suffix else: @@ -215,7 +207,7 @@ def visit_erased_type(self, t: ErasedType) -> Type: return t def visit_instance(self, t: Instance) -> Type: - args = self.expand_types_with_unpack(list(t.args))[0] + args = self.expand_types_with_unpack(list(t.args)) if isinstance(t.type, FakeInfo): # The type checker expands function definitions and bodies @@ -309,24 +301,23 @@ def visit_unpack_type(self, t: UnpackType) -> Type: # example is non-normalized types when called from semanal.py. return UnpackType(t.type.accept(self)) - # TODO: there must be a cleaner way to not discard `erased_typevartuple` - def expand_unpack(self, t: UnpackType) -> tuple[list[Type], bool]: + def expand_unpack(self, t: UnpackType) -> list[Type]: assert isinstance(t.type, TypeVarTupleType) repl = get_proper_type(self.variables.get(t.type.id, t.type)) if isinstance(repl, UnpackType): repl = get_proper_type(repl.type) if isinstance(repl, TupleType): - return repl.items, repl.erased_typevartuple + return repl.items elif ( isinstance(repl, Instance) and repl.type.fullname == "builtins.tuple" or isinstance(repl, TypeVarTupleType) ): - return [UnpackType(typ=repl)], False + return [UnpackType(typ=repl)] elif isinstance(repl, (AnyType, UninhabitedType)): # Replace *Ts = Any with *Ts = *tuple[Any, ...] and same for Never. # These types may appear here as a result of user error or failed inference. - return [UnpackType(t.type.tuple_fallback.copy_modified(args=[repl]))], False + return [UnpackType(t.type.tuple_fallback.copy_modified(args=[repl]))] else: raise RuntimeError(f"Invalid type replacement to expand: {repl}") @@ -348,12 +339,13 @@ def interpolate_args_for_unpack(self, t: CallableType, var_arg: UnpackType) -> l expanded_tuple = var_arg_type.accept(self) assert isinstance(expanded_tuple, ProperType) and isinstance(expanded_tuple, TupleType) expanded_items = expanded_tuple.items - new_unpack = UnpackType(var_arg_type.copy_modified(items=expanded_items)) + fallback = var_arg_type.partial_fallback + new_unpack = UnpackType(TupleType(expanded_items, fallback)) elif isinstance(var_arg_type, TypeVarTupleType): # We have plain Unpack[Ts] fallback = var_arg_type.tuple_fallback - expanded_items, etv = self.expand_unpack(var_arg) - new_unpack = UnpackType(TupleType(expanded_items, fallback, erased_typevartuple=etv)) + expanded_items = self.expand_unpack(var_arg) + new_unpack = UnpackType(TupleType(expanded_items, fallback)) else: # We have invalid type in Unpack. This can happen when expanding aliases # to Callable[[*Invalid], Ret] @@ -433,23 +425,18 @@ def visit_overloaded(self, t: Overloaded) -> Type: items.append(new_item) return Overloaded(items) - def expand_types_with_unpack(self, typs: Sequence[Type]) -> tuple[list[Type], bool]: + def expand_types_with_unpack(self, typs: Sequence[Type]) -> list[Type]: """Expands a list of types that has an unpack.""" items: list[Type] = [] - met_erased_typevartuple = False # not sure this is the right behavior. for item in typs: if isinstance(item, UnpackType) and isinstance(item.type, TypeVarTupleType): - its, etv = self.expand_unpack(item) - met_erased_typevartuple = met_erased_typevartuple or etv - items.extend(its) + items.extend(self.expand_unpack(item)) else: items.append(item.accept(self)) - - assert not met_erased_typevartuple or len(typs) == 1 - return items, met_erased_typevartuple + return items def visit_tuple_type(self, t: TupleType) -> Type: - items, etv = self.expand_types_with_unpack(t.items) + items = self.expand_types_with_unpack(t.items) if len(items) == 1: # Normalize Tuple[*Tuple[X, ...]] -> Tuple[X, ...] item = items[0] @@ -464,7 +451,7 @@ def visit_tuple_type(self, t: TupleType) -> Type: return unpacked fallback = t.partial_fallback.accept(self) assert isinstance(fallback, ProperType) and isinstance(fallback, Instance) - return t.copy_modified(items=items, fallback=fallback, erased_typevartuple=etv) + return t.copy_modified(items=items, fallback=fallback) def visit_typeddict_type(self, t: TypedDictType) -> Type: fallback = t.fallback.accept(self) @@ -503,7 +490,7 @@ def visit_type_type(self, t: TypeType) -> Type: def visit_type_alias_type(self, t: TypeAliasType) -> Type: # Target of the type alias cannot contain type variables (not bound by the type # alias itself), so we just expand the arguments. - args = self.expand_types_with_unpack(t.args)[0] + args = self.expand_types_with_unpack(t.args) # TODO: normalize if target is Tuple, and args are [*tuple[X, ...]]? return t.copy_modified(args=args) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 3295e7f7743e..63592560a717 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -462,9 +462,6 @@ def visit_instance(self, left: Instance) -> bool: if isinstance(right, TupleType) and right.partial_fallback.type.is_enum: return self._is_subtype(left, mypy.typeops.tuple_fallback(right)) if isinstance(right, TupleType): - if right.erased_typevartuple: - return True # treat it like Any - if len(right.items) == 1: # Non-normalized Tuple type (may be left after semantic analysis # because semanal_typearg visitor is not a type translator). @@ -787,8 +784,6 @@ def visit_tuple_type(self, left: TupleType) -> bool: return True return False elif isinstance(right, TupleType): - if right.erased_typevartuple: - return True # treat it like Any # If right has a variadic unpack this needs special handling. If there is a TypeVarTuple # unpack, item count must coincide. If the left has variadic unpack but right # doesn't have one, we will fall through to False down the line. @@ -829,8 +824,6 @@ def variadic_tuple_subtype(self, left: TupleType, right: TupleType) -> bool: right_unpack = right.items[right_unpack_index] assert isinstance(right_unpack, UnpackType) right_unpacked = get_proper_type(right_unpack.type) - if isinstance(right_unpacked, TupleType) and right_unpacked.erased_typevartuple: - return True # treat it as Any if not isinstance(right_unpacked, Instance): # This case should be handled by the caller. return False @@ -1609,12 +1602,15 @@ def are_parameters_compatible( if are_trivial_parameters(right) and not is_proper_subtype: return True trivial_suffix = is_trivial_suffix(right) and not is_proper_subtype - # erased typevartuples, like erased paramspecs or erased typevars are trivial + + # tuple[Any, ...] allows any number of arguments, not just infinite. if right_star and isinstance(right_star.typ, UnpackType): right_star_inner_type = get_proper_type(right_star.typ.type) trivial_varargs = ( - isinstance(right_star_inner_type, TupleType) - and right_star_inner_type.erased_typevartuple + isinstance(right_star_inner_type, Instance) + and right_star_inner_type.type.fullname == "builtins.tuple" + and len(right_star_inner_type.args) == 1 + and isinstance(right_star_inner_type.args[0], AnyType) ) else: trivial_varargs = False diff --git a/mypy/test/testtypes.py b/mypy/test/testtypes.py index d69350d59e46..0218d33cc124 100644 --- a/mypy/test/testtypes.py +++ b/mypy/test/testtypes.py @@ -1560,7 +1560,7 @@ def make_call(*items: tuple[str, str | None]) -> CallExpr: class TestExpandTypeLimitGetProperType(TestCase): # WARNING: do not increase this number unless absolutely necessary, # and you understand what you are doing. - ALLOWED_GET_PROPER_TYPES = 10 + ALLOWED_GET_PROPER_TYPES = 9 @skipUnless(mypy.expandtype.__file__.endswith(".py"), "Skip for compiled mypy") def test_count_get_proper_type(self) -> None: diff --git a/mypy/types.py b/mypy/types.py index 5f99d282acb0..91b40536f1cf 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -2128,8 +2128,6 @@ def with_normalized_var_args(self) -> Self: # this should be done once in semanal_typeargs.py for user-defined types, # and we ourselves rarely construct such type. return self - if unpacked.erased_typevartuple: - return self unpack_index = find_unpack_in_list(unpacked.items) if unpack_index == 0 and len(unpacked.items) > 1: # Already normalized. @@ -2356,16 +2354,13 @@ class TupleType(ProperType): a tuple base class. Use mypy.typeops.tuple_fallback to calculate the precise fallback type derived from item types. implicit: If True, derived from a tuple expression (t,....) instead of Tuple[t, ...] - erased_typevartuple: If True, this came from a (now-erased) TypeVarTuple. This - indicates that this tuple should act more like an Any. """ - __slots__ = ("items", "partial_fallback", "implicit", "erased_typevartuple") + __slots__ = ("items", "partial_fallback", "implicit") items: list[Type] partial_fallback: Instance implicit: bool - erased_typevartuple: bool def __init__( self, @@ -2374,13 +2369,11 @@ def __init__( line: int = -1, column: int = -1, implicit: bool = False, - erased_typevartuple: bool = False, ) -> None: super().__init__(line, column) self.partial_fallback = fallback self.items = items self.implicit = implicit - self.erased_typevartuple = erased_typevartuple def can_be_true_default(self) -> bool: if self.can_be_any_bool(): @@ -2421,16 +2414,12 @@ def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_tuple_type(self) def __hash__(self) -> int: - return hash((tuple(self.items), self.partial_fallback, self.erased_typevartuple)) + return hash((tuple(self.items), self.partial_fallback)) def __eq__(self, other: object) -> bool: if not isinstance(other, TupleType): return NotImplemented - return ( - self.items == other.items - and self.partial_fallback == other.partial_fallback - and self.erased_typevartuple == other.erased_typevartuple - ) + return self.items == other.items and self.partial_fallback == other.partial_fallback def serialize(self) -> JsonDict: return { @@ -2438,7 +2427,6 @@ def serialize(self) -> JsonDict: "items": [t.serialize() for t in self.items], "partial_fallback": self.partial_fallback.serialize(), "implicit": self.implicit, - "erased_typevartuple": self.erased_typevartuple, } @classmethod @@ -2448,25 +2436,16 @@ def deserialize(cls, data: JsonDict) -> TupleType: [deserialize_type(t) for t in data["items"]], Instance.deserialize(data["partial_fallback"]), implicit=data["implicit"], - erased_typevartuple=data["erased_typevartuple"], ) def copy_modified( - self, - *, - fallback: Instance | None = None, - items: list[Type] | None = None, - erased_typevartuple: bool | None = None, + self, *, fallback: Instance | None = None, items: list[Type] | None = None ) -> TupleType: if fallback is None: fallback = self.partial_fallback if items is None: items = self.items - if erased_typevartuple is None: - erased_typevartuple = self.erased_typevartuple - return TupleType( - items, fallback, self.line, self.column, erased_typevartuple=erased_typevartuple - ) + return TupleType(items, fallback, self.line, self.column) def slice( self, begin: int | None, end: int | None, stride: int | None, *, fallback: Instance | None @@ -2519,9 +2498,7 @@ def slice( return None else: slice_items = self.items[begin:end:stride] - return TupleType( - slice_items, fallback, self.line, self.column, self.implicit, self.erased_typevartuple - ) + return TupleType(slice_items, fallback, self.line, self.column, self.implicit) class TypedDictType(ProperType): @@ -3400,8 +3377,6 @@ def visit_overloaded(self, t: Overloaded) -> str: return f"Overload({', '.join(a)})" def visit_tuple_type(self, t: TupleType) -> str: - if t.erased_typevartuple: - return "tuple[...]" s = self.list_str(t.items) or "()" tuple_name = "tuple" if self.options.use_lowercase_names() else "Tuple" if t.partial_fallback and t.partial_fallback.type: From 16d4dcff68d2804ce1c19ea62ebe7c1cd5dd9e55 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Wed, 28 Aug 2024 07:33:10 +0900 Subject: [PATCH 09/11] Fix isinstance argument --- mypy/subtypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 63592560a717..ad9d46afa990 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -1610,7 +1610,7 @@ def are_parameters_compatible( isinstance(right_star_inner_type, Instance) and right_star_inner_type.type.fullname == "builtins.tuple" and len(right_star_inner_type.args) == 1 - and isinstance(right_star_inner_type.args[0], AnyType) + and isinstance(get_proper_type(right_star_inner_type.args[0]), AnyType) ) else: trivial_varargs = False From 9f5509c4f4ebc49b050e2b35cc5badf4344cf900 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Wed, 28 Aug 2024 15:50:25 +0900 Subject: [PATCH 10/11] Remove erroneous comment --- test-data/unit/check-typevar-tuple.test | 1 - 1 file changed, 1 deletion(-) diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index b8d05dcb659f..f49e1b3c6613 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -2136,7 +2136,6 @@ get_items(b) # E: Argument 1 to "get_items" has incompatible type "Bad"; expect # N: def items(self) -> Tuple[Never, ...] \ # N: Got: \ # N: def items(self) -> List[int] -# TODO: this *should* work. match(b) # E: Argument 1 to "match" has incompatible type "Bad"; expected "PC[Unpack[Tuple[Never, ...]]]" \ # N: Following member(s) of "Bad" have conflicts: \ # N: Expected: \ From 08748f2e9c97a788b0ad7a89987c617ff414a6f6 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sat, 28 Sep 2024 01:19:20 +0900 Subject: [PATCH 11/11] Copy over correctness changes --- mypy/subtypes.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 73a261efb880..45e940315390 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -1603,14 +1603,13 @@ def are_parameters_compatible( return True trivial_suffix = is_trivial_suffix(right) and not is_proper_subtype - # tuple[Any, ...] allows any number of arguments, not just infinite. + # def _(*a: Unpack[tuple[object, ...]]) allows any number of arguments, not just infinite. if right_star and isinstance(right_star.typ, UnpackType): right_star_inner_type = get_proper_type(right_star.typ.type) trivial_varargs = ( isinstance(right_star_inner_type, Instance) and right_star_inner_type.type.fullname == "builtins.tuple" and len(right_star_inner_type.args) == 1 - and isinstance(get_proper_type(right_star_inner_type.args[0]), AnyType) ) else: trivial_varargs = False @@ -1652,14 +1651,17 @@ def are_parameters_compatible( # Furthermore, if we're checking for compatibility in all cases, # we confirm that if R accepts an infinite number of arguments, # L must accept the same. - def _incompatible(left_arg: FormalArgument | None, right_arg: FormalArgument | None) -> bool: + def _incompatible( + left_arg: FormalArgument | None, right_arg: FormalArgument | None, varargs: bool + ) -> bool: if right_arg is None: return False if left_arg is None: - return not allow_partial_overlap and not trivial_suffix and not trivial_varargs + return not (allow_partial_overlap or trivial_suffix or (varargs and trivial_varargs)) + return not is_compat(right_arg.typ, left_arg.typ) - if _incompatible(left_star, right_star) or _incompatible(left_star2, right_star2): + if _incompatible(left_star, right_star, True) or _incompatible(left_star2, right_star2, False): return False # Phase 1b: Check non-star args: for every arg right can accept, left must @@ -1684,7 +1686,6 @@ def _incompatible(left_arg: FormalArgument | None, right_arg: FormalArgument | N # Phase 1c: Check var args. Right has an infinite series of optional positional # arguments. Get all further positional args of left, and make sure # they're more general than the corresponding member in right. - # TODO: are we handling UnpackType correctly here? if right_star is not None and not trivial_suffix and not trivial_varargs: # Synthesize an anonymous formal argument for the right right_by_position = right.try_synthesizing_arg_from_vararg(None)