diff --git a/mypy/applytype.py b/mypy/applytype.py index 6abe7f0022f8..884be287e33d 100644 --- a/mypy/applytype.py +++ b/mypy/applytype.py @@ -3,15 +3,13 @@ from typing import Callable, Sequence import mypy.subtypes -from mypy.expandtype import expand_type, expand_unpack_with_variables -from mypy.nodes import ARG_STAR, Context +from mypy.expandtype import expand_type +from mypy.nodes import Context from mypy.types import ( AnyType, CallableType, - Instance, ParamSpecType, PartialType, - TupleType, Type, TypeVarId, TypeVarLikeType, @@ -21,7 +19,6 @@ UnpackType, get_proper_type, ) -from mypy.typevartuples import find_unpack_in_list, replace_starargs def get_target_type( @@ -107,6 +104,8 @@ def apply_generic_arguments( if target_type is not None: id_to_type[tvar.id] = target_type + # TODO: validate arg_kinds/arg_names for ParamSpec and TypeVarTuple replacements, + # not just type variable bounds above. param_spec = callable.param_spec() if param_spec is not None: nt = id_to_type.get(param_spec.id) @@ -122,55 +121,9 @@ def apply_generic_arguments( # Apply arguments to argument types. var_arg = callable.var_arg() if var_arg is not None and isinstance(var_arg.typ, UnpackType): - star_index = callable.arg_kinds.index(ARG_STAR) - callable = callable.copy_modified( - arg_types=( - [expand_type(at, id_to_type) for at in callable.arg_types[:star_index]] - + [callable.arg_types[star_index]] - + [expand_type(at, id_to_type) for at in callable.arg_types[star_index + 1 :]] - ) - ) - - unpacked_type = get_proper_type(var_arg.typ.type) - if isinstance(unpacked_type, TupleType): - # Assuming for now that because we convert prefixes to positional arguments, - # the first argument is always an unpack. - expanded_tuple = expand_type(unpacked_type, id_to_type) - if isinstance(expanded_tuple, TupleType): - # TODO: handle the case where the tuple has an unpack. This will - # hit an assert below. - expanded_unpack = find_unpack_in_list(expanded_tuple.items) - if expanded_unpack is not None: - callable = callable.copy_modified( - arg_types=( - callable.arg_types[:star_index] - + [expanded_tuple] - + callable.arg_types[star_index + 1 :] - ) - ) - else: - callable = replace_starargs(callable, expanded_tuple.items) - else: - # TODO: handle the case for if we get a variable length tuple. - assert False, f"mypy bug: unimplemented case, {expanded_tuple}" - elif isinstance(unpacked_type, TypeVarTupleType): - expanded_tvt = expand_unpack_with_variables(var_arg.typ, id_to_type) - if isinstance(expanded_tvt, list): - for t in expanded_tvt: - assert not isinstance(t, UnpackType) - callable = replace_starargs(callable, expanded_tvt) - else: - assert isinstance(expanded_tvt, Instance) - assert expanded_tvt.type.fullname == "builtins.tuple" - callable = callable.copy_modified( - arg_types=( - callable.arg_types[:star_index] - + [expanded_tvt.args[0]] - + callable.arg_types[star_index + 1 :] - ) - ) - else: - assert False, "mypy bug: unhandled case applying unpack" + callable = expand_type(callable, id_to_type) + assert isinstance(callable, CallableType) + return callable.copy_modified(variables=[tv for tv in tvars if tv.id not in id_to_type]) else: callable = callable.copy_modified( arg_types=[expand_type(at, id_to_type) for at in callable.arg_types] @@ -183,6 +136,9 @@ def apply_generic_arguments( type_guard = None # The callable may retain some type vars if only some were applied. + # TODO: move apply_poly() logic from checkexpr.py here when new inference + # becomes universally used (i.e. in all passes + in unification). + # With this new logic we can actually *add* some new free variables. remaining_tvars = [tv for tv in tvars if tv.id not in id_to_type] return callable.copy_modified( diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 797473f7f58f..420cfd990820 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -2373,11 +2373,15 @@ def check_argument_types( ] actual_kinds = [nodes.ARG_STAR] + [nodes.ARG_POS] * (len(actuals) - 1) - assert isinstance(orig_callee_arg_type, TupleType) - assert orig_callee_arg_type.items - callee_arg_types = orig_callee_arg_type.items + # TODO: can we really assert this? What if formal is just plain Unpack[Ts]? + assert isinstance(orig_callee_arg_type, UnpackType) + assert isinstance(orig_callee_arg_type.type, ProperType) and isinstance( + orig_callee_arg_type.type, TupleType + ) + assert orig_callee_arg_type.type.items + callee_arg_types = orig_callee_arg_type.type.items callee_arg_kinds = [nodes.ARG_STAR] + [nodes.ARG_POS] * ( - len(orig_callee_arg_type.items) - 1 + len(orig_callee_arg_type.type.items) - 1 ) expanded_tuple = True @@ -5853,8 +5857,9 @@ def visit_param_spec(self, t: ParamSpecType) -> Type: return super().visit_param_spec(t) def visit_type_var_tuple(self, t: TypeVarTupleType) -> Type: - # TODO: Support polymorphic apply for TypeVarTuple. - raise PolyTranslationError() + if t in self.poly_tvars and t not in self.bound_tvars: + raise PolyTranslationError() + return super().visit_type_var_tuple(t) def visit_type_alias_type(self, t: TypeAliasType) -> Type: if not t.args: @@ -5888,7 +5893,6 @@ def visit_instance(self, t: Instance) -> Type: return t.copy_modified(args=new_args) # There is the same problem with callback protocols as with aliases # (callback protocols are essentially more flexible aliases to callables). - # Note: consider supporting bindings in instances, e.g. LRUCache[[x: T], T]. if t.args and t.type.is_protocol and t.type.protocol_members == ["__call__"]: if t.type in self.seen_aliases: raise PolyTranslationError() @@ -5923,6 +5927,12 @@ def __init__(self) -> None: def visit_type_var(self, t: TypeVarType) -> bool: return True + def visit_param_spec(self, t: ParamSpecType) -> bool: + return True + + def visit_type_var_tuple(self, t: TypeVarTupleType) -> bool: + return True + def has_erased_component(t: Type | None) -> bool: return t is not None and t.accept(HasErasedComponentsQuery()) diff --git a/mypy/constraints.py b/mypy/constraints.py index 04c3378ce16b..26504ed06b3e 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -9,7 +9,16 @@ from mypy.argmap import ArgTypeExpander from mypy.erasetype import erase_typevars from mypy.maptype import map_instance_to_supertype -from mypy.nodes import ARG_OPT, ARG_POS, ARG_STAR, ARG_STAR2, CONTRAVARIANT, COVARIANT, ArgKind +from mypy.nodes import ( + ARG_OPT, + ARG_POS, + ARG_STAR, + ARG_STAR2, + CONTRAVARIANT, + COVARIANT, + ArgKind, + TypeInfo, +) from mypy.types import ( TUPLE_LIKE_INSTANCE_NAMES, AnyType, @@ -70,6 +79,8 @@ class Constraint: def __init__(self, type_var: TypeVarLikeType, op: int, target: Type) -> None: self.type_var = type_var.id self.op = op + # TODO: should we add "assert not isinstance(target, UnpackType)"? + # UnpackType is a synthetic type, and is never valid as a constraint target. self.target = target self.origin_type_var = type_var # These are additional type variables that should be solved for together with type_var. @@ -940,17 +951,20 @@ def visit_callable_type(self, template: CallableType) -> list[Constraint]: if not template.is_ellipsis_args: unpack_present = find_unpack_in_list(template.arg_types) if unpack_present is not None: - ( - unpack_constraints, - cactual_args_t, - template_args_t, - ) = find_and_build_constraints_for_unpack( - tuple(cactual.arg_types), tuple(template.arg_types), self.direction + # We need to re-normalize args to the form they appear in tuples, + # for callables we always pack the suffix inside another tuple. + unpack = template.arg_types[unpack_present] + assert isinstance(unpack, UnpackType) + tuple_type = get_tuple_fallback_from_unpack(unpack) + template_types = repack_callable_args(template, tuple_type) + actual_types = repack_callable_args(cactual, tuple_type) + # Now we can use the same general helper as for tuple types. + unpack_constraints = build_constraints_for_simple_unpack( + template_types, actual_types, neg_op(self.direction) ) - template_args = list(template_args_t) - cactual_args = list(cactual_args_t) + template_args = [] + cactual_args = [] res.extend(unpack_constraints) - assert len(template_args) == len(cactual_args) else: template_args = template.arg_types cactual_args = cactual.arg_types @@ -961,7 +975,7 @@ def visit_callable_type(self, template: CallableType) -> list[Constraint]: # branches), and in Callable vs Callable inference (two branches). for t, a in zip(template_args, cactual_args): # This avoids bogus constraints like T <: P.args - if isinstance(a, ParamSpecType): + if isinstance(a, (ParamSpecType, UnpackType)): # TODO: can we infer something useful for *T vs P? continue # Negate direction due to function argument type contravariance. @@ -1093,13 +1107,11 @@ def visit_tuple_type(self, template: TupleType) -> list[Constraint]: return [Constraint(type_var=unpacked_type, op=self.direction, target=actual)] else: assert isinstance(actual, TupleType) - ( - unpack_constraints, - actual_items, - template_items, - ) = find_and_build_constraints_for_unpack( - tuple(actual.items), tuple(template.items), self.direction + unpack_constraints = build_constraints_for_simple_unpack( + template.items, actual.items, self.direction ) + actual_items: tuple[Type, ...] = () + template_items: tuple[Type, ...] = () res.extend(unpack_constraints) elif isinstance(actual, TupleType): actual_items = tuple(actual.items) @@ -1232,28 +1244,132 @@ def find_matching_overload_items( return res -def find_and_build_constraints_for_unpack( - mapped: tuple[Type, ...], template: tuple[Type, ...], direction: int -) -> tuple[list[Constraint], tuple[Type, ...], tuple[Type, ...]]: - mapped_prefix_len = find_unpack_in_list(mapped) - if mapped_prefix_len is not None: - mapped_suffix_len: int | None = len(mapped) - mapped_prefix_len - 1 +def get_tuple_fallback_from_unpack(unpack: UnpackType) -> TypeInfo | None: + """Get builtins.tuple type from available types to construct homogeneous tuples.""" + tp = get_proper_type(unpack.type) + if isinstance(tp, Instance) and tp.type.fullname == "builtins.tuple": + return tp.type + if isinstance(tp, TypeVarTupleType): + return tp.tuple_fallback.type + if isinstance(tp, TupleType): + for base in tp.partial_fallback.type.mro: + if base.fullname == "builtins.tuple": + return base + return None + + +def repack_callable_args(callable: CallableType, tuple_type: TypeInfo | None) -> list[Type]: + """Present callable with star unpack in a normalized form. + + Since positional arguments cannot follow star argument, they are packed in a suffix, + while prefix is represented as individual positional args. We want to put all in a single + list with unpack in the middle, and prefix/suffix on the sides (as they would appear + in e.g. a TupleType). + """ + if ARG_STAR not in callable.arg_kinds: + return callable.arg_types + star_index = callable.arg_kinds.index(ARG_STAR) + arg_types = callable.arg_types[:star_index] + star_type = callable.arg_types[star_index] + suffix_types = [] + if not isinstance(star_type, UnpackType): + if tuple_type is not None: + # Re-normalize *args: X -> *args: *tuple[X, ...] + star_type = UnpackType(Instance(tuple_type, [star_type])) + else: + # This is unfortunate, something like tuple[Any, ...] would be better. + star_type = UnpackType(AnyType(TypeOfAny.from_error)) else: - mapped_suffix_len = None + tp = get_proper_type(star_type.type) + if isinstance(tp, TupleType): + assert isinstance(tp.items[0], UnpackType) + star_type = tp.items[0] + suffix_types = tp.items[1:] + return arg_types + [star_type] + suffix_types - template_prefix_len = find_unpack_in_list(template) - assert template_prefix_len is not None - template_suffix_len = len(template) - template_prefix_len - 1 - return build_constraints_for_unpack( - mapped, - mapped_prefix_len, - mapped_suffix_len, - template, - template_prefix_len, - template_suffix_len, - direction, +def build_constraints_for_simple_unpack( + template_args: list[Type], actual_args: list[Type], direction: int +) -> list[Constraint]: + """Infer constraints between two lists of types with variadic items. + + This function is only supposed to be called when a variadic item is present in templates. + If there is no variadic item the actuals, we simply use split_with_prefix_and_suffix() + and infer prefix <: prefix, suffix <: suffix, variadic <: middle. If there is a variadic + item in the actuals we need to be more careful, only common prefix/suffix can generate + constraints, also we can only infer constraints for variadic template item, if template + prefix/suffix are shorter that actual ones, otherwise there may be partial overlap + between variadic items, for example if template prefix is longer: + + templates: T1, T2, Ts, Ts, Ts, ... + actuals: A1, As, As, As, ... + + Note: this function can only be called for builtin variadic constructors: Tuple and Callable, + for Instances variance depends on position, and a much more complex function + build_constraints_for_unpack() should be used. + """ + template_unpack = find_unpack_in_list(template_args) + assert template_unpack is not None + template_prefix = template_unpack + template_suffix = len(template_args) - template_prefix - 1 + + t_unpack = None + res = [] + + actual_unpack = find_unpack_in_list(actual_args) + if actual_unpack is None: + t_unpack = template_args[template_unpack] + if template_prefix + template_suffix > len(actual_args): + # These can't be subtypes of each-other, return fast. + assert isinstance(t_unpack, UnpackType) + if isinstance(t_unpack.type, TypeVarTupleType): + # Set TypeVarTuple to empty to improve error messages. + return [ + Constraint( + t_unpack.type, direction, TupleType([], t_unpack.type.tuple_fallback) + ) + ] + else: + return [] + common_prefix = template_prefix + common_suffix = template_suffix + else: + actual_prefix = actual_unpack + actual_suffix = len(actual_args) - actual_prefix - 1 + common_prefix = min(template_prefix, actual_prefix) + common_suffix = min(template_suffix, actual_suffix) + if actual_prefix >= template_prefix and actual_suffix >= template_suffix: + # This is the only case where we can guarantee there will be no partial overlap. + t_unpack = template_args[template_unpack] + + # Handle constraints from prefixes/suffixes first. + start, middle, end = split_with_prefix_and_suffix( + tuple(actual_args), common_prefix, common_suffix ) + for t, a in zip(template_args[:common_prefix], start): + res.extend(infer_constraints(t, a, direction)) + if common_suffix: + for t, a in zip(template_args[-common_suffix:], end): + res.extend(infer_constraints(t, a, direction)) + + if t_unpack is not None: + # Add constraint(s) for variadic item when possible. + assert isinstance(t_unpack, UnpackType) + tp = get_proper_type(t_unpack.type) + if isinstance(tp, Instance) and tp.type.fullname == "builtins.tuple": + # Homogeneous case *tuple[T, ...] <: [X, Y, Z, ...]. + for a in middle: + # TODO: should we use union instead of join here? + if not isinstance(a, UnpackType): + res.extend(infer_constraints(tp.args[0], a, direction)) + else: + a_tp = get_proper_type(a.type) + # This is the case *tuple[T, ...] <: *tuple[A, ...]. + if isinstance(a_tp, Instance) and a_tp.type.fullname == "builtins.tuple": + res.extend(infer_constraints(tp.args[0], a_tp.args[0], direction)) + elif isinstance(tp, TypeVarTupleType): + res.append(Constraint(tp, direction, TupleType(list(middle), tp.tuple_fallback))) + return res def build_constraints_for_unpack( @@ -1268,6 +1384,10 @@ def build_constraints_for_unpack( template_suffix_len: int, direction: int, ) -> tuple[list[Constraint], tuple[Type, ...], tuple[Type, ...]]: + # TODO: this function looks broken: + # a) it should take into account variances, but it doesn't + # b) it looks like both call sites always pass identical values to args (2, 3) and (5, 6) + # because after map_instance_to_supertype() both template and actual have same TypeInfo. if mapped_prefix_len is None: mapped_prefix_len = template_prefix_len if mapped_suffix_len is None: @@ -1314,4 +1434,4 @@ def build_constraints_for_unpack( if len(template_unpack.items) == len(mapped_middle): for template_arg, item in zip(template_unpack.items, mapped_middle): res.extend(infer_constraints(template_arg, item, direction)) - return (res, mapped_prefix + mapped_suffix, template_prefix + template_suffix) + return res, mapped_prefix + mapped_suffix, template_prefix + template_suffix diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 01d9c4463174..6f69e09936db 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -257,7 +257,7 @@ def visit_param_spec(self, t: ParamSpecType) -> Type: def visit_type_var_tuple(self, t: TypeVarTupleType) -> Type: # Sometimes solver may need to expand a type variable with (a copy of) itself # (usually together with other TypeVars, but it is hard to filter out TypeVarTuples). - repl = self.variables[t.id] + repl = self.variables.get(t.id, t) if isinstance(repl, TypeVarTupleType): return repl raise NotImplementedError @@ -269,45 +269,54 @@ def visit_unpack_type(self, t: UnpackType) -> Type: # Relevant sections that can call unpack should call expand_unpack() # instead. # However, if the item is a variadic tuple, we can simply carry it over. + # In particular, if we expand A[*tuple[T, ...]] with substitutions {T: str}, # it is hard to assert this without getting proper type. return UnpackType(t.type.accept(self)) - def expand_unpack(self, t: UnpackType) -> list[Type] | Instance | AnyType | None: - return expand_unpack_with_variables(t, self.variables) + def expand_unpack(self, t: UnpackType) -> list[Type] | AnyType | UninhabitedType: + assert isinstance(t.type, TypeVarTupleType) + repl = get_proper_type(self.variables.get(t.type.id, t.type)) + if isinstance(repl, TupleType): + return repl.items + elif ( + isinstance(repl, Instance) + and repl.type.fullname == "builtins.tuple" + or isinstance(repl, TypeVarTupleType) + ): + return [UnpackType(typ=repl)] + elif isinstance(repl, (AnyType, UninhabitedType)): + # tuple[Any, ...] for Any would be better, but we don't have + # the type info to construct that type here. + return repl + else: + raise RuntimeError(f"Invalid type replacement to expand: {repl}") def visit_parameters(self, t: Parameters) -> Type: return t.copy_modified(arg_types=self.expand_types(t.arg_types)) + # TODO: can we simplify this method? It is too long. def interpolate_args_for_unpack( self, t: CallableType, var_arg: UnpackType ) -> tuple[list[str | None], list[ArgKind], list[Type]]: star_index = t.arg_kinds.index(ARG_STAR) - # We have something like Unpack[Tuple[X1, X2, Unpack[Ts], Y1, Y2]] var_arg_type = get_proper_type(var_arg.type) + # We have something like Unpack[Tuple[Unpack[Ts], X1, X2]] if isinstance(var_arg_type, TupleType): expanded_tuple = var_arg_type.accept(self) - # TODO: handle the case that expanded_tuple is a variable length tuple. assert isinstance(expanded_tuple, ProperType) and isinstance(expanded_tuple, TupleType) expanded_items = expanded_tuple.items else: + # We have plain Unpack[Ts] expanded_items_res = self.expand_unpack(var_arg) if isinstance(expanded_items_res, list): expanded_items = expanded_items_res - elif ( - isinstance(expanded_items_res, Instance) - and expanded_items_res.type.fullname == "builtins.tuple" - ): - # TODO: We shouldnt't simply treat this as a *arg because of suffix handling - # (there cannot be positional args after a *arg) + else: + # We got Any or arg_types = ( - t.arg_types[:star_index] - + [expanded_items_res.args[0]] - + t.arg_types[star_index + 1 :] + t.arg_types[:star_index] + [expanded_items_res] + t.arg_types[star_index + 1 :] ) - return (t.arg_names, t.arg_kinds, arg_types) - else: - return (t.arg_names, t.arg_kinds, t.arg_types) + return t.arg_names, t.arg_kinds, arg_types expanded_unpack_index = find_unpack_in_list(expanded_items) # This is the case where we just have Unpack[Tuple[X1, X2, X3]] @@ -337,13 +346,14 @@ def interpolate_args_for_unpack( expanded_unpack = expanded_items[expanded_unpack_index] assert isinstance(expanded_unpack, UnpackType) - # Extract the typevartuple so we can get a tuple fallback from it. + # Extract the TypeVarTuple, so we can get a tuple fallback from it. expanded_unpacked_tvt = expanded_unpack.type if isinstance(expanded_unpacked_tvt, TypeVarTupleType): fallback = expanded_unpacked_tvt.tuple_fallback else: # This can happen when tuple[Any, ...] is used to "patch" a variadic - # generic type without type arguments provided. + # generic type without type arguments provided, or when substitution is + # homogeneous tuple. assert isinstance(expanded_unpacked_tvt, ProperType) assert isinstance(expanded_unpacked_tvt, Instance) assert expanded_unpacked_tvt.type.fullname == "builtins.tuple" @@ -354,18 +364,31 @@ def interpolate_args_for_unpack( arg_kinds = ( t.arg_kinds[:star_index] + [ARG_POS] * prefix_len + t.arg_kinds[star_index:] ) - arg_types = ( - self.expand_types(t.arg_types[:star_index]) - + expanded_items[:prefix_len] - # Constructing the Unpack containing the tuple without the prefix. - + [ - UnpackType(TupleType(expanded_items[prefix_len:], fallback)) - if len(expanded_items) - prefix_len > 1 - else expanded_items[0] - ] - + self.expand_types(t.arg_types[star_index + 1 :]) - ) - return (arg_names, arg_kinds, arg_types) + if ( + len(expanded_items) == 1 + and isinstance(expanded_unpack.type, ProperType) + and isinstance(expanded_unpack.type, Instance) + ): + assert expanded_unpack.type.type.fullname == "builtins.tuple" + # Normalize *args: *tuple[X, ...] -> *args: X + arg_types = ( + self.expand_types(t.arg_types[:star_index]) + + [expanded_unpack.type.args[0]] + + self.expand_types(t.arg_types[star_index + 1 :]) + ) + else: + arg_types = ( + self.expand_types(t.arg_types[:star_index]) + + expanded_items[:prefix_len] + # Constructing the Unpack containing the tuple without the prefix. + + [ + UnpackType(TupleType(expanded_items[prefix_len:], fallback)) + if len(expanded_items) - prefix_len > 1 + else expanded_items[prefix_len] + ] + + self.expand_types(t.arg_types[star_index + 1 :]) + ) + return arg_names, arg_kinds, arg_types def visit_callable_type(self, t: CallableType) -> CallableType: param_spec = t.param_spec() @@ -430,7 +453,7 @@ def visit_overloaded(self, t: Overloaded) -> Type: def expand_types_with_unpack( self, typs: Sequence[Type] - ) -> list[Type] | AnyType | UninhabitedType | Instance: + ) -> list[Type] | AnyType | UninhabitedType: """Expands a list of types that has an unpack. In corner cases, this can return a type rather than a list, in which case this @@ -444,15 +467,8 @@ def expand_types_with_unpack( for item in typs: if isinstance(item, UnpackType) and isinstance(item.type, TypeVarTupleType): unpacked_items = self.expand_unpack(item) - if unpacked_items is None: - # TODO: better error, something like tuple of unknown? - return UninhabitedType() - elif isinstance(unpacked_items, Instance): - if len(typs) == 1: - return unpacked_items - else: - assert False, "Invalid unpack of variable length tuple" - elif isinstance(unpacked_items, AnyType): + if isinstance(unpacked_items, (AnyType, UninhabitedType)): + # TODO: better error for , something like tuple of unknown? return unpacked_items else: items.extend(unpacked_items) @@ -464,6 +480,14 @@ def expand_types_with_unpack( def visit_tuple_type(self, t: TupleType) -> Type: items = self.expand_types_with_unpack(t.items) if isinstance(items, list): + if len(items) == 1: + # Normalize Tuple[*Tuple[X, ...]] -> Tuple[X, ...] + item = items[0] + if isinstance(item, UnpackType): + assert isinstance(item.type, ProperType) + if isinstance(item.type, Instance): + assert item.type.type.fullname == "builtins.tuple" + return item.type fallback = t.partial_fallback.accept(self) assert isinstance(fallback, ProperType) and isinstance(fallback, Instance) return t.copy_modified(items=items, fallback=fallback) @@ -509,6 +533,7 @@ def visit_type_alias_type(self, t: TypeAliasType) -> Type: # alias itself), so we just expand the arguments. args = self.expand_types_with_unpack(t.args) if isinstance(args, list): + # TODO: normalize if target is Tuple, and args are [*tuple[X, ...]]? return t.copy_modified(args=args) else: return args @@ -520,34 +545,6 @@ def expand_types(self, types: Iterable[Type]) -> list[Type]: return a -def expand_unpack_with_variables( - t: UnpackType, variables: Mapping[TypeVarId, Type] -) -> list[Type] | Instance | AnyType | None: - """May return either a list of types to unpack to, any, or a single - variable length tuple. The latter may not be valid in all contexts. - """ - if isinstance(t.type, TypeVarTupleType): - repl = get_proper_type(variables.get(t.type.id, t)) - if isinstance(repl, TupleType): - return repl.items - elif isinstance(repl, Instance) and repl.type.fullname == "builtins.tuple": - return repl - elif isinstance(repl, AnyType): - # tuple[Any, ...] would be better, but we don't have - # the type info to construct that type here. - return repl - elif isinstance(repl, TypeVarTupleType): - return [UnpackType(typ=repl)] - elif isinstance(repl, UnpackType): - return [repl] - elif isinstance(repl, UninhabitedType): - return None - else: - raise NotImplementedError(f"Invalid type replacement to expand: {repl}") - else: - raise NotImplementedError(f"Invalid type to expand: {t.type}") - - @overload def expand_self_type(var: Var, typ: ProperType, replacement: ProperType) -> ProperType: ... diff --git a/mypy/solve.py b/mypy/solve.py index 4b2b899c2a8d..5945d97ed85a 100644 --- a/mypy/solve.py +++ b/mypy/solve.py @@ -3,7 +3,7 @@ from __future__ import annotations from collections import defaultdict -from typing import Iterable, Sequence +from typing import Iterable, Sequence, Tuple from typing_extensions import TypeAlias as _TypeAlias from mypy.constraints import SUBTYPE_OF, SUPERTYPE_OF, Constraint, infer_constraints @@ -19,13 +19,16 @@ NoneType, ParamSpecType, ProperType, + TupleType, Type, TypeOfAny, TypeVarId, TypeVarLikeType, + TypeVarTupleType, TypeVarType, UninhabitedType, UnionType, + UnpackType, get_proper_type, ) from mypy.typestate import type_state @@ -330,6 +333,23 @@ def is_trivial_bound(tp: ProperType) -> bool: return isinstance(tp, Instance) and tp.type.fullname == "builtins.object" +def find_linear(c: Constraint) -> Tuple[bool, TypeVarId | None]: + """Find out if this constraint represent a linear relationship, return target id if yes.""" + if isinstance(c.origin_type_var, TypeVarType): + if isinstance(c.target, TypeVarType): + return True, c.target.id + if isinstance(c.origin_type_var, ParamSpecType): + if isinstance(c.target, ParamSpecType) and not c.target.prefix.arg_types: + return True, c.target.id + if isinstance(c.origin_type_var, TypeVarTupleType): + target = get_proper_type(c.target) + if isinstance(target, TupleType) and len(target.items) == 1: + item = target.items[0] + if isinstance(item, UnpackType) and isinstance(item.type, TypeVarTupleType): + return True, item.type.id + return False, None + + def transitive_closure( tvars: list[TypeVarId], constraints: list[Constraint] ) -> tuple[Graph, Bounds, Bounds]: @@ -361,16 +381,15 @@ def transitive_closure( c = remaining.pop() # Note that ParamSpec constraint P <: Q may be considered linear only if Q has no prefix, # for cases like P <: Concatenate[T, Q] we should consider this non-linear and put {P} and - # {T, Q} into separate SCCs. - if ( - isinstance(c.target, TypeVarType) - or isinstance(c.target, ParamSpecType) - and not c.target.prefix.arg_types - ) and c.target.id in tvars: + # {T, Q} into separate SCCs. Similarly, Ts <: Tuple[*Us] considered linear, while + # Ts <: Tuple[*Us, U] is non-linear. + is_linear, target_id = find_linear(c) + if is_linear and target_id in tvars: + assert target_id is not None if c.op == SUBTYPE_OF: - lower, upper = c.type_var, c.target.id + lower, upper = c.type_var, target_id else: - lower, upper = c.target.id, c.type_var + lower, upper = target_id, c.type_var if (lower, upper) in graph: continue graph |= { diff --git a/mypy/typeops.py b/mypy/typeops.py index d746ea701fde..22dbd9e9f42e 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -973,6 +973,9 @@ def visit_type_var(self, t: TypeVarType) -> list[TypeVarLikeType]: def visit_param_spec(self, t: ParamSpecType) -> list[TypeVarLikeType]: return [t] if self.include_all else [] + def visit_type_var_tuple(self, t: TypeVarTupleType) -> list[TypeVarLikeType]: + return [t] if self.include_all else [] + def custom_special_method(typ: Type, name: str, check_all: bool = False) -> bool: """Does this type have a custom special method such as __format__() or __eq__()? diff --git a/mypy/types.py b/mypy/types.py index 359ca713616b..d4e2fc7cb63c 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1045,7 +1045,8 @@ class UnpackType(ProperType): or unpacking * syntax. The inner type should be either a TypeVarTuple, a constant size - tuple, or a variable length tuple, or a union of one of those. + tuple, or a variable length tuple. Type aliases to these are not allowed, + except during semantic analysis. """ __slots__ = ["type"] @@ -2260,6 +2261,10 @@ def __init__( ) -> None: super().__init__(line, column) self.partial_fallback = fallback + # TODO: flatten/normalize unpack items (very similar to unions) here. + # Probably also for instances, type aliases, callables, and Unpack itself. For example, + # tuple[*tuple[X, ...], ...] -> tuple[X, ...] and Tuple[*tuple[X, ...]] -> tuple[X, ...]. + # Currently normalization happens in expand_type() et al., which is sub-optimal. self.items = items self.implicit = implicit diff --git a/mypy/typevartuples.py b/mypy/typevartuples.py index ac5f4e43c3bf..29c800140eec 100644 --- a/mypy/typevartuples.py +++ b/mypy/typevartuples.py @@ -4,9 +4,7 @@ from typing import Sequence -from mypy.nodes import ARG_POS, ARG_STAR from mypy.types import ( - CallableType, Instance, ProperType, Type, @@ -179,20 +177,3 @@ def extract_unpack(types: Sequence[Type]) -> ProperType | None: if isinstance(types[0], UnpackType): return get_proper_type(types[0].type) return None - - -def replace_starargs(callable: CallableType, types: list[Type]) -> CallableType: - star_index = callable.arg_kinds.index(ARG_STAR) - arg_kinds = ( - callable.arg_kinds[:star_index] - + [ARG_POS] * len(types) - + callable.arg_kinds[star_index + 1 :] - ) - arg_names = ( - callable.arg_names[:star_index] - + [None] * len(types) - + callable.arg_names[star_index + 1 :] - ) - arg_types = callable.arg_types[:star_index] + types + callable.arg_types[star_index + 1 :] - - return callable.copy_modified(arg_types=arg_types, arg_names=arg_names, arg_kinds=arg_kinds) diff --git a/test-data/unit/check-generics.test b/test-data/unit/check-generics.test index 1fac42b492a8..95a7bdd2b2cd 100644 --- a/test-data/unit/check-generics.test +++ b/test-data/unit/check-generics.test @@ -3144,7 +3144,7 @@ def pair(x: U) -> Callable[[V], Tuple[V, U]]: ... reveal_type(dec(id)) # N: Revealed type is "def [T] (T`2) -> T`2" reveal_type(dec(either)) # N: Revealed type is "def [T] (T`5, x: T`5) -> T`5" reveal_type(dec(pair)) # N: Revealed type is "def [T, U] (T`8, x: U`-1) -> Tuple[T`8, U`-1]" -# This is counter-intuitive but looks correct, dec matches itself only if P is empty +# This is counter-intuitive but looks correct, dec matches itself only if P can be empty reveal_type(dec(dec)) # N: Revealed type is "def [T, S] (T`11, f: def () -> def (T`11) -> S`12) -> S`12" [builtins fixtures/list.pyi] @@ -3179,7 +3179,6 @@ P = ParamSpec('P') Q = ParamSpec('Q') class Foo(Generic[P]): ... -class Bar(Generic[P, T]): ... def dec(f: Callable[P, int]) -> Callable[P, Foo[P]]: ... h: Callable[Concatenate[T, Q], int] @@ -3263,3 +3262,144 @@ def pop_off(fn: Callable[Concatenate[T1, P], T2]) -> Callable[P, Callable[[T1], def test(command: Foo[Q]) -> Foo[Q]: ... reveal_type(test) # N: Revealed type is "def () -> def [Q] (__main__.Foo[Q`-1]) -> __main__.Foo[Q`-1]" [builtins fixtures/tuple.pyi] + +[case testInferenceAgainstGenericVariadicBasicInList] +# flags: --new-type-inference +from typing import Tuple, TypeVar, List, Callable +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +Ts = TypeVarTuple("Ts") +def dec(f: Callable[[Unpack[Ts]], T]) -> Callable[[Unpack[Ts]], List[T]]: ... + +U = TypeVar("U") +V = TypeVar("V") +def id(x: U) -> U: ... +def either(x: U, y: U) -> U: ... +def pair(x: U, y: V) -> Tuple[U, V]: ... + +reveal_type(dec(id)) # N: Revealed type is "def [T] (T`2) -> builtins.list[T`2]" +reveal_type(dec(either)) # N: Revealed type is "def [T] (T`4, T`4) -> builtins.list[T`4]" +reveal_type(dec(pair)) # N: Revealed type is "def [U, V] (U`-1, V`-2) -> builtins.list[Tuple[U`-1, V`-2]]" +[builtins fixtures/tuple.pyi] + +[case testInferenceAgainstGenericVariadicBasicDeList] +# flags: --new-type-inference +from typing import Tuple, TypeVar, List, Callable +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +Ts = TypeVarTuple("Ts") +def dec(f: Callable[[Unpack[Ts]], List[T]]) -> Callable[[Unpack[Ts]], T]: ... + +U = TypeVar("U") +V = TypeVar("V") +def id(x: U) -> U: ... +def either(x: U, y: U) -> U: ... + +reveal_type(dec(id)) # N: Revealed type is "def [T] (builtins.list[T`2]) -> T`2" +reveal_type(dec(either)) # N: Revealed type is "def [T] (builtins.list[T`4], builtins.list[T`4]) -> T`4" +[builtins fixtures/tuple.pyi] + +[case testInferenceAgainstGenericVariadicPopOff] +# flags: --new-type-inference +from typing import TypeVar, Callable, List, Tuple +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") +def dec(f: Callable[[T, Unpack[Ts]], S]) -> Callable[[Unpack[Ts]], Callable[[T], S]]: ... + +U = TypeVar("U") +V = TypeVar("V") +def id(x: U) -> U: ... +def either(x: U, y: U) -> U: ... +def pair(x: U, y: V) -> Tuple[U, V]: ... + +reveal_type(dec(id)) # N: Revealed type is "def () -> def [T] (T`1) -> T`1" +reveal_type(dec(either)) # N: Revealed type is "def [T] (T`4) -> def (T`4) -> T`4" +reveal_type(dec(pair)) # N: Revealed type is "def [V] (V`-2) -> def [T] (T`7) -> Tuple[T`7, V`-2]" +reveal_type(dec(dec)) # N: Revealed type is "def () -> def [T, Ts, S] (def (T`-1, *Unpack[Ts`-2]) -> S`-3) -> def (*Unpack[Ts`-2]) -> def (T`-1) -> S`-3" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericVariadicPopOn] +# flags: --new-type-inference +from typing import TypeVar, Callable, List, Tuple +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") +def dec(f: Callable[[Unpack[Ts]], Callable[[T], S]]) -> Callable[[T, Unpack[Ts]], S]: ... + +U = TypeVar("U") +V = TypeVar("V") +def id() -> Callable[[U], U]: ... +def either(x: U) -> Callable[[U], U]: ... +def pair(x: U) -> Callable[[V], Tuple[V, U]]: ... + +reveal_type(dec(id)) # N: Revealed type is "def [T] (T`2) -> T`2" +reveal_type(dec(either)) # N: Revealed type is "def [T] (T`5, T`5) -> T`5" +reveal_type(dec(pair)) # N: Revealed type is "def [T, U] (T`8, U`-1) -> Tuple[T`8, U`-1]" +# This is counter-intuitive but looks correct, dec matches itself only if Ts is empty +reveal_type(dec(dec)) # N: Revealed type is "def [T, S] (T`11, def () -> def (T`11) -> S`12) -> S`12" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericVariadicVsVariadic] +# flags: --new-type-inference +from typing import TypeVar, Callable, List, Generic +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") +Us = TypeVarTuple("Us") + +class Foo(Generic[Unpack[Ts]]): ... +class Bar(Generic[Unpack[Ts], T]): ... + +def dec(f: Callable[[Unpack[Ts]], T]) -> Callable[[Unpack[Ts]], List[T]]: ... +# TODO: do not crash on Foo[Us] (with missing Unpack), instead give an error. +def f(*args: Unpack[Us]) -> Foo[Unpack[Us]]: ... +reveal_type(dec(f)) # N: Revealed type is "def [Ts] (*Unpack[Ts`1]) -> builtins.list[__main__.Foo[Unpack[Ts`1]]]" +g: Callable[[Unpack[Us]], Foo[Unpack[Us]]] +reveal_type(dec(g)) # N: Revealed type is "def [Ts] (*Unpack[Ts`3]) -> builtins.list[__main__.Foo[Unpack[Ts`3]]]" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericVariadicVsVariadicConcatenate] +# flags: --new-type-inference +from typing import TypeVar, Callable, Generic +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +S = TypeVar("S") +Ts = TypeVarTuple("Ts") +Us = TypeVarTuple("Us") + +class Foo(Generic[Unpack[Ts]]): ... + +def dec(f: Callable[[Unpack[Ts]], int]) -> Callable[[Unpack[Ts]], Foo[Unpack[Ts]]]: ... +h: Callable[[T, Unpack[Us]], int] +g: Callable[[T, Unpack[Us]], int] +h = g +reveal_type(dec(h)) # N: Revealed type is "def [T, Us] (T`-1, *Unpack[Us`-2]) -> __main__.Foo[T`-1, Unpack[Us`-2]]" +[builtins fixtures/list.pyi] + +[case testInferenceAgainstGenericVariadicSecondary] +# flags: --new-type-inference +from typing import TypeVar, Callable, Generic +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +Ts = TypeVarTuple("Ts") +Us = TypeVarTuple("Us") + +class Foo(Generic[Unpack[Ts]]): ... + +def dec(f: Callable[[Unpack[Ts]], Foo[Unpack[Ts]]]) -> Callable[[Unpack[Ts]], Foo[Unpack[Ts]]]: ... +g: Callable[[T], Foo[int]] +reveal_type(dec(g)) # N: Revealed type is "def (builtins.int) -> __main__.Foo[builtins.int]" +h: Callable[[Unpack[Us]], Foo[int]] +reveal_type(dec(g)) # N: Revealed type is "def (builtins.int) -> __main__.Foo[builtins.int]" +[builtins fixtures/list.pyi] diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index e822cea9304f..b28b2ead45e7 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -479,18 +479,18 @@ vargs: Tuple[int, ...] vargs_str: Tuple[str, ...] call(target=func, args=(0, 'foo')) -call(target=func, args=('bar', 'foo')) # E: Argument "target" to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[object, str], None]" -call(target=func, args=(True, 'foo', 0)) # E: Argument "target" to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[VarArg(object)], None]" -call(target=func, args=(0, 0, 'foo')) # E: Argument "target" to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[VarArg(object)], None]" -call(target=func, args=vargs) # E: Argument "target" to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[VarArg(object)], None]" +call(target=func, args=('bar', 'foo')) # E: Argument "target" to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[str, str], None]" +call(target=func, args=(True, 'foo', 0)) # E: Argument "target" to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[bool, str, int], None]" +call(target=func, args=(0, 0, 'foo')) # E: Argument "target" to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[int, int, str], None]" +call(target=func, args=vargs) # E: Argument "target" to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[VarArg(int)], None]" # NOTE: This behavior may be a bit contentious, it is maybe inconsistent with our handling of # PEP646 but consistent with our handling of callable constraints. call(target=func2, args=vargs) # E: Argument "target" to "call" has incompatible type "Callable[[int, int], None]"; expected "Callable[[VarArg(int)], None]" call(target=func3, args=vargs) call(target=func3, args=(0,1)) -call(target=func3, args=(0,'foo')) # E: Argument "target" to "call" has incompatible type "Callable[[VarArg(int)], None]"; expected "Callable[[VarArg(object)], None]" -call(target=func3, args=vargs_str) # E: Argument "target" to "call" has incompatible type "Callable[[VarArg(int)], None]"; expected "Callable[[VarArg(object)], None]" +call(target=func3, args=(0,'foo')) # E: Argument "target" to "call" has incompatible type "Callable[[VarArg(int)], None]"; expected "Callable[[int, str], None]" +call(target=func3, args=vargs_str) # E: Argument "target" to "call" has incompatible type "Callable[[VarArg(int)], None]"; expected "Callable[[VarArg(str)], None]" [builtins fixtures/tuple.pyi] [case testTypeVarTuplePep646CallableWithPrefixSuffix] @@ -561,11 +561,11 @@ class A: vargs: Tuple[int, ...] vargs_str: Tuple[str, ...] -call(A().func) # E: Argument 1 to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[VarArg(object)], None]" +call(A().func) # E: Argument 1 to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[], None]" call(A().func, 0, 'foo') -call(A().func, 0, 'foo', 0) # E: Argument 1 to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[VarArg(object)], None]" -call(A().func, 0) # E: Argument 1 to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[VarArg(object)], None]" -call(A().func, 0, 1) # E: Argument 1 to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[int, object], None]" +call(A().func, 0, 'foo', 0) # E: Argument 1 to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[int, str, int], None]" +call(A().func, 0) # E: Argument 1 to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[int], None]" +call(A().func, 0, 1) # E: Argument 1 to "call" has incompatible type "Callable[[int, str], None]"; expected "Callable[[int, int], None]" call(A().func2, 0, 0) call(A().func3, 0, 1, 2) call(A().func3) @@ -689,7 +689,7 @@ reveal_type(x) # N: Revealed type is "builtins.list[Tuple[Any, Unpack[builtins. B = Callable[[T, Unpack[Ts]], int] y: B -reveal_type(y) # N: Revealed type is "def (Any, *Unpack[builtins.tuple[Any, ...]]) -> builtins.int" +reveal_type(y) # N: Revealed type is "def (Any, *Any) -> builtins.int" C = G[T, Unpack[Ts], T] z: C @@ -711,7 +711,7 @@ reveal_type(x) # N: Revealed type is "builtins.list[Tuple[Any, Unpack[builtins. B = Callable[[T, S, Unpack[Ts]], int] y: B[int] # E: Bad number of arguments for type alias, expected: at least 2, given: 1 -reveal_type(y) # N: Revealed type is "def (Any, Any, *Unpack[builtins.tuple[Any, ...]]) -> builtins.int" +reveal_type(y) # N: Revealed type is "def (Any, Any, *Any) -> builtins.int" C = G[T, Unpack[Ts], S] z: C[int] # E: Bad number of arguments for type alias, expected: at least 2, given: 1