From 7cb733ad42eccaccd29380d46d5c222ccc2788cb Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 19 Jun 2024 23:48:35 +0100 Subject: [PATCH] Re-work overload overlap logic (#17392) Fixes https://github.com/python/mypy/issues/5510 OK, so I noticed during last couple years, that every other time I change something about type variables, a few unsafe overload overlap errors either appears or disappears. At some point I almost stopped looking at them. The problem is that unsafe overload overlap detection for generic callables is currently ad-hoc. However, as I started working on it, I discovered a bunch of foundational problems (and few smaller issues), so I decided to re-work the unsafe overload overlap detection. Here is a detailed summary: * Currently return type compatibility is decided using regular subtype check. Although it is technically correct, in most cases there is nothing wrong if first overload returns `list[Subtype]` and second returns `list[Supertype]`. All the unsafe overload story is about runtime values, not static types, so we should use `is_subset()` instead of `is_subtype()`, which is IIUC easy to implement: we simply need to consider all invariant types covariant. * Current implementation only checks for overlap between parameters, i.e. it checks if there are some calls that are valid for both overloads. But we also need to check that those common calls will not be always caught by the first overload. I assume it was not checked because, naively, we already check elsewhere that first overload doesn't completely shadow the second one. But this is not the same: first overload may be not more general overall, but when narrowed to common calls, it may be more general. Example of such false-positive (this is an oversimplified version of what is often used in situations with many optional positional arguments): ```python @overload def foo(x: object) -> object: ... @overload def foo(x: int = ...) -> int: ... ``` * Currently overlap for generic callables is decided using some weird two-way unification procedure, where we actually keep going on (with non-unified variables, and/or ``) if the right to left unification fails. TBH I never understood this. What we need is to find some set of type variable values that makes two overloads unsafely overlapping. Constraint inference may be used as a (good) source of such guesses, but is not decisive in any way. So instead I simply try all combinations of upper bounds and values. The main benefit of such approach is that it is guaranteed false-positive free. If such algorithm finds an overlap it is definitely an overlap. There are however false negatives, but we can incrementally tighten them in the future. * I am making `Any` overlap nothing when considering overloads. Currently it overlaps everything (i.e. it is not different from `object`), but this violates the rule that replacing a precise type with `Any` should not generate an error. IOW I essentially treat `Any` as "too dynamic or not imported". * I extend `None` special-casing to be more uniform. Now essentially it only overlaps with explicitly optional types. This is important for descriptor-like signatures. * Finally, I did a cleanup in `is_overlapping_types()`, most notably flags were not passed down to various (recursive) helpers, and `ParamSpec`/`Parameters` were treated a bit arbitrary. Pros/cons of the outcome: * Pro: simple (even if not 100% accurate) mental model * Pro: all major classes of false positives eliminated * Pro: couple minor false negatives fixed * Con: two new false negatives added, more details below So here a two new false negatives and motivation on why I think they are OK. First example is ```python T = TypeVar("T") @overload def foo(x: str) -> int: ... @overload def foo(x: T) -> T: ... def foo(x): if isinstance(x, str): return 0 return x ``` This is obviously unsafe (consider `T = float`), but not flagged after this PR. I think this is ~fine for two reasons: * There is no good alternative for a user, the error is not very actionable. Using types like `(str | T) -> int | T` is a bad idea because unions with type variables are not only imprecise, but also highly problematic for inference. * The false negative is mostly affecting unbounded type variables, if a "suspicious" bound is used (like `bound=float` in this example), the error will be still reported. Second example is signatures like ```python @overload def foo(x: str, y: str) -> str: ... @overload def foo(*args: str) -> int: ... @overload def bar(*, x: str, y: str) -> str: ... @overload def bar(**kwds: str) -> int: ... ``` These are also unsafe because one can fool mypy with `x: tuple[str, ...] = ("x", "y"); foo(*x)` and `x: dict[str, str] = {"x": "x", "y": "y"}; bar(**x)`. I think this is OK because while such unsafe calls are quite rare, this kind of catch-all fallback as last overload is relatively common. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- mypy/checker.py | 195 +++++++++++------- mypy/constraints.py | 2 +- mypy/expandtype.py | 2 +- mypy/meet.py | 111 +++++----- mypy/messages.py | 10 +- mypy/semanal.py | 9 +- mypy/subtypes.py | 68 ++---- mypy/types.py | 9 +- mypy/typeshed/stdlib/builtins.pyi | 4 +- test-data/unit/check-async-await.test | 4 +- test-data/unit/check-classes.test | 18 +- test-data/unit/check-generics.test | 14 +- test-data/unit/check-overloading.test | 144 ++++++++----- .../unit/check-parameter-specification.test | 20 +- test-data/unit/check-selftype.test | 6 +- test-data/unit/pythoneval.test | 11 +- 16 files changed, 352 insertions(+), 275 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index bf739e7d1242..3a7f231ebf1d 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -170,7 +170,6 @@ false_only, fixup_partial_type, function_type, - get_type_vars, is_literal_type_like, is_singleton_type, make_simplified_union, @@ -787,7 +786,16 @@ def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None: type_vars = current_class.defn.type_vars if current_class else [] with state.strict_optional_set(True): if is_unsafe_overlapping_overload_signatures(sig1, sig2, type_vars): - self.msg.overloaded_signatures_overlap(i + 1, i + j + 2, item.func) + flip_note = ( + j == 0 + and not is_unsafe_overlapping_overload_signatures( + sig2, sig1, type_vars + ) + and not overload_can_never_match(sig2, sig1) + ) + self.msg.overloaded_signatures_overlap( + i + 1, i + j + 2, flip_note, item.func + ) if impl_type is not None: assert defn.impl is not None @@ -1764,6 +1772,8 @@ def is_unsafe_overlapping_op( # second operand is the right argument -- we switch the order of # the arguments of the reverse method. + # TODO: this manipulation is dangerous if callables are generic. + # Shuffling arguments between callables can create meaningless types. forward_tweaked = forward_item.copy_modified( arg_types=[forward_base_erased, forward_item.arg_types[0]], arg_kinds=[nodes.ARG_POS] * 2, @@ -1790,7 +1800,9 @@ def is_unsafe_overlapping_op( current_class = self.scope.active_class() type_vars = current_class.defn.type_vars if current_class else [] - return is_unsafe_overlapping_overload_signatures(first, second, type_vars) + return is_unsafe_overlapping_overload_signatures( + first, second, type_vars, partial_only=False + ) def check_inplace_operator_method(self, defn: FuncBase) -> None: """Check an inplace operator method such as __iadd__. @@ -2185,7 +2197,7 @@ def get_op_other_domain(self, tp: FunctionLike) -> Type | None: if isinstance(tp, CallableType): if tp.arg_kinds and tp.arg_kinds[0] == ARG_POS: # For generic methods, domain comparison is tricky, as a first - # approximation erase all remaining type variables to bounds. + # approximation erase all remaining type variables. return erase_typevars(tp.arg_types[0], {v.id for v in tp.variables}) return None elif isinstance(tp, Overloaded): @@ -7827,68 +7839,112 @@ def are_argument_counts_overlapping(t: CallableType, s: CallableType) -> bool: return min_args <= max_args +def expand_callable_variants(c: CallableType) -> list[CallableType]: + """Expand a generic callable using all combinations of type variables' values/bounds.""" + for tv in c.variables: + # We need to expand self-type before other variables, because this is the only + # type variable that can have other type variables in the upper bound. + if tv.id.is_self(): + c = expand_type(c, {tv.id: tv.upper_bound}).copy_modified( + variables=[v for v in c.variables if not v.id.is_self()] + ) + break + + if not c.is_generic(): + # Fast path. + return [c] + + tvar_values = [] + for tvar in c.variables: + if isinstance(tvar, TypeVarType) and tvar.values: + tvar_values.append(tvar.values) + else: + tvar_values.append([tvar.upper_bound]) + + variants = [] + for combination in itertools.product(*tvar_values): + tvar_map = {tv.id: subst for (tv, subst) in zip(c.variables, combination)} + variants.append(expand_type(c, tvar_map).copy_modified(variables=[])) + return variants + + def is_unsafe_overlapping_overload_signatures( - signature: CallableType, other: CallableType, class_type_vars: list[TypeVarLikeType] + signature: CallableType, + other: CallableType, + class_type_vars: list[TypeVarLikeType], + partial_only: bool = True, ) -> bool: """Check if two overloaded signatures are unsafely overlapping or partially overlapping. - We consider two functions 's' and 't' to be unsafely overlapping if both - of the following are true: + We consider two functions 's' and 't' to be unsafely overlapping if three + conditions hold: + + 1. s's parameters are partially overlapping with t's. i.e. there are calls that are + valid for both signatures. + 2. for these common calls, some of t's parameters types are wider that s's. + 3. s's return type is NOT a subset of t's. - 1. s's parameters are all more precise or partially overlapping with t's - 2. s's return type is NOT a subtype of t's. + Note that we use subset rather than subtype relationship in these checks because: + * Overload selection happens at runtime, not statically. + * This results in more lenient behavior. + This can cause false negatives (e.g. if overloaded function returns an externally + visible attribute with invariant type), but such situations are rare. In general, + overloads in Python are generally unsafe, so we intentionally try to avoid giving + non-actionable errors (see more details in comments below). Assumes that 'signature' appears earlier in the list of overload alternatives then 'other' and that their argument counts are overlapping. """ # Try detaching callables from the containing class so that all TypeVars - # are treated as being free. - # - # This lets us identify cases where the two signatures use completely - # incompatible types -- e.g. see the testOverloadingInferUnionReturnWithMixedTypevars - # test case. + # are treated as being free, i.e. the signature is as seen from inside the class, + # where "self" is not yet bound to anything. signature = detach_callable(signature, class_type_vars) other = detach_callable(other, class_type_vars) - # Note: We repeat this check twice in both directions due to a slight - # asymmetry in 'is_callable_compatible'. When checking for partial overlaps, - # we attempt to unify 'signature' and 'other' both against each other. - # - # If 'signature' cannot be unified with 'other', we end early. However, - # if 'other' cannot be modified with 'signature', the function continues - # using the older version of 'other'. - # - # This discrepancy is unfortunately difficult to get rid of, so we repeat the - # checks twice in both directions for now. - # - # Note that we ignore possible overlap between type variables and None. This - # is technically unsafe, but unsafety is tiny and this prevents some common - # use cases like: - # @overload - # def foo(x: None) -> None: .. - # @overload - # def foo(x: T) -> Foo[T]: ... - return is_callable_compatible( - signature, - other, - is_compat=is_overlapping_types_no_promote_no_uninhabited_no_none, - is_proper_subtype=False, - is_compat_return=lambda l, r: not is_subtype_no_promote(l, r), - ignore_return=False, - check_args_covariantly=True, - allow_partial_overlap=True, - no_unify_none=True, - ) or is_callable_compatible( - other, - signature, - is_compat=is_overlapping_types_no_promote_no_uninhabited_no_none, - is_proper_subtype=False, - is_compat_return=lambda l, r: not is_subtype_no_promote(r, l), - ignore_return=False, - check_args_covariantly=False, - allow_partial_overlap=True, - no_unify_none=True, - ) + # Note: We repeat this check twice in both directions compensate for slight + # asymmetries in 'is_callable_compatible'. + + for sig_variant in expand_callable_variants(signature): + for other_variant in expand_callable_variants(other): + # Using only expanded callables may cause false negatives, we can add + # more variants (e.g. using inference between callables) in the future. + if is_subset_no_promote(sig_variant.ret_type, other_variant.ret_type): + continue + if not ( + is_callable_compatible( + sig_variant, + other_variant, + is_compat=is_overlapping_types_for_overload, + check_args_covariantly=False, + is_proper_subtype=False, + is_compat_return=lambda l, r: not is_subset_no_promote(l, r), + allow_partial_overlap=True, + ) + or is_callable_compatible( + other_variant, + sig_variant, + is_compat=is_overlapping_types_for_overload, + check_args_covariantly=True, + is_proper_subtype=False, + is_compat_return=lambda l, r: not is_subset_no_promote(r, l), + allow_partial_overlap=True, + ) + ): + continue + # Using the same `allow_partial_overlap` flag as before, can cause false + # negatives in case where star argument is used in a catch-all fallback overload. + # But again, practicality beats purity here. + if not partial_only or not is_callable_compatible( + other_variant, + sig_variant, + is_compat=is_subset_no_promote, + check_args_covariantly=True, + is_proper_subtype=False, + ignore_return=True, + allow_partial_overlap=True, + ): + return True + return False def detach_callable(typ: CallableType, class_type_vars: list[TypeVarLikeType]) -> CallableType: @@ -7897,21 +7953,11 @@ def detach_callable(typ: CallableType, class_type_vars: list[TypeVarLikeType]) - A callable normally keeps track of the type variables it uses within its 'variables' field. However, if the callable is from a method and that method is using a class type variable, the callable will not keep track of that type variable since it belongs to the class. - - This function will traverse the callable and find all used type vars and add them to the - variables field if it isn't already present. - - The caller can then unify on all type variables whether the callable is originally from - the class or not.""" + """ if not class_type_vars: # Fast path, nothing to update. return typ - seen_type_vars = set() - for t in typ.arg_types + [typ.ret_type]: - seen_type_vars |= set(get_type_vars(t)) - return typ.copy_modified( - variables=list(typ.variables) + [tv for tv in class_type_vars if tv in seen_type_vars] - ) + return typ.copy_modified(variables=list(typ.variables) + class_type_vars) def overload_can_never_match(signature: CallableType, other: CallableType) -> bool: @@ -8388,21 +8434,24 @@ def get_property_type(t: ProperType) -> ProperType: return t -def is_subtype_no_promote(left: Type, right: Type) -> bool: - return is_subtype(left, right, ignore_promotions=True) +def is_subset_no_promote(left: Type, right: Type) -> bool: + return is_subtype(left, right, ignore_promotions=True, always_covariant=True) -def is_overlapping_types_no_promote_no_uninhabited_no_none(left: Type, right: Type) -> bool: - # For the purpose of unsafe overload checks we consider list[Never] and list[int] - # non-overlapping. This is consistent with how we treat list[int] and list[str] as - # non-overlapping, despite [] belongs to both. Also this will prevent false positives - # for failed type inference during unification. +def is_overlapping_types_for_overload(left: Type, right: Type) -> bool: + # Note that among other effects 'overlap_for_overloads' flag will effectively + # ignore possible overlap between type variables and None. This is technically + # unsafe, but unsafety is tiny and this prevents some common use cases like: + # @overload + # def foo(x: None) -> None: .. + # @overload + # def foo(x: T) -> Foo[T]: ... return is_overlapping_types( left, right, ignore_promotions=True, - ignore_uninhabited=True, prohibit_none_typevar_overlap=True, + overlap_for_overloads=True, ) diff --git a/mypy/constraints.py b/mypy/constraints.py index 56ca51d19486..316f481ac870 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -1055,7 +1055,7 @@ def visit_callable_type(self, template: CallableType) -> list[Constraint]: # like U -> U, should be Callable[..., Any], but if U is a self-type, we can # allow it to leak, to be later bound to self. A bunch of existing code # depends on this old behaviour. - and not any(tv.id.raw_id == 0 for tv in cactual.variables) + and not any(tv.id.is_self() for tv in cactual.variables) ): # If the actual callable is generic, infer constraints in the opposite # direction, and indicate to the solver there are extra type variables diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 86875bc6079a..bff23c53defd 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -221,7 +221,7 @@ def visit_instance(self, t: Instance) -> Type: def visit_type_var(self, t: TypeVarType) -> Type: # Normally upper bounds can't contain other type variables, the only exception is # special type variable Self`0 <: C[T, S], where C is the class where Self is used. - if t.id.raw_id == 0: + if t.id.is_self(): t = t.copy_modified(upper_bound=t.upper_bound.accept(self)) repl = self.variables.get(t.id, t) if isinstance(repl, ProperType) and isinstance(repl, Instance): diff --git a/mypy/meet.py b/mypy/meet.py index 401200a11cc1..91abf43c0877 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -7,6 +7,7 @@ from mypy.maptype import map_instance_to_supertype from mypy.state import state from mypy.subtypes import ( + are_parameters_compatible, find_member, is_callable_compatible, is_equivalent, @@ -257,12 +258,16 @@ def is_literal_in_union(x: ProperType, y: ProperType) -> bool: ) +def is_object(t: ProperType) -> bool: + return isinstance(t, Instance) and t.type.fullname == "builtins.object" + + def is_overlapping_types( left: Type, right: Type, ignore_promotions: bool = False, prohibit_none_typevar_overlap: bool = False, - ignore_uninhabited: bool = False, + overlap_for_overloads: bool = False, seen_types: set[tuple[Type, Type]] | None = None, ) -> bool: """Can a value of type 'left' also be of type 'right' or vice-versa? @@ -270,6 +275,9 @@ def is_overlapping_types( If 'ignore_promotions' is True, we ignore promotions while checking for overlaps. If 'prohibit_none_typevar_overlap' is True, we disallow None from overlapping with TypeVars (in both strict-optional and non-strict-optional mode). + If 'overlap_for_overloads' is True, we check for overlaps more strictly (to avoid false + positives), for example: None only overlaps with explicitly optional types, Any + doesn't overlap with anything except object, we don't ignore positional argument names. """ if isinstance(left, TypeGuardedType) or isinstance( # type: ignore[misc] right, TypeGuardedType @@ -296,7 +304,7 @@ def _is_overlapping_types(left: Type, right: Type) -> bool: right, ignore_promotions=ignore_promotions, prohibit_none_typevar_overlap=prohibit_none_typevar_overlap, - ignore_uninhabited=ignore_uninhabited, + overlap_for_overloads=overlap_for_overloads, seen_types=seen_types.copy(), ) @@ -325,7 +333,7 @@ def _is_overlapping_types(left: Type, right: Type) -> bool: # 'Any' may or may not be overlapping with the other type if isinstance(left, AnyType) or isinstance(right, AnyType): - return True + return not overlap_for_overloads or is_object(left) or is_object(right) # We check for complete overlaps next as a general-purpose failsafe. # If this check fails, we start checking to see if there exists a @@ -345,11 +353,25 @@ def _is_overlapping_types(left: Type, right: Type) -> bool: ): return True - if is_proper_subtype( - left, right, ignore_promotions=ignore_promotions, ignore_uninhabited=ignore_uninhabited - ) or is_proper_subtype( - right, left, ignore_promotions=ignore_promotions, ignore_uninhabited=ignore_uninhabited - ): + def is_none_object_overlap(t1: Type, t2: Type) -> bool: + t1, t2 = get_proper_types((t1, t2)) + return ( + isinstance(t1, NoneType) + and isinstance(t2, Instance) + and t2.type.fullname == "builtins.object" + ) + + if overlap_for_overloads: + if is_none_object_overlap(left, right) or is_none_object_overlap(right, left): + return False + + def _is_subtype(left: Type, right: Type) -> bool: + if overlap_for_overloads: + return is_proper_subtype(left, right, ignore_promotions=ignore_promotions) + else: + return is_subtype(left, right, ignore_promotions=ignore_promotions) + + if _is_subtype(left, right) or _is_subtype(right, left): return True # See the docstring for 'get_possible_variants' for more info on what the @@ -358,21 +380,6 @@ def _is_overlapping_types(left: Type, right: Type) -> bool: left_possible = get_possible_variants(left) right_possible = get_possible_variants(right) - # First handle special cases relating to PEP 612: - # - comparing a `Parameters` to a `Parameters` - # - comparing a `Parameters` to a `ParamSpecType` - # - comparing a `ParamSpecType` to a `ParamSpecType` - # - # These should all always be considered overlapping equality checks. - # These need to be done before we move on to other TypeVarLike comparisons. - if isinstance(left, (Parameters, ParamSpecType)) and isinstance( - right, (Parameters, ParamSpecType) - ): - return True - # A `Parameters` does not overlap with anything else, however - if isinstance(left, Parameters) or isinstance(right, Parameters): - return False - # Now move on to checking multi-variant types like Unions. We also perform # the same logic if either type happens to be a TypeVar/ParamSpec/TypeVarTuple. # @@ -422,7 +429,7 @@ def is_none_typevarlike_overlap(t1: Type, t2: Type) -> bool: # into their 'Instance' fallbacks. if isinstance(left, TypedDictType) and isinstance(right, TypedDictType): - return are_typed_dicts_overlapping(left, right, ignore_promotions=ignore_promotions) + return are_typed_dicts_overlapping(left, right, _is_overlapping_types) elif typed_dict_mapping_pair(left, right): # Overlaps between TypedDicts and Mappings require dedicated logic. return typed_dict_mapping_overlap(left, right, overlapping=_is_overlapping_types) @@ -432,7 +439,7 @@ def is_none_typevarlike_overlap(t1: Type, t2: Type) -> bool: right = right.fallback if is_tuple(left) and is_tuple(right): - return are_tuples_overlapping(left, right, ignore_promotions=ignore_promotions) + return are_tuples_overlapping(left, right, _is_overlapping_types) elif isinstance(left, TupleType): left = tuple_fallback(left) elif isinstance(right, TupleType): @@ -469,13 +476,26 @@ def _type_object_overlap(left: Type, right: Type) -> bool: if isinstance(left, TypeType) or isinstance(right, TypeType): return _type_object_overlap(left, right) or _type_object_overlap(right, left) + if isinstance(left, Parameters) and isinstance(right, Parameters): + return are_parameters_compatible( + left, + right, + is_compat=_is_overlapping_types, + is_proper_subtype=False, + ignore_pos_arg_names=not overlap_for_overloads, + allow_partial_overlap=True, + ) + # A `Parameters` does not overlap with anything else, however + if isinstance(left, Parameters) or isinstance(right, Parameters): + return False + if isinstance(left, CallableType) and isinstance(right, CallableType): return is_callable_compatible( left, right, is_compat=_is_overlapping_types, is_proper_subtype=False, - ignore_pos_arg_names=True, + ignore_pos_arg_names=not overlap_for_overloads, allow_partial_overlap=True, ) @@ -514,11 +534,7 @@ def _type_object_overlap(left: Type, right: Type) -> bool: if isinstance(left, Instance) and isinstance(right, Instance): # First we need to handle promotions and structural compatibility for instances # that came as fallbacks, so simply call is_subtype() to avoid code duplication. - if is_subtype( - left, right, ignore_promotions=ignore_promotions, ignore_uninhabited=ignore_uninhabited - ) or is_subtype( - right, left, ignore_promotions=ignore_promotions, ignore_uninhabited=ignore_uninhabited - ): + if _is_subtype(left, right) or _is_subtype(right, left): return True if right.type.fullname == "builtins.int" and left.type.fullname in MYPYC_NATIVE_INT_NAMES: @@ -578,32 +594,21 @@ def is_overlapping_erased_types( def are_typed_dicts_overlapping( - left: TypedDictType, - right: TypedDictType, - *, - ignore_promotions: bool = False, - prohibit_none_typevar_overlap: bool = False, + left: TypedDictType, right: TypedDictType, is_overlapping: Callable[[Type, Type], bool] ) -> bool: """Returns 'true' if left and right are overlapping TypeDictTypes.""" # All required keys in left are present and overlapping with something in right for key in left.required_keys: if key not in right.items: return False - if not is_overlapping_types( - left.items[key], - right.items[key], - ignore_promotions=ignore_promotions, - prohibit_none_typevar_overlap=prohibit_none_typevar_overlap, - ): + if not is_overlapping(left.items[key], right.items[key]): return False # Repeat check in the other direction for key in right.required_keys: if key not in left.items: return False - if not is_overlapping_types( - left.items[key], right.items[key], ignore_promotions=ignore_promotions - ): + if not is_overlapping(left.items[key], right.items[key]): return False # The presence of any additional optional keys does not affect whether the two @@ -613,11 +618,7 @@ def are_typed_dicts_overlapping( def are_tuples_overlapping( - left: Type, - right: Type, - *, - ignore_promotions: bool = False, - prohibit_none_typevar_overlap: bool = False, + left: Type, right: Type, is_overlapping: Callable[[Type, Type], bool] ) -> bool: """Returns true if left and right are overlapping tuples.""" left, right = get_proper_types((left, right)) @@ -640,15 +641,7 @@ def are_tuples_overlapping( if len(left.items) != len(right.items): return False - return all( - is_overlapping_types( - l, - r, - ignore_promotions=ignore_promotions, - prohibit_none_typevar_overlap=prohibit_none_typevar_overlap, - ) - for l, r in zip(left.items, right.items) - ) + return all(is_overlapping(l, r) for l, r in zip(left.items, right.items)) def expand_tuple_if_possible(tup: TupleType, target: int) -> TupleType: diff --git a/mypy/messages.py b/mypy/messages.py index f01b0a726584..c3a34bd41aba 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -1624,13 +1624,21 @@ def overload_inconsistently_applies_decorator(self, decorator: str, context: Con context, ) - def overloaded_signatures_overlap(self, index1: int, index2: int, context: Context) -> None: + def overloaded_signatures_overlap( + self, index1: int, index2: int, flip_note: bool, context: Context + ) -> None: self.fail( "Overloaded function signatures {} and {} overlap with " "incompatible return types".format(index1, index2), context, code=codes.OVERLOAD_OVERLAP, ) + if flip_note: + self.note( + "Flipping the order of overloads will fix this error", + context, + code=codes.OVERLOAD_OVERLAP, + ) def overloaded_signature_will_never_match( self, index1: int, index2: int, context: Context diff --git a/mypy/semanal.py b/mypy/semanal.py index 03e6172bb325..c7a22d20aac6 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1739,10 +1739,14 @@ def analyze_type_param( fullname = self.qualified_name(type_param.name) if type_param.upper_bound: upper_bound = self.anal_type(type_param.upper_bound) + # TODO: we should validate the upper bound is valid for a given kind. if upper_bound is None: return None else: - upper_bound = self.named_type("builtins.object") + if type_param.kind == TYPE_VAR_TUPLE_KIND: + upper_bound = self.named_type("builtins.tuple", [self.object_type()]) + else: + upper_bound = self.object_type() default = AnyType(TypeOfAny.from_omitted_generics) if type_param.kind == TYPE_VAR_KIND: values = [] @@ -1777,8 +1781,7 @@ def analyze_type_param( return TypeVarTupleExpr( name=type_param.name, fullname=fullname, - # Upper bound for *Ts is *tuple[object, ...], it can never be object. - upper_bound=tuple_fallback.copy_modified(), + upper_bound=upper_bound, tuple_fallback=tuple_fallback, default=default, is_new_style=True, diff --git a/mypy/subtypes.py b/mypy/subtypes.py index a5d1d5d8194a..649cbae4c831 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -92,8 +92,8 @@ def __init__( ignore_pos_arg_names: bool = False, ignore_declared_variance: bool = False, # Supported for both proper and non-proper + always_covariant: bool = False, ignore_promotions: bool = False, - ignore_uninhabited: bool = False, # Proper subtype flags erase_instances: bool = False, keep_erased_types: bool = False, @@ -102,8 +102,8 @@ def __init__( self.ignore_type_params = ignore_type_params self.ignore_pos_arg_names = ignore_pos_arg_names self.ignore_declared_variance = ignore_declared_variance + self.always_covariant = always_covariant self.ignore_promotions = ignore_promotions - self.ignore_uninhabited = ignore_uninhabited self.erase_instances = erase_instances self.keep_erased_types = keep_erased_types self.options = options @@ -125,8 +125,8 @@ def is_subtype( ignore_type_params: bool = False, ignore_pos_arg_names: bool = False, ignore_declared_variance: bool = False, + always_covariant: bool = False, ignore_promotions: bool = False, - ignore_uninhabited: bool = False, options: Options | None = None, ) -> bool: """Is 'left' subtype of 'right'? @@ -145,8 +145,8 @@ def is_subtype( ignore_type_params=ignore_type_params, ignore_pos_arg_names=ignore_pos_arg_names, ignore_declared_variance=ignore_declared_variance, + always_covariant=always_covariant, ignore_promotions=ignore_promotions, - ignore_uninhabited=ignore_uninhabited, options=options, ) else: @@ -155,8 +155,8 @@ def is_subtype( ignore_type_params, ignore_pos_arg_names, ignore_declared_variance, + always_covariant, ignore_promotions, - ignore_uninhabited, options, } ), "Don't pass both context and individual flags" @@ -191,7 +191,6 @@ def is_proper_subtype( *, subtype_context: SubtypeContext | None = None, ignore_promotions: bool = False, - ignore_uninhabited: bool = False, erase_instances: bool = False, keep_erased_types: bool = False, ) -> bool: @@ -207,19 +206,12 @@ def is_proper_subtype( if subtype_context is None: subtype_context = SubtypeContext( ignore_promotions=ignore_promotions, - ignore_uninhabited=ignore_uninhabited, erase_instances=erase_instances, keep_erased_types=keep_erased_types, ) else: assert not any( - { - ignore_promotions, - ignore_uninhabited, - erase_instances, - keep_erased_types, - ignore_uninhabited, - } + {ignore_promotions, erase_instances, keep_erased_types} ), "Don't pass both context and individual flags" if type_state.is_assumed_proper_subtype(left, right): return True @@ -409,6 +401,7 @@ def build_subtype_kind(subtype_context: SubtypeContext, proper_subtype: bool) -> subtype_context.ignore_type_params, subtype_context.ignore_pos_arg_names, subtype_context.ignore_declared_variance, + subtype_context.always_covariant, subtype_context.ignore_promotions, subtype_context.erase_instances, subtype_context.keep_erased_types, @@ -447,11 +440,7 @@ def visit_none_type(self, left: NoneType) -> bool: return True def visit_uninhabited_type(self, left: UninhabitedType) -> bool: - # We ignore this for unsafe overload checks, so that and empty list and - # a list of int will be considered non-overlapping. - if isinstance(self.right, UninhabitedType): - return True - return not self.subtype_context.ignore_uninhabited + return True def visit_erased_type(self, left: ErasedType) -> bool: # This may be encountered during type inference. The result probably doesn't @@ -590,12 +579,15 @@ def visit_instance(self, left: Instance) -> bool: if tvar.variance == VARIANCE_NOT_READY and not tried_infer: infer_class_variances(right.type) tried_infer = True + if ( + self.subtype_context.always_covariant + and tvar.variance == INVARIANT + ): + variance = COVARIANT + else: + variance = tvar.variance if not check_type_parameter( - lefta, - righta, - tvar.variance, - self.proper_subtype, - self.subtype_context, + lefta, righta, variance, self.proper_subtype, self.subtype_context ): nominal = False else: @@ -687,6 +679,8 @@ def visit_parameters(self, left: Parameters) -> bool: is_proper_subtype=False, ignore_pos_arg_names=self.subtype_context.ignore_pos_arg_names, ) + elif isinstance(self.right, Instance): + return self.right.type.fullname == "builtins.object" else: return False @@ -1417,7 +1411,6 @@ def is_callable_compatible( check_args_covariantly: bool = False, allow_partial_overlap: bool = False, strict_concatenate: bool = False, - no_unify_none: bool = False, ) -> bool: """Is the left compatible with the right, using the provided compatibility check? @@ -1438,7 +1431,7 @@ def is_callable_compatible( configurable. For example, when checking the validity of overloads, it's useful to see if - the first overload alternative has more precise arguments then the second. + the first overload alternative has more precise arguments than the second. We would want to check the arguments covariantly in that case. Note! The following two function calls are NOT equivalent: @@ -1534,26 +1527,11 @@ def g(x: int) -> int: ... # (below) treats type variables on the two sides as independent. if left.variables: # Apply generic type variables away in left via type inference. - unified = unify_generic_callable( - left, right, ignore_return=ignore_return, no_unify_none=no_unify_none - ) + unified = unify_generic_callable(left, right, ignore_return=ignore_return) if unified is None: return False left = unified - # If we allow partial overlaps, we don't need to leave R generic: - # if we can find even just a single typevar assignment which - # would make these callables compatible, we should return True. - - # So, we repeat the above checks in the opposite direction. This also - # lets us preserve the 'symmetry' property of allow_partial_overlap. - if allow_partial_overlap and right.variables: - unified = unify_generic_callable( - right, left, ignore_return=ignore_return, no_unify_none=no_unify_none - ) - if unified is not None: - right = unified - # Check return types. if not ignore_return and not is_compat_return(left.ret_type, right.ret_type): return False @@ -1856,8 +1834,6 @@ def unify_generic_callable( target: NormalizedCallableType, ignore_return: bool, return_constraint_direction: int | None = None, - *, - no_unify_none: bool = False, ) -> NormalizedCallableType | None: """Try to unify a generic callable type with another callable type. @@ -1888,10 +1864,6 @@ def unify_generic_callable( type.ret_type, target.ret_type, return_constraint_direction ) constraints.extend(c) - if no_unify_none: - constraints = [ - c for c in constraints if not isinstance(get_proper_type(c.target), NoneType) - ] inferred_vars, _ = mypy.solve.solve_constraints( type.variables, constraints, allow_polymorphic=True ) diff --git a/mypy/types.py b/mypy/types.py index 0f8c48c8cb7d..3f764a5cc49e 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -501,7 +501,7 @@ class TypeVarId: # function type variables. # Metavariables are allocated unique ids starting from 1. - raw_id: int = 0 + raw_id: int # Level of the variable in type inference. Currently either 0 for # declared types, or 1 for type inference metavariables. @@ -545,6 +545,10 @@ def __hash__(self) -> int: def is_meta_var(self) -> bool: return self.meta_level > 0 + def is_self(self) -> bool: + # This is a special value indicating typing.Self variable. + return self.raw_id == 0 + class TypeVarLikeType(ProperType): __slots__ = ("name", "fullname", "id", "upper_bound", "default") @@ -3095,8 +3099,7 @@ def get_proper_type(typ: Type | None) -> ProperType | None: @overload -def get_proper_types(types: list[Type] | tuple[Type, ...]) -> list[ProperType]: # type: ignore[overload-overlap] - ... +def get_proper_types(types: list[Type] | tuple[Type, ...]) -> list[ProperType]: ... @overload diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi index 4a6c4bbcae45..42c0b27baf68 100644 --- a/mypy/typeshed/stdlib/builtins.pyi +++ b/mypy/typeshed/stdlib/builtins.pyi @@ -1572,9 +1572,9 @@ def pow(base: float, exp: complex | _SupportsSomeKindOfPow, mod: None = None) -> @overload def pow(base: complex, exp: complex | _SupportsSomeKindOfPow, mod: None = None) -> complex: ... @overload -def pow(base: _SupportsPow2[_E, _T_co], exp: _E, mod: None = None) -> _T_co: ... +def pow(base: _SupportsPow2[_E, _T_co], exp: _E, mod: None = None) -> _T_co: ... # type: ignore[overload-overlap] @overload -def pow(base: _SupportsPow3NoneOnly[_E, _T_co], exp: _E, mod: None = None) -> _T_co: ... +def pow(base: _SupportsPow3NoneOnly[_E, _T_co], exp: _E, mod: None = None) -> _T_co: ... # type: ignore[overload-overlap] @overload def pow(base: _SupportsPow3[_E, _M, _T_co], exp: _E, mod: _M) -> _T_co: ... @overload diff --git a/test-data/unit/check-async-await.test b/test-data/unit/check-async-await.test index 713c82c752ce..876fe0c6be15 100644 --- a/test-data/unit/check-async-await.test +++ b/test-data/unit/check-async-await.test @@ -765,7 +765,8 @@ class Task(Future[T]): @overload def wait(fs: Iterable[FT]) -> Future[Tuple[List[FT], List[FT]]]: ... \ - # E: Overloaded function signatures 1 and 2 overlap with incompatible return types + # E: Overloaded function signatures 1 and 2 overlap with incompatible return types \ + # N: Flipping the order of overloads will fix this error @overload def wait(fs: Iterable[Awaitable[T]]) -> Future[Tuple[List[Task[T]], List[Task[T]]]]: ... def wait(fs: Any) -> Any: @@ -789,6 +790,7 @@ async def precise2(futures: Iterable[Awaitable[int]]) -> None: done, pending = await wait(futures) reveal_type(done) # N: Revealed type is "builtins.list[__main__.Task[builtins.int]]" + [builtins fixtures/async_await.pyi] [typing fixtures/typing-async.pyi] diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 427133eca10b..e66eab5e2927 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -2427,10 +2427,10 @@ class B: [builtins fixtures/tuple.pyi] [case testReverseOperatorTypeVar1] -from typing import TypeVar, Any +from typing import TypeVar T = TypeVar("T", bound='Real') class Real: - def __add__(self, other: Any) -> str: ... + def __add__(self, other: object) -> str: ... class Fraction(Real): def __radd__(self, other: T) -> T: ... # E: Signatures of "__radd__" of "Fraction" and "__add__" of "T" are unsafely overlapping @@ -2465,7 +2465,7 @@ reveal_type(Real() + Fraction()) # N: Revealed type is "__main__.Real" reveal_type(Fraction() + Fraction()) # N: Revealed type is "builtins.str" [case testReverseOperatorTypeVar3] -from typing import TypeVar, Any +from typing import TypeVar T = TypeVar("T", bound='Real') class Real: def __add__(self, other: FractionChild) -> str: ... @@ -2701,14 +2701,12 @@ class X: [out] tmp/foo.pyi:6: error: Signatures of "__radd__" of "B" and "__add__" of "X" are unsafely overlapping -[case testUnsafeOverlappingWithLineNo] +[case testUnsafeOverlappingNotWithAny] from typing import TypeVar class Real: def __add__(self, other) -> str: ... class Fraction(Real): def __radd__(self, other: Real) -> Real: ... -[out] -main:5: error: Signatures of "__radd__" of "Fraction" and "__add__" of "Real" are unsafely overlapping [case testOverlappingNormalAndInplaceOperatorMethod] import typing @@ -4042,10 +4040,16 @@ def f(a: Type[User]) -> int: pass # E: Overloaded function signatures 1 and 2 o @overload def f(a: object) -> str: pass +# Note: plain type is equivalent to Type[Any] so no error here @overload -def g(a: Type[User]) -> int: pass # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def g(a: Type[User]) -> int: pass @overload def g(a: type) -> str: pass + +@overload +def h(a: Type[User]) -> int: pass # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +@overload +def h(a: Type[object]) -> str: pass [builtins fixtures/classmethod.pyi] [out] diff --git a/test-data/unit/check-generics.test b/test-data/unit/check-generics.test index abcb2a4bbc48..d46d19946098 100644 --- a/test-data/unit/check-generics.test +++ b/test-data/unit/check-generics.test @@ -3028,7 +3028,7 @@ def dec(f: Callable[[T], S], g: Callable[[T], U]) -> Callable[[T], Tuple[S, U]]: def id(x: V) -> V: ... -reveal_type(dec(id, id)) # N: Revealed type is "def [T] (T`7) -> Tuple[T`7, T`7]" +reveal_type(dec(id, id)) # N: Revealed type is "def [T] (T`1) -> Tuple[T`1, T`1]" [builtins fixtures/tuple.pyi] [case testInferenceAgainstGenericEllipsisSelfSpecialCase] @@ -3264,8 +3264,8 @@ def transform( def dec(f: Callable[W, U]) -> Callable[W, U]: ... def dec2(f: Callable[Concatenate[str, W], U]) -> Callable[Concatenate[bytes, W], U]: ... -reveal_type(transform(dec)) # N: Revealed type is "def [P, T] (def (builtins.int, *P.args, **P.kwargs) -> T`9) -> def (builtins.int, *P.args, **P.kwargs) -> T`9" -reveal_type(transform(dec2)) # N: Revealed type is "def [W, T] (def (builtins.int, builtins.str, *W.args, **W.kwargs) -> T`13) -> def (builtins.int, builtins.bytes, *W.args, **W.kwargs) -> T`13" +reveal_type(transform(dec)) # N: Revealed type is "def [P, T] (def (builtins.int, *P.args, **P.kwargs) -> T`3) -> def (builtins.int, *P.args, **P.kwargs) -> T`3" +reveal_type(transform(dec2)) # N: Revealed type is "def [W, T] (def (builtins.int, builtins.str, *W.args, **W.kwargs) -> T`7) -> def (builtins.int, builtins.bytes, *W.args, **W.kwargs) -> T`7" [builtins fixtures/tuple.pyi] [case testNoAccidentalVariableClashInNestedGeneric] @@ -3319,8 +3319,8 @@ def id(x: U) -> U: ... def either(x: U, y: U) -> U: ... def pair(x: U, y: V) -> Tuple[U, V]: ... -reveal_type(dec(id)) # N: Revealed type is "def [T] (T`9) -> builtins.list[T`9]" -reveal_type(dec(either)) # N: Revealed type is "def [T] (T`11, T`11) -> builtins.list[T`11]" +reveal_type(dec(id)) # N: Revealed type is "def [T] (T`3) -> builtins.list[T`3]" +reveal_type(dec(either)) # N: Revealed type is "def [T] (T`5, T`5) -> builtins.list[T`5]" reveal_type(dec(pair)) # N: Revealed type is "def [U, V] (U`-1, V`-2) -> builtins.list[Tuple[U`-1, V`-2]]" [builtins fixtures/tuple.pyi] @@ -3338,8 +3338,8 @@ V = TypeVar("V") def id(x: U) -> U: ... def either(x: U, y: U) -> U: ... -reveal_type(dec(id)) # N: Revealed type is "def [T] (builtins.list[T`9]) -> T`9" -reveal_type(dec(either)) # N: Revealed type is "def [T] (builtins.list[T`11], builtins.list[T`11]) -> T`11" +reveal_type(dec(id)) # N: Revealed type is "def [T] (builtins.list[T`3]) -> T`3" +reveal_type(dec(either)) # N: Revealed type is "def [T] (builtins.list[T`5], builtins.list[T`5]) -> T`5" [builtins fixtures/tuple.pyi] [case testInferenceAgainstGenericVariadicPopOff] diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test index bcb775ba5dac..03863b2978ba 100644 --- a/test-data/unit/check-overloading.test +++ b/test-data/unit/check-overloading.test @@ -373,7 +373,8 @@ def foo(t, s): pass class Wrapper(Generic[T]): @overload - def foo(self, t: List[T], s: T) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types + def foo(self, t: List[T], s: T) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types \ + # N: Flipping the order of overloads will fix this error @overload def foo(self, t: T, s: T) -> str: ... def foo(self, t, s): pass @@ -384,7 +385,8 @@ class Dummy(Generic[T]): pass # cause the constraint solver to not infer T = object like it did in the # first example? @overload -def bar(d: Dummy[T], t: List[T], s: T) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def bar(d: Dummy[T], t: List[T], s: T) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types \ + # N: Flipping the order of overloads will fix this error @overload def bar(d: Dummy[T], t: T, s: T) -> str: ... def bar(d: Dummy[T], t, s): pass @@ -1325,8 +1327,9 @@ def h(x: Sequence[str]) -> int: pass @overload def h(x: Sequence[T]) -> None: pass # E: Overloaded function signature 2 will never be matched: signature 1's parameter type(s) are the same or broader +# Safety of this highly depends on the implementation, so we lean towards being silent. @overload -def i(x: List[str]) -> int: pass # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def i(x: List[str]) -> int: pass @overload def i(x: List[T]) -> None: pass [builtins fixtures/list.pyi] @@ -1752,14 +1755,11 @@ reveal_type(f(d)) # N: Revealed type is "builtins.list[builtins.int]" from typing import overload, Any @overload -def f(*, x: int = 3, y: int = 3) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def f(*, x: int = 3, y: int = 3) -> int: ... @overload def f(**kwargs: str) -> str: ... def f(*args, **kwargs): pass -# Checking an overload flagged as unsafe is a bit weird, but this is the -# cleanest way to make sure 'Any' ambiguity checks work correctly with -# keyword arguments. a: Any i: int reveal_type(f(x=a, y=i)) # N: Revealed type is "builtins.int" @@ -2163,8 +2163,9 @@ from wrapper import * [file wrapper.pyi] from typing import overload +# Safety of this highly depends on the implementation, so we lean towards being silent. @overload -def foo1(*x: int) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo1(*x: int) -> int: ... @overload def foo1(x: int, y: int, z: int) -> str: ... @@ -2173,8 +2174,9 @@ def foo2(*x: int) -> int: ... @overload def foo2(x: int, y: str, z: int) -> str: ... +# Note: this is technically unsafe, but we don't report this for now. @overload -def bar1(x: int, y: int, z: int) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def bar1(x: int, y: int, z: int) -> str: ... @overload def bar1(*x: int) -> int: ... @@ -2248,7 +2250,7 @@ from wrapper import * from typing import overload @overload -def foo1(x: str) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo1(x: str) -> str: ... @overload def foo1(x: str, y: str = ...) -> int: ... @@ -2268,12 +2270,12 @@ from wrapper import * from typing import overload @overload -def foo1(*args: int) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo1(*args: int) -> int: ... @overload def foo1(**kwargs: int) -> str: ... @overload -def foo2(**kwargs: int) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo2(**kwargs: int) -> str: ... @overload def foo2(*args: int) -> int: ... [builtins fixtures/dict.pyi] @@ -2314,13 +2316,14 @@ def foo2(x: int, *args: int) -> str: ... @overload def foo2(*args2: str) -> int: ... +# The two examples are unsafe, but this is hard to detect. @overload -def foo3(*args: int) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo3(*args: int) -> int: ... @overload def foo3(x: int, *args2: int) -> str: ... @overload -def foo4(x: int, *args: int) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo4(x: int, *args: int) -> str: ... @overload def foo4(*args2: int) -> int: ... [builtins fixtures/tuple.pyi] @@ -2357,13 +2360,13 @@ def foo4(x: Other = ..., *args: str) -> int: ... from typing import overload @overload -def foo1(x: int = 0, y: int = 0) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo1(x: int = 0, y: int = 0) -> int: ... @overload def foo1(*xs: int) -> str: ... def foo1(*args): pass @overload -def foo2(*xs: int) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo2(*xs: int) -> str: ... @overload def foo2(x: int = 0, y: int = 0) -> int: ... def foo2(*args): pass @@ -2412,12 +2415,12 @@ from wrapper import * from typing import overload @overload -def foo1(x: str, y: str = ..., z: str = ...) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo1(x: str, y: str = ..., z: str = ...) -> str: ... @overload def foo1(*x: str) -> int: ... @overload -def foo2(*x: str) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo2(*x: str) -> int: ... @overload def foo2(x: str, y: str = ..., z: str = ...) -> str: ... @@ -2433,12 +2436,12 @@ from wrapper import * from typing import overload @overload -def foo1(x: str, y: str = ..., z: int = ...) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo1(x: str, y: str = ..., z: int = ...) -> str: ... @overload def foo1(*x: str) -> int: ... @overload -def foo2(x: str, y: str = ..., z: int = ...) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo2(x: str, y: str = ..., z: int = ...) -> str: ... @overload def foo2(*x: str) -> int: ... [builtins fixtures/tuple.pyi] @@ -2449,7 +2452,7 @@ from wrapper import * from typing import overload @overload -def foo1(*, x: str, y: str, z: str) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo1(*, x: str, y: str, z: str) -> str: ... @overload def foo1(**x: str) -> int: ... @@ -2481,12 +2484,12 @@ def foo2(**x: str) -> int: ... def foo2(*, x: str, y: str, z: int) -> str: ... @overload -def foo3(*, x: str, y: str, z: int = ...) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo3(*, x: str, y: str, z: int = ...) -> str: ... @overload def foo3(**x: str) -> int: ... @overload -def foo4(**x: str) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo4(**x: str) -> int: ... @overload def foo4(*, x: str, y: str, z: int = ...) -> str: ... [builtins fixtures/dict.pyi] @@ -2497,12 +2500,13 @@ from wrapper import * from typing import overload @overload -def foo1(x: str, *, y: str, z: str) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo1(x: str, *, y: str, z: str) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types \ + # N: Flipping the order of overloads will fix this error @overload def foo1(**x: str) -> int: ... @overload -def foo2(**x: str) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def foo2(**x: str) -> int: ... @overload def foo2(x: str, *, y: str, z: str) -> str: ... @@ -2798,7 +2802,8 @@ def h(x: List[Union[C, D]]) -> str: ... def h(x): ... @overload -def i(x: List[Union[A, B]]) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def i(x: List[Union[A, B]]) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types \ + # N: Flipping the order of overloads will fix this error @overload def i(x: List[Union[A, B, C]]) -> str: ... def i(x): ... @@ -2810,8 +2815,9 @@ from typing import TypeVar, overload T = TypeVar('T') +# Note: this is unsafe, but it is hard to detect. @overload -def f(x: int) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def f(x: int) -> str: ... @overload def f(x: T) -> T: ... def f(x): ... @@ -2827,14 +2833,15 @@ from typing import TypeVar, overload, List T = TypeVar('T') +# Note: first two examples are unsafe, but it is hard to detect. @overload -def f1(x: List[int]) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def f1(x: List[int]) -> str: ... @overload def f1(x: List[T]) -> T: ... def f1(x): ... @overload -def f2(x: List[int]) -> List[str]: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def f2(x: List[int]) -> List[str]: ... @overload def f2(x: List[T]) -> List[T]: ... def f2(x): ... @@ -2859,17 +2866,15 @@ from typing import TypeVar, overload, Generic T = TypeVar('T') class Wrapper(Generic[T]): + # Similar to above: this is unsafe, but it is hard to detect. @overload - def f(self, x: int) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types + def f(self, x: int) -> str: ... @overload def f(self, x: T) -> T: ... def f(self, x): ... - # TODO: This shouldn't trigger an error message? - # Related to testTypeCheckOverloadImplementationTypeVarDifferingUsage2? - # See https://github.com/python/mypy/issues/5510 @overload - def g(self, x: int) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types + def g(self, x: int) -> int: ... @overload def g(self, x: T) -> T: ... def g(self, x): ... @@ -2880,28 +2885,27 @@ from typing import TypeVar, overload, Generic, List T = TypeVar('T') class Wrapper(Generic[T]): + # Similar to above: first two examples are unsafe, but it is hard to detect. @overload - def f1(self, x: List[int]) -> str: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types + def f1(self, x: List[int]) -> str: ... @overload def f1(self, x: List[T]) -> T: ... def f1(self, x): ... @overload - def f2(self, x: List[int]) -> List[str]: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types + def f2(self, x: List[int]) -> List[str]: ... @overload def f2(self, x: List[T]) -> List[T]: ... def f2(self, x): ... - # TODO: This shouldn't trigger an error message? - # See https://github.com/python/mypy/issues/5510 @overload - def g1(self, x: List[int]) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types + def g1(self, x: List[int]) -> int: ... @overload def g1(self, x: List[T]) -> T: ... def g1(self, x): ... @overload - def g2(self, x: List[int]) -> List[int]: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types + def g2(self, x: List[int]) -> List[int]: ... @overload def g2(self, x: List[T]) -> List[T]: ... def g2(self, x): ... @@ -3078,13 +3082,14 @@ class C: pass S = TypeVar('S', A, B) @overload -def f(x: S) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def f(x: S) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types \ + # N: Flipping the order of overloads will fix this error @overload def f(x: Union[B, C]) -> str: ... def f(x): pass @overload -def g(x: Union[B, C]) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def g(x: Union[B, C]) -> int: ... @overload def g(x: S) -> str: ... def g(x): pass @@ -3607,7 +3612,7 @@ def test(x: T) -> T: from typing import overload, Optional @overload -def f(x: None) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def f(x: None) -> int: ... @overload def f(x: object) -> str: ... def f(x): ... @@ -3632,7 +3637,7 @@ reveal_type(g(c)) # N: Revealed type is "builtins.str" from typing import overload, Optional @overload -def f(x: None) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def f(x: None) -> int: ... @overload def f(x: object) -> str: ... def f(x): ... @@ -3978,7 +3983,7 @@ from typing import overload, Any, Optional, Union class FakeAttribute: @overload - def dummy(self, instance: None, owner: Any) -> 'FakeAttribute': ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types + def dummy(self, instance: None, owner: Any) -> 'FakeAttribute': ... @overload def dummy(self, instance: object, owner: Any) -> int: ... def dummy(self, instance: Optional[object], owner: Any) -> Union['FakeAttribute', int]: ... @@ -4545,7 +4550,7 @@ reveal_type(Child().foo(3).child_only()) # N: Revealed type is "builtins.in [case testOverloadAndSelfTypesGenericNoOverlap] from typing import Generic, TypeVar, Any, overload, Self, Union -T = TypeVar("T", bound=Any) +T = TypeVar("T") class C(Generic[T]): @overload def get(self, obj: None) -> Self: ... @@ -4903,7 +4908,7 @@ T = TypeVar('T') def f() -> None: @overload - def g(x: str) -> int: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types + def g(x: str) -> int: ... @overload def g(x: T) -> T: ... def g(x): @@ -4944,7 +4949,7 @@ x: Any reveal_type(attr(x)) # N: Revealed type is "Any" attr("hi", 1) # E: No overload variant of "attr" matches argument types "str", "int" \ # N: Possible overload variants: \ - # N: def [T in (int, float)] attr(default: T = ..., blah: int = ...) -> T \ + # N: def [T in (int, float)] attr(default: T, blah: int = ...) -> T \ # N: def attr(default: Any = ...) -> int [file lib.pyi] from typing import overload, Any, TypeVar @@ -4952,7 +4957,7 @@ from typing import overload, Any, TypeVar T = TypeVar('T', int, float) @overload -def attr(default: T = ..., blah: int = ...) -> T: ... +def attr(default: T, blah: int = ...) -> T: ... @overload def attr(default: Any = ...) -> int: ... [out] @@ -5008,7 +5013,7 @@ children: List[Child] parents: List[Parent] @overload -def f(x: Child) -> List[Child]: pass # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def f(x: Child) -> List[Child]: pass @overload def f(x: Parent) -> List[Parent]: pass def f(x: Union[Child, Parent]) -> Union[List[Child], List[Parent]]: @@ -5319,7 +5324,7 @@ def f1(g: G[A, B]) -> B: ... def f1(g: Any) -> Any: ... @overload -def f2(g: G[A, Any]) -> A: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def f2(g: G[A, Any]) -> A: ... @overload def f2(g: G[A, B], x: int = ...) -> B: ... def f2(g: Any, x: int = ...) -> Any: ... @@ -6500,7 +6505,7 @@ P = ParamSpec("P") R = TypeVar("R") @overload -def func(x: Callable[Concatenate[Any, P], R]) -> Callable[P, R]: ... # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def func(x: Callable[Concatenate[Any, P], R]) -> Callable[P, R]: ... @overload def func(x: Callable[P, R]) -> Callable[Concatenate[str, P], R]: ... def func(x: Callable[..., R]) -> Callable[..., R]: ... @@ -6710,3 +6715,38 @@ class B: def f(self, *args, **kwargs): pass [builtins fixtures/tuple.pyi] + +[case testOverloadsSafeOverlapAllowed] +from lib import * +[file lib.pyi] +from typing import overload + +@overload +def bar(x: object) -> object: ... +@overload +def bar(x: int = ...) -> int: ... + +[case testOverloadsInvariantOverlapAllowed] +from lib import * +[file lib.pyi] +from typing import overload, List + +@overload +def bar(x: List[int]) -> List[int]: ... +@overload +def bar(x: List[object]) -> List[object]: ... + +[case testOverloadsNoneAnyOverlapAllowed] +from lib import * +[file lib.pyi] +from typing import overload, Any + +@overload +def foo(x: None) -> int: ... +@overload +def foo(x: object) -> str: ... + +@overload +def bar(x: int) -> int: ... +@overload +def bar(x: Any) -> str: ... diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 37916c2155fe..e6d8cec3f0b0 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -901,8 +901,8 @@ class A: def func(self, action: Callable[_P, _R], *args: _P.args, **kwargs: _P.kwargs) -> _R: ... -reveal_type(A.func) # N: Revealed type is "def [_P, _R] (self: __main__.A, action: def (*_P.args, **_P.kwargs) -> _R`15, *_P.args, **_P.kwargs) -> _R`15" -reveal_type(A().func) # N: Revealed type is "def [_P, _R] (action: def (*_P.args, **_P.kwargs) -> _R`19, *_P.args, **_P.kwargs) -> _R`19" +reveal_type(A.func) # N: Revealed type is "def [_P, _R] (self: __main__.A, action: def (*_P.args, **_P.kwargs) -> _R`6, *_P.args, **_P.kwargs) -> _R`6" +reveal_type(A().func) # N: Revealed type is "def [_P, _R] (action: def (*_P.args, **_P.kwargs) -> _R`10, *_P.args, **_P.kwargs) -> _R`10" def f(x: int) -> int: ... @@ -933,8 +933,8 @@ class A: def func(self, action: Job[_P, None]) -> Job[_P, None]: ... -reveal_type(A.func) # N: Revealed type is "def [_P] (self: __main__.A, action: __main__.Job[_P`13, None]) -> __main__.Job[_P`13, None]" -reveal_type(A().func) # N: Revealed type is "def [_P] (action: __main__.Job[_P`15, None]) -> __main__.Job[_P`15, None]" +reveal_type(A.func) # N: Revealed type is "def [_P] (self: __main__.A, action: __main__.Job[_P`4, None]) -> __main__.Job[_P`4, None]" +reveal_type(A().func) # N: Revealed type is "def [_P] (action: __main__.Job[_P`6, None]) -> __main__.Job[_P`6, None]" reveal_type(A().func(Job(lambda x: x))) # N: Revealed type is "__main__.Job[[x: Any], None]" def f(x: int, y: int) -> None: ... @@ -1096,7 +1096,7 @@ j = Job(generic_f) reveal_type(j) # N: Revealed type is "__main__.Job[[x: _T`-1]]" jf = j.into_callable() -reveal_type(jf) # N: Revealed type is "def [_T] (x: _T`13)" +reveal_type(jf) # N: Revealed type is "def [_T] (x: _T`4)" reveal_type(jf(1)) # N: Revealed type is "None" [builtins fixtures/paramspec.pyi] @@ -1115,10 +1115,10 @@ class Job(Generic[_P, _T]): def generic_f(x: _T) -> _T: ... j = Job(generic_f) -reveal_type(j) # N: Revealed type is "__main__.Job[[x: _T`12], _T`12]" +reveal_type(j) # N: Revealed type is "__main__.Job[[x: _T`3], _T`3]" jf = j.into_callable() -reveal_type(jf) # N: Revealed type is "def [_T] (x: _T`13) -> _T`13" +reveal_type(jf) # N: Revealed type is "def [_T] (x: _T`4) -> _T`4" reveal_type(jf(1)) # N: Revealed type is "builtins.int" [builtins fixtures/paramspec.pyi] @@ -1600,7 +1600,7 @@ from typing_extensions import Concatenate, ParamSpec P = ParamSpec("P") @overload -def command() -> Callable[[Callable[Concatenate[object, object, P], object]], None]: # E: Overloaded function signatures 1 and 2 overlap with incompatible return types +def command() -> Callable[[Callable[Concatenate[object, object, P], object]], None]: ... @overload @@ -1640,13 +1640,13 @@ U = TypeVar("U") def dec(f: Callable[P, T]) -> Callable[P, List[T]]: ... def test(x: U) -> U: ... reveal_type(dec) # N: Revealed type is "def [P, T] (f: def (*P.args, **P.kwargs) -> T`-2) -> def (*P.args, **P.kwargs) -> builtins.list[T`-2]" -reveal_type(dec(test)) # N: Revealed type is "def [T] (x: T`12) -> builtins.list[T`12]" +reveal_type(dec(test)) # N: Revealed type is "def [T] (x: T`3) -> builtins.list[T`3]" class A: ... TA = TypeVar("TA", bound=A) def test_with_bound(x: TA) -> TA: ... -reveal_type(dec(test_with_bound)) # N: Revealed type is "def [T <: __main__.A] (x: T`14) -> builtins.list[T`14]" +reveal_type(dec(test_with_bound)) # N: Revealed type is "def [T <: __main__.A] (x: T`5) -> builtins.list[T`5]" dec(test_with_bound)(0) # E: Value of type variable "T" of function cannot be "int" dec(test_with_bound)(A()) # OK [builtins fixtures/paramspec.pyi] diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index fdd628b0271b..1480c83b2272 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -1793,7 +1793,7 @@ class C: def bar(self) -> Self: ... def foo(self, x: S) -> Tuple[Self, S]: ... -reveal_type(C.foo) # N: Revealed type is "def [Self <: __main__.C, S] (self: Self`7, x: S`8) -> Tuple[Self`7, S`8]" +reveal_type(C.foo) # N: Revealed type is "def [Self <: __main__.C, S] (self: Self`1, x: S`2) -> Tuple[Self`1, S`2]" reveal_type(C().foo(42)) # N: Revealed type is "Tuple[__main__.C, builtins.int]" [builtins fixtures/tuple.pyi] @@ -1807,7 +1807,7 @@ class C: def bar(self) -> Self: ... foo: Callable[[S, Self], Tuple[Self, S]] -reveal_type(C().foo) # N: Revealed type is "def [S] (S`7, __main__.C) -> Tuple[__main__.C, S`7]" +reveal_type(C().foo) # N: Revealed type is "def [S] (S`1, __main__.C) -> Tuple[__main__.C, S`1]" reveal_type(C().foo(42, C())) # N: Revealed type is "Tuple[__main__.C, builtins.int]" class This: ... [builtins fixtures/tuple.pyi] @@ -2032,7 +2032,7 @@ class Ben(Object): } @classmethod def doit(cls) -> Foo: - reveal_type(cls.MY_MAP) # N: Revealed type is "builtins.dict[builtins.str, def [Self <: __main__.Foo] (self: Self`10) -> Self`10]" + reveal_type(cls.MY_MAP) # N: Revealed type is "builtins.dict[builtins.str, def [Self <: __main__.Foo] (self: Self`4) -> Self`4]" foo_method = cls.MY_MAP["foo"] return foo_method(Foo()) [builtins fixtures/isinstancelist.pyi] diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index a76d3abd7114..3bf8613d2478 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -444,7 +444,7 @@ False [case testOverlappingOperatorMethods] class X: pass class A: - def __add__(self, x) -> int: + def __add__(self, x: object) -> int: if isinstance(x, X): return 1 return NotImplemented @@ -1942,13 +1942,13 @@ class Bar(Generic[P]): ... def bad(foo: Foo[[int]], bar: Bar[[int]]) -> bool: return foo == bar -def good1(foo1: Foo[[int]], foo2: Foo[[str]]) -> bool: +def bad1(foo1: Foo[[int]], foo2: Foo[[str]]) -> bool: return foo1 == foo2 -def good2(foo1: Foo[[int, str]], foo2: Foo[[int, bytes]]) -> bool: +def bad2(foo1: Foo[[int, str]], foo2: Foo[[int, bytes]]) -> bool: return foo1 == foo2 -def good3(foo1: Foo[[int]], foo2: Foo[[int, int]]) -> bool: +def bad3(foo1: Foo[[int]], foo2: Foo[[int, int]]) -> bool: return foo1 == foo2 def good4(foo1: Foo[[int]], foo2: Foo[[int]]) -> bool: @@ -1971,6 +1971,9 @@ def good9(foo1: Foo[Concatenate[int, P]], foo2: Foo[[int, str, bytes]], *args: P [out] _testStrictEqualitywithParamSpec.py:11: error: Non-overlapping equality check (left operand type: "Foo[[int]]", right operand type: "Bar[[int]]") +_testStrictEqualitywithParamSpec.py:14: error: Non-overlapping equality check (left operand type: "Foo[[int]]", right operand type: "Foo[[str]]") +_testStrictEqualitywithParamSpec.py:17: error: Non-overlapping equality check (left operand type: "Foo[[int, str]]", right operand type: "Foo[[int, bytes]]") +_testStrictEqualitywithParamSpec.py:20: error: Non-overlapping equality check (left operand type: "Foo[[int]]", right operand type: "Foo[[int, int]]") [case testInferenceOfDunderDictOnClassObjects] class Foo: ...