From 725a22465fb63cf61bd7d8dc317d9c6518056111 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 28 Dec 2023 14:21:06 +0000 Subject: [PATCH 01/42] WIP parse type variable definitions in function --- mypy/fastparse.py | 28 ++++++++++++++++++++-------- mypy/nodes.py | 10 +++++++--- mypy/options.py | 3 ++- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index e208e4d0b7d9..40b20c2c6076 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -87,7 +87,7 @@ YieldFromExpr, check_arg_names, ) -from mypy.options import Options +from mypy.options import Options, NEW_GENERIC_SYNTAX from mypy.patterns import ( AsPattern, ClassPattern, @@ -884,6 +884,8 @@ def do_func_def( arg_kinds = [arg.kind for arg in args] arg_names = [None if arg.pos_only else arg.variable.name for arg in args] + # Type parameters, if using new syntax for generics (PEP 695) + explicit_type_params: list[tuple[str, Type | None]] | None = None arg_types: list[Type | None] = [] if no_type_check: @@ -937,12 +939,22 @@ def do_func_def( return_type = AnyType(TypeOfAny.from_error) else: if sys.version_info >= (3, 12) and n.type_params: - self.fail( - ErrorMessage("PEP 695 generics are not yet supported", code=codes.VALID_TYPE), - n.type_params[0].lineno, - n.type_params[0].col_offset, - blocker=False, - ) + if NEW_GENERIC_SYNTAX in self.options.enable_incomplete_feature: + explicit_type_params = [] + for p in n.type_params: + if p.bound is None: + bound = None + else: + bound = TypeConverter(self.errors, line=p.lineno).visit(p.bound) + explicit_type_params.append((p.name, bound)) + else: + self.fail( + ErrorMessage("PEP 695 generics are not yet supported", + code=codes.VALID_TYPE), + n.type_params[0].lineno, + n.type_params[0].col_offset, + blocker=False, + ) arg_types = [a.type_annotation for a in args] return_type = TypeConverter( @@ -986,7 +998,7 @@ def do_func_def( self.class_and_function_stack.pop() self.class_and_function_stack.append("F") body = self.as_required_block(n.body, can_strip=True, is_coroutine=is_coroutine) - func_def = FuncDef(n.name, args, body, func_type) + func_def = FuncDef(n.name, args, body, func_type, explicit_type_params) if isinstance(func_def.type, CallableType): # semanal.py does some in-place modifications we want to avoid func_def.unanalyzed_type = func_def.type.copy_modified() diff --git a/mypy/nodes.py b/mypy/nodes.py index bb278d92392d..ac8d77e3be83 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -667,11 +667,12 @@ class FuncItem(FuncBase): __slots__ = ( "arguments", # Note that can be unset if deserialized (type is a lie!) - "arg_names", # Names of arguments - "arg_kinds", # Kinds of arguments + "arg_names", # Names of parameters + "arg_kinds", # Kinds of parameters "min_args", # Minimum number of arguments "max_pos", # Maximum number of positional arguments, -1 if no explicit # limit (*args not included) + "type_args", # New-style type parameters (PEP 695) "body", # Body of the function "is_overload", # Is this an overload variant of function with more than # one overload variant? @@ -689,12 +690,14 @@ def __init__( arguments: list[Argument] | None = None, body: Block | None = None, typ: mypy.types.FunctionLike | None = None, + type_args: list[tuple[str, mypy.types.Type | None]] | None = None, ) -> None: super().__init__() self.arguments = arguments or [] self.arg_names = [None if arg.pos_only else arg.variable.name for arg in self.arguments] self.arg_kinds: list[ArgKind] = [arg.kind for arg in self.arguments] self.max_pos: int = self.arg_kinds.count(ARG_POS) + self.arg_kinds.count(ARG_OPT) + self.type_args = type_args self.body: Block = body or Block([]) self.type = typ self.unanalyzed_type = typ @@ -761,8 +764,9 @@ def __init__( arguments: list[Argument] | None = None, body: Block | None = None, typ: mypy.types.FunctionLike | None = None, + type_args: list[tuple[str, mypy.types.Type | None]] | None = None, ) -> None: - super().__init__(arguments, body, typ) + super().__init__(arguments, body, typ, type_args) self._name = name self.is_decorated = False self.is_conditional = False # Defined conditionally (within block)? diff --git a/mypy/options.py b/mypy/options.py index bf9c09f1bf4b..6ae527b50f9e 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -73,7 +73,8 @@ class BuildType: TYPE_VAR_TUPLE: Final = "TypeVarTuple" UNPACK: Final = "Unpack" PRECISE_TUPLE_TYPES: Final = "PreciseTupleTypes" -INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES,)) +NEW_GENERIC_SYNTAX: Final = "NewGenericSyntax" +INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES, NEW_GENERIC_SYNTAX)) COMPLETE_FEATURES: Final = frozenset((TYPE_VAR_TUPLE, UNPACK)) From 3058b8400ada6ded402bf7c09ecf9ff3918ec0f8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 28 Dec 2023 15:57:06 +0000 Subject: [PATCH 02/42] WIP make function type vars work kind of --- mypy/semanal.py | 15 +++++++++++++++ test-data/unit/check-python312.test | 12 ++++++++++++ 2 files changed, 27 insertions(+) diff --git a/mypy/semanal.py b/mypy/semanal.py index 91a6b1808987..94c7e3e6393d 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -837,6 +837,16 @@ def visit_func_def(self, defn: FuncDef) -> None: def analyze_func_def(self, defn: FuncDef) -> None: self.function_stack.append(defn) + for tv in defn.type_args if defn.type_args else []: + tve = TypeVarExpr( + name=tv[0], + fullname=tv[0], + values=[], + upper_bound=self.named_type("builtins.object"), + default=AnyType(TypeOfAny.from_omitted_generics), + ) + self.add_symbol(tv[0], tve, defn) + if defn.type: assert isinstance(defn.type, CallableType) has_self_type = self.update_function_type_variables(defn.type, defn) @@ -943,6 +953,11 @@ def analyze_func_def(self, defn: FuncDef) -> None: defn.type = defn.type.copy_modified(ret_type=ret_type) self.wrapped_coro_return_types[defn] = defn.type + for tv in defn.type_args if defn.type_args else []: + names = self.current_symbol_table() + del names[tv[0]] + + def remove_unpack_kwargs(self, defn: FuncDef, typ: CallableType) -> CallableType: if not typ.arg_kinds or typ.arg_kinds[-1] is not ArgKind.ARG_STAR2: return typ diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 2b99a42628b1..cfb2350fe3df 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -82,3 +82,15 @@ reveal_type(ba2) # N: Revealed type is "def (*Any) -> builtins.str" [builtins fixtures/tuple.pyi] [typing fixtures/typing-full.pyi] + +[case testPEP695GenericFunctionSyntax] +# flags: --enable-incomplete-feature=NewGenericSyntax + +def ident[T](x: T) -> T: + y: T = x + y = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "T") + return x + +reveal_type(ident(1)) # N: Revealed type is "builtins.int" + +a: T # E: Name "T" is not defined From fc23bcf7830b71208c3cfd5981e4dbce3f7de910 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 29 Dec 2023 12:56:31 +0000 Subject: [PATCH 03/42] WIP parse new-style type params in classes --- mypy/fastparse.py | 37 ++++++++++++++++++++++++------------- mypy/nodes.py | 7 +++++++ 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 40b20c2c6076..30f0515ec7c8 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -940,13 +940,7 @@ def do_func_def( else: if sys.version_info >= (3, 12) and n.type_params: if NEW_GENERIC_SYNTAX in self.options.enable_incomplete_feature: - explicit_type_params = [] - for p in n.type_params: - if p.bound is None: - bound = None - else: - bound = TypeConverter(self.errors, line=p.lineno).visit(p.bound) - explicit_type_params.append((p.name, bound)) + explicit_type_params = self.translate_type_params(n.type_params) else: self.fail( ErrorMessage("PEP 695 generics are not yet supported", @@ -1132,13 +1126,19 @@ def visit_ClassDef(self, n: ast3.ClassDef) -> ClassDef: self.class_and_function_stack.append("C") keywords = [(kw.arg, self.visit(kw.value)) for kw in n.keywords if kw.arg] + # Type parameters, if using new syntax for generics (PEP 695) + explicit_type_params: list[tuple[str, Type | None]] | None = None + if sys.version_info >= (3, 12) and n.type_params: - self.fail( - ErrorMessage("PEP 695 generics are not yet supported", code=codes.VALID_TYPE), - n.type_params[0].lineno, - n.type_params[0].col_offset, - blocker=False, - ) + if NEW_GENERIC_SYNTAX in self.options.enable_incomplete_feature: + explicit_type_params = self.translate_type_params(n.type_params) + else: + self.fail( + ErrorMessage("PEP 695 generics are not yet supported", code=codes.VALID_TYPE), + n.type_params[0].lineno, + n.type_params[0].col_offset, + blocker=False, + ) cdef = ClassDef( n.name, @@ -1147,6 +1147,7 @@ def visit_ClassDef(self, n: ast3.ClassDef) -> ClassDef: self.translate_expr_list(n.bases), metaclass=dict(keywords).get("metaclass"), keywords=keywords, + type_args=explicit_type_params, ) cdef.decorators = self.translate_expr_list(n.decorator_list) # Set lines to match the old mypy 0.700 lines, in order to keep @@ -1162,6 +1163,16 @@ def visit_ClassDef(self, n: ast3.ClassDef) -> ClassDef: self.class_and_function_stack.pop() return cdef + def translate_type_params(self, type_params: Any) -> list[tuple[str, Type | None]]: + explicit_type_params = [] + for p in type_params: + if p.bound is None: + bound = None + else: + bound = TypeConverter(self.errors, line=p.lineno).visit(p.bound) + explicit_type_params.append((p.name, bound)) + return explicit_type_params + # Return(expr? value) def visit_Return(self, n: ast3.Return) -> ReturnStmt: node = ReturnStmt(self.visit(n.value)) diff --git a/mypy/nodes.py b/mypy/nodes.py index ac8d77e3be83..4d31c5f23821 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1074,6 +1074,7 @@ class ClassDef(Statement): "name", "_fullname", "defs", + "type_args", "type_vars", "base_type_exprs", "removed_base_type_exprs", @@ -1093,6 +1094,10 @@ class ClassDef(Statement): name: str # Name of the class without module prefix _fullname: str # Fully qualified name of the class defs: Block + # New-style type parameters (PEP 695), unanalyzed + type_args: list[tuple[str, mypy.types.Type | None]] | None + # Semantically analyzed type parameters (all syntax variants) + # TODO: Move these to TypeInfo? These partially duplicate type_args. type_vars: list[mypy.types.TypeVarLikeType] # Base class expressions (not semantically analyzed -- can be arbitrary expressions) base_type_exprs: list[Expression] @@ -1115,12 +1120,14 @@ def __init__( base_type_exprs: list[Expression] | None = None, metaclass: Expression | None = None, keywords: list[tuple[str, Expression]] | None = None, + type_args: list[tuple[str, mypy.types.Type | None]] | None = None ) -> None: super().__init__() self.name = name self._fullname = "" self.defs = defs self.type_vars = type_vars or [] + self.type_args = type_args self.base_type_exprs = base_type_exprs or [] self.removed_base_type_exprs = [] self.info = CLASSDEF_NO_INFO From 61e971d1ef497c074f756091fb282bf6516f9b89 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 29 Dec 2023 13:48:28 +0000 Subject: [PATCH 04/42] WIP make class type vars work partially --- mypy/semanal.py | 47 +++++++++++++++++++++-------- test-data/unit/check-python312.test | 21 +++++++++++++ 2 files changed, 55 insertions(+), 13 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 94c7e3e6393d..e92242439e19 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -787,6 +787,7 @@ def file_context( self.num_incomplete_refs = 0 if active_type: + self.push_type_args(active_type.defn.type_args, active_type.defn) self.incomplete_type_stack.append(False) scope.enter_class(active_type) self.enter_class(active_type.defn.info) @@ -800,6 +801,7 @@ def file_context( self.leave_class() self._type = None self.incomplete_type_stack.pop() + self.pop_type_args(active_type.defn.type_args) del self.options # @@ -837,15 +839,7 @@ def visit_func_def(self, defn: FuncDef) -> None: def analyze_func_def(self, defn: FuncDef) -> None: self.function_stack.append(defn) - for tv in defn.type_args if defn.type_args else []: - tve = TypeVarExpr( - name=tv[0], - fullname=tv[0], - values=[], - upper_bound=self.named_type("builtins.object"), - default=AnyType(TypeOfAny.from_omitted_generics), - ) - self.add_symbol(tv[0], tve, defn) + self.push_type_args(defn.type_args, defn) if defn.type: assert isinstance(defn.type, CallableType) @@ -953,10 +947,7 @@ def analyze_func_def(self, defn: FuncDef) -> None: defn.type = defn.type.copy_modified(ret_type=ret_type) self.wrapped_coro_return_types[defn] = defn.type - for tv in defn.type_args if defn.type_args else []: - names = self.current_symbol_table() - del names[tv[0]] - + self.pop_type_args(defn.type_args) def remove_unpack_kwargs(self, defn: FuncDef, typ: CallableType) -> CallableType: if not typ.arg_kinds or typ.arg_kinds[-1] is not ArgKind.ARG_STAR2: @@ -1633,9 +1624,32 @@ def visit_class_def(self, defn: ClassDef) -> None: self.incomplete_type_stack.append(not defn.info) namespace = self.qualified_name(defn.name) with self.tvar_scope_frame(self.tvar_scope.class_frame(namespace)): + self.push_type_args(defn.type_args, defn) self.analyze_class(defn) + self.pop_type_args(defn.type_args) self.incomplete_type_stack.pop() + def push_type_args(self, type_args: list[tuple[str, Type | None]] | None, + context: Context) -> None: + if not type_args: + return + for tv in type_args: + tve = TypeVarExpr( + name=tv[0], + fullname=tv[0], + values=[], + upper_bound=self.named_type("builtins.object"), + default=AnyType(TypeOfAny.from_omitted_generics), + ) + self.add_symbol(tv[0], tve, context) + + def pop_type_args(self, type_args: list[tuple[str, Type | None]] | None) -> None: + if not type_args: + return + for tv in type_args: + names = self.current_symbol_table() + del names[tv[0]] + def analyze_class(self, defn: ClassDef) -> None: fullname = self.qualified_name(defn.name) if not defn.info and not self.is_core_builtin_class(defn): @@ -1929,6 +1943,13 @@ class Foo(Bar, Generic[T]): ... removed: list[int] = [] declared_tvars: TypeVarLikeList = [] is_protocol = False + if defn.type_args is not None: + for n, _ in defn.type_args: + node = self.lookup(n, context) + assert node is not None + assert isinstance(node.node, TypeVarLikeExpr) + declared_tvars.append((n, node.node)) + for i, base_expr in enumerate(base_type_exprs): if isinstance(base_expr, StarExpr): base_expr.valid = True diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index cfb2350fe3df..943321feb7e1 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -94,3 +94,24 @@ def ident[T](x: T) -> T: reveal_type(ident(1)) # N: Revealed type is "builtins.int" a: T # E: Name "T" is not defined + +[case testPEP695GenericClassSyntax] +# flags: --enable-incomplete-feature=NewGenericSyntax + +class C[T]: + x: T + + def __init__(self, x: T) -> None: + self.x = x + + def ident(self, x: T) -> T: + y: T = x + if int(): + return self.x + else: + return y + +reveal_type(C("x")) # N: Revealed type is "__main__.C[builtins.str]" +c: C[int] = C(1) +reveal_type(c.x) # N: Revealed type is "builtins.int" +reveal_type(c.ident(1)) # N: Revealed type is "builtins.int" From 47cb82df1e6eb410767cdd4839617d54205523ec Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 29 Dec 2023 17:47:03 +0000 Subject: [PATCH 05/42] WIP variance test case --- test-data/unit/check-python312.test | 39 +++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 943321feb7e1..d97389097500 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -115,3 +115,42 @@ reveal_type(C("x")) # N: Revealed type is "__main__.C[builtins.str]" c: C[int] = C(1) reveal_type(c.x) # N: Revealed type is "builtins.int" reveal_type(c.ident(1)) # N: Revealed type is "builtins.int" + +[case testPEP695InferVarianceSimple] +# flags: --enable-incomplete-feature=NewGenericSyntax + +class Invariant[T]: + def f(self, x: T) -> None: + pass + + def g(self) -> T | None: + return None + +a: Invariant[object] +b: Invariant[int] +if int(): + a = b # E: Incompatible types in assignment (expression has type "Invariant[int]", variable has type "Invariant[object]") +if int(): + b = a # E: Incompatible types in assignment (expression has type "Invariant[object]", variable has type "Invariant[int]") + +class Covariant[T]: + def g(self) -> T | None: + return None + +c: Covariant[object] +d: Covariant[int] +if int(): + c = d +if int(): + d = c # E: Incompatible types in assignment (expression has type "Covariant[object]", variable has type "Covariant[int]") + +class Contravariant[T]: + def f(self, x: T) -> None: + pass + +e: Contravariant[object] +f: Contravariant[int] +if int(): + e = f # E: Incompatible types in assignment (expression has type "Contravariant[int]", variable has type "Contravariant[object]") +if int(): + f = e From e56dd9649983d67077c07412830ba52f6f294c48 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 21 Mar 2024 14:59:56 +0000 Subject: [PATCH 06/42] WIP first attempt at inference of variance --- mypy/checker.py | 45 +++++++++++++++++++++++++++++ test-data/unit/check-python312.test | 36 +++++++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/mypy/checker.py b/mypy/checker.py index 9c10cd2fc30d..f8c71a49d79e 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2396,6 +2396,7 @@ def visit_class_def(self, defn: ClassDef) -> None: self.check_protocol_variance(defn) if not defn.has_incompatible_baseclass and defn.info.is_enum: self.check_enum(defn) + infer_class_variances(defn, self.named_type("builtins.object")) def check_final_deletable(self, typ: TypeInfo) -> None: # These checks are only for mypyc. Only perform some checks that are easier @@ -8443,3 +8444,47 @@ def visit_starred_pattern(self, p: StarredPattern) -> None: self.lvalue = True p.capture.accept(self) self.lvalue = False + + +def infer_variance(info: TypeInfo, i: int, object_type: ProperType) -> None: + """Infer the variance of the ith type variable of a generic class.""" + for variance in COVARIANT, CONTRAVARIANT, INVARIANT: + tv = info.defn.type_vars[i] + assert isinstance(tv, TypeVarType) + tv.variance = variance + co = True + contra = True + tvar = info.defn.type_vars[i] + for member, sym in info.names.items(): + node = sym.node + typ = None + if isinstance(node, FuncDef): + typ = node.type + if isinstance(typ, FunctionLike): + typ = bind_self(typ) + + if typ: + typ2 = expand_type(typ, {tvar.id: object_type}) + print(member, typ, typ2) + if not is_subtype(typ, typ2): + co = False + if not is_subtype(typ2, typ): + contra = False + if co: + v = COVARIANT + elif contra: + v = CONTRAVARIANT + else: + v = INVARIANT + if v == variance: + break + tv.variance = INVARIANT + + +def infer_class_variances(defn: ClassDef, obj: ProperType) -> None: + if not defn.type_args: + return + tvs = defn.type_vars + for i, tv in enumerate(tvs): + if isinstance(tv, TypeVarType): + infer_variance(defn.info, i, obj) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index d97389097500..192edd4d2999 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -154,3 +154,39 @@ if int(): e = f # E: Incompatible types in assignment (expression has type "Contravariant[int]", variable has type "Contravariant[object]") if int(): f = e + +[case testPEP695InferVarianceRecursive] +# flags: --enable-incomplete-feature=NewGenericSyntax + +class Invariant[T]: + def f(self, x: Invariant[T]) -> Invariant[T]: + return x + +class Covariant[T]: + def f(self) -> Covariant[T]: + return self + +class Contravariant[T]: + def f(self, x: Contravariant[T]) -> None: + pass + +a: Invariant[object] +b: Invariant[int] +if int(): + a = b # E: Incompatible types in assignment (expression has type "Invariant[int]", variable has type "Invariant[object]") +if int(): + b = a # E: Incompatible types in assignment (expression has type "Invariant[object]", variable has type "Invariant[int]") + +c: Covariant[object] +d: Covariant[int] +if int(): + c = d +if int(): + d = c # E: Incompatible types in assignment (expression has type "Covariant[object]", variable has type "Covariant[int]") + +e: Contravariant[object] +f: Contravariant[int] +if int(): + e = f # E: Incompatible types in assignment (expression has type "Contravariant[int]", variable has type "Contravariant[object]") +if int(): + f = e From 21c3eb6ccb9416de8d6853a5a5aa764a8ff1e5fd Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 21 Mar 2024 16:50:41 +0000 Subject: [PATCH 07/42] WIP infer variance on demand --- mypy/checker.py | 48 ++----------------------- mypy/nodes.py | 1 + mypy/semanal.py | 3 +- mypy/subtypes.py | 54 +++++++++++++++++++++++++++-- test-data/unit/check-python312.test | 17 +++++++++ 5 files changed, 74 insertions(+), 49 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index f8c71a49d79e..b5abe8dac921 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -153,7 +153,7 @@ is_same_type, is_subtype, restrict_subtype_away, - unify_generic_callable, + unify_generic_callable, infer_class_variances, ) from mypy.traverser import TraverserVisitor, all_return_statements, has_return_statement from mypy.treetransform import TransformVisitor @@ -2396,7 +2396,7 @@ def visit_class_def(self, defn: ClassDef) -> None: self.check_protocol_variance(defn) if not defn.has_incompatible_baseclass and defn.info.is_enum: self.check_enum(defn) - infer_class_variances(defn, self.named_type("builtins.object")) + infer_class_variances(defn.info) def check_final_deletable(self, typ: TypeInfo) -> None: # These checks are only for mypyc. Only perform some checks that are easier @@ -8444,47 +8444,3 @@ def visit_starred_pattern(self, p: StarredPattern) -> None: self.lvalue = True p.capture.accept(self) self.lvalue = False - - -def infer_variance(info: TypeInfo, i: int, object_type: ProperType) -> None: - """Infer the variance of the ith type variable of a generic class.""" - for variance in COVARIANT, CONTRAVARIANT, INVARIANT: - tv = info.defn.type_vars[i] - assert isinstance(tv, TypeVarType) - tv.variance = variance - co = True - contra = True - tvar = info.defn.type_vars[i] - for member, sym in info.names.items(): - node = sym.node - typ = None - if isinstance(node, FuncDef): - typ = node.type - if isinstance(typ, FunctionLike): - typ = bind_self(typ) - - if typ: - typ2 = expand_type(typ, {tvar.id: object_type}) - print(member, typ, typ2) - if not is_subtype(typ, typ2): - co = False - if not is_subtype(typ2, typ): - contra = False - if co: - v = COVARIANT - elif contra: - v = CONTRAVARIANT - else: - v = INVARIANT - if v == variance: - break - tv.variance = INVARIANT - - -def infer_class_variances(defn: ClassDef, obj: ProperType) -> None: - if not defn.type_args: - return - tvs = defn.type_vars - for i, tv in enumerate(tvs): - if isinstance(tv, TypeVarType): - infer_variance(defn.info, i, obj) diff --git a/mypy/nodes.py b/mypy/nodes.py index 4d31c5f23821..7ec69e7f2b2d 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2453,6 +2453,7 @@ def accept(self, visitor: ExpressionVisitor[T]) -> T: INVARIANT: Final = 0 COVARIANT: Final = 1 CONTRAVARIANT: Final = 2 +VARIANCE_NOT_READY: Final = 3 # Variance hasn't been inferred (using Python 3.12 syntax) class TypeVarLikeExpr(SymbolNode, Expression): diff --git a/mypy/semanal.py b/mypy/semanal.py index e92242439e19..91c7575532bb 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -177,7 +177,7 @@ is_final_node, type_aliases, type_aliases_source_versions, - typing_extensions_aliases, + typing_extensions_aliases, VARIANCE_NOT_READY, ) from mypy.options import Options from mypy.patterns import ( @@ -1640,6 +1640,7 @@ def push_type_args(self, type_args: list[tuple[str, Type | None]] | None, values=[], upper_bound=self.named_type("builtins.object"), default=AnyType(TypeOfAny.from_omitted_generics), + variance=VARIANCE_NOT_READY, ) self.add_symbol(tv[0], tve, context) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 4d5e7335b14f..67c6438ea11f 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -8,7 +8,7 @@ import mypy.constraints import mypy.typeops from mypy.erasetype import erase_type -from mypy.expandtype import expand_self_type, expand_type_by_instance +from mypy.expandtype import expand_self_type, expand_type_by_instance, expand_type from mypy.maptype import map_instance_to_supertype # Circular import; done in the function instead. @@ -23,7 +23,7 @@ FuncBase, OverloadedFuncDef, TypeInfo, - Var, + Var, VARIANCE_NOT_READY, ) from mypy.options import Options from mypy.state import state @@ -577,6 +577,8 @@ def visit_instance(self, left: Instance) -> bool: if not self.subtype_context.ignore_type_params: for lefta, righta, tvar in type_params: if isinstance(tvar, TypeVarType): + if tvar.variance ==VARIANCE_NOT_READY: + infer_class_variances(right.type) if not check_type_parameter( lefta, righta, @@ -1978,3 +1980,51 @@ def is_more_precise(left: Type, right: Type, *, ignore_promotions: bool = False) if isinstance(right, AnyType): return True return is_proper_subtype(left, right, ignore_promotions=ignore_promotions) + + +def infer_variance(info: TypeInfo, i: int) -> None: + """Infer the variance of the ith type variable of a generic class.""" + object_type = Instance(info.mro[-1], []) + from mypy.typeops import bind_self + for variance in COVARIANT, CONTRAVARIANT, INVARIANT: + tv = info.defn.type_vars[i] + assert isinstance(tv, TypeVarType) + if tv.variance != VARIANCE_NOT_READY: + continue + tv.variance = variance + co = True + contra = True + tvar = info.defn.type_vars[i] + for member, sym in info.names.items(): + node = sym.node + typ = None + if isinstance(node, FuncBase): + typ = node.type + if isinstance(typ, FunctionLike): + typ = bind_self(typ) + + if typ: + typ2 = expand_type(typ, {tvar.id: object_type}) + print(member, typ, typ2) + if not is_subtype(typ, typ2): + co = False + if not is_subtype(typ2, typ): + contra = False + if co: + v = COVARIANT + elif contra: + v = CONTRAVARIANT + else: + v = INVARIANT + if v == variance: + break + tv.variance = VARIANCE_NOT_READY + + +def infer_class_variances(info: TypeInfo) -> None: + if not info.defn.type_args: + return + tvs = info.defn.type_vars + for i, tv in enumerate(tvs): + if isinstance(tv, TypeVarType): + infer_variance(info, i) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 192edd4d2999..af97c438686b 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -190,3 +190,20 @@ if int(): e = f # E: Incompatible types in assignment (expression has type "Contravariant[int]", variable has type "Contravariant[object]") if int(): f = e + +[case testPEP695InferVarianceCalculateOnDemand] +# flags: --enable-incomplete-feature=NewGenericSyntax + +class Covariant[T]: + def __init__(self) -> None: + self.x = [1] + + def f(self) -> None: + c = Covariant[int]() + # We need to know that T is covariant here + self.g(c) + c2 = Covariant[object]() + self.h(c2) # E: Argument 1 to "h" of "Covariant" has incompatible type "Covariant[object]"; expected "Covariant[int]" + + def g(self, x: Covariant[object]) -> None: pass + def h(self, x: Covariant[int]) -> None: pass From 7e15173fed9b0e9038ebe638a77f48b005213829 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 22 Mar 2024 13:47:27 +0000 Subject: [PATCH 08/42] Fall back to covariance if variance not ready, and support attributes --- mypy/subtypes.py | 37 ++++++++++--- mypy/typestate.py | 5 +- test-data/unit/check-python312.test | 80 ++++++++++++++++++++++++++++- 3 files changed, 113 insertions(+), 9 deletions(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 67c6438ea11f..7ed6ad4d40e0 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -361,7 +361,10 @@ def check_type_parameter( p_left = get_proper_type(left) if isinstance(p_left, UninhabitedType) and p_left.ambiguous: variance = COVARIANT - if variance == COVARIANT: + # If variance hasn't been inferred yet, we are lenient and default to + # covariance. This shouldn't happen often, but it's very difficult to + # avoid these cases altogether. + if variance == COVARIANT or variance == VARIANCE_NOT_READY: if proper_subtype: return is_proper_subtype(left, right, subtype_context=subtype_context) else: @@ -575,10 +578,12 @@ def visit_instance(self, left: Instance) -> bool: else: type_params = zip(t.args, right.args, right.type.defn.type_vars) if not self.subtype_context.ignore_type_params: + tried_infer = False for lefta, righta, tvar in type_params: if isinstance(tvar, TypeVarType): - if tvar.variance ==VARIANCE_NOT_READY: + if tvar.variance == VARIANCE_NOT_READY and not tried_infer: infer_class_variances(right.type) + tried_infer = True if not check_type_parameter( lefta, righta, @@ -1982,10 +1987,15 @@ def is_more_precise(left: Type, right: Type, *, ignore_promotions: bool = False) return is_proper_subtype(left, right, ignore_promotions=ignore_promotions) -def infer_variance(info: TypeInfo, i: int) -> None: - """Infer the variance of the ith type variable of a generic class.""" +def infer_variance(info: TypeInfo, i: int) -> bool: + """Infer the variance of the ith type variable of a generic class. + + Return True if successful. This can fail if some inferred types aren't ready. + """ object_type = Instance(info.mro[-1], []) + from mypy.typeops import bind_self + for variance in COVARIANT, CONTRAVARIANT, INVARIANT: tv = info.defn.type_vars[i] assert isinstance(tv, TypeVarType) @@ -1996,20 +2006,30 @@ def infer_variance(info: TypeInfo, i: int) -> None: contra = True tvar = info.defn.type_vars[i] for member, sym in info.names.items(): + if member in ("__init__", "__new__"): + continue node = sym.node typ = None + settable = False if isinstance(node, FuncBase): typ = node.type if isinstance(typ, FunctionLike): typ = bind_self(typ) + elif isinstance(node, Var): + settable = True + typ = node.type + if typ is None: + tv.variance = VARIANCE_NOT_READY + return False if typ: typ2 = expand_type(typ, {tvar.id: object_type}) - print(member, typ, typ2) if not is_subtype(typ, typ2): co = False if not is_subtype(typ2, typ): contra = False + if settable: + co = False if co: v = COVARIANT elif contra: @@ -2019,12 +2039,15 @@ def infer_variance(info: TypeInfo, i: int) -> None: if v == variance: break tv.variance = VARIANCE_NOT_READY + return True -def infer_class_variances(info: TypeInfo) -> None: +def infer_class_variances(info: TypeInfo) -> bool: if not info.defn.type_args: return tvs = info.defn.type_vars for i, tv in enumerate(tvs): if isinstance(tv, TypeVarType): - infer_variance(info, i) + if not infer_variance(info, i): + return False + return True diff --git a/mypy/typestate.py b/mypy/typestate.py index c5a5da03eae5..b16fb7c21fed 100644 --- a/mypy/typestate.py +++ b/mypy/typestate.py @@ -8,7 +8,7 @@ from typing import Dict, Final, Set, Tuple from typing_extensions import TypeAlias as _TypeAlias -from mypy.nodes import TypeInfo +from mypy.nodes import TypeInfo, VARIANCE_NOT_READY from mypy.server.trigger import make_trigger from mypy.types import Instance, Type, TypeVarId, get_proper_type @@ -192,6 +192,9 @@ def record_subtype_cache_entry( # These are unlikely to match, due to the large space of # possible values. Avoid uselessly increasing cache sizes. return + if any(tv.variance == VARIANCE_NOT_READY for tv in right.type.defn.type_vars): + # Variance indeterminate -- don't know the result + return cache = self._subtype_caches.setdefault(right.type, {}) cache.setdefault(kind, set()).add((left, right)) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index af97c438686b..9d16afddf651 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -116,7 +116,7 @@ c: C[int] = C(1) reveal_type(c.x) # N: Revealed type is "builtins.int" reveal_type(c.ident(1)) # N: Revealed type is "builtins.int" -[case testPEP695InferVarianceSimple] +[case testPEP695InferVarianceSimpleFromMethod] # flags: --enable-incomplete-feature=NewGenericSyntax class Invariant[T]: @@ -155,6 +155,42 @@ if int(): if int(): f = e +[case testPEP695InferVarianceSimpleFromAttribute] +# flags: --enable-incomplete-feature=NewGenericSyntax + +class Invariant1[T]: + def __init__(self, x: T) -> None: + self.x = x + +a: Invariant1[object] +b: Invariant1[int] +if int(): + a = b # E: Incompatible types in assignment (expression has type "Invariant1[int]", variable has type "Invariant1[object]") +if int(): + b = a # E: Incompatible types in assignment (expression has type "Invariant1[object]", variable has type "Invariant1[int]") + +class Invariant2[T]: + def __init__(self) -> None: + self.x: list[T] = [] + +a2: Invariant2[object] +b2: Invariant2[int] +if int(): + a2 = b2 # E: Incompatible types in assignment (expression has type "Invariant2[int]", variable has type "Invariant2[object]") +if int(): + b2 = a2 # E: Incompatible types in assignment (expression has type "Invariant2[object]", variable has type "Invariant2[int]") + +class Invariant3[T]: + def __init__(self) -> None: + self.x: T | None = None + +a3: Invariant3[object] +b3: Invariant3[int] +if int(): + a3 = b3 # E: Incompatible types in assignment (expression has type "Invariant3[int]", variable has type "Invariant3[object]") +if int(): + b3 = a3 # E: Incompatible types in assignment (expression has type "Invariant3[object]", variable has type "Invariant3[int]") + [case testPEP695InferVarianceRecursive] # flags: --enable-incomplete-feature=NewGenericSyntax @@ -207,3 +243,45 @@ class Covariant[T]: def g(self, x: Covariant[object]) -> None: pass def h(self, x: Covariant[int]) -> None: pass + +[case testPEP695InferVarianceNotReadyWhenNeeded] +# flags: --enable-incomplete-feature=NewGenericSyntax + +class Covariant[T]: + def f(self) -> None: + c = Covariant[int]() + # We need to know that T is covariant here + self.g(c) + c2 = Covariant[object]() + self.h(c2) # E: Argument 1 to "h" of "Covariant" has incompatible type "Covariant[object]"; expected "Covariant[int]" + + def g(self, x: Covariant[object]) -> None: pass + def h(self, x: Covariant[int]) -> None: pass + + def __init__(self) -> None: + self.x = [1] + +class Invariant[T]: + def f(self) -> None: + c = Invariant(1) + # We need to know that T is invariant here, and for this we need the type + # of self.x, which won't be available on the first type checking pass, + # since __init__ is defined later in the file. In this case we fall back + # covariance. + self.g(c) + c2 = Invariant(object()) + self.h(c2) # E: Argument 1 to "h" of "Invariant" has incompatible type "Invariant[object]"; expected "Invariant[int]" + + def g(self, x: Invariant[object]) -> None: pass + def h(self, x: Invariant[int]) -> None: pass + + def __init__(self, x: T) -> None: + self.x = x + +# Now we should have the variance correct. +a: Invariant[object] +b: Invariant[int] +if int(): + a = b # E: Incompatible types in assignment (expression has type "Invariant[int]", variable has type "Invariant[object]") +if int(): + b = a # E: Incompatible types in assignment (expression has type "Invariant[object]", variable has type "Invariant[int]") From b76e7eb2ea0b9a0712cb960d71b9d410592114a8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 22 Mar 2024 15:08:08 +0000 Subject: [PATCH 09/42] Fix join --- mypy/join.py | 4 ++-- test-data/unit/check-python312.test | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/mypy/join.py b/mypy/join.py index 3603e9fefb7a..7e0ff301ebf8 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -6,7 +6,7 @@ import mypy.typeops from mypy.maptype import map_instance_to_supertype -from mypy.nodes import CONTRAVARIANT, COVARIANT, INVARIANT +from mypy.nodes import CONTRAVARIANT, COVARIANT, INVARIANT, VARIANCE_NOT_READY from mypy.state import state from mypy.subtypes import ( SubtypeContext, @@ -97,7 +97,7 @@ def join_instances(self, t: Instance, s: Instance) -> ProperType: elif isinstance(sa_proper, AnyType): new_type = AnyType(TypeOfAny.from_another_any, sa_proper) elif isinstance(type_var, TypeVarType): - if type_var.variance == COVARIANT: + if type_var.variance in (COVARIANT, VARIANCE_NOT_READY): new_type = join_types(ta, sa, self) if len(type_var.values) != 0 and new_type not in type_var.values: self.seen_instances.pop() diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 9d16afddf651..76e5e9b9afde 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -285,3 +285,16 @@ if int(): a = b # E: Incompatible types in assignment (expression has type "Invariant[int]", variable has type "Invariant[object]") if int(): b = a # E: Incompatible types in assignment (expression has type "Invariant[object]", variable has type "Invariant[int]") + +[case testPEP695InferVarianceNotReadyForJoin] +# flags: --enable-incomplete-feature=NewGenericSyntax + +class Invariant[T]: + def f(self) -> None: + reveal_type([Invariant(1), Invariant(object())]) \ + # N: Revealed type is "builtins.list[__main__.Invariant[builtins.object]]" + + def __init__(self, x: T) -> None: + self.x = x + +reveal_type([Invariant(1), Invariant(object())]) # N: Revealed type is "builtins.list[builtins.object]" From 650468b036e023d14ae425773aede356718703e3 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 22 Mar 2024 15:24:27 +0000 Subject: [PATCH 10/42] Test meet when not ready --- test-data/unit/check-python312.test | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 76e5e9b9afde..1a4b63b54995 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -291,6 +291,7 @@ if int(): class Invariant[T]: def f(self) -> None: + # Assume covariance if variance us not ready reveal_type([Invariant(1), Invariant(object())]) \ # N: Revealed type is "builtins.list[__main__.Invariant[builtins.object]]" @@ -298,3 +299,23 @@ class Invariant[T]: self.x = x reveal_type([Invariant(1), Invariant(object())]) # N: Revealed type is "builtins.list[builtins.object]" + +[case testPEP695InferVarianceNotReadyForMeet] +# flags: --enable-incomplete-feature=NewGenericSyntax + +from typing import TypeVar, Callable + +S = TypeVar("S") +def c(a: Callable[[S], None], b: Callable[[S], None]) -> S: ... + +def a1(x: Invariant[int]) -> None: pass +def a2(x: Invariant[object]) -> None: pass + +class Invariant[T]: + def f(self) -> None: + reveal_type(c(a1, a2)) # N: Revealed type is "__main__.Invariant[builtins.int]" + + def __init__(self, x: T) -> None: + self.x = x + +reveal_type(c(a1, a2)) # N: Revealed type is "Never" From 639a85bc9f47cb3e97e3dc0d05e904da132fba1c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 3 May 2024 13:44:07 +0100 Subject: [PATCH 11/42] WIP minimal support for inheritance --- mypy/checker.py | 2 +- mypy/subtypes.py | 34 ++++++++++++++--------------- test-data/unit/check-python312.test | 23 +++++++++++++++++++ 3 files changed, 41 insertions(+), 18 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index b5abe8dac921..ba84ab007a0e 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2374,7 +2374,7 @@ def visit_class_def(self, defn: ClassDef) -> None: self.allow_abstract_call = old_allow_abstract_call # TODO: Apply the sig to the actual TypeInfo so we can handle decorators # that completely swap out the type. (e.g. Callable[[Type[A]], Type[B]]) - if typ.defn.type_vars: + if typ.defn.type_vars and typ.defn.type_args is None: for base_inst in typ.bases: for base_tvar, base_decl_tvar in zip( base_inst.args, base_inst.type.defn.type_vars diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 7ed6ad4d40e0..b3b6de5e0bf7 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -10,6 +10,7 @@ from mypy.erasetype import erase_type from mypy.expandtype import expand_self_type, expand_type_by_instance, expand_type from mypy.maptype import map_instance_to_supertype +from mypy.typevars import fill_typevars # Circular import; done in the function instead. # import mypy.solve @@ -1987,6 +1988,13 @@ def is_more_precise(left: Type, right: Type, *, ignore_promotions: bool = False) return is_proper_subtype(left, right, ignore_promotions=ignore_promotions) +def all_members(info: TypeInfo) -> set[str]: + members = set(info.names) + for base in info.mro[1:]: + members.update(base.names) + return members + + def infer_variance(info: TypeInfo, i: int) -> bool: """Infer the variance of the ith type variable of a generic class. @@ -1994,8 +2002,6 @@ def infer_variance(info: TypeInfo, i: int) -> bool: """ object_type = Instance(info.mro[-1], []) - from mypy.typeops import bind_self - for variance in COVARIANT, CONTRAVARIANT, INVARIANT: tv = info.defn.type_vars[i] assert isinstance(tv, TypeVarType) @@ -2005,23 +2011,17 @@ def infer_variance(info: TypeInfo, i: int) -> bool: co = True contra = True tvar = info.defn.type_vars[i] - for member, sym in info.names.items(): + self_type = fill_typevars(info) + for member in all_members(info): if member in ("__init__", "__new__"): continue - node = sym.node - typ = None - settable = False - if isinstance(node, FuncBase): - typ = node.type - if isinstance(typ, FunctionLike): - typ = bind_self(typ) - elif isinstance(node, Var): - settable = True - typ = node.type - if typ is None: - tv.variance = VARIANCE_NOT_READY - return False - + node = info[member].node + if isinstance(node, Var) and node.type is None: + tv.variance = VARIANCE_NOT_READY + return False + flags = get_member_flags(member, self_type) + typ = find_member(member, self_type, self_type) + settable = IS_SETTABLE in flags if typ: typ2 = expand_type(typ, {tvar.id: object_type}) if not is_subtype(typ, typ2): diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 1a4b63b54995..9fefee4aaa48 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -319,3 +319,26 @@ class Invariant[T]: self.x = x reveal_type(c(a1, a2)) # N: Revealed type is "Never" + +[case testPEP695InheritInvariant] +# flags: --enable-incomplete-feature=NewGenericSyntax + +class Invariant[T]: + x: T + +class Subclass[T](Invariant[T]): + pass + +x: Invariant[int] +y: Invariant[object] +if int(): + x = y # E: Incompatible types in assignment (expression has type "Invariant[object]", variable has type "Invariant[int]") +if int(): + y = x # E: Incompatible types in assignment (expression has type "Invariant[int]", variable has type "Invariant[object]") + +a: Subclass[int] +b: Subclass[object] +if int(): + a = b # E: Incompatible types in assignment (expression has type "Subclass[object]", variable has type "Subclass[int]") +if int(): + b = a # E: Incompatible types in assignment (expression has type "Subclass[int]", variable has type "Subclass[object]") From 2b4dbe10266c6f2390dfbaa3955a55a44fa0548f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 3 May 2024 14:57:12 +0100 Subject: [PATCH 12/42] Add inheritance test cases --- test-data/unit/check-python312.test | 41 +++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 9fefee4aaa48..e56043a7e113 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -342,3 +342,44 @@ if int(): a = b # E: Incompatible types in assignment (expression has type "Subclass[object]", variable has type "Subclass[int]") if int(): b = a # E: Incompatible types in assignment (expression has type "Subclass[int]", variable has type "Subclass[object]") + +[case testPEP695InheritanceMakesInvariant] +# flags: --enable-incomplete-feature=NewGenericSyntax +class Covariant[T]: + def f(self) -> T: + ... + +class Subclass[T](Covariant[list[T]]): + pass + +x: Covariant[int] = Covariant[object]() # E: Incompatible types in assignment (expression has type "Covariant[object]", variable has type "Covariant[int]") +y: Covariant[object] = Covariant[int]() + +a: Subclass[int] = Subclass[object]() # E: Incompatible types in assignment (expression has type "Subclass[object]", variable has type "Subclass[int]") +b: Subclass[object] = Subclass[int]() # E: Incompatible types in assignment (expression has type "Subclass[int]", variable has type "Subclass[object]") + +[case testPEP695InheritCoOrContravariant] +# flags: --enable-incomplete-feature=NewGenericSyntax +class Contravariant[T]: + def f(self, x: T) -> None: pass + +class CovSubclass[T](Contravariant[T]): + pass + +a: CovSubclass[int] = CovSubclass[object]() +b: CovSubclass[object] = CovSubclass[int]() # E: Incompatible types in assignment (expression has type "CovSubclass[int]", variable has type "CovSubclass[object]") + +class Covariant[T]: + def f(self) -> T: ... + +class CoSubclass[T](Covariant[T]): + pass + +c: CoSubclass[int] = CoSubclass[object]() # E: Incompatible types in assignment (expression has type "CoSubclass[object]", variable has type "CoSubclass[int]") +d: CoSubclass[object] = CoSubclass[int]() + +class InvSubclass[T](Covariant[T]): + def g(self, x: T) -> None: pass + +e: InvSubclass[int] = InvSubclass[object]() # E: Incompatible types in assignment (expression has type "InvSubclass[object]", variable has type "InvSubclass[int]") +f: InvSubclass[object] = InvSubclass[int]() # E: Incompatible types in assignment (expression has type "InvSubclass[int]", variable has type "InvSubclass[object]") From 5081cb9a07ffb789ea0bd0370adf3ab3536ef265 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 3 May 2024 15:04:48 +0100 Subject: [PATCH 13/42] Add test cases --- test-data/unit/check-python312.test | 42 +++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index e56043a7e113..ad4bc50760d2 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -383,3 +383,45 @@ class InvSubclass[T](Covariant[T]): e: InvSubclass[int] = InvSubclass[object]() # E: Incompatible types in assignment (expression has type "InvSubclass[object]", variable has type "InvSubclass[int]") f: InvSubclass[object] = InvSubclass[int]() # E: Incompatible types in assignment (expression has type "InvSubclass[int]", variable has type "InvSubclass[object]") + +[case testPEP695FinalAttribute] +# flags: --enable-incomplete-feature=NewGenericSyntax +from typing import Final + +class C[T]: + def __init__(self, x: T) -> None: + self.x: Final = x + +a: C[int] = C[object](1) # E: Incompatible types in assignment (expression has type "C[object]", variable has type "C[int]") +b: C[object] = C[int](1) + +[case testPEP695TwoTypeVariables] +# flags: --enable-incomplete-feature=NewGenericSyntax + +class C[T, S]: + def f(self, x: T) -> None: ... + def g(self) -> S: ... + +a: C[int, int] = C[object, int]() +b: C[object, int] = C[int, int]() # E: Incompatible types in assignment (expression has type "C[int, int]", variable has type "C[object, int]") +c: C[int, int] = C[int, object]() # E: Incompatible types in assignment (expression has type "C[int, object]", variable has type "C[int, int]") +d: C[int, object] = C[int, int]() + +[case testPEP695Properties] +# flags: --enable-incomplete-feature=NewGenericSyntax + +class R[T]: + @property + def p(self) -> T: ... + +class RW[T]: + @property + def p(self) -> T: ... + @p.setter + def p(self, x: T) -> None: ... + +a: R[int] = R[object]() # E: Incompatible types in assignment (expression has type "R[object]", variable has type "R[int]") +b: R[object] = R[int]() +c: RW[int] = RW[object]() # E: Incompatible types in assignment (expression has type "RW[object]", variable has type "RW[int]") +d: RW[object] = RW[int]() # E: Incompatible types in assignment (expression has type "RW[int]", variable has type "RW[object]") +[builtins fixtures/property.pyi] From d42026e6d5b7d114849b2b6cbf609a3a5d47fd9f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 7 May 2024 11:40:11 +0100 Subject: [PATCH 14/42] Support protocols --- mypy/checker.py | 3 +++ test-data/unit/check-python312.test | 37 +++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/mypy/checker.py b/mypy/checker.py index ba84ab007a0e..5e1218999c59 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2567,6 +2567,9 @@ def check_protocol_variance(self, defn: ClassDef) -> None: if they are actually covariant/contravariant, since this may break transitivity of subtyping, see PEP 544. """ + if defn.type_args is not None: + # Using new-style syntax (PEP 695), so variance will be inferred + return info = defn.info object_type = Instance(info.mro[-1], []) tvars = info.defn.type_vars diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index ad4bc50760d2..4cda76d64297 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -425,3 +425,40 @@ b: R[object] = R[int]() c: RW[int] = RW[object]() # E: Incompatible types in assignment (expression has type "RW[object]", variable has type "RW[int]") d: RW[object] = RW[int]() # E: Incompatible types in assignment (expression has type "RW[int]", variable has type "RW[object]") [builtins fixtures/property.pyi] + +[case testPEP695Protocol] +# flags: --enable-incomplete-feature=NewGenericSyntax +from typing import Protocol + +class PContra[T](Protocol): + def f(self, x: T) -> None: ... + +PContra() # E: Cannot instantiate protocol class "PContra" +a: PContra[int] +b: PContra[object] +if int(): + a = b +if int(): + b = a # E: Incompatible types in assignment (expression has type "PContra[int]", variable has type "PContra[object]") + +class PCov[T](Protocol): + def f(self) -> T: ... + +PCov() # E: Cannot instantiate protocol class "PCov" +c: PCov[int] +d: PCov[object] +if int(): + c = d # E: Incompatible types in assignment (expression has type "PCov[object]", variable has type "PCov[int]") +if int(): + d = c + +class PInv[T](Protocol): + def f(self, x: T) -> T: ... + +PInv() # E: Cannot instantiate protocol class "PInv" +e: PInv[int] +f: PInv[object] +if int(): + e = f # E: Incompatible types in assignment (expression has type "PInv[object]", variable has type "PInv[int]") +if int(): + f = e # E: Incompatible types in assignment (expression has type "PInv[int]", variable has type "PInv[object]") From 9745e5bd0eba4641ca0a2d72283857a4b91ed32b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 7 May 2024 12:48:58 +0100 Subject: [PATCH 15/42] Add generic function test cases --- test-data/unit/check-python312.test | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 4cda76d64297..8f2457fb7a4f 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -86,14 +86,22 @@ reveal_type(ba2) # N: Revealed type is "def (*Any) -> builtins.str" [case testPEP695GenericFunctionSyntax] # flags: --enable-incomplete-feature=NewGenericSyntax -def ident[T](x: T) -> T: - y: T = x - y = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "T") +def ident[TV](x: TV) -> TV: + y: TV = x + y = 1 # E: Incompatible types in assignment (expression has type "int", variable has type "TV") return x reveal_type(ident(1)) # N: Revealed type is "builtins.int" +reveal_type(ident('x')) # N: Revealed type is "builtins.str" -a: T # E: Name "T" is not defined +a: TV # E: Name "TV" is not defined + +def tup[T, S](x: T, y: S) -> tuple[T, S]: + reveal_type((x, y)) # N: Revealed type is "Tuple[T`-1, S`-2]" + return (x, y) + +reveal_type(tup(1, 'x')) # N: Revealed type is "Tuple[builtins.int, builtins.str]" +[builtins fixtures/tuple.pyi] [case testPEP695GenericClassSyntax] # flags: --enable-incomplete-feature=NewGenericSyntax @@ -116,6 +124,17 @@ c: C[int] = C(1) reveal_type(c.x) # N: Revealed type is "builtins.int" reveal_type(c.ident(1)) # N: Revealed type is "builtins.int" +[case testPEP695GenericMethodInGenericClass] +# flags: --enable-incomplete-feature=NewGenericSyntax + +class C[T]: + def m[S](self, x: S) -> T: ... + +a: C[int] = C[object]() # E: Incompatible types in assignment (expression has type "C[object]", variable has type "C[int]") +b: C[object] = C[int]() + +reveal_type(C[str]().m(1)) # N: Revealed type is "builtins.str" + [case testPEP695InferVarianceSimpleFromMethod] # flags: --enable-incomplete-feature=NewGenericSyntax From ddb4f587e93a691d264cea6b3ad239d0fa572bee Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 8 May 2024 11:52:45 +0100 Subject: [PATCH 16/42] Basic support for PEP 695 type aliases --- mypy/fastparse.py | 26 +++++--- mypy/nodes.py | 19 ++++++ mypy/semanal.py | 74 ++++++++++++++++++++++- mypy/strconv.py | 11 ++++ mypy/test/testparse.py | 4 ++ mypy/visitor.py | 4 ++ test-data/unit/check-python312.test | 92 +++++++++++++++++++++++++++++ test-data/unit/parse-python312.test | 12 ++++ 8 files changed, 232 insertions(+), 10 deletions(-) create mode 100644 test-data/unit/parse-python312.test diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 30f0515ec7c8..95d6501e7233 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -79,6 +79,7 @@ TempNode, TryStmt, TupleExpr, + TypeAliasStmt, UnaryExpr, Var, WhileStmt, @@ -1758,15 +1759,22 @@ def visit_MatchOr(self, n: MatchOr) -> OrPattern: node = OrPattern([self.visit(pattern) for pattern in n.patterns]) return self.set_line(node, n) - def visit_TypeAlias(self, n: ast_TypeAlias) -> AssignmentStmt: - self.fail( - ErrorMessage("PEP 695 type aliases are not yet supported", code=codes.VALID_TYPE), - n.lineno, - n.col_offset, - blocker=False, - ) - node = AssignmentStmt([NameExpr(n.name.id)], self.visit(n.value)) - return self.set_line(node, n) + # TypeAlias(identifier name, type_param* type_params, expr value) + def visit_TypeAlias(self, n: ast_TypeAlias) -> TypeAliasStmt | AssignmentStmt: + if NEW_GENERIC_SYNTAX in self.options.enable_incomplete_feature: + type_params = self.translate_type_params(n.type_params) + value = self.visit(n.value) + node = TypeAliasStmt(self.visit_Name(n.name), type_params, value) + return self.set_line(node, n) + else: + self.fail( + ErrorMessage("PEP 695 type aliases are not yet supported", code=codes.VALID_TYPE), + n.lineno, + n.col_offset, + blocker=False, + ) + node = AssignmentStmt([NameExpr(n.name.id)], self.visit(n.value)) + return self.set_line(node, n) class TypeConverter: diff --git a/mypy/nodes.py b/mypy/nodes.py index 7ec69e7f2b2d..22aaa0ef07de 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1618,6 +1618,25 @@ def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_match_stmt(self) +class TypeAliasStmt(Statement): + __slots__ = ("name", "type_args", "value") + + __match_args__ = ("name", "type_args", "value") + + name: NameExpr + type_args: list[tuple[str, mypy.types.Type | None]] + value: Expression # mypy.types.Type + + def __init__(self, name: NameExpr, type_args: list[tuple[str, mypy.types.Type | None]], value: Expression) -> None: + super().__init__() + self.name = name + self.type_args = type_args + self.value = value + + def accept(self, visitor: StatementVisitor[T]) -> T: + return visitor.visit_type_alias_stmt(self) + + # Expressions diff --git a/mypy/semanal.py b/mypy/semanal.py index 91c7575532bb..f8483113dc58 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -159,6 +159,7 @@ TupleExpr, TypeAlias, TypeAliasExpr, + TypeAliasStmt, TypeApplication, TypedDictExpr, TypeInfo, @@ -1636,7 +1637,7 @@ def push_type_args(self, type_args: list[tuple[str, Type | None]] | None, for tv in type_args: tve = TypeVarExpr( name=tv[0], - fullname=tv[0], + fullname=self.qualified_name(tv[0]), values=[], upper_bound=self.named_type("builtins.object"), default=AnyType(TypeOfAny.from_omitted_generics), @@ -5162,6 +5163,77 @@ def visit_match_stmt(self, s: MatchStmt) -> None: guard.accept(self) self.visit_block(s.bodies[i]) + def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: + self.statement = s + type_params: TypeVarLikeList = [] + all_type_params_names = [] + for name, bound in s.type_args: + upper_bound = bound or self.object_type() + fullname = self.qualified_name(name) + type_params.append((name, TypeVarExpr(name, fullname, [], upper_bound, + default=AnyType(TypeOfAny.from_omitted_generics)))) + all_type_params_names.append(name) + + self.push_type_args(s.type_args, s) + try: + tag = self.track_incomplete_refs() + res, alias_tvars, depends_on, qualified_tvars, empty_tuple_index = self.analyze_alias( + s.name.name, + s.value, + allow_placeholder=True, + declared_type_vars=type_params, + all_declared_type_params_names=all_type_params_names, + ) + if not res: + res = AnyType(TypeOfAny.from_error) + + if not self.is_func_scope(): + # Only marking incomplete for top-level placeholders makes recursive aliases like + # `A = Sequence[str | A]` valid here, similar to how we treat base classes in class + # definitions, allowing `class str(Sequence[str]): ...` + incomplete_target = isinstance(res, ProperType) and isinstance( + res, PlaceholderType + ) + else: + incomplete_target = has_placeholder(res) + + if self.found_incomplete_ref(tag) or incomplete_target: + # Since we have got here, we know this must be a type alias (incomplete refs + # may appear in nested positions), therefore use becomes_typeinfo=True. + self.mark_incomplete(s.name.name, s.value, becomes_typeinfo=True) + return + + self.add_type_alias_deps(depends_on) + # In addition to the aliases used, we add deps on unbound + # type variables, since they are erased from target type. + self.add_type_alias_deps(qualified_tvars) + # The above are only direct deps on other aliases. + # For subscripted aliases, type deps from expansion are added in deps.py + # (because the type is stored). + check_for_explicit_any(res, self.options, self.is_typeshed_stub_file, self.msg, context=s) + # When this type alias gets "inlined", the Any is not explicit anymore, + # so we need to replace it with non-explicit Anys. + res = make_any_non_explicit(res) + #if isinstance(res, ProperType) and isinstance(res, Instance): + # if not validate_instance(res, self.fail, empty_tuple_index): + # fix_instance(res, self.fail, self.note, disallow_any=False, options=self.options) + eager = self.is_func_scope() + alias_node = TypeAlias( + res, + self.qualified_name(s.name.name), + s.line, + s.column, + alias_tvars=alias_tvars, + no_args=False, + eager=eager, + ) + + self.add_symbol(s.name.name, alias_node, s) + + # TODO: Check if existing + finally: + self.pop_type_args(s.type_args) + # # Expressions # diff --git a/mypy/strconv.py b/mypy/strconv.py index 42a07c7f62fa..5b18099c965c 100644 --- a/mypy/strconv.py +++ b/mypy/strconv.py @@ -323,6 +323,17 @@ def visit_match_stmt(self, o: mypy.nodes.MatchStmt) -> str: a.append(("Body", o.bodies[i].body)) return self.dump(a, o) + def visit_type_alias_stmt(self, o: mypy.nodes.TypeAliasStmt) -> str: + a: list[Any] = [o.name] + for n, t in o.type_args: + aa = [n] + if t: + aa.append(t) + a.append(("TypeArg", aa)) + a.append(o.value) + + return self.dump(a, o) + # Expressions # Simple expressions diff --git a/mypy/test/testparse.py b/mypy/test/testparse.py index e33fa7e53ff0..e215920a6797 100644 --- a/mypy/test/testparse.py +++ b/mypy/test/testparse.py @@ -23,6 +23,8 @@ class ParserSuite(DataSuite): if sys.version_info < (3, 10): files.remove("parse-python310.test") + if sys.version_info < (3, 12): + files.remove("parse-python312.test") def run_case(self, testcase: DataDrivenTestCase) -> None: test_parser(testcase) @@ -39,6 +41,8 @@ def test_parser(testcase: DataDrivenTestCase) -> None: if testcase.file.endswith("python310.test"): options.python_version = (3, 10) + elif testcase.file.endswith("python312.test"): + options.python_version = (3, 12) else: options.python_version = defaults.PYTHON3_VERSION diff --git a/mypy/visitor.py b/mypy/visitor.py index c5aa3caa8295..be7b9dd5a82d 100644 --- a/mypy/visitor.py +++ b/mypy/visitor.py @@ -309,6 +309,10 @@ def visit_try_stmt(self, o: mypy.nodes.TryStmt) -> T: def visit_match_stmt(self, o: mypy.nodes.MatchStmt) -> T: pass + @abstractmethod + def visit_type_alias_stmt(self, o: mypy.nodes.TypeAliasStmt) -> T: + pass + @trait @mypyc_attr(allow_interpreted_subclasses=True) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 8f2457fb7a4f..30daf33052d2 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -481,3 +481,95 @@ if int(): e = f # E: Incompatible types in assignment (expression has type "PInv[object]", variable has type "PInv[int]") if int(): f = e # E: Incompatible types in assignment (expression has type "PInv[int]", variable has type "PInv[object]") + +[case testPEP695TypeAlias] +# flags: --enable-incomplete-feature=NewGenericSyntax + +class C[T]: pass +class D[T, S]: pass + +type A[S] = C[S] + +a: A[int] +reveal_type(a) # N: Revealed type is "__main__.C[builtins.int]" + +type A2[T] = C[C[T]] +a2: A2[str] +reveal_type(a2) # N: Revealed type is "__main__.C[__main__.C[builtins.str]]" + +type A3[T, S] = D[S, C[T]] +a3: A3[int, str] +reveal_type(a3) # N: Revealed type is "__main__.D[builtins.str, __main__.C[builtins.int]]" + +type A4 = int | str +a4: A4 +reveal_type(a4) # N: Revealed type is "Union[builtins.int, builtins.str]" + +[case testPEP695TypeAliasWithUnusedTypeParams] +# flags: --enable-incomplete-feature=NewGenericSyntax +type A[T] = int +a: A[str] +reveal_type(a) # N: Revealed type is "builtins.int" + +[case testPEP695TypeAliasForwardReference1] +# flags: --enable-incomplete-feature=NewGenericSyntax + +type A[T] = C[T] + +a: A[int] +reveal_type(a) # N: Revealed type is "__main__.C[builtins.int]" + +class C[T]: pass + +[case testPEP695TypeAliasForwardReference2] +# flags: --enable-incomplete-feature=NewGenericSyntax + +type X = C +type A = X + +a: A +reveal_type(a) # N: Revealed type is "__main__.C" + +class C: pass + +[case testPEP695TypeAliasForwardReference3] +# flags: --enable-incomplete-feature=NewGenericSyntax + +type X = D +type A = C[X] + +a: A +reveal_type(a) # N: Revealed type is "__main__.C[__main__.D]" + +class C[T]: pass +class D: pass + +[case testPEP695TypeAliasForwardReference4] +# flags: --enable-incomplete-feature=NewGenericSyntax + +type A = C + +# Note that this doesn't actually work at runtime, but we currently don't +# keep track whether a type alias is valid in various runtime type contexts. +class D(A): + pass + +class C: pass + +x: C = D() +y: D = C() # E: Incompatible types in assignment (expression has type "C", variable has type "D") + +[case testPEP695TypeAliasWithUndefineName] +# flags: --enable-incomplete-feature=NewGenericSyntax +type A[T] = XXX # E: Name "XXX" is not defined +a: A[int] +reveal_type(a) # N: Revealed type is "Any" + +[case testPEP695TypeAliasInvalidType] +# flags: --enable-incomplete-feature=NewGenericSyntax +type A = int | 1 # E: Invalid type: try using Literal[1] instead? +a: A +reveal_type(a) # N: Revealed type is "Union[builtins.int, Any]" +type B = int + str # E: Invalid type alias: expression is not a valid type +b: B +reveal_type(b) # N: Revealed type is "Any" diff --git a/test-data/unit/parse-python312.test b/test-data/unit/parse-python312.test new file mode 100644 index 000000000000..57d49e2d2314 --- /dev/null +++ b/test-data/unit/parse-python312.test @@ -0,0 +1,12 @@ +[case testPEP695TypeAlias] +# mypy: enable-incomplete-feature=NewGenericSyntax +type A[T] = C[T] +[out] +MypyFile:1( + TypeAliasStmt:2( + NameExpr(A) + TypeArg( + T) + IndexExpr:2( + NameExpr(C) + NameExpr(T)))) From a1ea4877476374a6b52a48b585ac020edcea5e39 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 8 May 2024 12:09:00 +0100 Subject: [PATCH 17/42] Partial support for upper bounds --- mypy/semanal.py | 15 ++++++++++----- test-data/unit/check-python312.test | 24 ++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index f8483113dc58..d4e56e68b9f2 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1634,16 +1634,21 @@ def push_type_args(self, type_args: list[tuple[str, Type | None]] | None, context: Context) -> None: if not type_args: return - for tv in type_args: + for name, upper_bound in type_args: + if upper_bound: + upper_bound = self.anal_type(upper_bound) + else: + upper_bound = self.named_type("builtins.object") + tve = TypeVarExpr( - name=tv[0], - fullname=self.qualified_name(tv[0]), + name=name, + fullname=self.qualified_name(name), values=[], - upper_bound=self.named_type("builtins.object"), + upper_bound=upper_bound, default=AnyType(TypeOfAny.from_omitted_generics), variance=VARIANCE_NOT_READY, ) - self.add_symbol(tv[0], tve, context) + self.add_symbol(name, tve, context) def pop_type_args(self, type_args: list[tuple[str, Type | None]] | None) -> None: if not type_args: diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 30daf33052d2..35c9b93092be 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -573,3 +573,27 @@ reveal_type(a) # N: Revealed type is "Union[builtins.int, Any]" type B = int + str # E: Invalid type alias: expression is not a valid type b: B reveal_type(b) # N: Revealed type is "Any" + +[case testPEP695UpperBound] +# flags: --enable-incomplete-feature=NewGenericSyntax + +class D: + x: int +class E(D): pass + +class C[T: D]: pass + +a: C[D] +b: C[E] +reveal_type(a) # N: Revealed type is "__main__.C[__main__.D]" +reveal_type(b) # N: Revealed type is "__main__.C[__main__.E]" + +c: C[int] # E: Type argument "int" of "C" must be a subtype of "D" + +def f[T: D](a: T) -> T: + reveal_type(a.x) # N: Revealed type is "builtins.int" + return a + +reveal_type(f(D())) # N: Revealed type is "__main__.D" +reveal_type(f(E())) # N: Revealed type is "__main__.E" +f(1) # E: Value of type variable "T" of "f" cannot be "int" From 0e41f88928628d16b78dc20e05f768c7b4f25dfe Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 8 May 2024 12:16:00 +0100 Subject: [PATCH 18/42] Ssupport forward refs in upper bounds --- mypy/semanal.py | 34 +++++++++++------ test-data/unit/check-python312.test | 59 +++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 11 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index d4e56e68b9f2..ed9b05cf276f 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1625,30 +1625,42 @@ def visit_class_def(self, defn: ClassDef) -> None: self.incomplete_type_stack.append(not defn.info) namespace = self.qualified_name(defn.name) with self.tvar_scope_frame(self.tvar_scope.class_frame(namespace)): - self.push_type_args(defn.type_args, defn) + if not self.push_type_args(defn.type_args, defn): + self.mark_incomplete(defn.name, defn) + return + self.analyze_class(defn) self.pop_type_args(defn.type_args) self.incomplete_type_stack.pop() def push_type_args(self, type_args: list[tuple[str, Type | None]] | None, - context: Context) -> None: + context: Context) -> bool: if not type_args: - return + return True + tvs = [] for name, upper_bound in type_args: if upper_bound: upper_bound = self.anal_type(upper_bound) + if upper_bound is None: + return False else: upper_bound = self.named_type("builtins.object") - tve = TypeVarExpr( - name=name, - fullname=self.qualified_name(name), - values=[], - upper_bound=upper_bound, - default=AnyType(TypeOfAny.from_omitted_generics), - variance=VARIANCE_NOT_READY, + tvs.append( + (name, TypeVarExpr( + name=name, + fullname=self.qualified_name(name), + values=[], + upper_bound=upper_bound, + default=AnyType(TypeOfAny.from_omitted_generics), + variance=VARIANCE_NOT_READY, + )) ) - self.add_symbol(name, tve, context) + + for name, tv in tvs: + self.add_symbol(name, tv, context) + + return True def pop_type_args(self, type_args: list[tuple[str, Type | None]] | None) -> None: if not type_args: diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 35c9b93092be..4889eae8cbec 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -597,3 +597,62 @@ def f[T: D](a: T) -> T: reveal_type(f(D())) # N: Revealed type is "__main__.D" reveal_type(f(E())) # N: Revealed type is "__main__.E" f(1) # E: Value of type variable "T" of "f" cannot be "int" + +[case testPEP695UpperBoundForwardReference1] +# flags: --enable-incomplete-feature=NewGenericSyntax + +class C[T: D]: pass + +a: C[D] +b: C[E] +reveal_type(a) # N: Revealed type is "__main__.C[__main__.D]" +reveal_type(b) # N: Revealed type is "__main__.C[__main__.E]" + +c: C[int] # E: Type argument "int" of "C" must be a subtype of "D" + +class D: pass +class E(D): pass + +[case testPEP695UpperBoundForwardReference2] +# flags: --enable-incomplete-feature=NewGenericSyntax + +type A = D +class C[T: A]: pass + +class D: pass +class E(D): pass + +a: C[D] +b: C[E] +reveal_type(a) # N: Revealed type is "__main__.C[__main__.D]" +reveal_type(b) # N: Revealed type is "__main__.C[__main__.E]" + +c: C[int] # E: Type argument "int" of "C" must be a subtype of "D" + +[case testPEP695UpperBoundForwardReference3] +# flags: --enable-incomplete-feature=NewGenericSyntax + +class D[T]: pass +class E[T](D[T]): pass + +type A = D[X] + +class C[T: A]: pass + +class X: pass + +a: C[D[X]] +b: C[E[X]] +reveal_type(a) # N: Revealed type is "__main__.C[__main__.D[__main__.X]]" +reveal_type(b) # N: Revealed type is "__main__.C[__main__.E[__main__.X]]" + +c: C[D[int]] # E: Type argument "D[int]" of "C" must be a subtype of "D[X]" + +[case testPEP695UpperBoundUndefinedName] +# flags: --enable-incomplete-feature=NewGenericSyntax + +class C[T: XX]: # E: Name "XX" is not defined + pass + +a: C[int] + From 16ffbfb3438f9c3e17c17fc2fb87a7d41b502862 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 8 May 2024 12:32:45 +0100 Subject: [PATCH 19/42] Fix for invalid upper bound in generic functions --- mypy/semanal.py | 6 ++++-- test-data/unit/check-python312.test | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index ed9b05cf276f..a4a007c499f3 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -838,9 +838,11 @@ def visit_func_def(self, defn: FuncDef) -> None: self.analyze_func_def(defn) def analyze_func_def(self, defn: FuncDef) -> None: - self.function_stack.append(defn) + if not self.push_type_args(defn.type_args, defn): + self.defer(defn) + return - self.push_type_args(defn.type_args, defn) + self.function_stack.append(defn) if defn.type: assert isinstance(defn.type, CallableType) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 4889eae8cbec..7d520f29f574 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -648,6 +648,21 @@ reveal_type(b) # N: Revealed type is "__main__.C[__main__.E[__main__.X]]" c: C[D[int]] # E: Type argument "D[int]" of "C" must be a subtype of "D[X]" +[case testPEP695UpperBoundForwardReference4] +# flags: --enable-incomplete-feature=NewGenericSyntax + +def f[T: D](a: T) -> T: + reveal_type(a.x) # N: Revealed type is "builtins.int" + return a + +class D: + x: int +class E(D): pass + +reveal_type(f(D())) # N: Revealed type is "__main__.D" +reveal_type(f(E())) # N: Revealed type is "__main__.E" +f(1) # E: Value of type variable "T" of "f" cannot be "int" + [case testPEP695UpperBoundUndefinedName] # flags: --enable-incomplete-feature=NewGenericSyntax @@ -656,3 +671,6 @@ class C[T: XX]: # E: Name "XX" is not defined a: C[int] +def f[T: YY](x: T) -> T: # E: Name "YY" is not defined + return x +reveal_type(f) # N: Revealed type is "def [T <: Any] (x: T`-1) -> T`-1" From c6bbe70f19dc43d57f1cc074ad06ad00effd34dd Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 8 May 2024 13:41:40 +0100 Subject: [PATCH 20/42] Add test case --- test-data/unit/check-python312.test | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 7d520f29f574..7f770ecb6f14 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -674,3 +674,22 @@ a: C[int] def f[T: YY](x: T) -> T: # E: Name "YY" is not defined return x reveal_type(f) # N: Revealed type is "def [T <: Any] (x: T`-1) -> T`-1" + +[case testPEP695UpperBoundWithMultipleParams] +# flags: --enable-incomplete-feature=NewGenericSyntax + +class C[T, S: int]: pass +class D[A: int, B]: pass + +def f[T: int, S: int | str](x: T, y: S) -> T | S: + return x + +C[str, int]() +C[str, str]() # E: Value of type variable "S" of "C" cannot be "str" +D[int, str]() +D[str, str]() # E: Value of type variable "A" of "D" cannot be "str" +f(1, 1) +u: int | str +f(1, u) +f('x', None) # E: Value of type variable "T" of "f" cannot be "str" \ + # E: Value of type variable "S" of "f" cannot be "None" From 59434fa4584c8c9f2a4b8c13095c59ab4aadba12 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 8 May 2024 14:16:05 +0100 Subject: [PATCH 21/42] Add missing visit methods --- mypy/traverser.py | 6 ++++++ mypy/visitor.py | 3 +++ 2 files changed, 9 insertions(+) diff --git a/mypy/traverser.py b/mypy/traverser.py index d11dd395f978..225de27e7002 100644 --- a/mypy/traverser.py +++ b/mypy/traverser.py @@ -71,6 +71,7 @@ TupleExpr, TypeAlias, TypeAliasExpr, + TypeAliasStmt, TypeApplication, TypedDictExpr, TypeVarExpr, @@ -243,6 +244,11 @@ def visit_match_stmt(self, o: MatchStmt) -> None: guard.accept(self) o.bodies[i].accept(self) + def visit_type_alias_stmt(self, o: TypeAliasStmt) -> None: + o.name.accept(self) + # TODO: params + o.value.accept(self) + def visit_member_expr(self, o: MemberExpr) -> None: o.expr.accept(self) diff --git a/mypy/visitor.py b/mypy/visitor.py index be7b9dd5a82d..340e1af64e00 100644 --- a/mypy/visitor.py +++ b/mypy/visitor.py @@ -464,6 +464,9 @@ def visit_with_stmt(self, o: mypy.nodes.WithStmt) -> T: def visit_match_stmt(self, o: mypy.nodes.MatchStmt) -> T: pass + def visit_type_alias_stmt(self, o: mypy.nodes.TypeAliasStmt) -> T: + pass + # Expressions (default no-op implementation) def visit_int_expr(self, o: mypy.nodes.IntExpr) -> T: From 84e3f11004704abc9eb7b3850fd0e968648d64c2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 8 May 2024 14:16:15 +0100 Subject: [PATCH 22/42] Generate error in mypyc if compiling a type alias statement --- mypyc/irbuild/visitor.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mypyc/irbuild/visitor.py b/mypyc/irbuild/visitor.py index 12e186fd40d8..e7256f036e4c 100644 --- a/mypyc/irbuild/visitor.py +++ b/mypyc/irbuild/visitor.py @@ -70,6 +70,7 @@ TryStmt, TupleExpr, TypeAliasExpr, + TypeAliasStmt, TypeApplication, TypedDictExpr, TypeVarExpr, @@ -249,6 +250,9 @@ def visit_nonlocal_decl(self, stmt: NonlocalDecl) -> None: def visit_match_stmt(self, stmt: MatchStmt) -> None: transform_match_stmt(self.builder, stmt) + def visit_type_alias_stmt(self, stmt: TypeAliasStmt) -> None: + self.bail('The "type" statement is not yet supported by mypyc', stmt.line) + # Expressions def visit_name_expr(self, expr: NameExpr) -> Value: From 586eb644b3dfc6c932ae6711b0905dd2817119d9 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 9 May 2024 10:25:44 +0100 Subject: [PATCH 23/42] Fix type alias self check --- mypy/fastparse.py | 1 + mypy/strconv.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 95d6501e7233..a8ac08ae045f 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -1761,6 +1761,7 @@ def visit_MatchOr(self, n: MatchOr) -> OrPattern: # TypeAlias(identifier name, type_param* type_params, expr value) def visit_TypeAlias(self, n: ast_TypeAlias) -> TypeAliasStmt | AssignmentStmt: + node: TypeAliasStmt | AssignmentStmt if NEW_GENERIC_SYNTAX in self.options.enable_incomplete_feature: type_params = self.translate_type_params(n.type_params) value = self.visit(n.value) diff --git a/mypy/strconv.py b/mypy/strconv.py index 5b18099c965c..ad4f8433673f 100644 --- a/mypy/strconv.py +++ b/mypy/strconv.py @@ -326,7 +326,7 @@ def visit_match_stmt(self, o: mypy.nodes.MatchStmt) -> str: def visit_type_alias_stmt(self, o: mypy.nodes.TypeAliasStmt) -> str: a: list[Any] = [o.name] for n, t in o.type_args: - aa = [n] + aa: list[Any] = [n] if t: aa.append(t) a.append(("TypeArg", aa)) From 6bbc046bb13e9e37339efe9ccc73ce7097cf6bf7 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 9 May 2024 10:26:31 +0100 Subject: [PATCH 24/42] Fix variance not ready handling in typestate --- mypy/typestate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/typestate.py b/mypy/typestate.py index b16fb7c21fed..3fcaab426ca2 100644 --- a/mypy/typestate.py +++ b/mypy/typestate.py @@ -10,7 +10,7 @@ from mypy.nodes import TypeInfo, VARIANCE_NOT_READY from mypy.server.trigger import make_trigger -from mypy.types import Instance, Type, TypeVarId, get_proper_type +from mypy.types import Instance, Type, TypeVarId, TypeVarType, get_proper_type MAX_NEGATIVE_CACHE_TYPES: Final = 1000 MAX_NEGATIVE_CACHE_ENTRIES: Final = 10000 @@ -192,7 +192,7 @@ def record_subtype_cache_entry( # These are unlikely to match, due to the large space of # possible values. Avoid uselessly increasing cache sizes. return - if any(tv.variance == VARIANCE_NOT_READY for tv in right.type.defn.type_vars): + if any((isinstance(tv, TypeVarType) and tv.variance == VARIANCE_NOT_READY) for tv in right.type.defn.type_vars): # Variance indeterminate -- don't know the result return cache = self._subtype_caches.setdefault(right.type, {}) From ea39e96afadd0410a83da44f5ad905a8aed83b38 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 9 May 2024 10:27:06 +0100 Subject: [PATCH 25/42] Fix missing return value in infer_class_variances --- mypy/subtypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index b3b6de5e0bf7..e3ce1e48e085 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -2044,7 +2044,7 @@ def infer_variance(info: TypeInfo, i: int) -> bool: def infer_class_variances(info: TypeInfo) -> bool: if not info.defn.type_args: - return + return True tvs = info.defn.type_vars for i, tv in enumerate(tvs): if isinstance(tv, TypeVarType): From ad8c062c81beabcfcb087334b846c47de072b2da Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 9 May 2024 10:48:41 +0100 Subject: [PATCH 26/42] Fix false positives about undefined names --- mypy/partially_defined.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mypy/partially_defined.py b/mypy/partially_defined.py index b7f577110fa8..da0bb517189a 100644 --- a/mypy/partially_defined.py +++ b/mypy/partially_defined.py @@ -36,6 +36,7 @@ SymbolTable, TryStmt, TupleExpr, + TypeAliasStmt, WhileStmt, WithStmt, implicit_module_attrs, @@ -673,3 +674,7 @@ def visit_import_from(self, o: ImportFrom) -> None: name = mod self.tracker.record_definition(name) super().visit_import_from(o) + + def visit_type_alias_stmt(self, o: TypeAliasStmt) -> None: + # Type alias target may contain forward references + self.tracker.record_definition(o.name.name) From 6138266f562e70429232ef44ed2154fb035facd5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 9 May 2024 11:31:51 +0100 Subject: [PATCH 27/42] Basic support for tuple subclasses --- mypy/subtypes.py | 3 +++ test-data/unit/check-python312.test | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index e3ce1e48e085..43e238219887 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -2019,6 +2019,9 @@ def infer_variance(info: TypeInfo, i: int) -> bool: if isinstance(node, Var) and node.type is None: tv.variance = VARIANCE_NOT_READY return False + if isinstance(self_type, TupleType): + self_type =mypy.typeops.tuple_fallback(self_type) + flags = get_member_flags(member, self_type) typ = find_member(member, self_type, self_type) settable = IS_SETTABLE in flags diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 7f770ecb6f14..0275599ab1ca 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -693,3 +693,25 @@ u: int | str f(1, u) f('x', None) # E: Value of type variable "T" of "f" cannot be "str" \ # E: Value of type variable "S" of "f" cannot be "None" + +[case testPEP695InferVarianceOfTupleType] +# flags: --enable-incomplete-feature=NewGenericSyntax + +class Cov[T](tuple[int, str]): + def f(self) -> T: pass + +class Cov2[T](tuple[T, T]): + pass + +class Contra[T](tuple[int, str]): + def f(self, x: T) -> None: pass + +a: Cov[object] = Cov[int]() +b: Cov[int] = Cov[object]() # E: Incompatible types in assignment (expression has type "Cov[object]", variable has type "Cov[int]") + +c: Cov2[object] = Cov2[int]() +d: Cov2[int] = Cov2[object]() # E: Incompatible types in assignment (expression has type "Cov2[object]", variable has type "Cov2[int]") + +e: Contra[int] = Contra[object]() +f: Contra[object] = Contra[int]() # E: Incompatible types in assignment (expression has type "Contra[int]", variable has type "Contra[object]") +[builtins fixtures/tuple-simple.pyi] From 98bd612eac27abf0643f25b99327091ac3864627 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 9 May 2024 11:52:19 +0100 Subject: [PATCH 28/42] Black --- mypy/checker.py | 3 ++- mypy/fastparse.py | 7 ++++--- mypy/nodes.py | 11 ++++++++--- mypy/semanal.py | 47 ++++++++++++++++++++++++++++++++--------------- mypy/subtypes.py | 10 +++++----- mypy/typestate.py | 7 +++++-- 6 files changed, 56 insertions(+), 29 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 5e1218999c59..ef96f8dfa309 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -146,6 +146,7 @@ from mypy.state import state from mypy.subtypes import ( find_member, + infer_class_variances, is_callable_compatible, is_equivalent, is_more_precise, @@ -153,7 +154,7 @@ is_same_type, is_subtype, restrict_subtype_away, - unify_generic_callable, infer_class_variances, + unify_generic_callable, ) from mypy.traverser import TraverserVisitor, all_return_statements, has_return_statement from mypy.treetransform import TransformVisitor diff --git a/mypy/fastparse.py b/mypy/fastparse.py index a8ac08ae045f..b8ae9288049a 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -88,7 +88,7 @@ YieldFromExpr, check_arg_names, ) -from mypy.options import Options, NEW_GENERIC_SYNTAX +from mypy.options import NEW_GENERIC_SYNTAX, Options from mypy.patterns import ( AsPattern, ClassPattern, @@ -944,8 +944,9 @@ def do_func_def( explicit_type_params = self.translate_type_params(n.type_params) else: self.fail( - ErrorMessage("PEP 695 generics are not yet supported", - code=codes.VALID_TYPE), + ErrorMessage( + "PEP 695 generics are not yet supported", code=codes.VALID_TYPE + ), n.type_params[0].lineno, n.type_params[0].col_offset, blocker=False, diff --git a/mypy/nodes.py b/mypy/nodes.py index 22aaa0ef07de..b2c0972de5f5 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1120,7 +1120,7 @@ def __init__( base_type_exprs: list[Expression] | None = None, metaclass: Expression | None = None, keywords: list[tuple[str, Expression]] | None = None, - type_args: list[tuple[str, mypy.types.Type | None]] | None = None + type_args: list[tuple[str, mypy.types.Type | None]] | None = None, ) -> None: super().__init__() self.name = name @@ -1625,9 +1625,14 @@ class TypeAliasStmt(Statement): name: NameExpr type_args: list[tuple[str, mypy.types.Type | None]] - value: Expression # mypy.types.Type + value: Expression # mypy.types.Type - def __init__(self, name: NameExpr, type_args: list[tuple[str, mypy.types.Type | None]], value: Expression) -> None: + def __init__( + self, + name: NameExpr, + type_args: list[tuple[str, mypy.types.Type | None]], + value: Expression, + ) -> None: super().__init__() self.name = name self.type_args = type_args diff --git a/mypy/semanal.py b/mypy/semanal.py index a4a007c499f3..24096c359e63 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -84,6 +84,7 @@ REVEAL_LOCALS, REVEAL_TYPE, RUNTIME_PROTOCOL_DECOS, + VARIANCE_NOT_READY, ArgKind, AssertStmt, AssertTypeExpr, @@ -178,7 +179,7 @@ is_final_node, type_aliases, type_aliases_source_versions, - typing_extensions_aliases, VARIANCE_NOT_READY, + typing_extensions_aliases, ) from mypy.options import Options from mypy.patterns import ( @@ -1635,8 +1636,9 @@ def visit_class_def(self, defn: ClassDef) -> None: self.pop_type_args(defn.type_args) self.incomplete_type_stack.pop() - def push_type_args(self, type_args: list[tuple[str, Type | None]] | None, - context: Context) -> bool: + def push_type_args( + self, type_args: list[tuple[str, Type | None]] | None, context: Context + ) -> bool: if not type_args: return True tvs = [] @@ -1649,14 +1651,17 @@ def push_type_args(self, type_args: list[tuple[str, Type | None]] | None, upper_bound = self.named_type("builtins.object") tvs.append( - (name, TypeVarExpr( - name=name, - fullname=self.qualified_name(name), - values=[], - upper_bound=upper_bound, - default=AnyType(TypeOfAny.from_omitted_generics), - variance=VARIANCE_NOT_READY, - )) + ( + name, + TypeVarExpr( + name=name, + fullname=self.qualified_name(name), + values=[], + upper_bound=upper_bound, + default=AnyType(TypeOfAny.from_omitted_generics), + variance=VARIANCE_NOT_READY, + ), + ) ) for name, tv in tvs: @@ -5189,8 +5194,18 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: for name, bound in s.type_args: upper_bound = bound or self.object_type() fullname = self.qualified_name(name) - type_params.append((name, TypeVarExpr(name, fullname, [], upper_bound, - default=AnyType(TypeOfAny.from_omitted_generics)))) + type_params.append( + ( + name, + TypeVarExpr( + name, + fullname, + [], + upper_bound, + default=AnyType(TypeOfAny.from_omitted_generics), + ), + ) + ) all_type_params_names.append(name) self.push_type_args(s.type_args, s) @@ -5229,11 +5244,13 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: # The above are only direct deps on other aliases. # For subscripted aliases, type deps from expansion are added in deps.py # (because the type is stored). - check_for_explicit_any(res, self.options, self.is_typeshed_stub_file, self.msg, context=s) + check_for_explicit_any( + res, self.options, self.is_typeshed_stub_file, self.msg, context=s + ) # When this type alias gets "inlined", the Any is not explicit anymore, # so we need to replace it with non-explicit Anys. res = make_any_non_explicit(res) - #if isinstance(res, ProperType) and isinstance(res, Instance): + # if isinstance(res, ProperType) and isinstance(res, Instance): # if not validate_instance(res, self.fail, empty_tuple_index): # fix_instance(res, self.fail, self.note, disallow_any=False, options=self.options) eager = self.is_func_scope() diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 43e238219887..5aa6a08c0994 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -8,9 +8,8 @@ import mypy.constraints import mypy.typeops from mypy.erasetype import erase_type -from mypy.expandtype import expand_self_type, expand_type_by_instance, expand_type +from mypy.expandtype import expand_self_type, expand_type, expand_type_by_instance from mypy.maptype import map_instance_to_supertype -from mypy.typevars import fill_typevars # Circular import; done in the function instead. # import mypy.solve @@ -20,11 +19,12 @@ CONTRAVARIANT, COVARIANT, INVARIANT, + VARIANCE_NOT_READY, Decorator, FuncBase, OverloadedFuncDef, TypeInfo, - Var, VARIANCE_NOT_READY, + Var, ) from mypy.options import Options from mypy.state import state @@ -67,7 +67,7 @@ ) from mypy.types_utils import flatten_types from mypy.typestate import SubtypeKind, type_state -from mypy.typevars import fill_typevars_with_any +from mypy.typevars import fill_typevars, fill_typevars_with_any # Flags for detected protocol members IS_SETTABLE: Final = 1 @@ -2020,7 +2020,7 @@ def infer_variance(info: TypeInfo, i: int) -> bool: tv.variance = VARIANCE_NOT_READY return False if isinstance(self_type, TupleType): - self_type =mypy.typeops.tuple_fallback(self_type) + self_type = mypy.typeops.tuple_fallback(self_type) flags = get_member_flags(member, self_type) typ = find_member(member, self_type, self_type) diff --git a/mypy/typestate.py b/mypy/typestate.py index 3fcaab426ca2..0082c5564705 100644 --- a/mypy/typestate.py +++ b/mypy/typestate.py @@ -8,7 +8,7 @@ from typing import Dict, Final, Set, Tuple from typing_extensions import TypeAlias as _TypeAlias -from mypy.nodes import TypeInfo, VARIANCE_NOT_READY +from mypy.nodes import VARIANCE_NOT_READY, TypeInfo from mypy.server.trigger import make_trigger from mypy.types import Instance, Type, TypeVarId, TypeVarType, get_proper_type @@ -192,7 +192,10 @@ def record_subtype_cache_entry( # These are unlikely to match, due to the large space of # possible values. Avoid uselessly increasing cache sizes. return - if any((isinstance(tv, TypeVarType) and tv.variance == VARIANCE_NOT_READY) for tv in right.type.defn.type_vars): + if any( + (isinstance(tv, TypeVarType) and tv.variance == VARIANCE_NOT_READY) + for tv in right.type.defn.type_vars + ): # Variance indeterminate -- don't know the result return cache = self._subtype_caches.setdefault(right.type, {}) From a1720274742d4be87865d5d0067e37128c6fe640 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 9 May 2024 12:20:12 +0100 Subject: [PATCH 29/42] Refactor AST --- mypy/fastparse.py | 10 ++++---- mypy/nodes.py | 21 +++++++++++----- mypy/semanal.py | 37 +++++++++++++++-------------- mypy/strconv.py | 17 +++++++++---- test-data/unit/parse-python312.test | 23 +++++++++++++++++- 5 files changed, 74 insertions(+), 34 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index b8ae9288049a..549651365a3e 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -80,6 +80,8 @@ TryStmt, TupleExpr, TypeAliasStmt, + TypeParam, + TypeVarExpr, UnaryExpr, Var, WhileStmt, @@ -886,7 +888,7 @@ def do_func_def( arg_kinds = [arg.kind for arg in args] arg_names = [None if arg.pos_only else arg.variable.name for arg in args] # Type parameters, if using new syntax for generics (PEP 695) - explicit_type_params: list[tuple[str, Type | None]] | None = None + explicit_type_params: list[TypeParam] | None = None arg_types: list[Type | None] = [] if no_type_check: @@ -1129,7 +1131,7 @@ def visit_ClassDef(self, n: ast3.ClassDef) -> ClassDef: keywords = [(kw.arg, self.visit(kw.value)) for kw in n.keywords if kw.arg] # Type parameters, if using new syntax for generics (PEP 695) - explicit_type_params: list[tuple[str, Type | None]] | None = None + explicit_type_params: list[TypeParam] | None = None if sys.version_info >= (3, 12) and n.type_params: if NEW_GENERIC_SYNTAX in self.options.enable_incomplete_feature: @@ -1165,14 +1167,14 @@ def visit_ClassDef(self, n: ast3.ClassDef) -> ClassDef: self.class_and_function_stack.pop() return cdef - def translate_type_params(self, type_params: Any) -> list[tuple[str, Type | None]]: + def translate_type_params(self, type_params: list[Any]) -> list[TypeParam]: explicit_type_params = [] for p in type_params: if p.bound is None: bound = None else: bound = TypeConverter(self.errors, line=p.lineno).visit(p.bound) - explicit_type_params.append((p.name, bound)) + explicit_type_params.append(TypeParam(p.name, bound, [])) return explicit_type_params # Return(expr? value) diff --git a/mypy/nodes.py b/mypy/nodes.py index b2c0972de5f5..ee53153ae2b5 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -653,6 +653,15 @@ def set_line( self.variable.set_line(self.line, self.column, self.end_line, self.end_column) +class TypeParam: + __slots__ = ("name", "upper_bound", "values") + + def __init__(self, name: str, upper_bound: mypy.types.Type | None, values: list[mypy.types.Type]) -> None: + self.name = name + self.upper_bound = upper_bound + self.values = values + + FUNCITEM_FLAGS: Final = FUNCBASE_FLAGS + [ "is_overload", "is_generator", @@ -690,7 +699,7 @@ def __init__( arguments: list[Argument] | None = None, body: Block | None = None, typ: mypy.types.FunctionLike | None = None, - type_args: list[tuple[str, mypy.types.Type | None]] | None = None, + type_args: list[TypeParam] | None = None, ) -> None: super().__init__() self.arguments = arguments or [] @@ -764,7 +773,7 @@ def __init__( arguments: list[Argument] | None = None, body: Block | None = None, typ: mypy.types.FunctionLike | None = None, - type_args: list[tuple[str, mypy.types.Type | None]] | None = None, + type_args: list[TypeParam] | None = None, ) -> None: super().__init__(arguments, body, typ, type_args) self._name = name @@ -1095,7 +1104,7 @@ class ClassDef(Statement): _fullname: str # Fully qualified name of the class defs: Block # New-style type parameters (PEP 695), unanalyzed - type_args: list[tuple[str, mypy.types.Type | None]] | None + type_args: list[TypeParam] | None # Semantically analyzed type parameters (all syntax variants) # TODO: Move these to TypeInfo? These partially duplicate type_args. type_vars: list[mypy.types.TypeVarLikeType] @@ -1120,7 +1129,7 @@ def __init__( base_type_exprs: list[Expression] | None = None, metaclass: Expression | None = None, keywords: list[tuple[str, Expression]] | None = None, - type_args: list[tuple[str, mypy.types.Type | None]] | None = None, + type_args: list[TypeParam] | None = None, ) -> None: super().__init__() self.name = name @@ -1624,13 +1633,13 @@ class TypeAliasStmt(Statement): __match_args__ = ("name", "type_args", "value") name: NameExpr - type_args: list[tuple[str, mypy.types.Type | None]] + type_args: list[TypeParam] value: Expression # mypy.types.Type def __init__( self, name: NameExpr, - type_args: list[tuple[str, mypy.types.Type | None]], + type_args: list[TypeParam], value: Expression, ) -> None: super().__init__() diff --git a/mypy/semanal.py b/mypy/semanal.py index 24096c359e63..0b85e031b380 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -162,6 +162,7 @@ TypeAliasExpr, TypeAliasStmt, TypeApplication, + TypeParam, TypedDictExpr, TypeInfo, TypeVarExpr, @@ -1637,14 +1638,14 @@ def visit_class_def(self, defn: ClassDef) -> None: self.incomplete_type_stack.pop() def push_type_args( - self, type_args: list[tuple[str, Type | None]] | None, context: Context + self, type_args: list[TypeParam] | None, context: Context ) -> bool: if not type_args: return True tvs = [] - for name, upper_bound in type_args: - if upper_bound: - upper_bound = self.anal_type(upper_bound) + for p in type_args: + if p.upper_bound: + upper_bound = self.anal_type(p.upper_bound) if upper_bound is None: return False else: @@ -1652,10 +1653,10 @@ def push_type_args( tvs.append( ( - name, + p.name, TypeVarExpr( - name=name, - fullname=self.qualified_name(name), + name=p.name, + fullname=self.qualified_name(p.name), values=[], upper_bound=upper_bound, default=AnyType(TypeOfAny.from_omitted_generics), @@ -1669,12 +1670,12 @@ def push_type_args( return True - def pop_type_args(self, type_args: list[tuple[str, Type | None]] | None) -> None: + def pop_type_args(self, type_args: list[TypeParam] | None) -> None: if not type_args: return for tv in type_args: names = self.current_symbol_table() - del names[tv[0]] + del names[tv.name] def analyze_class(self, defn: ClassDef) -> None: fullname = self.qualified_name(defn.name) @@ -1970,11 +1971,11 @@ class Foo(Bar, Generic[T]): ... declared_tvars: TypeVarLikeList = [] is_protocol = False if defn.type_args is not None: - for n, _ in defn.type_args: - node = self.lookup(n, context) + for p in defn.type_args: + node = self.lookup(p.name, context) assert node is not None assert isinstance(node.node, TypeVarLikeExpr) - declared_tvars.append((n, node.node)) + declared_tvars.append((p.name, node.node)) for i, base_expr in enumerate(base_type_exprs): if isinstance(base_expr, StarExpr): @@ -5191,14 +5192,14 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: self.statement = s type_params: TypeVarLikeList = [] all_type_params_names = [] - for name, bound in s.type_args: - upper_bound = bound or self.object_type() - fullname = self.qualified_name(name) + for p in s.type_args: + upper_bound = p.upper_bound or self.object_type() + fullname = self.qualified_name(p.name) type_params.append( ( - name, + p.name, TypeVarExpr( - name, + p.name, fullname, [], upper_bound, @@ -5206,7 +5207,7 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: ), ) ) - all_type_params_names.append(name) + all_type_params_names.append(p.name) self.push_type_args(s.type_args, s) try: diff --git a/mypy/strconv.py b/mypy/strconv.py index ad4f8433673f..084f83cf0b96 100644 --- a/mypy/strconv.py +++ b/mypy/strconv.py @@ -86,6 +86,9 @@ def func_helper(self, o: mypy.nodes.FuncItem) -> list[object]: elif kind == mypy.nodes.ARG_STAR2: extra.append(("DictVarArg", [arg.variable])) a: list[Any] = [] + if o.type_args: + for p in o.type_args: + self.type_param(a, p) if args: a.append(("Args", args)) if o.type: @@ -325,15 +328,19 @@ def visit_match_stmt(self, o: mypy.nodes.MatchStmt) -> str: def visit_type_alias_stmt(self, o: mypy.nodes.TypeAliasStmt) -> str: a: list[Any] = [o.name] - for n, t in o.type_args: - aa: list[Any] = [n] - if t: - aa.append(t) - a.append(("TypeArg", aa)) + for p in o.type_args: + self.type_param(a, p) a.append(o.value) return self.dump(a, o) + def type_param(self, a: list[Any], p: mypy.nodes.TypeParam) -> None: + aa: list[Any] = [] + aa.append(p.name) + if p.upper_bound: + aa.append(p.upper_bound) + a.append(("TypeParam", aa)) + # Expressions # Simple expressions diff --git a/test-data/unit/parse-python312.test b/test-data/unit/parse-python312.test index 57d49e2d2314..5422bb87854c 100644 --- a/test-data/unit/parse-python312.test +++ b/test-data/unit/parse-python312.test @@ -5,8 +5,29 @@ type A[T] = C[T] MypyFile:1( TypeAliasStmt:2( NameExpr(A) - TypeArg( + TypeParam( T) IndexExpr:2( NameExpr(C) NameExpr(T)))) + +[case testPEP695GenericFunction] +# mypy: enable-incomplete-feature=NewGenericSyntax + +def f[T](): pass +def g[T: str](): pass +[out] +MypyFile:1( + FuncDef:3( + f + TypeParam( + T) + Block:3( + PassStmt:3())) + FuncDef:4( + g + TypeParam( + T + str?) + Block:4( + PassStmt:4()))) From c46550ab2b7de158b7a504564847fb00b7499033 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 9 May 2024 17:54:56 +0100 Subject: [PATCH 30/42] Support value restriction --- mypy/fastparse.py | 11 ++++--- mypy/semanal.py | 9 +++++- mypy/strconv.py | 2 ++ test-data/unit/check-python312.test | 45 +++++++++++++++++++++++++++++ test-data/unit/parse-python312.test | 12 +++++++- 5 files changed, 73 insertions(+), 6 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 549651365a3e..7c6801f46259 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -1170,11 +1170,14 @@ def visit_ClassDef(self, n: ast3.ClassDef) -> ClassDef: def translate_type_params(self, type_params: list[Any]) -> list[TypeParam]: explicit_type_params = [] for p in type_params: - if p.bound is None: - bound = None - else: + bound = None + values: list[Type] = [] + if isinstance(p.bound, ast3.Tuple): + conv = TypeConverter(self.errors, line=p.lineno) + values = [conv.visit(t) for t in p.bound.elts] + elif p.bound is not None: bound = TypeConverter(self.errors, line=p.lineno).visit(p.bound) - explicit_type_params.append(TypeParam(p.name, bound, [])) + explicit_type_params.append(TypeParam(p.name, bound, values)) return explicit_type_params # Return(expr? value) diff --git a/mypy/semanal.py b/mypy/semanal.py index 0b85e031b380..b3e218000c92 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1644,12 +1644,19 @@ def push_type_args( return True tvs = [] for p in type_args: + values = [] if p.upper_bound: upper_bound = self.anal_type(p.upper_bound) if upper_bound is None: return False else: upper_bound = self.named_type("builtins.object") + if p.values: + for value in p.values: + analyzed = self.anal_type(value) + if analyzed is None: + return False + values.append(analyzed) tvs.append( ( @@ -1657,7 +1664,7 @@ def push_type_args( TypeVarExpr( name=p.name, fullname=self.qualified_name(p.name), - values=[], + values=values, upper_bound=upper_bound, default=AnyType(TypeOfAny.from_omitted_generics), variance=VARIANCE_NOT_READY, diff --git a/mypy/strconv.py b/mypy/strconv.py index 084f83cf0b96..b0ac5d34a86a 100644 --- a/mypy/strconv.py +++ b/mypy/strconv.py @@ -339,6 +339,8 @@ def type_param(self, a: list[Any], p: mypy.nodes.TypeParam) -> None: aa.append(p.name) if p.upper_bound: aa.append(p.upper_bound) + if p.values: + aa.append(("Values", p.values)) a.append(("TypeParam", aa)) # Expressions diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 0275599ab1ca..066371085936 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -715,3 +715,48 @@ d: Cov2[int] = Cov2[object]() # E: Incompatible types in assignment (expression e: Contra[int] = Contra[object]() f: Contra[object] = Contra[int]() # E: Incompatible types in assignment (expression has type "Contra[int]", variable has type "Contra[object]") [builtins fixtures/tuple-simple.pyi] + +[case testPEP695ValueRestiction] +# flags: --enable-incomplete-feature=NewGenericSyntax + +def f[T: (int, str)](x: T) -> T: + reveal_type(x) # N: Revealed type is "builtins.int" \ + # N: Revealed type is "builtins.str" + return x + +reveal_type(f(1)) # N: Revealed type is "builtins.int" +reveal_type(f('x')) # N: Revealed type is "builtins.str" +f(None) # E: Value of type variable "T" of "f" cannot be "None" + +class C[T: (object, None)]: pass + +a: C[object] +b: C[None] +c: C[int] # E: Value of type variable "T" of "C" cannot be "int" + +[case testPEP695ValueRestictionForwardReference] +# flags: --enable-incomplete-feature=NewGenericSyntax + +class C[T: (int, D)]: + def __init__(self, x: T) -> None: + a = x + if int(): + a = 'x' # E: Incompatible types in assignment (expression has type "str", variable has type "int") \ + # E: Incompatible types in assignment (expression has type "str", variable has type "D") + self.x: T = x + +reveal_type(C(1).x) # N: Revealed type is "builtins.int" +C(None) # E: Value of type variable "T" of "C" cannot be "None" + +class D: pass + +C(D()) + +[case testPEP695ValueRestictionUndefinedName] +# flags: --enable-incomplete-feature=NewGenericSyntax + +class C[T: (int, XX)]: # E: Name "XX" is not defined + pass + +def f[S: (int, YY)](x: S) -> S: # E: Name "YY" is not defined + return x diff --git a/test-data/unit/parse-python312.test b/test-data/unit/parse-python312.test index 5422bb87854c..de824b26f3ea 100644 --- a/test-data/unit/parse-python312.test +++ b/test-data/unit/parse-python312.test @@ -16,6 +16,7 @@ MypyFile:1( def f[T](): pass def g[T: str](): pass +def h[T: (int, str)](): pass [out] MypyFile:1( FuncDef:3( @@ -30,4 +31,13 @@ MypyFile:1( T str?) Block:4( - PassStmt:4()))) + PassStmt:4())) + FuncDef:5( + h + TypeParam( + T + Values( + int? + str?)) + Block:5( + PassStmt:5()))) From 1a56647e1bac1a9b4c430515c632eae0a6e6f182 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 10 May 2024 10:57:27 +0100 Subject: [PATCH 31/42] Support ParamSpec --- mypy/checker.py | 4 +-- mypy/fastparse.py | 22 ++++++++---- mypy/nodes.py | 24 ++++++++----- mypy/semanal.py | 52 ++++++++++++++++++----------- mypy/strconv.py | 25 +++++++++----- test-data/unit/check-python312.test | 22 ++++++++++++ test-data/unit/parse-python312.test | 22 ++++++++++++ 7 files changed, 127 insertions(+), 44 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index ef96f8dfa309..3daf64daaac4 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3417,8 +3417,8 @@ def check_final(self, s: AssignmentStmt | OperatorAssignmentStmt | AssignmentExp if ( lv.node.final_unset_in_class and not lv.node.final_set_in_init - and not self.is_stub - and # It is OK to skip initializer in stub files. + and not self.is_stub # It is OK to skip initializer in stub files. + and # Avoid extra error messages, if there is no type in Final[...], # then we already reported the error about missing r.h.s. isinstance(s, AssignmentStmt) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 7c6801f46259..175e33c21728 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -17,6 +17,8 @@ ARG_POS, ARG_STAR, ARG_STAR2, + PARAM_SPEC_KIND, + TYPE_VAR_KIND, ArgKind, Argument, AssertStmt, @@ -81,7 +83,6 @@ TupleExpr, TypeAliasStmt, TypeParam, - TypeVarExpr, UnaryExpr, Var, WhileStmt, @@ -178,6 +179,10 @@ def ast3_parse( TryStar = ast3.TryStar else: TryStar = Any +if sys.version_info >= (3, 12): + ParamSpec = ast3.ParamSpec +else: + ParamSpec = Any N = TypeVar("N", bound=Node) @@ -1172,12 +1177,15 @@ def translate_type_params(self, type_params: list[Any]) -> list[TypeParam]: for p in type_params: bound = None values: list[Type] = [] - if isinstance(p.bound, ast3.Tuple): - conv = TypeConverter(self.errors, line=p.lineno) - values = [conv.visit(t) for t in p.bound.elts] - elif p.bound is not None: - bound = TypeConverter(self.errors, line=p.lineno).visit(p.bound) - explicit_type_params.append(TypeParam(p.name, bound, values)) + if isinstance(p, ParamSpec): # type: ignore[misc] + explicit_type_params.append(TypeParam(p.name, PARAM_SPEC_KIND, None, [])) + else: + if isinstance(p.bound, ast3.Tuple): + conv = TypeConverter(self.errors, line=p.lineno) + values = [conv.visit(t) for t in p.bound.elts] + elif p.bound is not None: + bound = TypeConverter(self.errors, line=p.lineno).visit(p.bound) + explicit_type_params.append(TypeParam(p.name, TYPE_VAR_KIND, bound, values)) return explicit_type_params # Return(expr? value) diff --git a/mypy/nodes.py b/mypy/nodes.py index ee53153ae2b5..5190fc1bf0ca 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -653,11 +653,24 @@ def set_line( self.variable.set_line(self.line, self.column, self.end_line, self.end_column) +# These specify the kind of a TypeParam +TYPE_VAR_KIND: Final = 0 +PARAM_SPEC_KIND: Final = 1 +TYPE_VAR_TUPLE_KIND: Final = 2 + + class TypeParam: - __slots__ = ("name", "upper_bound", "values") + __slots__ = ("name", "kind", "upper_bound", "values") - def __init__(self, name: str, upper_bound: mypy.types.Type | None, values: list[mypy.types.Type]) -> None: + def __init__( + self, + name: str, + kind: int, + upper_bound: mypy.types.Type | None, + values: list[mypy.types.Type], + ) -> None: self.name = name + self.kind = kind self.upper_bound = upper_bound self.values = values @@ -1636,12 +1649,7 @@ class TypeAliasStmt(Statement): type_args: list[TypeParam] value: Expression # mypy.types.Type - def __init__( - self, - name: NameExpr, - type_args: list[TypeParam], - value: Expression, - ) -> None: + def __init__(self, name: NameExpr, type_args: list[TypeParam], value: Expression) -> None: super().__init__() self.name = name self.type_args = type_args diff --git a/mypy/semanal.py b/mypy/semanal.py index b3e218000c92..0a797edf07dd 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -81,9 +81,11 @@ LDEF, MDEF, NOT_ABSTRACT, + PARAM_SPEC_KIND, REVEAL_LOCALS, REVEAL_TYPE, RUNTIME_PROTOCOL_DECOS, + TYPE_VAR_KIND, VARIANCE_NOT_READY, ArgKind, AssertStmt, @@ -162,9 +164,9 @@ TypeAliasExpr, TypeAliasStmt, TypeApplication, - TypeParam, TypedDictExpr, TypeInfo, + TypeParam, TypeVarExpr, TypeVarLikeExpr, TypeVarTupleExpr, @@ -1637,40 +1639,52 @@ def visit_class_def(self, defn: ClassDef) -> None: self.pop_type_args(defn.type_args) self.incomplete_type_stack.pop() - def push_type_args( - self, type_args: list[TypeParam] | None, context: Context - ) -> bool: + def push_type_args(self, type_args: list[TypeParam] | None, context: Context) -> bool: if not type_args: return True - tvs = [] + tvs: list[tuple[str, TypeVarLikeExpr]] = [] for p in type_args: - values = [] + fullname = self.qualified_name(p.name) if p.upper_bound: upper_bound = self.anal_type(p.upper_bound) if upper_bound is None: return False else: upper_bound = self.named_type("builtins.object") + default = AnyType(TypeOfAny.from_omitted_generics) + if p.kind == TYPE_VAR_KIND: + values = [] if p.values: for value in p.values: analyzed = self.anal_type(value) if analyzed is None: return False values.append(analyzed) - - tvs.append( - ( - p.name, - TypeVarExpr( - name=p.name, - fullname=self.qualified_name(p.name), - values=values, - upper_bound=upper_bound, - default=AnyType(TypeOfAny.from_omitted_generics), - variance=VARIANCE_NOT_READY, - ), + tvs.append( + ( + p.name, + TypeVarExpr( + name=p.name, + fullname=fullname, + values=values, + upper_bound=upper_bound, + default=default, + variance=VARIANCE_NOT_READY, + ), + ) + ) + elif p.kind == PARAM_SPEC_KIND: + tvs.append( + ( + p.name, + ParamSpecExpr( + name=p.name, + fullname=fullname, + upper_bound=upper_bound, + default=default, + ), + ) ) - ) for name, tv in tvs: self.add_symbol(name, tv, context) diff --git a/mypy/strconv.py b/mypy/strconv.py index b0ac5d34a86a..c81dba5a03ee 100644 --- a/mypy/strconv.py +++ b/mypy/strconv.py @@ -88,7 +88,7 @@ def func_helper(self, o: mypy.nodes.FuncItem) -> list[object]: a: list[Any] = [] if o.type_args: for p in o.type_args: - self.type_param(a, p) + a.append(self.type_param(p)) if args: a.append(("Args", args)) if o.type: @@ -190,6 +190,9 @@ def visit_class_def(self, o: mypy.nodes.ClassDef) -> str: a.insert(1, ("TupleType", [o.info.tuple_type])) if o.info and o.info.fallback_to_any: a.insert(1, "FallbackToAny") + if o.type_args: + for p in reversed(o.type_args): + a.insert(1, self.type_param(p)) return self.dump(a, o) def visit_var(self, o: mypy.nodes.Var) -> str: @@ -329,19 +332,25 @@ def visit_match_stmt(self, o: mypy.nodes.MatchStmt) -> str: def visit_type_alias_stmt(self, o: mypy.nodes.TypeAliasStmt) -> str: a: list[Any] = [o.name] for p in o.type_args: - self.type_param(a, p) + a.append(self.type_param(p)) a.append(o.value) return self.dump(a, o) - def type_param(self, a: list[Any], p: mypy.nodes.TypeParam) -> None: - aa: list[Any] = [] - aa.append(p.name) + def type_param(self, p: mypy.nodes.TypeParam) -> list[Any]: + a: list[Any] = [] + if p.kind == mypy.nodes.PARAM_SPEC_KIND: + prefix = "**" + elif p.kind == mypy.nodes.TYPE_VAR_TUPLE_KIND: + prefix = "*" + else: + prefix = "" + a.append(prefix + p.name) if p.upper_bound: - aa.append(p.upper_bound) + a.append(p.upper_bound) if p.values: - aa.append(("Values", p.values)) - a.append(("TypeParam", aa)) + a.append(("Values", p.values)) + return [("TypeParam", a)] # Expressions diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 066371085936..d8077b08c8c9 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -760,3 +760,25 @@ class C[T: (int, XX)]: # E: Name "XX" is not defined def f[S: (int, YY)](x: S) -> S: # E: Name "YY" is not defined return x + +[case testPEP695ParamSpec] +# flags: --enable-incomplete-feature=NewGenericSyntax +from typing import Callable + +def g[**P](f: Callable[P, None], *args: P.args, **kwargs: P.kwargs) -> None: + f(*args, **kwargs) + f(1, *args, **kwargs) # E: Argument 1 has incompatible type "int"; expected "P.args" + +def h(x: int, y: str) -> None: pass + +g(h, 1, y='x') +g(h, 1, x=1) # E: "g" gets multiple values for keyword argument "x" \ + # E: Missing positional argument "y" in call to "g" + +class C[**P, T]: + def m(self, *args: P.args, **kwargs: P.kwargs) -> T: ... + +a: C[[int, str], None] +reveal_type(a) # N: Revealed type is "__main__.C[[builtins.int, builtins.str], None]" +reveal_type(a.m) # N: Revealed type is "def (builtins.int, builtins.str)" +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/parse-python312.test b/test-data/unit/parse-python312.test index de824b26f3ea..c8b7cb82524f 100644 --- a/test-data/unit/parse-python312.test +++ b/test-data/unit/parse-python312.test @@ -41,3 +41,25 @@ MypyFile:1( str?)) Block:5( PassStmt:5()))) + +[case testPEP695ParamSpec] +# mypy: enable-incomplete-feature=NewGenericSyntax + +def f[**P](): pass +class C[T: int, **P]: pass +[out] +MypyFile:1( + FuncDef:3( + f + TypeParam( + **P) + Block:3( + PassStmt:3())) + ClassDef:4( + C + TypeParam( + T + int?) + TypeParam( + **P) + PassStmt:4())) From 80e8c793c87bcd771a7b17f042acad6286e8bd50 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 10 May 2024 11:18:30 +0100 Subject: [PATCH 32/42] Support TypeVarTuple --- mypy/fastparse.py | 14 +++++++++----- mypy/semanal.py | 17 +++++++++++++++++ mypy/strconv.py | 1 - test-data/unit/check-python312.test | 20 ++++++++++++++++++++ test-data/unit/parse-python312.test | 22 ++++++++++++++++++++++ 5 files changed, 68 insertions(+), 6 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 175e33c21728..f76ee5ddf4fb 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -19,6 +19,7 @@ ARG_STAR2, PARAM_SPEC_KIND, TYPE_VAR_KIND, + TYPE_VAR_TUPLE_KIND, ArgKind, Argument, AssertStmt, @@ -148,11 +149,6 @@ def ast3_parse( NamedExpr = ast3.NamedExpr Constant = ast3.Constant -if sys.version_info >= (3, 12): - ast_TypeAlias = ast3.TypeAlias -else: - ast_TypeAlias = Any - if sys.version_info >= (3, 10): Match = ast3.Match MatchValue = ast3.MatchValue @@ -175,14 +171,20 @@ def ast3_parse( MatchAs = Any MatchOr = Any AstNode = Union[ast3.expr, ast3.stmt, ast3.ExceptHandler] + if sys.version_info >= (3, 11): TryStar = ast3.TryStar else: TryStar = Any + if sys.version_info >= (3, 12): + ast_TypeAlias = ast3.TypeAlias ParamSpec = ast3.ParamSpec + TypeVarTuple = ast3.TypeVarTuple else: + ast_TypeAlias = Any ParamSpec = Any + TypeVarTuple = Any N = TypeVar("N", bound=Node) @@ -1179,6 +1181,8 @@ def translate_type_params(self, type_params: list[Any]) -> list[TypeParam]: values: list[Type] = [] if isinstance(p, ParamSpec): # type: ignore[misc] explicit_type_params.append(TypeParam(p.name, PARAM_SPEC_KIND, None, [])) + elif isinstance(p, TypeVarTuple): # type: ignore[misc] + explicit_type_params.append(TypeParam(p.name, TYPE_VAR_TUPLE_KIND, None, [])) else: if isinstance(p.bound, ast3.Tuple): conv = TypeConverter(self.errors, line=p.lineno) diff --git a/mypy/semanal.py b/mypy/semanal.py index 0a797edf07dd..66572a979c9c 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -86,6 +86,7 @@ REVEAL_TYPE, RUNTIME_PROTOCOL_DECOS, TYPE_VAR_KIND, + TYPE_VAR_TUPLE_KIND, VARIANCE_NOT_READY, ArgKind, AssertStmt, @@ -1685,6 +1686,22 @@ def push_type_args(self, type_args: list[TypeParam] | None, context: Context) -> ), ) ) + else: + assert p.kind == TYPE_VAR_TUPLE_KIND + tuple_fallback = self.named_type("builtins.tuple", [self.object_type()]) + tvs.append( + ( + p.name, + TypeVarTupleExpr( + name=p.name, + fullname=fullname, + # Upper bound for *Ts is *tuple[object, ...], it can never be object. + upper_bound=tuple_fallback.copy_modified(), + tuple_fallback=tuple_fallback, + default=default, + ), + ) + ) for name, tv in tvs: self.add_symbol(name, tv, context) diff --git a/mypy/strconv.py b/mypy/strconv.py index c81dba5a03ee..a96a27c45d75 100644 --- a/mypy/strconv.py +++ b/mypy/strconv.py @@ -334,7 +334,6 @@ def visit_type_alias_stmt(self, o: mypy.nodes.TypeAliasStmt) -> str: for p in o.type_args: a.append(self.type_param(p)) a.append(o.value) - return self.dump(a, o) def type_param(self, p: mypy.nodes.TypeParam) -> list[Any]: diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index d8077b08c8c9..7774a1756c77 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -782,3 +782,23 @@ a: C[[int, str], None] reveal_type(a) # N: Revealed type is "__main__.C[[builtins.int, builtins.str], None]" reveal_type(a.m) # N: Revealed type is "def (builtins.int, builtins.str)" [builtins fixtures/tuple.pyi] + +[case testPEP695TypeVarTuple] +# flags: --enable-incomplete-feature=NewGenericSyntax + +def f[*Ts](t: tuple[*Ts]) -> tuple[*Ts]: + reveal_type(t) # N: Revealed type is "Tuple[Unpack[Ts`-1]]" + return t + +reveal_type(f((1, 'x'))) # N: Revealed type is "Tuple[Literal[1]?, Literal['x']?]" +a: tuple[int, ...] +reveal_type(f(a)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" + +class C[T, *Ts]: + pass + +b: C[int, str, None] +reveal_type(b) # N: Revealed type is "__main__.C[builtins.int, builtins.str, None]" +c: C[str] +reveal_type(c) # N: Revealed type is "__main__.C[builtins.str]" +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/parse-python312.test b/test-data/unit/parse-python312.test index c8b7cb82524f..28204ccd647b 100644 --- a/test-data/unit/parse-python312.test +++ b/test-data/unit/parse-python312.test @@ -63,3 +63,25 @@ MypyFile:1( TypeParam( **P) PassStmt:4())) + +[case testPEP695TypeVarTuple] +# mypy: enable-incomplete-feature=NewGenericSyntax + +def f[*Ts](): pass +class C[T: int, *Ts]: pass +[out] +MypyFile:1( + FuncDef:3( + f + TypeParam( + *Ts) + Block:3( + PassStmt:3())) + ClassDef:4( + C + TypeParam( + T + int?) + TypeParam( + *Ts) + PassStmt:4())) From 4838121f11680af029ebe46a4dac602007d1d4d3 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 10 May 2024 14:51:32 +0100 Subject: [PATCH 33/42] Add some incremental mode tests --- test-data/unit/check-python312.test | 108 ++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 7774a1756c77..3e3a906e3ce6 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -802,3 +802,111 @@ reveal_type(b) # N: Revealed type is "__main__.C[builtins.int, builtins.str, No c: C[str] reveal_type(c) # N: Revealed type is "__main__.C[builtins.str]" [builtins fixtures/tuple.pyi] + +[case testPEP695IncrementalFunction] +# flags: --enable-incomplete-feature=NewGenericSyntax +import a + +[file a.py] +import b + +[file a.py.2] +import b +reveal_type(b.f(1)) +reveal_type(b.g(1, 'x')) +b.g('x', 'x') +b.g(1, 2) + +[file b.py] +def f[T](x: T) -> T: + return x + +def g[T: int, S: (str, None)](x: T, y: S) -> T | S: + return x + +[out2] +tmp/a.py:2: note: Revealed type is "builtins.int" +tmp/a.py:3: note: Revealed type is "Union[builtins.int, builtins.str]" +tmp/a.py:4: error: Value of type variable "T" of "g" cannot be "str" +tmp/a.py:5: error: Value of type variable "S" of "g" cannot be "int" + +[case testPEP695IncrementalClass] +# flags: --enable-incomplete-feature=NewGenericSyntax +import a + +[file a.py] +import b + +[file a.py.2] +from b import C, D +x: C[int] +reveal_type(x) + +class N(int): pass +class SS(str): pass + +y1: D[int, str] +y2: D[N, str] +y3: D[int, None] +y4: D[int, None] +y5: D[int, SS] # Error +y6: D[object, str] # Error + +[file b.py] +class C[T]: pass + +class D[T: int, S: (str, None)]: + pass + +[out2] +tmp/a.py:3: note: Revealed type is "b.C[builtins.int]" +tmp/a.py:12: error: Value of type variable "S" of "D" cannot be "SS" +tmp/a.py:13: error: Type argument "object" of "D" must be a subtype of "int" + +[case testPEP695IncrementalParamSpecAndTypeVarTuple] +# flags: --enable-incomplete-feature=NewGenericSyntax +import a + +[file a.py] +import b + +[file a.py.2] +from b import C, D +x1: C[()] +x2: C[int] +x3: C[int, str] +y: D[[int, str]] +reveal_type(y.m) + +[file b.py] +class C[*Ts]: pass +class D[**P]: + def m(self, *args: P.args, **kwargs: P.kwargs) -> None: pass + +[builtins fixtures/tuple.pyi] +[out2] +tmp/a.py:6: note: Revealed type is "def (builtins.int, builtins.str)" + +[case testPEP695IncrementalTypeAlias] +# flags: --enable-incomplete-feature=NewGenericSyntax +import a + +[file a.py] +import b + +[file a.py.2] +from b import A, B +a: A +reveal_type(a) +b: B[int] +reveal_type(b) + +[file b.py] +type A = str +class Foo[T]: pass +type B[T] = Foo[T] + +[builtins fixtures/tuple.pyi] +[out2] +tmp/a.py:3: note: Revealed type is "builtins.str" +tmp/a.py:5: note: Revealed type is "b.Foo[builtins.int]" From d88a03b637f1e8ecaedd679d2f74c0fca0d70e25 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 10 May 2024 16:18:13 +0100 Subject: [PATCH 34/42] Fix issue with forward references in type aliases --- mypy/semanal.py | 10 +++++----- test-data/unit/check-python312.test | 12 ++++++++++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 66572a979c9c..405a34c7cb10 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -5289,9 +5289,6 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: # When this type alias gets "inlined", the Any is not explicit anymore, # so we need to replace it with non-explicit Anys. res = make_any_non_explicit(res) - # if isinstance(res, ProperType) and isinstance(res, Instance): - # if not validate_instance(res, self.fail, empty_tuple_index): - # fix_instance(res, self.fail, self.note, disallow_any=False, options=self.options) eager = self.is_func_scope() alias_node = TypeAlias( res, @@ -5303,9 +5300,12 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: eager=eager, ) - self.add_symbol(s.name.name, alias_node, s) + existing = self.current_symbol_table().get(s.name.name) + if existing and isinstance(existing.node, (PlaceholderNode, TypeAlias)) and existing.node.line == s.line: + existing.node = alias_node + else: + self.add_symbol(s.name.name, alias_node, s) - # TODO: Check if existing finally: self.pop_type_args(s.type_args) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 3e3a906e3ce6..6babe6218439 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -559,6 +559,18 @@ class C: pass x: C = D() y: D = C() # E: Incompatible types in assignment (expression has type "C", variable has type "D") +[case testPEP695TypeAliasForwardReference5] +# flags: --enable-incomplete-feature=NewGenericSyntax +type A = str +type B[T] = C[T] +class C[T]: pass +a: A +b: B[int] +c: C[str] +reveal_type(a) # N: Revealed type is "builtins.str" +reveal_type(b) # N: Revealed type is "__main__.C[builtins.int]" +reveal_type(c) # N: Revealed type is "__main__.C[builtins.str]" + [case testPEP695TypeAliasWithUndefineName] # flags: --enable-incomplete-feature=NewGenericSyntax type A[T] = XXX # E: Name "XXX" is not defined From d8291f31404f246344ea6d50e7318ea915c277fb Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 10 May 2024 16:50:49 +0100 Subject: [PATCH 35/42] Minor tweaks --- mypy/nodes.py | 9 ++++----- mypy/semanal.py | 6 +++++- test-data/unit/check-python312.test | 1 + 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 5190fc1bf0ca..4c83d8081f6c 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -689,8 +689,8 @@ class FuncItem(FuncBase): __slots__ = ( "arguments", # Note that can be unset if deserialized (type is a lie!) - "arg_names", # Names of parameters - "arg_kinds", # Kinds of parameters + "arg_names", # Names of arguments + "arg_kinds", # Kinds of arguments "min_args", # Minimum number of arguments "max_pos", # Maximum number of positional arguments, -1 if no explicit # limit (*args not included) @@ -719,7 +719,7 @@ def __init__( self.arg_names = [None if arg.pos_only else arg.variable.name for arg in self.arguments] self.arg_kinds: list[ArgKind] = [arg.kind for arg in self.arguments] self.max_pos: int = self.arg_kinds.count(ARG_POS) + self.arg_kinds.count(ARG_OPT) - self.type_args = type_args + self.type_args: list[TypeParam] | None = type_args self.body: Block = body or Block([]) self.type = typ self.unanalyzed_type = typ @@ -1119,7 +1119,6 @@ class ClassDef(Statement): # New-style type parameters (PEP 695), unanalyzed type_args: list[TypeParam] | None # Semantically analyzed type parameters (all syntax variants) - # TODO: Move these to TypeInfo? These partially duplicate type_args. type_vars: list[mypy.types.TypeVarLikeType] # Base class expressions (not semantically analyzed -- can be arbitrary expressions) base_type_exprs: list[Expression] @@ -1647,7 +1646,7 @@ class TypeAliasStmt(Statement): name: NameExpr type_args: list[TypeParam] - value: Expression # mypy.types.Type + value: Expression # Will get translated into a type def __init__(self, name: NameExpr, type_args: list[TypeParam], value: Expression) -> None: super().__init__() diff --git a/mypy/semanal.py b/mypy/semanal.py index 405a34c7cb10..821fa945c83b 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -5301,7 +5301,11 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: ) existing = self.current_symbol_table().get(s.name.name) - if existing and isinstance(existing.node, (PlaceholderNode, TypeAlias)) and existing.node.line == s.line: + if ( + existing + and isinstance(existing.node, (PlaceholderNode, TypeAlias)) + and existing.node.line == s.line + ): existing.node = alias_node else: self.add_symbol(s.name.name, alias_node, s) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 6babe6218439..efe5d44a3b2c 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -813,6 +813,7 @@ b: C[int, str, None] reveal_type(b) # N: Revealed type is "__main__.C[builtins.int, builtins.str, None]" c: C[str] reveal_type(c) # N: Revealed type is "__main__.C[builtins.str]" +b = c # E: Incompatible types in assignment (expression has type "C[str]", variable has type "C[int, str, None]") [builtins fixtures/tuple.pyi] [case testPEP695IncrementalFunction] From 6c9f73caa2f6e95e608cb402f5e6ca120e912090 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 15 May 2024 11:02:40 +0100 Subject: [PATCH 36/42] Support *Ts and **P in type aliases --- mypy/semanal.py | 49 +++++++++++++++++++++++------ test-data/unit/check-python312.test | 20 ++++++++++++ 2 files changed, 59 insertions(+), 10 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 821fa945c83b..a129d51035b8 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -5233,18 +5233,47 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: for p in s.type_args: upper_bound = p.upper_bound or self.object_type() fullname = self.qualified_name(p.name) - type_params.append( - ( - p.name, - TypeVarExpr( + default = AnyType(TypeOfAny.from_omitted_generics) + if p.kind == PARAM_SPEC_KIND: + type_params.append( + ( p.name, - fullname, - [], - upper_bound, - default=AnyType(TypeOfAny.from_omitted_generics), - ), + ParamSpecExpr( + name=p.name, + fullname=fullname, + upper_bound=upper_bound, + default=default, + ), + ) + ) + elif p.kind == TYPE_VAR_TUPLE_KIND: + tuple_fallback = self.named_type("builtins.tuple", [self.object_type()]) + type_params.append( + ( + p.name, + TypeVarTupleExpr( + name=p.name, + fullname=fullname, + # Upper bound for *Ts is *tuple[object, ...], it can never be object. + upper_bound=tuple_fallback.copy_modified(), + tuple_fallback=tuple_fallback, + default=default, + ), + ) + ) + else: + type_params.append( + ( + p.name, + TypeVarExpr( + p.name, + fullname, + [], + upper_bound, + default=default, + ), + ) ) - ) all_type_params_names.append(p.name) self.push_type_args(s.type_args, s) diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index efe5d44a3b2c..0171ab582a06 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -795,6 +795,16 @@ reveal_type(a) # N: Revealed type is "__main__.C[[builtins.int, builtins.str], reveal_type(a.m) # N: Revealed type is "def (builtins.int, builtins.str)" [builtins fixtures/tuple.pyi] +[case testPEP695ParamSpecTypeAlias] +# flags: --enable-incomplete-feature=NewGenericSyntax +from typing import Callable + +type C[**P] = Callable[P, int] + +f: C[[str, int | None]] +reveal_type(f) # N: Revealed type is "def (builtins.str, Union[builtins.int, None]) -> builtins.int" +[builtins fixtures/tuple.pyi] + [case testPEP695TypeVarTuple] # flags: --enable-incomplete-feature=NewGenericSyntax @@ -816,6 +826,16 @@ reveal_type(c) # N: Revealed type is "__main__.C[builtins.str]" b = c # E: Incompatible types in assignment (expression has type "C[str]", variable has type "C[int, str, None]") [builtins fixtures/tuple.pyi] +[case testPEP695TypeVarTupleAlias] +# flags: --enable-incomplete-feature=NewGenericSyntax +from typing import Callable + +type C[*Ts] = tuple[*Ts, int] + +a: C[str, None] +reveal_type(a) # N: Revealed type is "Tuple[builtins.str, None, builtins.int]" +[builtins fixtures/tuple.pyi] + [case testPEP695IncrementalFunction] # flags: --enable-incomplete-feature=NewGenericSyntax import a From 2c7e147ee708fe3f67d19ecf698d8eabcbb81b04 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 15 May 2024 11:32:08 +0100 Subject: [PATCH 37/42] Refactor to share code --- mypy/semanal.py | 166 ++++++++++++++++-------------------------------- 1 file changed, 56 insertions(+), 110 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index a129d51035b8..3ab4b881856f 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -843,7 +843,7 @@ def visit_func_def(self, defn: FuncDef) -> None: self.analyze_func_def(defn) def analyze_func_def(self, defn: FuncDef) -> None: - if not self.push_type_args(defn.type_args, defn): + if self.push_type_args(defn.type_args, defn) is None: self.defer(defn) return @@ -1632,7 +1632,7 @@ def visit_class_def(self, defn: ClassDef) -> None: self.incomplete_type_stack.append(not defn.info) namespace = self.qualified_name(defn.name) with self.tvar_scope_frame(self.tvar_scope.class_frame(namespace)): - if not self.push_type_args(defn.type_args, defn): + if self.push_type_args(defn.type_args, defn) is None: self.mark_incomplete(defn.name, defn) return @@ -1640,73 +1640,63 @@ def visit_class_def(self, defn: ClassDef) -> None: self.pop_type_args(defn.type_args) self.incomplete_type_stack.pop() - def push_type_args(self, type_args: list[TypeParam] | None, context: Context) -> bool: + def push_type_args( + self, type_args: list[TypeParam] | None, context: Context + ) -> list[tuple[str, TypeVarLikeExpr]] | None: if not type_args: - return True + return [] tvs: list[tuple[str, TypeVarLikeExpr]] = [] for p in type_args: - fullname = self.qualified_name(p.name) - if p.upper_bound: - upper_bound = self.anal_type(p.upper_bound) - if upper_bound is None: - return False - else: - upper_bound = self.named_type("builtins.object") - default = AnyType(TypeOfAny.from_omitted_generics) - if p.kind == TYPE_VAR_KIND: - values = [] - if p.values: - for value in p.values: - analyzed = self.anal_type(value) - if analyzed is None: - return False - values.append(analyzed) - tvs.append( - ( - p.name, - TypeVarExpr( - name=p.name, - fullname=fullname, - values=values, - upper_bound=upper_bound, - default=default, - variance=VARIANCE_NOT_READY, - ), - ) - ) - elif p.kind == PARAM_SPEC_KIND: - tvs.append( - ( - p.name, - ParamSpecExpr( - name=p.name, - fullname=fullname, - upper_bound=upper_bound, - default=default, - ), - ) - ) - else: - assert p.kind == TYPE_VAR_TUPLE_KIND - tuple_fallback = self.named_type("builtins.tuple", [self.object_type()]) - tvs.append( - ( - p.name, - TypeVarTupleExpr( - name=p.name, - fullname=fullname, - # Upper bound for *Ts is *tuple[object, ...], it can never be object. - upper_bound=tuple_fallback.copy_modified(), - tuple_fallback=tuple_fallback, - default=default, - ), - ) - ) + tv = self.analyze_type_param(p) + if tv is None: + return None + tvs.append((p.name, tv)) for name, tv in tvs: self.add_symbol(name, tv, context) - return True + return tvs + + def analyze_type_param(self, type_param: TypeParam) -> TypeVarLikeExpr | None: + fullname = self.qualified_name(type_param.name) + if type_param.upper_bound: + upper_bound = self.anal_type(type_param.upper_bound) + if upper_bound is None: + return None + else: + upper_bound = self.named_type("builtins.object") + default = AnyType(TypeOfAny.from_omitted_generics) + if type_param.kind == TYPE_VAR_KIND: + values = [] + if type_param.values: + for value in type_param.values: + analyzed = self.anal_type(value) + if analyzed is None: + return None + values.append(analyzed) + return TypeVarExpr( + name=type_param.name, + fullname=fullname, + values=values, + upper_bound=upper_bound, + default=default, + variance=VARIANCE_NOT_READY, + ) + elif type_param.kind == PARAM_SPEC_KIND: + return ParamSpecExpr( + name=type_param.name, fullname=fullname, upper_bound=upper_bound, default=default + ) + else: + assert type_param.kind == TYPE_VAR_TUPLE_KIND + tuple_fallback = self.named_type("builtins.tuple", [self.object_type()]) + return TypeVarTupleExpr( + name=type_param.name, + fullname=fullname, + # Upper bound for *Ts is *tuple[object, ...], it can never be object. + upper_bound=tuple_fallback.copy_modified(), + tuple_fallback=tuple_fallback, + default=default, + ) def pop_type_args(self, type_args: list[TypeParam] | None) -> None: if not type_args: @@ -5228,55 +5218,11 @@ def visit_match_stmt(self, s: MatchStmt) -> None: def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: self.statement = s - type_params: TypeVarLikeList = [] - all_type_params_names = [] - for p in s.type_args: - upper_bound = p.upper_bound or self.object_type() - fullname = self.qualified_name(p.name) - default = AnyType(TypeOfAny.from_omitted_generics) - if p.kind == PARAM_SPEC_KIND: - type_params.append( - ( - p.name, - ParamSpecExpr( - name=p.name, - fullname=fullname, - upper_bound=upper_bound, - default=default, - ), - ) - ) - elif p.kind == TYPE_VAR_TUPLE_KIND: - tuple_fallback = self.named_type("builtins.tuple", [self.object_type()]) - type_params.append( - ( - p.name, - TypeVarTupleExpr( - name=p.name, - fullname=fullname, - # Upper bound for *Ts is *tuple[object, ...], it can never be object. - upper_bound=tuple_fallback.copy_modified(), - tuple_fallback=tuple_fallback, - default=default, - ), - ) - ) - else: - type_params.append( - ( - p.name, - TypeVarExpr( - p.name, - fullname, - [], - upper_bound, - default=default, - ), - ) - ) - all_type_params_names.append(p.name) + type_params = self.push_type_args(s.type_args, s) + # TODO: defer as needed + assert type_params is not None, "forward references in type aliases not implemented" + all_type_params_names = [p.name for p in s.type_args] - self.push_type_args(s.type_args, s) try: tag = self.track_incomplete_refs() res, alias_tvars, depends_on, qualified_tvars, empty_tuple_index = self.analyze_alias( From ef1b47385893bb9085500f005e067008b2f97581 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 15 May 2024 12:48:17 +0100 Subject: [PATCH 38/42] Address feedback --- mypy/fastparse.py | 12 ++++++------ test-data/unit/check-python312.test | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index f76ee5ddf4fb..ee042b96339f 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -179,12 +179,12 @@ def ast3_parse( if sys.version_info >= (3, 12): ast_TypeAlias = ast3.TypeAlias - ParamSpec = ast3.ParamSpec - TypeVarTuple = ast3.TypeVarTuple + ast_ParamSpec = ast3.ParamSpec + ast_TypeVarTuple = ast3.TypeVarTuple else: ast_TypeAlias = Any - ParamSpec = Any - TypeVarTuple = Any + ast_ParamSpec = Any + ast_TypeVarTuple = Any N = TypeVar("N", bound=Node) @@ -1179,9 +1179,9 @@ def translate_type_params(self, type_params: list[Any]) -> list[TypeParam]: for p in type_params: bound = None values: list[Type] = [] - if isinstance(p, ParamSpec): # type: ignore[misc] + if isinstance(p, ast_ParamSpec): # type: ignore[misc] explicit_type_params.append(TypeParam(p.name, PARAM_SPEC_KIND, None, [])) - elif isinstance(p, TypeVarTuple): # type: ignore[misc] + elif isinstance(p, ast_TypeVarTuple): # type: ignore[misc] explicit_type_params.append(TypeParam(p.name, TYPE_VAR_TUPLE_KIND, None, [])) else: if isinstance(p.bound, ast3.Tuple): diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 0171ab582a06..cea0be198e5a 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -128,12 +128,12 @@ reveal_type(c.ident(1)) # N: Revealed type is "builtins.int" # flags: --enable-incomplete-feature=NewGenericSyntax class C[T]: - def m[S](self, x: S) -> T: ... + def m[S](self, x: S) -> T | S: ... a: C[int] = C[object]() # E: Incompatible types in assignment (expression has type "C[object]", variable has type "C[int]") b: C[object] = C[int]() -reveal_type(C[str]().m(1)) # N: Revealed type is "builtins.str" +reveal_type(C[str]().m(1)) # N: Revealed type is "Union[builtins.str, builtins.int]" [case testPEP695InferVarianceSimpleFromMethod] # flags: --enable-incomplete-feature=NewGenericSyntax From 4819d603f53dd2fc317c132ceaff184f04adb780 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 15 May 2024 13:07:14 +0100 Subject: [PATCH 39/42] Fix forward reference in type alias bound --- mypy/semanal.py | 5 +++-- test-data/unit/check-python312.test | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 3ab4b881856f..a63e58ba54f2 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -5219,8 +5219,9 @@ def visit_match_stmt(self, s: MatchStmt) -> None: def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: self.statement = s type_params = self.push_type_args(s.type_args, s) - # TODO: defer as needed - assert type_params is not None, "forward references in type aliases not implemented" + if type_params is None: + self.defer(s) + return all_type_params_names = [p.name for p in s.type_args] try: diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index cea0be198e5a..2b85020e6609 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -586,6 +586,11 @@ type B = int + str # E: Invalid type alias: expression is not a valid type b: B reveal_type(b) # N: Revealed type is "Any" +[case testPEP695TypeAliasBoundForwardReference] +# mypy: enable-incomplete-feature=NewGenericSyntax +type B[T: Foo] = list[T] +class Foo: pass + [case testPEP695UpperBound] # flags: --enable-incomplete-feature=NewGenericSyntax From 196b2a32be9080b025146d3029fb95a2e579492f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 15 May 2024 16:44:17 +0100 Subject: [PATCH 40/42] Fix crash on undefined name in generic method --- mypy/semanal.py | 12 +++++++++--- test-data/unit/check-python312.test | 10 ++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index a63e58ba54f2..f92471c159de 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1653,7 +1653,7 @@ def push_type_args( tvs.append((p.name, tv)) for name, tv in tvs: - self.add_symbol(name, tv, context) + self.add_symbol(name, tv, context, no_progress=True) return tvs @@ -5967,6 +5967,7 @@ def lookup( for table in reversed(self.locals): if table is not None and name in table: return table[name] + # 4. Current file global scope if name in self.globals: return self.globals[name] @@ -6279,6 +6280,7 @@ def add_symbol( module_hidden: bool = False, can_defer: bool = True, escape_comprehensions: bool = False, + no_progress: bool = False, ) -> bool: """Add symbol to the currently active symbol table. @@ -6300,7 +6302,9 @@ def add_symbol( symbol = SymbolTableNode( kind, node, module_public=module_public, module_hidden=module_hidden ) - return self.add_symbol_table_node(name, symbol, context, can_defer, escape_comprehensions) + return self.add_symbol_table_node( + name, symbol, context, can_defer, escape_comprehensions, no_progress + ) def add_symbol_skip_local(self, name: str, node: SymbolNode) -> None: """Same as above, but skipping the local namespace. @@ -6331,6 +6335,7 @@ def add_symbol_table_node( context: Context | None = None, can_defer: bool = True, escape_comprehensions: bool = False, + no_progress: bool = False, ) -> bool: """Add symbol table node to the currently active symbol table. @@ -6379,7 +6384,8 @@ def add_symbol_table_node( self.name_already_defined(name, context, existing) elif name not in self.missing_names[-1] and "*" not in self.missing_names[-1]: names[name] = symbol - self.progress = True + if not no_progress: + self.progress = True return True return False diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 2b85020e6609..53656ae5e3fb 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -948,3 +948,13 @@ type B[T] = Foo[T] [out2] tmp/a.py:3: note: Revealed type is "builtins.str" tmp/a.py:5: note: Revealed type is "b.Foo[builtins.int]" + +[case testPEP695UndefinedNameInGenericFunction] +# mypy: enable-incomplete-feature=NewGenericSyntax + +def f[T](x: T) -> T: + return unknown() # E: Name "unknown" is not defined + +class C: + def m[T](self, x: T) -> T: + return unknown() # E: Name "unknown" is not defined From 369a4afc9f5eda263883f6b348b5f32fc9f4051c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 16 May 2024 11:21:09 +0100 Subject: [PATCH 41/42] Fix variance inference when using real typeshed stubs --- mypy/subtypes.py | 6 +++--- test-data/unit/pythoneval.test | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 5aa6a08c0994..e1609717e160 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -1988,9 +1988,9 @@ def is_more_precise(left: Type, right: Type, *, ignore_promotions: bool = False) return is_proper_subtype(left, right, ignore_promotions=ignore_promotions) -def all_members(info: TypeInfo) -> set[str]: +def all_non_object_members(info: TypeInfo) -> set[str]: members = set(info.names) - for base in info.mro[1:]: + for base in info.mro[1:-1]: members.update(base.names) return members @@ -2012,7 +2012,7 @@ def infer_variance(info: TypeInfo, i: int) -> bool: contra = True tvar = info.defn.type_vars[i] self_type = fill_typevars(info) - for member in all_members(info): + for member in all_non_object_members(info): if member in ("__init__", "__new__"): continue node = info[member].node diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index b51a965c95da..0ed3540b6bb9 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -2091,3 +2091,24 @@ def f(d: Description) -> None: reveal_type(d.name_fn) [out] _testDataclassStrictOptionalAlwaysSet.py:9: note: Revealed type is "def (Union[builtins.int, None]) -> Union[builtins.str, None]" + +[case testPEP695VarianceInference] +# flags: --python-version=3.12 --enable-incomplete-feature=NewGenericSyntax +from typing import Callable, Final + +class Job[_R_co]: + def __init__(self, target: Callable[[], _R_co]) -> None: + self.target: Final = target + +def func( + action: Job[int | None], + a1: Job[int | None], + a2: Job[int], + a3: Job[None], +) -> None: + action = a1 + action = a2 + action = a3 + a2 = action # Error +[out] +_testPEP695VarianceInference.py:17: error: Incompatible types in assignment (expression has type "Job[None]", variable has type "Job[int]") From dba87a89a968e506fbc2db7652f01f0804ca3760 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 16 May 2024 13:01:29 +0100 Subject: [PATCH 42/42] Try to infer any variances that can be inferred Also avoid inferring variances repeatedly. --- mypy/subtypes.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index e1609717e160..a5523fbe0d45 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -2049,8 +2049,9 @@ def infer_class_variances(info: TypeInfo) -> bool: if not info.defn.type_args: return True tvs = info.defn.type_vars + success = True for i, tv in enumerate(tvs): - if isinstance(tv, TypeVarType): + if isinstance(tv, TypeVarType) and tv.variance == VARIANCE_NOT_READY: if not infer_variance(info, i): - return False - return True + success = False + return success