diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 50a6ef65f4d0..906231dc7e42 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -1055,6 +1055,27 @@ List of currently incomplete/experimental features: # Without PreciseTupleTypes: tuple[int, ...] # With PreciseTupleTypes: tuple[()] | tuple[int] | tuple[int, int] +* ``NewGenericSyntax``: this feature enables support for syntax defined + by :pep:`695`. For example: + + .. code-block:: python + + class Container[T]: # defines a generic class + content: T + + def first[T](items: list[T]) -> T: # defines a generic function + return items[0] + + type Items[T] = list[tuple[T, T]] # defines a generic type alias + +* ``InlineTypedDict``: this feature enables non-standard syntax for inline + :ref:`TypedDicts `, for example: + + .. code-block:: python + + def test_values() -> {"int": int, "str": str}: + return {"int": 42, "str": "test"} + Miscellaneous ************* diff --git a/docs/source/typed_dict.rst b/docs/source/typed_dict.rst index e5ce2927db4d..c379b5449eae 100644 --- a/docs/source/typed_dict.rst +++ b/docs/source/typed_dict.rst @@ -248,3 +248,41 @@ section of the docs has a full description with an example, but in short, you wi need to give each TypedDict the same key where each value has a unique :ref:`Literal type `. Then, check that key to distinguish between your TypedDicts. + +Inline TypedDict types +---------------------- + +.. note:: + + This is an experimental (non-standard) feature. Use + ``--enable-incomplete-feature=InlineTypedDict`` to enable. + +Sometimes you may want to define a complex nested JSON schema, or annotate +a one-off function that returns a TypedDict. In such cases it may be convenient +to use inline TypedDict syntax. For example: + +.. code-block:: python + + def test_values() -> {"int": int, "str": str}: + return {"int": 42, "str": "test"} + + class Response(TypedDict): + status: int + msg: str + # Using inline syntax here avoids defining two additional TypedDicts. + content: {"items": list[{"key": str, "value": str}]} + +Inline TypedDicts can also by used as targets of type aliases, but due to +ambiguity with a regular variables it is only allowed for (newer) explicit +type alias forms: + +.. code-block:: python + + from typing import TypeAlias + + X = {"a": int, "b": int} # creates a variable with type dict[str, type[int]] + Y: TypeAlias = {"a": int, "b": int} # creates a type alias + type Z = {"a": int, "b": int} # same as above (Python 3.12+ only) + +Also, due to incompatibility with runtime type-checking it is strongly recommended +to *not* use inline syntax in union types. diff --git a/mypy/checker.py b/mypy/checker.py index 2df74cf7be8d..0ae499916ec6 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2971,7 +2971,8 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: self.msg.annotation_in_unchecked_function(context=s) def check_type_alias_rvalue(self, s: AssignmentStmt) -> None: - alias_type = self.expr_checker.accept(s.rvalue) + with self.msg.filter_errors(): + alias_type = self.expr_checker.accept(s.rvalue) self.store_type(s.lvalues[-1], alias_type) def check_assignment( @@ -5311,7 +5312,8 @@ def remove_capture_conflicts(self, type_map: TypeMap, inferred_types: dict[Var, del type_map[expr] def visit_type_alias_stmt(self, o: TypeAliasStmt) -> None: - self.expr_checker.accept(o.value) + with self.msg.filter_errors(): + self.expr_checker.accept(o.value) def make_fake_typeinfo( self, diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index d9bdf2e2b20b..92316d11926d 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -4,9 +4,11 @@ from mypy.fastparse import parse_type_string from mypy.nodes import ( + MISSING_FALLBACK, BytesExpr, CallExpr, ComplexExpr, + DictExpr, EllipsisExpr, Expression, FloatExpr, @@ -29,9 +31,11 @@ AnyType, CallableArgument, EllipsisType, + Instance, ProperType, RawExpressionType, Type, + TypedDictType, TypeList, TypeOfAny, UnboundType, @@ -55,7 +59,7 @@ def _extract_argument_name(expr: Expression) -> str | None: def expr_to_unanalyzed_type( expr: Expression, - options: Options | None = None, + options: Options, allow_new_syntax: bool = False, _parent: Expression | None = None, allow_unpack: bool = False, @@ -67,6 +71,8 @@ def expr_to_unanalyzed_type( If allow_new_syntax is True, allow all type syntax independent of the target Python version (used in stubs). + + # TODO: a lot of code here is duplicated in fastparse.py, refactor this. """ # The `parent` parameter is used in recursive calls to provide context for # understanding whether an CallableArgument is ok. @@ -116,7 +122,7 @@ def expr_to_unanalyzed_type( elif ( isinstance(expr, OpExpr) and expr.op == "|" - and ((options and options.python_version >= (3, 10)) or allow_new_syntax) + and ((options.python_version >= (3, 10)) or allow_new_syntax) ): return UnionType( [ @@ -206,5 +212,26 @@ def expr_to_unanalyzed_type( return UnpackType( expr_to_unanalyzed_type(expr.expr, options, allow_new_syntax), from_star_syntax=True ) + elif isinstance(expr, DictExpr): + if not expr.items: + raise TypeTranslationError() + items: dict[str, Type] = {} + extra_items_from = [] + for item_name, value in expr.items: + if not isinstance(item_name, StrExpr): + if item_name is None: + extra_items_from.append( + expr_to_unanalyzed_type(value, options, allow_new_syntax, expr) + ) + continue + raise TypeTranslationError() + items[item_name.value] = expr_to_unanalyzed_type( + value, options, allow_new_syntax, expr + ) + result = TypedDictType( + items, set(), Instance(MISSING_FALLBACK, ()), expr.line, expr.column + ) + result.extra_items_from = extra_items_from + return result else: raise TypeTranslationError() diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 01f6ed4733ae..75c4bd46550c 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -17,6 +17,7 @@ ARG_POS, ARG_STAR, ARG_STAR2, + MISSING_FALLBACK, PARAM_SPEC_KIND, TYPE_VAR_KIND, TYPE_VAR_TUPLE_KIND, @@ -42,7 +43,6 @@ EllipsisExpr, Expression, ExpressionStmt, - FakeInfo, FloatExpr, ForStmt, FuncDef, @@ -116,6 +116,7 @@ RawExpressionType, TupleType, Type, + TypedDictType, TypeList, TypeOfAny, UnboundType, @@ -190,7 +191,6 @@ def ast3_parse( # There is no way to create reasonable fallbacks at this stage, # they must be patched later. -MISSING_FALLBACK: Final = FakeInfo("fallback can't be filled out until semanal") _dummy_fallback: Final = Instance(MISSING_FALLBACK, [], -1) TYPE_IGNORE_PATTERN: Final = re.compile(r"[^#]*#\s*type:\s*ignore\s*(.*)") @@ -2106,6 +2106,22 @@ def visit_Tuple(self, n: ast3.Tuple) -> Type: column=self.convert_column(n.col_offset), ) + def visit_Dict(self, n: ast3.Dict) -> Type: + if not n.keys: + return self.invalid_type(n) + items: dict[str, Type] = {} + extra_items_from = [] + for item_name, value in zip(n.keys, n.values): + if not isinstance(item_name, ast3.Constant) or not isinstance(item_name.value, str): + if item_name is None: + extra_items_from.append(self.visit(value)) + continue + return self.invalid_type(n) + items[item_name.value] = self.visit(value) + result = TypedDictType(items, set(), _dummy_fallback, n.lineno, n.col_offset) + result.extra_items_from = extra_items_from + return result + # Attribute(expr value, identifier attr, expr_context ctx) def visit_Attribute(self, n: Attribute) -> Type: before_dot = self.visit(n.value) diff --git a/mypy/message_registry.py b/mypy/message_registry.py index befacc9e6182..06199e70d6b4 100644 --- a/mypy/message_registry.py +++ b/mypy/message_registry.py @@ -138,6 +138,7 @@ def with_additional_msg(self, info: str) -> ErrorMessage: TYPEDDICT_KEY_MUST_BE_STRING_LITERAL: Final = ErrorMessage( "Expected TypedDict key to be string literal" ) +TYPEDDICT_OVERRIDE_MERGE: Final = 'Overwriting TypedDict field "{}" while merging' MALFORMED_ASSERT: Final = ErrorMessage("Assertion is always true, perhaps remove parentheses?") DUPLICATE_TYPE_SIGNATURES: Final = ErrorMessage("Function has duplicate type signatures") DESCRIPTOR_SET_NOT_CALLABLE: Final = ErrorMessage("{}.__set__ is not callable") diff --git a/mypy/nodes.py b/mypy/nodes.py index 2eb39d4baaf6..4a5c7240fa83 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -3480,6 +3480,7 @@ def __getattribute__(self, attr: str) -> type: VAR_NO_INFO: Final[TypeInfo] = FakeInfo("Var is lacking info") CLASSDEF_NO_INFO: Final[TypeInfo] = FakeInfo("ClassDef is lacking info") FUNC_NO_INFO: Final[TypeInfo] = FakeInfo("FuncBase for non-methods lack info") +MISSING_FALLBACK: Final = FakeInfo("fallback can't be filled out until semanal") class TypeAlias(SymbolNode): diff --git a/mypy/options.py b/mypy/options.py index 5ef6bc2a35e7..bff096d82c15 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -74,7 +74,8 @@ class BuildType: UNPACK: Final = "Unpack" PRECISE_TUPLE_TYPES: Final = "PreciseTupleTypes" NEW_GENERIC_SYNTAX: Final = "NewGenericSyntax" -INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES, NEW_GENERIC_SYNTAX)) +INLINE_TYPEDDICT: Final = "InlineTypedDict" +INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES, NEW_GENERIC_SYNTAX, INLINE_TYPEDDICT)) COMPLETE_FEATURES: Final = frozenset((TYPE_VAR_TUPLE, UNPACK)) diff --git a/mypy/semanal_typeddict.py b/mypy/semanal_typeddict.py index 7b8d874337a2..e639871364ce 100644 --- a/mypy/semanal_typeddict.py +++ b/mypy/semanal_typeddict.py @@ -8,6 +8,7 @@ from mypy.errorcodes import ErrorCode from mypy.expandtype import expand_type from mypy.exprtotype import TypeTranslationError, expr_to_unanalyzed_type +from mypy.message_registry import TYPEDDICT_OVERRIDE_MERGE from mypy.messages import MessageBuilder from mypy.nodes import ( ARG_NAMED, @@ -216,7 +217,7 @@ def add_keys_and_types_from_base( valid_items = self.map_items_to_base(valid_items, tvars, base_args) for key in base_items: if key in keys: - self.fail(f'Overwriting TypedDict field "{key}" while merging', ctx) + self.fail(TYPEDDICT_OVERRIDE_MERGE.format(key), ctx) keys.extend(valid_items.keys()) types.extend(valid_items.values()) required_keys.update(base_typed_dict.required_keys) @@ -507,17 +508,7 @@ def parse_typeddict_fields_with_types( field_type_expr, self.options, self.api.is_stub_file ) except TypeTranslationError: - if ( - isinstance(field_type_expr, CallExpr) - and isinstance(field_type_expr.callee, RefExpr) - and field_type_expr.callee.fullname in TPDICT_NAMES - ): - self.fail_typeddict_arg( - "Inline TypedDict types not supported; use assignment to define TypedDict", - field_type_expr, - ) - else: - self.fail_typeddict_arg("Invalid field type", field_type_expr) + self.fail_typeddict_arg("Use dict literal for nested TypedDict", field_type_expr) return [], [], False analyzed = self.api.anal_type( type, diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 6651af7dad4f..f63aef30a09a 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -10,7 +10,11 @@ from mypy import errorcodes as codes, message_registry, nodes from mypy.errorcodes import ErrorCode from mypy.expandtype import expand_type -from mypy.message_registry import INVALID_PARAM_SPEC_LOCATION, INVALID_PARAM_SPEC_LOCATION_NOTE +from mypy.message_registry import ( + INVALID_PARAM_SPEC_LOCATION, + INVALID_PARAM_SPEC_LOCATION_NOTE, + TYPEDDICT_OVERRIDE_MERGE, +) from mypy.messages import ( MessageBuilder, format_type, @@ -25,6 +29,7 @@ ARG_POS, ARG_STAR, ARG_STAR2, + MISSING_FALLBACK, SYMBOL_FUNCBASE_TYPES, ArgKind, Context, @@ -43,7 +48,7 @@ check_arg_names, get_nongen_builtins, ) -from mypy.options import Options +from mypy.options import INLINE_TYPEDDICT, Options from mypy.plugin import AnalyzeTypeContext, Plugin, TypeAnalyzerPluginInterface from mypy.semanal_shared import ( SemanticAnalyzerCoreInterface, @@ -1220,10 +1225,45 @@ def visit_tuple_type(self, t: TupleType) -> Type: return TupleType(self.anal_array(t.items, allow_unpack=True), fallback, t.line) def visit_typeddict_type(self, t: TypedDictType) -> Type: - items = { - item_name: self.anal_type(item_type) for (item_name, item_type) in t.items.items() - } - return TypedDictType(items, set(t.required_keys), t.fallback) + req_keys = set() + items = {} + for item_name, item_type in t.items.items(): + analyzed = self.anal_type(item_type, allow_required=True) + if isinstance(analyzed, RequiredType): + if analyzed.required: + req_keys.add(item_name) + analyzed = analyzed.item + else: + # Keys are required by default. + req_keys.add(item_name) + items[item_name] = analyzed + if t.fallback.type is MISSING_FALLBACK: # anonymous/inline TypedDict + if INLINE_TYPEDDICT not in self.options.enable_incomplete_feature: + self.fail( + "Inline TypedDict is experimental," + " must be enabled with --enable-incomplete-feature=InlineTypedDict", + t, + ) + required_keys = req_keys + fallback = self.named_type("typing._TypedDict") + for typ in t.extra_items_from: + analyzed = self.analyze_type(typ) + p_analyzed = get_proper_type(analyzed) + if not isinstance(p_analyzed, TypedDictType): + if not isinstance(p_analyzed, (AnyType, PlaceholderType)): + self.fail("Can only merge-in other TypedDict", t, code=codes.VALID_TYPE) + continue + for sub_item_name, sub_item_type in p_analyzed.items.items(): + if sub_item_name in items: + self.fail(TYPEDDICT_OVERRIDE_MERGE.format(sub_item_name), t) + continue + items[sub_item_name] = sub_item_type + if sub_item_name in p_analyzed.required_keys: + req_keys.add(sub_item_name) + else: + required_keys = t.required_keys + fallback = t.fallback + return TypedDictType(items, required_keys, fallback, t.line, t.column) def visit_raw_expression_type(self, t: RawExpressionType) -> Type: # We should never see a bare Literal. We synthesize these raw literals @@ -1761,11 +1801,12 @@ def anal_type( allow_param_spec: bool = False, allow_unpack: bool = False, allow_ellipsis: bool = False, + allow_required: bool = False, ) -> Type: if nested: self.nesting_level += 1 old_allow_required = self.allow_required - self.allow_required = False + self.allow_required = allow_required old_allow_ellipsis = self.allow_ellipsis self.allow_ellipsis = allow_ellipsis old_allow_unpack = self.allow_unpack diff --git a/mypy/types.py b/mypy/types.py index 89609e8d0546..91b40536f1cf 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -2521,11 +2521,12 @@ class TypedDictType(ProperType): TODO: The fallback structure is perhaps overly complicated. """ - __slots__ = ("items", "required_keys", "fallback") + __slots__ = ("items", "required_keys", "fallback", "extra_items_from") items: dict[str, Type] # item_name -> item_type required_keys: set[str] fallback: Instance + extra_items_from: list[ProperType] # only used during semantic analysis def __init__( self, @@ -2541,6 +2542,7 @@ def __init__( self.fallback = fallback self.can_be_true = len(self.items) > 0 self.can_be_false = len(self.required_keys) == 0 + self.extra_items_from = [] def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_typeddict_type(self) diff --git a/test-data/unit/check-literal.test b/test-data/unit/check-literal.test index 8f8aaf6a3982..6d76ce176aaf 100644 --- a/test-data/unit/check-literal.test +++ b/test-data/unit/check-literal.test @@ -608,36 +608,35 @@ e: Literal[dummy()] # E: Invalid type: Literal[...] cannot contain a [case testLiteralDisallowCollections] from typing_extensions import Literal -a: Literal[{"a": 1, "b": 2}] # E: Invalid type: Literal[...] cannot contain arbitrary expressions +a: Literal[{"a": 1, "b": 2}] # E: Parameter 1 of Literal[...] is invalid b: Literal[{1, 2, 3}] # E: Invalid type: Literal[...] cannot contain arbitrary expressions -c: {"a": 1, "b": 2} # E: Invalid type comment or annotation +c: {"a": 1, "b": 2} # E: Inline TypedDict is experimental, must be enabled with --enable-incomplete-feature=InlineTypedDict \ + # E: Invalid type: try using Literal[1] instead? \ + # E: Invalid type: try using Literal[2] instead? d: {1, 2, 3} # E: Invalid type comment or annotation [builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] [case testLiteralDisallowCollections2] - from typing_extensions import Literal a: (1, 2, 3) # E: Syntax error in type annotation \ # N: Suggestion: Use Tuple[T1, ..., Tn] instead of (T1, ..., Tn) b: Literal[[1, 2, 3]] # E: Parameter 1 of Literal[...] is invalid c: [1, 2, 3] # E: Bracketed expression "[...]" is not valid as a type [builtins fixtures/tuple.pyi] -[out] [case testLiteralDisallowCollectionsTypeAlias] - from typing_extensions import Literal -at = Literal[{"a": 1, "b": 2}] # E: Invalid type alias: expression is not a valid type +at = Literal[{"a": 1, "b": 2}] # E: Parameter 1 of Literal[...] is invalid bt = {"a": 1, "b": 2} -a: at # E: Variable "__main__.at" is not valid as a type \ - # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +a: at +reveal_type(a) # N: Revealed type is "Any" b: bt # E: Variable "__main__.bt" is not valid as a type \ # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases [builtins fixtures/dict.pyi] -[out] +[typing fixtures/typing-typeddict.pyi] [case testLiteralDisallowCollectionsTypeAlias2] - from typing_extensions import Literal at = Literal[{1, 2, 3}] # E: Invalid type alias: expression is not a valid type bt = {1, 2, 3} diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 5307f47d539a..073ef7f4bdec 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -585,8 +585,7 @@ reveal_type(a) # N: Revealed type is "Any" [case testPEP695TypeAliasInvalidType] # flags: --enable-incomplete-feature=NewGenericSyntax -type A = int | 1 # E: Invalid type: try using Literal[1] instead? \ - # E: Unsupported operand types for | ("Type[int]" and "int") +type A = int | 1 # E: Invalid type: try using Literal[1] instead? a: A reveal_type(a) # N: Revealed type is "Union[builtins.int, Any]" @@ -1656,3 +1655,15 @@ type I2 = C[Any] | None type I3 = None | C[TD] [builtins fixtures/type.pyi] [typing fixtures/typing-full.pyi] + +[case testTypedDictInlineYesNewStyleAlias] +# flags: --enable-incomplete-feature=NewGenericSyntax --enable-incomplete-feature=InlineTypedDict +type X[T] = {"item": T, "other": X[T] | None} +x: X[str] +reveal_type(x) # N: Revealed type is "TypedDict({'item': builtins.str, 'other': Union[..., None]})" +if x["other"] is not None: + reveal_type(x["other"]["item"]) # N: Revealed type is "builtins.str" + +type Y[T] = {"item": T, **Y[T]} # E: Overwriting TypedDict field "item" while merging +[builtins fixtures/dict.pyi] +[typing fixtures/typing-full.pyi] diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index d35ec8ddd80e..6a5120159c2d 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -78,7 +78,7 @@ p = Point(x='meaning_of_life', y=1337) # E: Incompatible types (expression has [case testCannotCreateTypedDictInstanceWithInlineTypedDict] from mypy_extensions import TypedDict D = TypedDict('D', { - 'x': TypedDict('E', { # E: Inline TypedDict types not supported; use assignment to define TypedDict + 'x': TypedDict('E', { # E: Use dict literal for nested TypedDict 'y': int }) }) @@ -3570,3 +3570,59 @@ class Test: run(test2, other="yes", **params) run(test2, other=0, **params) # E: Argument "other" to "run" has incompatible type "int"; expected "str" [builtins fixtures/tuple.pyi] + +[case testTypedDictInlineNoOldStyleAlias] +# flags: --enable-incomplete-feature=InlineTypedDict +X = {"int": int, "str": str} +reveal_type(X) # N: Revealed type is "builtins.dict[builtins.str, def () -> builtins.object]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictInlineYesMidStyleAlias] +# flags: --enable-incomplete-feature=InlineTypedDict +from typing_extensions import TypeAlias +X: TypeAlias = {"int": int, "str": str} +x: X +reveal_type(x) # N: # N: Revealed type is "TypedDict({'int': builtins.int, 'str': builtins.str})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictInlineNoEmpty] +# flags: --enable-incomplete-feature=InlineTypedDict +x: {} # E: Invalid type comment or annotation +reveal_type(x) # N: Revealed type is "Any" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictInlineNotRequired] +# flags: --enable-incomplete-feature=InlineTypedDict +from typing import NotRequired + +x: {"one": int, "other": NotRequired[int]} +x = {"one": 1} # OK +y: {"one": int, "other": int} +y = {"one": 1} # E: Expected TypedDict keys ("one", "other") but found only key "one" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictInlineNestedSchema] +# flags: --enable-incomplete-feature=InlineTypedDict +def nested() -> {"one": str, "other": {"a": int, "b": int}}: + if bool(): + return {"one": "yes", "other": {"a": 1, "b": 2}} # OK + else: + return {"one": "no", "other": {"a": 1, "b": "2"}} # E: Incompatible types (expression has type "str", TypedDict item "b" has type "int") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictInlineMergeAnother] +# flags: --enable-incomplete-feature=InlineTypedDict +from typing import TypeVar +from typing_extensions import TypeAlias + +T = TypeVar("T") +X: TypeAlias = {"item": T} +x: {"a": int, **X[str], "b": int} +reveal_type(x) # N: Revealed type is "TypedDict({'a': builtins.int, 'b': builtins.int, 'item': builtins.str})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-full.pyi]