From 02d3667442e0c8e63989e0276d1e78754cc84b2a Mon Sep 17 00:00:00 2001 From: Danny Yang Date: Sun, 30 Jun 2024 16:15:34 -0400 Subject: [PATCH 01/36] Fix typechecking for async generators (#17452) Fixes #10534 This PR fixes a bug in typechecking asynchronous generators. Mypy currently typechecks a generator/comprehension as `AsyncGenerator` if the leftmost expression contains `await`, or if it contains an `async for`. However, there are other situations where we should get async generator: If there is an `await` expression in any of the conditions or in any sequence except for the leftmost one, the generator/comprehension should also be typechecked as `AsyncGenerator`. I've implemented this change in Mypy and added a test case to assert this behavior. If I enter the test cases into a regular repl, I can confirm that the runtime representation is generator/async_generator as the test case expects. According to the [language reference](https://docs.python.org/3/reference/expressions.html#grammar-token-python-grammar-comp_for): > If a comprehension contains either async for clauses or await expressions or other asynchronous comprehensions it is called an asynchronous comprehension. Confusingly, the documentation itself is actually not quite correct either, as pointed out in https://github.com/python/cpython/issues/114104 Alongside this change, I've made a PR to update the docs to be more precise: https://github.com/python/cpython/pull/121175 has more details. --- mypy/checkexpr.py | 9 +++++++-- test-data/unit/check-async-await.test | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index fdc0f94b3997..c4ab8a081acc 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -5585,8 +5585,13 @@ def visit_set_comprehension(self, e: SetComprehension) -> Type: def visit_generator_expr(self, e: GeneratorExpr) -> Type: # If any of the comprehensions use async for, the expression will return an async generator - # object, or if the left-side expression uses await. - if any(e.is_async) or has_await_expression(e.left_expr): + # object, or await is used anywhere but in the leftmost sequence. + if ( + any(e.is_async) + or has_await_expression(e.left_expr) + or any(has_await_expression(sequence) for sequence in e.sequences[1:]) + or any(has_await_expression(cond) for condlist in e.condlists for cond in condlist) + ): typ = "typing.AsyncGenerator" # received type is always None in async generator expressions additional_args: list[Type] = [NoneType()] diff --git a/test-data/unit/check-async-await.test b/test-data/unit/check-async-await.test index 876fe0c6be15..0ef08e5a0775 100644 --- a/test-data/unit/check-async-await.test +++ b/test-data/unit/check-async-await.test @@ -573,6 +573,25 @@ async def return_f() -> AsyncGenerator[int, None]: [builtins fixtures/dict.pyi] [typing fixtures/typing-async.pyi] +[case testImplicitAsyncGenerator] +from typing import List + +async def get_list() -> List[int]: + return [1] + +async def predicate() -> bool: + return True + +async def test_implicit_generators() -> None: + reveal_type(await predicate() for _ in [1]) # N: Revealed type is "typing.AsyncGenerator[builtins.bool, None]" + reveal_type(x for x in [1] if await predicate()) # N: Revealed type is "typing.AsyncGenerator[builtins.int, None]" + reveal_type(x for x in await get_list()) # N: Revealed type is "typing.Generator[builtins.int, None, None]" + reveal_type(x for _ in [1] for x in await get_list()) # N: Revealed type is "typing.AsyncGenerator[builtins.int, None]" + +[builtins fixtures/dict.pyi] +[typing fixtures/typing-async.pyi] + + -- The full matrix of coroutine compatibility -- ------------------------------------------ From c346c5425891c1eac5e4eba4c1eb4e84ceeb2a06 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 30 Jun 2024 17:14:04 -0700 Subject: [PATCH 02/36] Use Python 3.12 for mypy_primer (#17456) I'd like for us to cover projects that use Python 3.12 only syntax, like homeassistant --- .github/workflows/mypy_primer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mypy_primer.yml b/.github/workflows/mypy_primer.yml index 07a1d0863eb2..0c77d3a255d8 100644 --- a/.github/workflows/mypy_primer.yml +++ b/.github/workflows/mypy_primer.yml @@ -39,7 +39,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "3.12" - name: Install dependencies run: | python -m pip install -U pip From 98717718be97bf54a102f9d844ae7185aedaa7ac Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 30 Jun 2024 17:24:36 -0700 Subject: [PATCH 03/36] Sync typeshed (#17458) Source commit: https://github.com/python/typeshed/commit/dcab6e88883c629ede9637fb011958f8b4918f52 --- mypy/typeshed/stdlib/VERSIONS | 1 + mypy/typeshed/stdlib/_interpchannels.pyi | 84 +++++ mypy/typeshed/stdlib/argparse.pyi | 8 +- mypy/typeshed/stdlib/asyncio/events.pyi | 93 +++--- mypy/typeshed/stdlib/asyncio/tasks.pyi | 5 +- mypy/typeshed/stdlib/asyncio/unix_events.pyi | 290 ++++++++++-------- .../stdlib/asyncio/windows_events.pyi | 5 +- mypy/typeshed/stdlib/dataclasses.pyi | 2 +- mypy/typeshed/stdlib/posixpath.pyi | 16 +- mypy/typeshed/stdlib/subprocess.pyi | 8 + mypy/typeshed/stdlib/tarfile.pyi | 24 +- mypy/typeshed/stdlib/zipimport.pyi | 2 + 12 files changed, 347 insertions(+), 191 deletions(-) create mode 100644 mypy/typeshed/stdlib/_interpchannels.pyi diff --git a/mypy/typeshed/stdlib/VERSIONS b/mypy/typeshed/stdlib/VERSIONS index 7b9ce2864484..89754f65f3fa 100644 --- a/mypy/typeshed/stdlib/VERSIONS +++ b/mypy/typeshed/stdlib/VERSIONS @@ -34,6 +34,7 @@ _dummy_thread: 3.0-3.8 _dummy_threading: 3.0-3.8 _heapq: 3.0- _imp: 3.0- +_interpchannels: 3.13- _json: 3.0- _locale: 3.0- _lsprof: 3.0- diff --git a/mypy/typeshed/stdlib/_interpchannels.pyi b/mypy/typeshed/stdlib/_interpchannels.pyi new file mode 100644 index 000000000000..b77fe321a071 --- /dev/null +++ b/mypy/typeshed/stdlib/_interpchannels.pyi @@ -0,0 +1,84 @@ +from _typeshed import structseq +from typing import Final, Literal, SupportsIndex, final +from typing_extensions import Buffer, Self + +class ChannelError(RuntimeError): ... +class ChannelClosedError(ChannelError): ... +class ChannelEmptyError(ChannelError): ... +class ChannelNotEmptyError(ChannelError): ... +class ChannelNotFoundError(ChannelError): ... + +# Mark as final, since instantiating ChannelID is not supported. +@final +class ChannelID: + @property + def end(self) -> Literal["send", "recv", "both"]: ... + @property + def send(self) -> Self: ... + @property + def recv(self) -> Self: ... + def __eq__(self, other: object) -> bool: ... + def __ge__(self, other: ChannelID) -> bool: ... + def __gt__(self, other: ChannelID) -> bool: ... + def __hash__(self) -> int: ... + def __index__(self) -> int: ... + def __int__(self) -> int: ... + def __le__(self, other: ChannelID) -> bool: ... + def __lt__(self, other: ChannelID) -> bool: ... + def __ne__(self, other: object) -> bool: ... + +@final +class ChannelInfo(structseq[int], tuple[bool, bool, bool, int, int, int, int, int]): + __match_args__: Final = ( + "open", + "closing", + "closed", + "count", + "num_interp_send", + "num_interp_send_released", + "num_interp_recv", + "num_interp_recv_released", + ) + @property + def open(self) -> bool: ... + @property + def closing(self) -> bool: ... + @property + def closed(self) -> bool: ... + @property + def count(self) -> int: ... # type: ignore[override] + @property + def num_interp_send(self) -> int: ... + @property + def num_interp_send_released(self) -> int: ... + @property + def num_interp_recv(self) -> int: ... + @property + def num_interp_recv_released(self) -> int: ... + @property + def num_interp_both(self) -> int: ... + @property + def num_interp_both_recv_released(self) -> int: ... + @property + def num_interp_both_send_released(self) -> int: ... + @property + def num_interp_both_released(self) -> int: ... + @property + def recv_associated(self) -> bool: ... + @property + def recv_released(self) -> bool: ... + @property + def send_associated(self) -> bool: ... + @property + def send_released(self) -> bool: ... + +def create() -> ChannelID: ... +def destroy(cid: SupportsIndex) -> None: ... +def list_all() -> list[ChannelID]: ... +def list_interpreters(cid: SupportsIndex, *, send: bool) -> list[int]: ... +def send(cid: SupportsIndex, obj: object, *, blocking: bool = True, timeout: float | None = None) -> None: ... +def send_buffer(cid: SupportsIndex, obj: Buffer, *, blocking: bool = True, timeout: float | None = None) -> None: ... +def recv(cid: SupportsIndex, default: object = ...) -> object: ... +def close(cid: SupportsIndex, *, send: bool = False, recv: bool = False) -> None: ... +def get_info(cid: SupportsIndex) -> ChannelInfo: ... +def release(cid: SupportsIndex, *, send: bool = False, recv: bool = False, force: bool = False) -> None: ... diff --git a/mypy/typeshed/stdlib/argparse.pyi b/mypy/typeshed/stdlib/argparse.pyi index 1956d08c9933..bc781ec8e61d 100644 --- a/mypy/typeshed/stdlib/argparse.pyi +++ b/mypy/typeshed/stdlib/argparse.pyi @@ -32,6 +32,7 @@ _T = TypeVar("_T") _ActionT = TypeVar("_ActionT", bound=Action) _ArgumentParserT = TypeVar("_ArgumentParserT", bound=ArgumentParser) _N = TypeVar("_N") +_ActionType: TypeAlias = Callable[[str], Any] | FileType | str # more precisely, Literal["store", "store_const", "store_true", # "store_false", "append", "append_const", "count", "help", "version", # "extend"], but using this would make it hard to annotate callers @@ -89,7 +90,7 @@ class _ActionsContainer: nargs: int | _NArgsStr | _SUPPRESS_T | None = None, const: Any = ..., default: Any = ..., - type: Callable[[str], _T] | FileType = ..., + type: _ActionType = ..., choices: Iterable[_T] | None = ..., required: bool = ..., help: str | None = ..., @@ -313,7 +314,7 @@ class Action(_AttributeHolder): nargs: int | str | None const: Any default: Any - type: Callable[[str], Any] | FileType | None + type: _ActionType | None choices: Iterable[Any] | None required: bool help: str | None @@ -699,6 +700,7 @@ class _SubParsersAction(Action, Generic[_ArgumentParserT]): add_help: bool = ..., allow_abbrev: bool = ..., exit_on_error: bool = ..., + **kwargs: Any, # Accepting any additional kwargs for custom parser classes ) -> _ArgumentParserT: ... elif sys.version_info >= (3, 9): def add_parser( @@ -721,6 +723,7 @@ class _SubParsersAction(Action, Generic[_ArgumentParserT]): add_help: bool = ..., allow_abbrev: bool = ..., exit_on_error: bool = ..., + **kwargs: Any, # Accepting any additional kwargs for custom parser classes ) -> _ArgumentParserT: ... else: def add_parser( @@ -742,6 +745,7 @@ class _SubParsersAction(Action, Generic[_ArgumentParserT]): conflict_handler: str = ..., add_help: bool = ..., allow_abbrev: bool = ..., + **kwargs: Any, # Accepting any additional kwargs for custom parser classes ) -> _ArgumentParserT: ... def _get_subactions(self) -> list[Action]: ... diff --git a/mypy/typeshed/stdlib/asyncio/events.pyi b/mypy/typeshed/stdlib/asyncio/events.pyi index c0345eb1b5b5..8c2664666835 100644 --- a/mypy/typeshed/stdlib/asyncio/events.pyi +++ b/mypy/typeshed/stdlib/asyncio/events.pyi @@ -16,23 +16,40 @@ from .tasks import Task from .transports import BaseTransport, DatagramTransport, ReadTransport, SubprocessTransport, Transport, WriteTransport from .unix_events import AbstractChildWatcher -__all__ = ( - "AbstractEventLoopPolicy", - "AbstractEventLoop", - "AbstractServer", - "Handle", - "TimerHandle", - "get_event_loop_policy", - "set_event_loop_policy", - "get_event_loop", - "set_event_loop", - "new_event_loop", - "get_child_watcher", - "set_child_watcher", - "_set_running_loop", - "get_running_loop", - "_get_running_loop", -) +if sys.version_info >= (3, 14): + __all__ = ( + "AbstractEventLoopPolicy", + "AbstractEventLoop", + "AbstractServer", + "Handle", + "TimerHandle", + "get_event_loop_policy", + "set_event_loop_policy", + "get_event_loop", + "set_event_loop", + "new_event_loop", + "_set_running_loop", + "get_running_loop", + "_get_running_loop", + ) +else: + __all__ = ( + "AbstractEventLoopPolicy", + "AbstractEventLoop", + "AbstractServer", + "Handle", + "TimerHandle", + "get_event_loop_policy", + "set_event_loop_policy", + "get_event_loop", + "set_event_loop", + "new_event_loop", + "get_child_watcher", + "set_child_watcher", + "_set_running_loop", + "get_running_loop", + "_get_running_loop", + ) _T = TypeVar("_T") _Ts = TypeVarTuple("_Ts") @@ -541,18 +558,19 @@ class AbstractEventLoopPolicy: @abstractmethod def new_event_loop(self) -> AbstractEventLoop: ... # Child processes handling (Unix only). - if sys.version_info >= (3, 12): - @abstractmethod - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") - def get_child_watcher(self) -> AbstractChildWatcher: ... - @abstractmethod - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") - def set_child_watcher(self, watcher: AbstractChildWatcher) -> None: ... - else: - @abstractmethod - def get_child_watcher(self) -> AbstractChildWatcher: ... - @abstractmethod - def set_child_watcher(self, watcher: AbstractChildWatcher) -> None: ... + if sys.version_info < (3, 14): + if sys.version_info >= (3, 12): + @abstractmethod + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + def get_child_watcher(self) -> AbstractChildWatcher: ... + @abstractmethod + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + def set_child_watcher(self, watcher: AbstractChildWatcher) -> None: ... + else: + @abstractmethod + def get_child_watcher(self) -> AbstractChildWatcher: ... + @abstractmethod + def set_child_watcher(self, watcher: AbstractChildWatcher) -> None: ... class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy, metaclass=ABCMeta): def get_event_loop(self) -> AbstractEventLoop: ... @@ -565,15 +583,16 @@ def get_event_loop() -> AbstractEventLoop: ... def set_event_loop(loop: AbstractEventLoop | None) -> None: ... def new_event_loop() -> AbstractEventLoop: ... -if sys.version_info >= (3, 12): - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") - def get_child_watcher() -> AbstractChildWatcher: ... - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") - def set_child_watcher(watcher: AbstractChildWatcher) -> None: ... +if sys.version_info < (3, 14): + if sys.version_info >= (3, 12): + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + def get_child_watcher() -> AbstractChildWatcher: ... + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + def set_child_watcher(watcher: AbstractChildWatcher) -> None: ... -else: - def get_child_watcher() -> AbstractChildWatcher: ... - def set_child_watcher(watcher: AbstractChildWatcher) -> None: ... + else: + def get_child_watcher() -> AbstractChildWatcher: ... + def set_child_watcher(watcher: AbstractChildWatcher) -> None: ... def _set_running_loop(loop: AbstractEventLoop | None, /) -> None: ... def _get_running_loop() -> AbstractEventLoop: ... diff --git a/mypy/typeshed/stdlib/asyncio/tasks.pyi b/mypy/typeshed/stdlib/asyncio/tasks.pyi index c16a1919b7c8..4613bca70c1a 100644 --- a/mypy/typeshed/stdlib/asyncio/tasks.pyi +++ b/mypy/typeshed/stdlib/asyncio/tasks.pyi @@ -70,7 +70,10 @@ _T4 = TypeVar("_T4") _T5 = TypeVar("_T5") _T6 = TypeVar("_T6") _FT = TypeVar("_FT", bound=Future[Any]) -_FutureLike: TypeAlias = Future[_T] | Generator[Any, None, _T] | Awaitable[_T] +if sys.version_info >= (3, 12): + _FutureLike: TypeAlias = Future[_T] | Awaitable[_T] +else: + _FutureLike: TypeAlias = Future[_T] | Generator[Any, None, _T] | Awaitable[_T] _TaskYieldType: TypeAlias = Future[object] | None FIRST_COMPLETED = concurrent.futures.FIRST_COMPLETED diff --git a/mypy/typeshed/stdlib/asyncio/unix_events.pyi b/mypy/typeshed/stdlib/asyncio/unix_events.pyi index e9274b853290..3a2c62646121 100644 --- a/mypy/typeshed/stdlib/asyncio/unix_events.pyi +++ b/mypy/typeshed/stdlib/asyncio/unix_events.pyi @@ -13,51 +13,54 @@ _Ts = TypeVarTuple("_Ts") # This is also technically not available on Win, # but other parts of typeshed need this definition. # So, it is special cased. -if sys.version_info >= (3, 12): - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") - class AbstractChildWatcher: - @abstractmethod - def add_child_handler( - self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] - ) -> None: ... - @abstractmethod - def remove_child_handler(self, pid: int) -> bool: ... - @abstractmethod - def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... - @abstractmethod - def close(self) -> None: ... - @abstractmethod - def __enter__(self) -> Self: ... - @abstractmethod - def __exit__( - self, typ: type[BaseException] | None, exc: BaseException | None, tb: types.TracebackType | None - ) -> None: ... - @abstractmethod - def is_active(self) -> bool: ... - -else: - class AbstractChildWatcher: - @abstractmethod - def add_child_handler( - self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] - ) -> None: ... - @abstractmethod - def remove_child_handler(self, pid: int) -> bool: ... - @abstractmethod - def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... - @abstractmethod - def close(self) -> None: ... - @abstractmethod - def __enter__(self) -> Self: ... - @abstractmethod - def __exit__( - self, typ: type[BaseException] | None, exc: BaseException | None, tb: types.TracebackType | None - ) -> None: ... - @abstractmethod - def is_active(self) -> bool: ... +if sys.version_info < (3, 14): + if sys.version_info >= (3, 12): + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + class AbstractChildWatcher: + @abstractmethod + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + @abstractmethod + def remove_child_handler(self, pid: int) -> bool: ... + @abstractmethod + def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... + @abstractmethod + def close(self) -> None: ... + @abstractmethod + def __enter__(self) -> Self: ... + @abstractmethod + def __exit__( + self, typ: type[BaseException] | None, exc: BaseException | None, tb: types.TracebackType | None + ) -> None: ... + @abstractmethod + def is_active(self) -> bool: ... + + else: + class AbstractChildWatcher: + @abstractmethod + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + @abstractmethod + def remove_child_handler(self, pid: int) -> bool: ... + @abstractmethod + def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... + @abstractmethod + def close(self) -> None: ... + @abstractmethod + def __enter__(self) -> Self: ... + @abstractmethod + def __exit__( + self, typ: type[BaseException] | None, exc: BaseException | None, tb: types.TracebackType | None + ) -> None: ... + @abstractmethod + def is_active(self) -> bool: ... if sys.platform != "win32": - if sys.version_info >= (3, 9): + if sys.version_info >= (3, 14): + __all__ = ("SelectorEventLoop", "DefaultEventLoopPolicy") + elif sys.version_info >= (3, 9): __all__ = ( "SelectorEventLoop", "AbstractChildWatcher", @@ -79,118 +82,137 @@ if sys.platform != "win32": "DefaultEventLoopPolicy", ) - # Doesn't actually have ABCMeta metaclass at runtime, but mypy complains if we don't have it in the stub. - # See discussion in #7412 - class BaseChildWatcher(AbstractChildWatcher, metaclass=ABCMeta): - def close(self) -> None: ... - def is_active(self) -> bool: ... - def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... - - if sys.version_info >= (3, 12): - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") - class SafeChildWatcher(BaseChildWatcher): - def __enter__(self) -> Self: ... - def __exit__(self, a: type[BaseException] | None, b: BaseException | None, c: types.TracebackType | None) -> None: ... - def add_child_handler( - self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] - ) -> None: ... - def remove_child_handler(self, pid: int) -> bool: ... + if sys.version_info < (3, 14): + if sys.version_info >= (3, 12): + # Doesn't actually have ABCMeta metaclass at runtime, but mypy complains if we don't have it in the stub. + # See discussion in #7412 + class BaseChildWatcher(AbstractChildWatcher, metaclass=ABCMeta): + def close(self) -> None: ... + def is_active(self) -> bool: ... + def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") - class FastChildWatcher(BaseChildWatcher): - def __enter__(self) -> Self: ... - def __exit__(self, a: type[BaseException] | None, b: BaseException | None, c: types.TracebackType | None) -> None: ... - def add_child_handler( - self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] - ) -> None: ... - def remove_child_handler(self, pid: int) -> bool: ... + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + class SafeChildWatcher(BaseChildWatcher): + def __enter__(self) -> Self: ... + def __exit__( + self, a: type[BaseException] | None, b: BaseException | None, c: types.TracebackType | None + ) -> None: ... + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + def remove_child_handler(self, pid: int) -> bool: ... - else: - class SafeChildWatcher(BaseChildWatcher): - def __enter__(self) -> Self: ... - def __exit__(self, a: type[BaseException] | None, b: BaseException | None, c: types.TracebackType | None) -> None: ... - def add_child_handler( - self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] - ) -> None: ... - def remove_child_handler(self, pid: int) -> bool: ... + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + class FastChildWatcher(BaseChildWatcher): + def __enter__(self) -> Self: ... + def __exit__( + self, a: type[BaseException] | None, b: BaseException | None, c: types.TracebackType | None + ) -> None: ... + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + def remove_child_handler(self, pid: int) -> bool: ... - class FastChildWatcher(BaseChildWatcher): - def __enter__(self) -> Self: ... - def __exit__(self, a: type[BaseException] | None, b: BaseException | None, c: types.TracebackType | None) -> None: ... - def add_child_handler( - self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] - ) -> None: ... - def remove_child_handler(self, pid: int) -> bool: ... + else: + # Doesn't actually have ABCMeta metaclass at runtime, but mypy complains if we don't have it in the stub. + # See discussion in #7412 + class BaseChildWatcher(AbstractChildWatcher, metaclass=ABCMeta): + def close(self) -> None: ... + def is_active(self) -> bool: ... + def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... + + class SafeChildWatcher(BaseChildWatcher): + def __enter__(self) -> Self: ... + def __exit__( + self, a: type[BaseException] | None, b: BaseException | None, c: types.TracebackType | None + ) -> None: ... + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + def remove_child_handler(self, pid: int) -> bool: ... + + class FastChildWatcher(BaseChildWatcher): + def __enter__(self) -> Self: ... + def __exit__( + self, a: type[BaseException] | None, b: BaseException | None, c: types.TracebackType | None + ) -> None: ... + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + def remove_child_handler(self, pid: int) -> bool: ... class _UnixSelectorEventLoop(BaseSelectorEventLoop): ... class _UnixDefaultEventLoopPolicy(BaseDefaultEventLoopPolicy): - if sys.version_info >= (3, 12): - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") - def get_child_watcher(self) -> AbstractChildWatcher: ... - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") - def set_child_watcher(self, watcher: AbstractChildWatcher | None) -> None: ... - else: - def get_child_watcher(self) -> AbstractChildWatcher: ... - def set_child_watcher(self, watcher: AbstractChildWatcher | None) -> None: ... + if sys.version_info < (3, 14): + if sys.version_info >= (3, 12): + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + def get_child_watcher(self) -> AbstractChildWatcher: ... + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + def set_child_watcher(self, watcher: AbstractChildWatcher | None) -> None: ... + else: + def get_child_watcher(self) -> AbstractChildWatcher: ... + def set_child_watcher(self, watcher: AbstractChildWatcher | None) -> None: ... SelectorEventLoop = _UnixSelectorEventLoop DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy - if sys.version_info >= (3, 12): - @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") - class MultiLoopChildWatcher(AbstractChildWatcher): - def is_active(self) -> bool: ... - def close(self) -> None: ... - def __enter__(self) -> Self: ... - def __exit__( - self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: types.TracebackType | None - ) -> None: ... - def add_child_handler( - self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] - ) -> None: ... - def remove_child_handler(self, pid: int) -> bool: ... - def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... + if sys.version_info < (3, 14): + if sys.version_info >= (3, 12): + @deprecated("Deprecated as of Python 3.12; will be removed in Python 3.14") + class MultiLoopChildWatcher(AbstractChildWatcher): + def is_active(self) -> bool: ... + def close(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: types.TracebackType | None + ) -> None: ... + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + def remove_child_handler(self, pid: int) -> bool: ... + def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... - else: - class MultiLoopChildWatcher(AbstractChildWatcher): - def is_active(self) -> bool: ... + else: + class MultiLoopChildWatcher(AbstractChildWatcher): + def is_active(self) -> bool: ... + def close(self) -> None: ... + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: types.TracebackType | None + ) -> None: ... + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + def remove_child_handler(self, pid: int) -> bool: ... + def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... + + if sys.version_info < (3, 14): + class ThreadedChildWatcher(AbstractChildWatcher): + def is_active(self) -> Literal[True]: ... def close(self) -> None: ... def __enter__(self) -> Self: ... def __exit__( self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: types.TracebackType | None ) -> None: ... + def __del__(self) -> None: ... def add_child_handler( self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] ) -> None: ... def remove_child_handler(self, pid: int) -> bool: ... def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... - class ThreadedChildWatcher(AbstractChildWatcher): - def is_active(self) -> Literal[True]: ... - def close(self) -> None: ... - def __enter__(self) -> Self: ... - def __exit__( - self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: types.TracebackType | None - ) -> None: ... - def __del__(self) -> None: ... - def add_child_handler( - self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] - ) -> None: ... - def remove_child_handler(self, pid: int) -> bool: ... - def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... - - if sys.version_info >= (3, 9): - class PidfdChildWatcher(AbstractChildWatcher): - def __enter__(self) -> Self: ... - def __exit__( - self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: types.TracebackType | None - ) -> None: ... - def is_active(self) -> bool: ... - def close(self) -> None: ... - def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... - def add_child_handler( - self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] - ) -> None: ... - def remove_child_handler(self, pid: int) -> bool: ... + if sys.version_info >= (3, 9): + class PidfdChildWatcher(AbstractChildWatcher): + def __enter__(self) -> Self: ... + def __exit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: types.TracebackType | None + ) -> None: ... + def is_active(self) -> bool: ... + def close(self) -> None: ... + def attach_loop(self, loop: AbstractEventLoop | None) -> None: ... + def add_child_handler( + self, pid: int, callback: Callable[[int, int, Unpack[_Ts]], object], *args: Unpack[_Ts] + ) -> None: ... + def remove_child_handler(self, pid: int) -> bool: ... diff --git a/mypy/typeshed/stdlib/asyncio/windows_events.pyi b/mypy/typeshed/stdlib/asyncio/windows_events.pyi index 9c150ee16beb..97aa52ff8b9a 100644 --- a/mypy/typeshed/stdlib/asyncio/windows_events.pyi +++ b/mypy/typeshed/stdlib/asyncio/windows_events.pyi @@ -74,8 +74,9 @@ if sys.platform == "win32": class WindowsSelectorEventLoopPolicy(events.BaseDefaultEventLoopPolicy): _loop_factory: ClassVar[type[SelectorEventLoop]] - def get_child_watcher(self) -> NoReturn: ... - def set_child_watcher(self, watcher: Any) -> NoReturn: ... + if sys.version_info < (3, 14): + def get_child_watcher(self) -> NoReturn: ... + def set_child_watcher(self, watcher: Any) -> NoReturn: ... class WindowsProactorEventLoopPolicy(events.BaseDefaultEventLoopPolicy): _loop_factory: ClassVar[type[ProactorEventLoop]] diff --git a/mypy/typeshed/stdlib/dataclasses.pyi b/mypy/typeshed/stdlib/dataclasses.pyi index 30489e6f8b3d..626608e8a59d 100644 --- a/mypy/typeshed/stdlib/dataclasses.pyi +++ b/mypy/typeshed/stdlib/dataclasses.pyi @@ -108,7 +108,7 @@ class _DefaultFactory(Protocol[_T_co]): class Field(Generic[_T]): name: str - type: Type[_T] + type: Type[_T] | str | Any default: _T | Literal[_MISSING_TYPE.MISSING] default_factory: _DefaultFactory[_T] | Literal[_MISSING_TYPE.MISSING] repr: bool diff --git a/mypy/typeshed/stdlib/posixpath.pyi b/mypy/typeshed/stdlib/posixpath.pyi index e5f5fa0d813c..31406f8df950 100644 --- a/mypy/typeshed/stdlib/posixpath.pyi +++ b/mypy/typeshed/stdlib/posixpath.pyi @@ -77,11 +77,7 @@ pathsep: LiteralString defpath: LiteralString devnull: LiteralString -# Overloads are necessary to work around python/mypy#3644. -@overload -def abspath(path: PathLike[AnyStr]) -> AnyStr: ... -@overload -def abspath(path: AnyStr) -> AnyStr: ... +def abspath(path: PathLike[AnyStr] | AnyStr) -> AnyStr: ... @overload def basename(p: PathLike[AnyStr]) -> AnyStr: ... @overload @@ -90,14 +86,8 @@ def basename(p: AnyOrLiteralStr) -> AnyOrLiteralStr: ... def dirname(p: PathLike[AnyStr]) -> AnyStr: ... @overload def dirname(p: AnyOrLiteralStr) -> AnyOrLiteralStr: ... -@overload -def expanduser(path: PathLike[AnyStr]) -> AnyStr: ... -@overload -def expanduser(path: AnyStr) -> AnyStr: ... -@overload -def expandvars(path: PathLike[AnyStr]) -> AnyStr: ... -@overload -def expandvars(path: AnyStr) -> AnyStr: ... +def expanduser(path: PathLike[AnyStr] | AnyStr) -> AnyStr: ... +def expandvars(path: PathLike[AnyStr] | AnyStr) -> AnyStr: ... @overload def normcase(s: PathLike[AnyStr]) -> AnyStr: ... @overload diff --git a/mypy/typeshed/stdlib/subprocess.pyi b/mypy/typeshed/stdlib/subprocess.pyi index 6234ecc02b48..b01bac2455ce 100644 --- a/mypy/typeshed/stdlib/subprocess.pyi +++ b/mypy/typeshed/stdlib/subprocess.pyi @@ -889,6 +889,7 @@ if sys.version_info >= (3, 11): start_new_session: bool = False, pass_fds: Collection[int] = ..., *, + encoding: str | None = None, timeout: float | None = None, text: bool | None = None, user: str | int | None = None, @@ -920,6 +921,7 @@ elif sys.version_info >= (3, 10): start_new_session: bool = False, pass_fds: Collection[int] = ..., *, + encoding: str | None = None, timeout: float | None = None, text: bool | None = None, user: str | int | None = None, @@ -950,6 +952,7 @@ elif sys.version_info >= (3, 9): start_new_session: bool = False, pass_fds: Collection[int] = ..., *, + encoding: str | None = None, timeout: float | None = None, text: bool | None = None, user: str | int | None = None, @@ -978,6 +981,7 @@ else: start_new_session: bool = False, pass_fds: Collection[int] = ..., *, + encoding: str | None = None, timeout: float | None = None, text: bool | None = None, ) -> int: ... @@ -1005,6 +1009,7 @@ if sys.version_info >= (3, 11): pass_fds: Collection[int] = ..., timeout: float | None = ..., *, + encoding: str | None = None, text: bool | None = None, user: str | int | None = None, group: str | int | None = None, @@ -1036,6 +1041,7 @@ elif sys.version_info >= (3, 10): pass_fds: Collection[int] = ..., timeout: float | None = ..., *, + encoding: str | None = None, text: bool | None = None, user: str | int | None = None, group: str | int | None = None, @@ -1066,6 +1072,7 @@ elif sys.version_info >= (3, 9): pass_fds: Collection[int] = ..., timeout: float | None = ..., *, + encoding: str | None = None, text: bool | None = None, user: str | int | None = None, group: str | int | None = None, @@ -1094,6 +1101,7 @@ else: pass_fds: Collection[int] = ..., timeout: float | None = ..., *, + encoding: str | None = None, text: bool | None = None, ) -> int: ... diff --git a/mypy/typeshed/stdlib/tarfile.pyi b/mypy/typeshed/stdlib/tarfile.pyi index e52099464174..d6adf21c1900 100644 --- a/mypy/typeshed/stdlib/tarfile.pyi +++ b/mypy/typeshed/stdlib/tarfile.pyi @@ -103,10 +103,13 @@ PAX_NAME_FIELDS: set[str] ENCODING: str +_FileCreationModes: TypeAlias = Literal["a", "w", "x"] + +@overload def open( name: StrOrBytesPath | None = None, mode: str = "r", - fileobj: IO[bytes] | None = None, # depends on mode + fileobj: IO[bytes] | None = None, bufsize: int = 10240, *, format: int | None = ..., @@ -121,6 +124,25 @@ def open( compresslevel: int | None = ..., preset: Literal[0, 1, 2, 3, 4, 5, 6, 7, 8, 9] | None = ..., ) -> TarFile: ... +@overload +def open( + name: StrOrBytesPath | None = None, + mode: _FileCreationModes = ..., + fileobj: _Fileobj | None = None, + bufsize: int = 10240, + *, + format: int | None = ..., + tarinfo: type[TarInfo] | None = ..., + dereference: bool | None = ..., + ignore_zeros: bool | None = ..., + encoding: str | None = ..., + errors: str = ..., + pax_headers: Mapping[str, str] | None = ..., + debug: int | None = ..., + errorlevel: int | None = ..., + compresslevel: int | None = ..., + preset: int | None = ..., +) -> TarFile: ... class ExFileObject(io.BufferedReader): def __init__(self, tarfile: TarFile, tarinfo: TarInfo) -> None: ... diff --git a/mypy/typeshed/stdlib/zipimport.pyi b/mypy/typeshed/stdlib/zipimport.pyi index 158d573cac74..f53b09e188eb 100644 --- a/mypy/typeshed/stdlib/zipimport.pyi +++ b/mypy/typeshed/stdlib/zipimport.pyi @@ -28,5 +28,7 @@ class zipimporter: def is_package(self, fullname: str) -> bool: ... def load_module(self, fullname: str) -> ModuleType: ... if sys.version_info >= (3, 10): + def exec_module(self, module: ModuleType) -> None: ... + def create_module(self, spec: ModuleSpec) -> None: ... def find_spec(self, fullname: str, target: ModuleType | None = None) -> ModuleSpec | None: ... def invalidate_caches(self) -> None: ... From 4ae632b6f07b33dc5937a3b1dd28af98a080bc88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 2 Jul 2024 01:15:20 +0200 Subject: [PATCH 04/36] Fix type comments crash inside generic definitions (#16849) Closes https://github.com/python/mypy/issues/16649 It's the first time I am contributing to mypy so I am not very familiar with how it works entirely behind the scene. The issue that I had is that a crash happens when using tuple type comments inside functions/classes that depend on a *constrained* type variable. After investigation, the reason is that the type checker generates all possible definitions (since constraints are known) and expands the functions definitions and bodies accordingly. However, by doing so, a tuple type comment ('# type: (int, float)') would have a FakeInfo, so `ExpandTypeVisitor` would fail since it queries `t.type.fullname`. By the way, feel free to change where my test should lie. --- mypy/expandtype.py | 12 ++++++++++- test-data/unit/check-typevar-values.test | 26 ++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 5c4d6af9458e..9336be54437b 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -2,7 +2,7 @@ from typing import Final, Iterable, Mapping, Sequence, TypeVar, cast, overload -from mypy.nodes import ARG_STAR, Var +from mypy.nodes import ARG_STAR, FakeInfo, Var from mypy.state import state from mypy.types import ( ANY_STRATEGY, @@ -208,6 +208,16 @@ def visit_erased_type(self, t: ErasedType) -> Type: def visit_instance(self, t: Instance) -> Type: args = self.expand_types_with_unpack(list(t.args)) + + if isinstance(t.type, FakeInfo): + # The type checker expands function definitions and bodies + # if they depend on constrained type variables but the body + # might contain a tuple type comment (e.g., # type: (int, float)), + # in which case 't.type' is not yet available. + # + # See: https://github.com/python/mypy/issues/16649 + return t.copy_modified(args=args) + if t.type.fullname == "builtins.tuple": # Normalize Tuple[*Tuple[X, ...], ...] -> Tuple[X, ...] arg = args[0] diff --git a/test-data/unit/check-typevar-values.test b/test-data/unit/check-typevar-values.test index effaf620f1f0..8b961d88d23d 100644 --- a/test-data/unit/check-typevar-values.test +++ b/test-data/unit/check-typevar-values.test @@ -706,3 +706,29 @@ Func = Callable[[], T] class A: ... class B: ... + +[case testTypeCommentInGenericTypeWithConstrainedTypeVar] +from typing import Generic, TypeVar + +NT = TypeVar("NT", int, float) + +class Foo1(Generic[NT]): + p = 1 # type: int + +class Foo2(Generic[NT]): + p, q = 1, 2.0 # type: (int, float) + +class Foo3(Generic[NT]): + def bar(self) -> None: + p = 1 # type: int + +class Foo4(Generic[NT]): + def bar(self) -> None: + p, q = 1, 2.0 # type: (int, float) + +def foo3(x: NT) -> None: + p = 1 # type: int + +def foo4(x: NT) -> None: + p, q = 1, 2.0 # type: (int, float) +[builtins fixtures/tuple.pyi] From d1d3c780c7a2d30b2a038903289ea7487303a218 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 1 Jul 2024 16:28:56 -0700 Subject: [PATCH 05/36] Further improvements to functools.partial handling (#17425) - Fixes another crash case / type inference in that case - Fix a false positive when calling the partially applied function with kwargs - TypeTraverse / comment / daemon test follow up ilevkivskyi mentioned on the original PR See also https://github.com/python/mypy/pull/17423 --- mypy/plugins/functools.py | 31 ++++--- mypy/type_visitor.py | 1 + mypy/types.py | 3 +- test-data/unit/check-functools.test | 121 ++++++++++++++++++++++------ test-data/unit/fine-grained.test | 48 +++++++++++ 5 files changed, 169 insertions(+), 35 deletions(-) diff --git a/mypy/plugins/functools.py b/mypy/plugins/functools.py index 9589c6aeca8b..6650af637519 100644 --- a/mypy/plugins/functools.py +++ b/mypy/plugins/functools.py @@ -245,11 +245,14 @@ def partial_new_callback(ctx: mypy.plugin.FunctionContext) -> Type: partial_kinds.append(fn_type.arg_kinds[i]) partial_types.append(arg_type) partial_names.append(fn_type.arg_names[i]) - elif actuals: - if any(actual_arg_kinds[j] == ArgKind.ARG_POS for j in actuals): + else: + assert actuals + if any(actual_arg_kinds[j] in (ArgKind.ARG_POS, ArgKind.ARG_STAR) for j in actuals): + # Don't add params for arguments passed positionally continue + # Add defaulted params for arguments passed via keyword kind = actual_arg_kinds[actuals[0]] - if kind == ArgKind.ARG_NAMED: + if kind == ArgKind.ARG_NAMED or kind == ArgKind.ARG_STAR2: kind = ArgKind.ARG_NAMED_OPT partial_kinds.append(kind) partial_types.append(arg_type) @@ -286,15 +289,25 @@ def partial_call_callback(ctx: mypy.plugin.MethodContext) -> Type: if len(ctx.arg_types) != 2: # *args, **kwargs return ctx.default_return_type - args = [a for param in ctx.args for a in param] - arg_kinds = [a for param in ctx.arg_kinds for a in param] - arg_names = [a for param in ctx.arg_names for a in param] + # See comments for similar actual to formal code above + actual_args = [] + actual_arg_kinds = [] + actual_arg_names = [] + seen_args = set() + for i, param in enumerate(ctx.args): + for j, a in enumerate(param): + if a in seen_args: + continue + seen_args.add(a) + actual_args.append(a) + actual_arg_kinds.append(ctx.arg_kinds[i][j]) + actual_arg_names.append(ctx.arg_names[i][j]) result = ctx.api.expr_checker.check_call( callee=partial_type, - args=args, - arg_kinds=arg_kinds, - arg_names=arg_names, + args=actual_args, + arg_kinds=actual_arg_kinds, + arg_names=actual_arg_names, context=ctx.context, ) return result[0] diff --git a/mypy/type_visitor.py b/mypy/type_visitor.py index d0876629fc08..e685c49904bc 100644 --- a/mypy/type_visitor.py +++ b/mypy/type_visitor.py @@ -213,6 +213,7 @@ def visit_instance(self, t: Instance) -> Type: line=t.line, column=t.column, last_known_value=last_known_value, + extra_attrs=t.extra_attrs, ) def visit_type_var(self, t: TypeVarType) -> Type: diff --git a/mypy/types.py b/mypy/types.py index 52f8a8d63f09..2e7cbfd4e733 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1417,8 +1417,7 @@ def __init__( self._hash = -1 # Additional attributes defined per instance of this type. For example modules - # have different attributes per instance of types.ModuleType. This is intended - # to be "short-lived", we don't serialize it, and even don't store as variable type. + # have different attributes per instance of types.ModuleType. self.extra_attrs = extra_attrs def accept(self, visitor: TypeVisitor[T]) -> T: diff --git a/test-data/unit/check-functools.test b/test-data/unit/check-functools.test index e4b3e4cffdc1..710d3e66dfad 100644 --- a/test-data/unit/check-functools.test +++ b/test-data/unit/check-functools.test @@ -191,6 +191,7 @@ functools.partial(1) # E: "int" not callable \ [case testFunctoolsPartialStar] import functools +from typing import List def foo(a: int, b: str, *args: int, d: str, **kwargs: int) -> int: ... @@ -215,6 +216,13 @@ def bar(*a: bytes, **k: int): p1("a", **k) # E: Argument 2 to "foo" has incompatible type "**Dict[str, int]"; expected "str" p1(**k) # E: Argument 1 to "foo" has incompatible type "**Dict[str, int]"; expected "str" p1(*a) # E: List or tuple expected as variadic arguments + + +def baz(a: int, b: int) -> int: ... +def test_baz(xs: List[int]): + p3 = functools.partial(baz, *xs) + p3() + p3(1) # E: Too many arguments for "baz" [builtins fixtures/dict.pyi] [case testFunctoolsPartialGeneric] @@ -408,33 +416,83 @@ def foo(cls3: Type[B[T]]): from typing_extensions import TypedDict, Unpack from functools import partial -class Data(TypedDict, total=False): - x: int - -def f(**kwargs: Unpack[Data]) -> None: ... -def g(**kwargs: Unpack[Data]) -> None: - partial(f, **kwargs)() - -class MoreData(TypedDict, total=False): - x: int - y: int +class D1(TypedDict, total=False): + a1: int + +def fn1(a1: int) -> None: ... # N: "fn1" defined here +def main1(**d1: Unpack[D1]) -> None: + partial(fn1, **d1)() + partial(fn1, **d1)(**d1) + partial(fn1, **d1)(a1=1) + partial(fn1, **d1)(a1="asdf") # E: Argument "a1" to "fn1" has incompatible type "str"; expected "int" + partial(fn1, **d1)(oops=1) # E: Unexpected keyword argument "oops" for "fn1" + +def fn2(**kwargs: Unpack[D1]) -> None: ... # N: "fn2" defined here +def main2(**d1: Unpack[D1]) -> None: + partial(fn2, **d1)() + partial(fn2, **d1)(**d1) + partial(fn2, **d1)(a1=1) + partial(fn2, **d1)(a1="asdf") # E: Argument "a1" to "fn2" has incompatible type "str"; expected "int" + partial(fn2, **d1)(oops=1) # E: Unexpected keyword argument "oops" for "fn2" + +class D2(TypedDict, total=False): + a1: int + a2: str + +class A2Good(TypedDict, total=False): + a2: str +class A2Bad(TypedDict, total=False): + a2: int + +def fn3(a1: int, a2: str) -> None: ... # N: "fn3" defined here +def main3(a2good: A2Good, a2bad: A2Bad, **d2: Unpack[D2]) -> None: + partial(fn3, **d2)() + partial(fn3, **d2)(a1=1, a2="asdf") + + partial(fn3, **d2)(**d2) + + partial(fn3, **d2)(a1="asdf") # E: Argument "a1" to "fn3" has incompatible type "str"; expected "int" + partial(fn3, **d2)(a1=1, a2="asdf", oops=1) # E: Unexpected keyword argument "oops" for "fn3" + + partial(fn3, **d2)(**a2good) + partial(fn3, **d2)(**a2bad) # E: Argument "a2" to "fn3" has incompatible type "int"; expected "str" + +def fn4(**kwargs: Unpack[D2]) -> None: ... # N: "fn4" defined here +def main4(a2good: A2Good, a2bad: A2Bad, **d2: Unpack[D2]) -> None: + partial(fn4, **d2)() + partial(fn4, **d2)(a1=1, a2="asdf") + + partial(fn4, **d2)(**d2) + + partial(fn4, **d2)(a1="asdf") # E: Argument "a1" to "fn4" has incompatible type "str"; expected "int" + partial(fn4, **d2)(a1=1, a2="asdf", oops=1) # E: Unexpected keyword argument "oops" for "fn4" + + partial(fn3, **d2)(**a2good) + partial(fn3, **d2)(**a2bad) # E: Argument "a2" to "fn3" has incompatible type "int"; expected "str" + +def main5(**d2: Unpack[D2]) -> None: + partial(fn1, **d2)() # E: Extra argument "a2" from **args for "fn1" + partial(fn2, **d2)() # E: Extra argument "a2" from **args for "fn2" + +def main6(a2good: A2Good, a2bad: A2Bad, **d1: Unpack[D1]) -> None: + partial(fn3, **d1)() # E: Missing positional argument "a1" in call to "fn3" + partial(fn3, **d1)("asdf") # E: Too many positional arguments for "fn3" \ + # E: Too few arguments for "fn3" \ + # E: Argument 1 to "fn3" has incompatible type "str"; expected "int" + partial(fn3, **d1)(a2="asdf") + partial(fn3, **d1)(**a2good) + partial(fn3, **d1)(**a2bad) # E: Argument "a2" to "fn3" has incompatible type "int"; expected "str" + + partial(fn4, **d1)() + partial(fn4, **d1)("asdf") # E: Too many positional arguments for "fn4" \ + # E: Argument 1 to "fn4" has incompatible type "str"; expected "int" + partial(fn4, **d1)(a2="asdf") + partial(fn4, **d1)(**a2good) + partial(fn4, **d1)(**a2bad) # E: Argument "a2" to "fn4" has incompatible type "int"; expected "str" -def f_more(**kwargs: Unpack[MoreData]) -> None: ... -def g_more(**kwargs: Unpack[MoreData]) -> None: - partial(f_more, **kwargs)() - -class Good(TypedDict, total=False): - y: int -class Bad(TypedDict, total=False): - y: str - -def h(**kwargs: Unpack[Data]) -> None: - bad: Bad - partial(f_more, **kwargs)(**bad) # E: Argument "y" to "f_more" has incompatible type "str"; expected "int" - good: Good - partial(f_more, **kwargs)(**good) [builtins fixtures/dict.pyi] + [case testFunctoolsPartialNestedGeneric] from functools import partial from typing import Generic, TypeVar, List @@ -456,6 +514,21 @@ first_kw([1]) # E: Too many positional arguments for "get" \ # E: Argument 1 to "get" has incompatible type "List[int]"; expected "int" [builtins fixtures/list.pyi] +[case testFunctoolsPartialHigherOrder] +from functools import partial +from typing import Callable + +def fn(a: int, b: str, c: bytes) -> int: ... + +def callback1(fn: Callable[[str, bytes], int]) -> None: ... +def callback2(fn: Callable[[str, int], int]) -> None: ... + +callback1(partial(fn, 1)) +# TODO: false negative +# https://github.com/python/mypy/issues/17461 +callback2(partial(fn, 1)) +[builtins fixtures/tuple.pyi] + [case testFunctoolsPartialClassObjectMatchingPartial] from functools import partial diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 2a652e50b1e6..2ad31311a402 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -10497,3 +10497,51 @@ from pkg.sub import modb [out] == + +[case testFineGrainedFunctoolsPartial] +import m + +[file m.py] +from typing import Callable +from partial import p1 + +reveal_type(p1) +p1("a") +p1("a", 3) +p1("a", c=3) +p1(1, 3) +p1(1, "a", 3) +p1(a=1, b="a", c=3) +[builtins fixtures/dict.pyi] + +[file partial.py] +from typing import Callable +import functools + +def foo(a: int, b: str, c: int = 5) -> int: ... +p1 = foo + +[file partial.py.2] +from typing import Callable +import functools + +def foo(a: int, b: str, c: int = 5) -> int: ... +p1 = functools.partial(foo, 1) + +[out] +m.py:4: note: Revealed type is "def (a: builtins.int, b: builtins.str, c: builtins.int =) -> builtins.int" +m.py:5: error: Too few arguments +m.py:5: error: Argument 1 has incompatible type "str"; expected "int" +m.py:6: error: Argument 1 has incompatible type "str"; expected "int" +m.py:6: error: Argument 2 has incompatible type "int"; expected "str" +m.py:7: error: Too few arguments +m.py:7: error: Argument 1 has incompatible type "str"; expected "int" +m.py:8: error: Argument 2 has incompatible type "int"; expected "str" +== +m.py:4: note: Revealed type is "functools.partial[builtins.int]" +m.py:8: error: Argument 1 to "foo" has incompatible type "int"; expected "str" +m.py:9: error: Too many arguments for "foo" +m.py:9: error: Argument 1 to "foo" has incompatible type "int"; expected "str" +m.py:9: error: Argument 2 to "foo" has incompatible type "str"; expected "int" +m.py:10: error: Unexpected keyword argument "a" for "foo" +partial.py:4: note: "foo" defined here From f297917fc9eba13cc2a5b7e2e276394d94c03c8d Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Tue, 2 Jul 2024 08:51:36 -0700 Subject: [PATCH 06/36] Mention --enable-incomplete-feature=NewGenericSyntax (#17462) --- mypy/fastparse.py | 16 +++++++++++++--- test-data/unit/check-python312.test | 28 ++++++++++++++-------------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 342cf36d69e8..01f6ed4733ae 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -954,7 +954,9 @@ def do_func_def( else: self.fail( ErrorMessage( - "PEP 695 generics are not yet supported", code=codes.VALID_TYPE + "PEP 695 generics are not yet supported. " + "Use --enable-incomplete-feature=NewGenericSyntax for experimental support", + code=codes.VALID_TYPE, ), n.type_params[0].lineno, n.type_params[0].col_offset, @@ -1145,7 +1147,11 @@ def visit_ClassDef(self, n: ast3.ClassDef) -> ClassDef: explicit_type_params = self.translate_type_params(n.type_params) else: self.fail( - ErrorMessage("PEP 695 generics are not yet supported", code=codes.VALID_TYPE), + ErrorMessage( + "PEP 695 generics are not yet supported. " + "Use --enable-incomplete-feature=NewGenericSyntax for experimental support", + code=codes.VALID_TYPE, + ), n.type_params[0].lineno, n.type_params[0].col_offset, blocker=False, @@ -1801,7 +1807,11 @@ def visit_TypeAlias(self, n: ast_TypeAlias) -> TypeAliasStmt | AssignmentStmt: return self.set_line(node, n) else: self.fail( - ErrorMessage("PEP 695 type aliases are not yet supported", code=codes.VALID_TYPE), + ErrorMessage( + "PEP 695 type aliases are not yet supported. " + "Use --enable-incomplete-feature=NewGenericSyntax for experimental support", + code=codes.VALID_TYPE, + ), n.lineno, n.col_offset, blocker=False, diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 27027d30a684..5307f47d539a 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -1,10 +1,10 @@ [case test695TypeAlias] -type MyInt = int # E: PEP 695 type aliases are not yet supported +type MyInt = int # E: PEP 695 type aliases are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support def f(x: MyInt) -> MyInt: return reveal_type(x) # N: Revealed type is "builtins.int" -type MyList[T] = list[T] # E: PEP 695 type aliases are not yet supported \ +type MyList[T] = list[T] # E: PEP 695 type aliases are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support \ # E: Name "T" is not defined def g(x: MyList[int]) -> MyList[int]: # E: Variable "__main__.MyList" is not valid as a type \ @@ -17,7 +17,7 @@ def h(x: MyInt2) -> MyInt2: return reveal_type(x) # N: Revealed type is "builtins.int" [case test695Class] -class MyGen[T]: # E: PEP 695 generics are not yet supported +class MyGen[T]: # E: PEP 695 generics are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support def __init__(self, x: T) -> None: # E: Name "T" is not defined self.x = x @@ -25,13 +25,13 @@ def f(x: MyGen[int]): # E: "MyGen" expects no type arguments, but 1 given reveal_type(x.x) # N: Revealed type is "Any" [case test695Function] -def f[T](x: T) -> T: # E: PEP 695 generics are not yet supported \ +def f[T](x: T) -> T: # E: PEP 695 generics are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support \ # E: Name "T" is not defined return reveal_type(x) # N: Revealed type is "Any" reveal_type(f(1)) # N: Revealed type is "Any" -async def g[T](x: T) -> T: # E: PEP 695 generics are not yet supported \ +async def g[T](x: T) -> T: # E: PEP 695 generics are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support \ # E: Name "T" is not defined return reveal_type(x) # N: Revealed type is "Any" @@ -41,26 +41,26 @@ reveal_type(g(1)) # E: Value of type "Coroutine[Any, Any, Any]" must be used \ [case test695TypeVar] from typing import Callable -type Alias1[T: int] = list[T] # E: PEP 695 type aliases are not yet supported \ +type Alias1[T: int] = list[T] # E: PEP 695 type aliases are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support \ # E: Name "T" is not defined -type Alias2[**P] = Callable[P, int] # E: PEP 695 type aliases are not yet supported \ +type Alias2[**P] = Callable[P, int] # E: PEP 695 type aliases are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support \ # E: Value of type "int" is not indexable \ # E: Name "P" is not defined -type Alias3[*Ts] = tuple[*Ts] # E: PEP 695 type aliases are not yet supported \ +type Alias3[*Ts] = tuple[*Ts] # E: PEP 695 type aliases are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support \ # E: Name "Ts" is not defined -class Cls1[T: int]: ... # E: PEP 695 generics are not yet supported -class Cls2[**P]: ... # E: PEP 695 generics are not yet supported -class Cls3[*Ts]: ... # E: PEP 695 generics are not yet supported +class Cls1[T: int]: ... # E: PEP 695 generics are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support +class Cls2[**P]: ... # E: PEP 695 generics are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support +class Cls3[*Ts]: ... # E: PEP 695 generics are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support -def func1[T: int](x: T) -> T: ... # E: PEP 695 generics are not yet supported \ +def func1[T: int](x: T) -> T: ... # E: PEP 695 generics are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support \ # E: Name "T" is not defined -def func2[**P](x: Callable[P, int]) -> Callable[P, str]: ... # E: PEP 695 generics are not yet supported \ +def func2[**P](x: Callable[P, int]) -> Callable[P, str]: ... # E: PEP 695 generics are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support \ # E: The first argument to Callable must be a list of types, parameter specification, or "..." \ # N: See https://mypy.readthedocs.io/en/stable/kinds_of_types.html#callable-types-and-lambdas \ # E: Name "P" is not defined -def func3[*Ts](x: tuple[*Ts]) -> tuple[int, *Ts]: ... # E: PEP 695 generics are not yet supported \ +def func3[*Ts](x: tuple[*Ts]) -> tuple[int, *Ts]: ... # E: PEP 695 generics are not yet supported. Use --enable-incomplete-feature=NewGenericSyntax for experimental support \ # E: Name "Ts" is not defined [builtins fixtures/tuple.pyi] From 55a08120b7be56e7d33cf2e9d240d30a8f608f83 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 2 Jul 2024 19:36:16 +0100 Subject: [PATCH 07/36] Infer unions for ternary expressions (#17427) Ref https://github.com/python/mypy/issues/12056 cc @JukkaL Again, let's check the primer... --- mypy/checkexpr.py | 19 +++-- mypyc/test-data/irbuild-any.test | 4 +- test-data/unit/check-errorcodes.test | 2 +- test-data/unit/check-expressions.test | 17 ++-- test-data/unit/check-functions.test | 17 +++- test-data/unit/check-inference-context.test | 6 +- test-data/unit/check-inference.test | 10 ++- test-data/unit/check-optional.test | 2 +- test-data/unit/check-tuples.test | 87 ++++++++++++--------- 9 files changed, 99 insertions(+), 65 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index c4ab8a081acc..3532e18b93b2 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -5766,16 +5766,15 @@ def visit_conditional_expr(self, e: ConditionalExpr, allow_none_return: bool = F context=if_type_fallback, allow_none_return=allow_none_return, ) - - # Only create a union type if the type context is a union, to be mostly - # compatible with older mypy versions where we always did a join. - # - # TODO: Always create a union or at least in more cases? - if isinstance(get_proper_type(self.type_context[-1]), UnionType): - res: Type = make_simplified_union([if_type, full_context_else_type]) - else: - res = join.join_types(if_type, else_type) - + res: Type = make_simplified_union([if_type, else_type]) + if has_uninhabited_component(res) and not isinstance( + get_proper_type(self.type_context[-1]), UnionType + ): + # In rare cases with empty collections join may give a better result. + alternative = join.join_types(if_type, else_type) + p_alt = get_proper_type(alternative) + if not isinstance(p_alt, Instance) or p_alt.type.fullname != "builtins.object": + res = alternative return res def analyze_cond_branch( diff --git a/mypyc/test-data/irbuild-any.test b/mypyc/test-data/irbuild-any.test index 0d14e1a5dfc8..3bfb1587fb3b 100644 --- a/mypyc/test-data/irbuild-any.test +++ b/mypyc/test-data/irbuild-any.test @@ -151,7 +151,9 @@ def f4(a, n, b): a :: object n :: int b :: bool - r0, r1, r2, r3 :: object + r0 :: union[object, int] + r1, r2 :: object + r3 :: union[int, object] r4 :: int L0: if b goto L1 else goto L2 :: bool diff --git a/test-data/unit/check-errorcodes.test b/test-data/unit/check-errorcodes.test index 961815b11817..c4d72388fba9 100644 --- a/test-data/unit/check-errorcodes.test +++ b/test-data/unit/check-errorcodes.test @@ -460,7 +460,7 @@ a: D = {'x': ''} # E: Incompatible types (expression has type "str", TypedDict b: D = {'y': ''} # E: Missing key "x" for TypedDict "D" [typeddict-item] \ # E: Extra key "y" for TypedDict "D" [typeddict-unknown-key] c = D(x=0) if int() else E(x=0, y=0) -c = {} # E: Expected TypedDict key "x" but found no keys [typeddict-item] +c = {} # E: Missing key "x" for TypedDict "D" [typeddict-item] d: D = {'x': '', 'y': 1} # E: Extra key "y" for TypedDict "D" [typeddict-unknown-key] \ # E: Incompatible types (expression has type "str", TypedDict item "x" has type "int") [typeddict-item] diff --git a/test-data/unit/check-expressions.test b/test-data/unit/check-expressions.test index f9bd60f4dcc8..61cee1d00c58 100644 --- a/test-data/unit/check-expressions.test +++ b/test-data/unit/check-expressions.test @@ -1470,10 +1470,9 @@ if int(): [case testConditionalExpressionUnion] from typing import Union -reveal_type(1 if bool() else 2) # N: Revealed type is "builtins.int" -reveal_type(1 if bool() else '') # N: Revealed type is "builtins.object" -x: Union[int, str] = reveal_type(1 if bool() else '') \ - # N: Revealed type is "Union[Literal[1]?, Literal['']?]" +reveal_type(1 if bool() else 2) # N: Revealed type is "Union[Literal[1]?, Literal[2]?]" +reveal_type(1 if bool() else '') # N: Revealed type is "Union[Literal[1]?, Literal['']?]" +x: Union[int, str] = reveal_type(1 if bool() else '') # N: Revealed type is "Union[Literal[1]?, Literal['']?]" class A: pass class B(A): @@ -1487,17 +1486,17 @@ b = B() c = C() d = D() reveal_type(a if bool() else b) # N: Revealed type is "__main__.A" -reveal_type(b if bool() else c) # N: Revealed type is "builtins.object" -reveal_type(c if bool() else b) # N: Revealed type is "builtins.object" -reveal_type(c if bool() else a) # N: Revealed type is "builtins.object" -reveal_type(d if bool() else b) # N: Revealed type is "__main__.A" +reveal_type(b if bool() else c) # N: Revealed type is "Union[__main__.B, __main__.C]" +reveal_type(c if bool() else b) # N: Revealed type is "Union[__main__.C, __main__.B]" +reveal_type(c if bool() else a) # N: Revealed type is "Union[__main__.C, __main__.A]" +reveal_type(d if bool() else b) # N: Revealed type is "Union[__main__.D, __main__.B]" [builtins fixtures/bool.pyi] [case testConditionalExpressionUnionWithAny] from typing import Union, Any a: Any x: Union[int, str] = reveal_type(a if int() else 1) # N: Revealed type is "Union[Any, Literal[1]?]" -reveal_type(a if int() else 1) # N: Revealed type is "Any" +reveal_type(a if int() else 1) # N: Revealed type is "Union[Any, Literal[1]?]" [case testConditionalExpressionStatementNoReturn] from typing import List, Union diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index 93540e203c36..6c895c86e899 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -2250,13 +2250,26 @@ def dec(f: Callable[[A, str], None]) -> Callable[[A, int], None]: pass [out] [case testUnknownFunctionNotCallable] +from typing import TypeVar + def f() -> None: pass def g(x: int) -> None: pass h = f if bool() else g -reveal_type(h) # N: Revealed type is "builtins.function" -h(7) # E: Cannot call function of unknown type +reveal_type(h) # N: Revealed type is "Union[def (), def (x: builtins.int)]" +h(7) # E: Too many arguments for "f" + +T = TypeVar("T") +def join(x: T, y: T) -> T: ... + +h2 = join(f, g) +reveal_type(h2) # N: Revealed type is "builtins.function" +h2(7) # E: Cannot call function of unknown type + +h3 = join(g, f) +reveal_type(h3) # N: Revealed type is "builtins.function" +h3(7) # E: Cannot call function of unknown type [builtins fixtures/bool.pyi] [case testFunctionWithNameUnderscore] diff --git a/test-data/unit/check-inference-context.test b/test-data/unit/check-inference-context.test index afe6548df2d4..17ae6d9934b7 100644 --- a/test-data/unit/check-inference-context.test +++ b/test-data/unit/check-inference-context.test @@ -701,7 +701,7 @@ class A: pass class B(A): pass class C(A): pass def f(func: Callable[[T], S], *z: T, r: Optional[S] = None) -> S: pass -reveal_type(f(lambda x: 0 if isinstance(x, B) else 1)) # N: Revealed type is "builtins.int" +reveal_type(f(lambda x: 0 if isinstance(x, B) else 1)) # N: Revealed type is "Union[Literal[0]?, Literal[1]?]" f(lambda x: 0 if isinstance(x, B) else 1, A())() # E: "int" not callable f(lambda x: x if isinstance(x, B) else B(), A(), r=B())() # E: "B" not callable f( @@ -1391,7 +1391,7 @@ from typing import Union, List, Any def f(x: Union[List[str], Any]) -> None: a = x if x else [] - reveal_type(a) # N: Revealed type is "Union[builtins.list[Union[builtins.str, Any]], builtins.list[builtins.str], Any]" + reveal_type(a) # N: Revealed type is "Union[builtins.list[builtins.str], Any, builtins.list[Union[builtins.str, Any]]]" [builtins fixtures/list.pyi] [case testConditionalExpressionWithEmptyIteableAndUnionWithAny] @@ -1399,7 +1399,7 @@ from typing import Union, Iterable, Any def f(x: Union[Iterable[str], Any]) -> None: a = x if x else [] - reveal_type(a) # N: Revealed type is "Union[builtins.list[Union[builtins.str, Any]], typing.Iterable[builtins.str], Any]" + reveal_type(a) # N: Revealed type is "Union[typing.Iterable[builtins.str], Any, builtins.list[Union[builtins.str, Any]]]" [builtins fixtures/list.pyi] [case testInferMultipleAnyUnionCovariant] diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index fcd03f8efe01..0dbefbc774a3 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -1438,18 +1438,22 @@ class Wrapper: def f(cond: bool) -> Any: f = Wrapper if cond else lambda x: x - reveal_type(f) # N: Revealed type is "def (x: Any) -> Any" + reveal_type(f) # N: Revealed type is "Union[def (x: Any) -> __main__.Wrapper, def (x: Any) -> Any]" return f(3) def g(cond: bool) -> Any: f = lambda x: x if cond else Wrapper - reveal_type(f) # N: Revealed type is "def (x: Any) -> Any" + reveal_type(f) # N: Revealed type is "def (x: Any) -> Union[Any, def (x: Any) -> __main__.Wrapper]" + return f(3) + +def h(cond: bool) -> Any: + f = (lambda x: x) if cond else Wrapper + reveal_type(f) # N: Revealed type is "Union[def (x: Any) -> Any, def (x: Any) -> __main__.Wrapper]" return f(3) -- Boolean operators -- ----------------- - [case testOrOperationWithGenericOperands] from typing import List a: List[A] diff --git a/test-data/unit/check-optional.test b/test-data/unit/check-optional.test index 70f3c4486e14..f80aa5115bc3 100644 --- a/test-data/unit/check-optional.test +++ b/test-data/unit/check-optional.test @@ -395,7 +395,7 @@ def lookup_field(name, obj): attr = None [case testTernaryWithNone] -reveal_type(None if bool() else 0) # N: Revealed type is "Union[Literal[0]?, None]" +reveal_type(None if bool() else 0) # N: Revealed type is "Union[None, Literal[0]?]" [builtins fixtures/bool.pyi] [case testListWithNone] diff --git a/test-data/unit/check-tuples.test b/test-data/unit/check-tuples.test index bf36977b56e3..972bccf8c24b 100644 --- a/test-data/unit/check-tuples.test +++ b/test-data/unit/check-tuples.test @@ -1228,68 +1228,76 @@ x, y = g(z) # E: Argument 1 to "g" has incompatible type "int"; expected "Tuple[ [out] [case testFixedTupleJoinVarTuple] -from typing import Tuple +from typing import Tuple, TypeVar class A: pass class B(A): pass fixtup: Tuple[B, B] +T = TypeVar("T") +def join(x: T, y: T) -> T: ... + vartup_b: Tuple[B, ...] -reveal_type(fixtup if int() else vartup_b) # N: Revealed type is "builtins.tuple[__main__.B, ...]" -reveal_type(vartup_b if int() else fixtup) # N: Revealed type is "builtins.tuple[__main__.B, ...]" +reveal_type(join(fixtup, vartup_b)) # N: Revealed type is "builtins.tuple[__main__.B, ...]" +reveal_type(join(vartup_b, fixtup)) # N: Revealed type is "builtins.tuple[__main__.B, ...]" vartup_a: Tuple[A, ...] -reveal_type(fixtup if int() else vartup_a) # N: Revealed type is "builtins.tuple[__main__.A, ...]" -reveal_type(vartup_a if int() else fixtup) # N: Revealed type is "builtins.tuple[__main__.A, ...]" - +reveal_type(join(fixtup, vartup_a)) # N: Revealed type is "builtins.tuple[__main__.A, ...]" +reveal_type(join(vartup_a, fixtup)) # N: Revealed type is "builtins.tuple[__main__.A, ...]" [builtins fixtures/tuple.pyi] [out] [case testFixedTupleJoinList] -from typing import Tuple, List +from typing import Tuple, List, TypeVar class A: pass class B(A): pass fixtup: Tuple[B, B] +T = TypeVar("T") +def join(x: T, y: T) -> T: ... + lst_b: List[B] -reveal_type(fixtup if int() else lst_b) # N: Revealed type is "typing.Sequence[__main__.B]" -reveal_type(lst_b if int() else fixtup) # N: Revealed type is "typing.Sequence[__main__.B]" +reveal_type(join(fixtup, lst_b)) # N: Revealed type is "typing.Sequence[__main__.B]" +reveal_type(join(lst_b, fixtup)) # N: Revealed type is "typing.Sequence[__main__.B]" lst_a: List[A] -reveal_type(fixtup if int() else lst_a) # N: Revealed type is "typing.Sequence[__main__.A]" -reveal_type(lst_a if int() else fixtup) # N: Revealed type is "typing.Sequence[__main__.A]" +reveal_type(join(fixtup, lst_a)) # N: Revealed type is "typing.Sequence[__main__.A]" +reveal_type(join(lst_a, fixtup)) # N: Revealed type is "typing.Sequence[__main__.A]" [builtins fixtures/tuple.pyi] [out] [case testEmptyTupleJoin] -from typing import Tuple, List +from typing import Tuple, List, TypeVar class A: pass empty = () +T = TypeVar("T") +def join(x: T, y: T) -> T: ... + fixtup: Tuple[A] -reveal_type(fixtup if int() else empty) # N: Revealed type is "builtins.tuple[__main__.A, ...]" -reveal_type(empty if int() else fixtup) # N: Revealed type is "builtins.tuple[__main__.A, ...]" +reveal_type(join(fixtup, empty)) # N: Revealed type is "builtins.tuple[__main__.A, ...]" +reveal_type(join(empty, fixtup)) # N: Revealed type is "builtins.tuple[__main__.A, ...]" vartup: Tuple[A, ...] -reveal_type(empty if int() else vartup) # N: Revealed type is "builtins.tuple[__main__.A, ...]" -reveal_type(vartup if int() else empty) # N: Revealed type is "builtins.tuple[__main__.A, ...]" +reveal_type(join(vartup, empty)) # N: Revealed type is "builtins.tuple[__main__.A, ...]" +reveal_type(join(empty, vartup)) # N: Revealed type is "builtins.tuple[__main__.A, ...]" lst: List[A] -reveal_type(empty if int() else lst) # N: Revealed type is "typing.Sequence[__main__.A]" -reveal_type(lst if int() else empty) # N: Revealed type is "typing.Sequence[__main__.A]" +reveal_type(join(empty, lst)) # N: Revealed type is "typing.Sequence[__main__.A]" +reveal_type(join(lst, empty)) # N: Revealed type is "typing.Sequence[__main__.A]" [builtins fixtures/tuple.pyi] [out] [case testTupleSubclassJoin] -from typing import Tuple, NamedTuple +from typing import Tuple, NamedTuple, TypeVar class NTup(NamedTuple): a: bool @@ -1302,32 +1310,38 @@ ntup: NTup subtup: SubTuple vartup: SubVarTuple -reveal_type(ntup if int() else vartup) # N: Revealed type is "builtins.tuple[builtins.int, ...]" -reveal_type(subtup if int() else vartup) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +T = TypeVar("T") +def join(x: T, y: T) -> T: ... + +reveal_type(join(ntup, vartup)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(join(subtup, vartup)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" [builtins fixtures/tuple.pyi] [out] [case testTupleJoinIrregular] -from typing import Tuple +from typing import Tuple, TypeVar tup1: Tuple[bool, int] tup2: Tuple[bool] -reveal_type(tup1 if int() else tup2) # N: Revealed type is "builtins.tuple[builtins.int, ...]" -reveal_type(tup2 if int() else tup1) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +T = TypeVar("T") +def join(x: T, y: T) -> T: ... + +reveal_type(join(tup1, tup2)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(join(tup2, tup1)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" -reveal_type(tup1 if int() else ()) # N: Revealed type is "builtins.tuple[builtins.int, ...]" -reveal_type(() if int() else tup1) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(join(tup1, ())) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(join((), tup1)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" -reveal_type(tup2 if int() else ()) # N: Revealed type is "builtins.tuple[builtins.bool, ...]" -reveal_type(() if int() else tup2) # N: Revealed type is "builtins.tuple[builtins.bool, ...]" +reveal_type(join(tup2, ())) # N: Revealed type is "builtins.tuple[builtins.bool, ...]" +reveal_type(join((), tup2)) # N: Revealed type is "builtins.tuple[builtins.bool, ...]" [builtins fixtures/tuple.pyi] [out] [case testTupleSubclassJoinIrregular] -from typing import Tuple, NamedTuple +from typing import Tuple, NamedTuple, TypeVar class NTup1(NamedTuple): a: bool @@ -1342,14 +1356,17 @@ tup1: NTup1 tup2: NTup2 subtup: SubTuple -reveal_type(tup1 if int() else tup2) # N: Revealed type is "builtins.tuple[builtins.bool, ...]" -reveal_type(tup2 if int() else tup1) # N: Revealed type is "builtins.tuple[builtins.bool, ...]" +T = TypeVar("T") +def join(x: T, y: T) -> T: ... + +reveal_type(join(tup1, tup2)) # N: Revealed type is "builtins.tuple[builtins.bool, ...]" +reveal_type(join(tup2, tup1)) # N: Revealed type is "builtins.tuple[builtins.bool, ...]" -reveal_type(tup1 if int() else subtup) # N: Revealed type is "builtins.tuple[builtins.int, ...]" -reveal_type(subtup if int() else tup1) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(join(tup1, subtup)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(join(subtup, tup1)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" -reveal_type(tup2 if int() else subtup) # N: Revealed type is "builtins.tuple[builtins.int, ...]" -reveal_type(subtup if int() else tup2) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(join(tup2, subtup)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(join(subtup, tup2)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" [builtins fixtures/tuple.pyi] [out] From 45afd73974ef9c558f753d3a55e2997610b9827a Mon Sep 17 00:00:00 2001 From: Raphael Krupinski Date: Wed, 3 Jul 2024 14:56:52 +0200 Subject: [PATCH 08/36] Fix help message for --no-namespace-packages. (#17472) Fixes #17466 Change help message of '--no-namespace-packages' option Co-authored-by: Raphael Krupinski <10319569-mattesilver@users.noreply.gitlab.com> --- mypy/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/main.py b/mypy/main.py index 489ef8fd9a7b..05044335ecee 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -565,7 +565,7 @@ def add_invertible_flag( "--no-namespace-packages", dest="namespace_packages", default=True, - help="Support namespace packages (PEP 420, __init__.py-less)", + help="Disable support for namespace packages (PEP 420, __init__.py-less)", group=imports_group, ) imports_group.add_argument( From cb7b96d32d7ca2ba791106f61e1adf5c0d7b0ea4 Mon Sep 17 00:00:00 2001 From: Max Muoto Date: Wed, 3 Jul 2024 12:05:40 -0500 Subject: [PATCH 09/36] Add `__replace__` for dataclasses in 3.13 (#17469) Fixes https://github.com/python/mypy/issues/17471 --- mypy/plugins/dataclasses.py | 15 ++++++++++++ test-data/unit/check-dataclasses.test | 34 +++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index dd2eceab217f..edfc6840fc37 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -385,6 +385,9 @@ def transform(self) -> bool: self._add_dataclass_fields_magic_attribute() self._add_internal_replace_method(attributes) + if self._api.options.python_version >= (3, 13): + self._add_dunder_replace(attributes) + if "__post_init__" in info.names: self._add_internal_post_init_method(attributes) @@ -395,6 +398,18 @@ def transform(self) -> bool: return True + def _add_dunder_replace(self, attributes: list[DataclassAttribute]) -> None: + """Add a `__replace__` method to the class, which is used to replace attributes in the `copy` module.""" + args = [attr.to_argument(self._cls.info, of="replace") for attr in attributes] + type_vars = [tv for tv in self._cls.type_vars] + add_method_to_class( + self._api, + self._cls, + "__replace__", + args=args, + return_type=Instance(self._cls.info, type_vars), + ) + def _add_internal_replace_method(self, attributes: list[DataclassAttribute]) -> None: """ Stashes the signature of 'dataclasses.replace(...)' for this specific dataclass diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index f26ccd9a4854..0f726242b25b 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -2489,3 +2489,37 @@ class Base: class Child(Base): y: int [builtins fixtures/dataclasses.pyi] + +[case testDunderReplacePresent] +# flags: --python-version 3.13 +from dataclasses import dataclass + +@dataclass +class Coords: + x: int + y: int + + +replaced = Coords(2, 4).__replace__(x=2, y=5) +reveal_type(replaced) # N: Revealed type is "__main__.Coords" + +replaced = Coords(2, 4).__replace__(x=2) +reveal_type(replaced) # N: Revealed type is "__main__.Coords" + +Coords(2, 4).__replace__(x="asdf") # E: Argument "x" to "__replace__" of "Coords" has incompatible type "str"; expected "int" +Coords(2, 4).__replace__(23) # E: Too many positional arguments for "__replace__" of "Coords" +Coords(2, 4).__replace__(23, 25) # E: Too many positional arguments for "__replace__" of "Coords" +Coords(2, 4).__replace__(x=23, y=25, z=42) # E: Unexpected keyword argument "z" for "__replace__" of "Coords" + +from typing import Generic, TypeVar +T = TypeVar('T') + +@dataclass +class Gen(Generic[T]): + x: T + +replaced_2 = Gen(2).__replace__(x=2) +reveal_type(replaced_2) # N: Revealed type is "__main__.Gen[builtins.int]" +Gen(2).__replace__(x="not an int") # E: Argument "x" to "__replace__" of "Gen" has incompatible type "str"; expected "int" + +[builtins fixtures/tuple.pyi] From 1882ed78aed75da6882145bb6c4747688b72eec0 Mon Sep 17 00:00:00 2001 From: Max Muoto Date: Wed, 3 Jul 2024 12:37:14 -0500 Subject: [PATCH 10/36] Have namedtuple `__replace__` return `Self` (#17475) --- mypy/semanal_namedtuple.py | 4 +++- test-data/unit/check-namedtuple.test | 16 +++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index 768dd265b338..bf526a1ee990 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -57,6 +57,7 @@ TYPED_NAMEDTUPLE_NAMES, AnyType, CallableType, + Instance, LiteralType, TupleType, Type, @@ -631,9 +632,10 @@ def add_method( args=[Argument(var, var.type, EllipsisExpr(), ARG_NAMED_OPT) for var in vars], ) if self.options.python_version >= (3, 13): + type_vars = [tv for tv in info.defn.type_vars] add_method( "__replace__", - ret=None, + ret=Instance(info, type_vars), args=[Argument(var, var.type, EllipsisExpr(), ARG_NAMED_OPT) for var in vars], ) diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index e9d156754d9c..f10217b9aa5f 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1407,9 +1407,23 @@ from typing import NamedTuple class A(NamedTuple): x: int -A(x=0).__replace__(x=1) +replaced = A(x=0).__replace__(x=1) +reveal_type(replaced) # N: Revealed type is "__main__.A" + A(x=0).__replace__(x="asdf") # E: Argument "x" to "__replace__" of "A" has incompatible type "str"; expected "int" A(x=0).__replace__(y=1) # E: Unexpected keyword argument "y" for "__replace__" of "A" + +from typing import TypeVar, Generic + +T = TypeVar("T") + +class GenericA(NamedTuple, Generic[T]): + x: T + +replaced_2 = GenericA(x=0).__replace__(x=1) +reveal_type(replaced_2) # N: Revealed type is "__main__.GenericA" +GenericA(x=0).__replace__(x="abc") # E: Argument "x" to "__replace__" of "GenericA" has incompatible type "str"; expected "int" + [builtins fixtures/tuple.pyi] [typing fixtures/typing-namedtuple.pyi] From 606971807fad1de26ebc575d327d4c1c33f71c0e Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Thu, 4 Jul 2024 11:17:14 +0200 Subject: [PATCH 11/36] Bump version to 1.12.0+dev (#17467) The release branch has been cut: https://github.com/python/mypy/tree/release-1.11 --- mypy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/version.py b/mypy/version.py index f2615b77109d..8e00b4cce702 100644 --- a/mypy/version.py +++ b/mypy/version.py @@ -8,7 +8,7 @@ # - Release versions have the form "1.2.3". # - Dev versions have the form "1.2.3+dev" (PLUS sign to conform to PEP 440). # - Before 1.0 we had the form "0.NNN". -__version__ = "1.11.0+dev" +__version__ = "1.12.0+dev" base_version = __version__ mypy_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) From ec00fb8db7f5c72faccde55a7f48dc6023c65411 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Fri, 5 Jul 2024 19:03:16 +0300 Subject: [PATCH 12/36] Refactor: remove temporary `def` from `semanal_classprop.check_protocol_status` (#17486) There's no need to create and call a temporary function, when we can just call an existing method: faster and simplier. --- mypy/semanal_classprop.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mypy/semanal_classprop.py b/mypy/semanal_classprop.py index b5f1b2181761..c5ad34122f6c 100644 --- a/mypy/semanal_classprop.py +++ b/mypy/semanal_classprop.py @@ -122,11 +122,12 @@ def check_protocol_status(info: TypeInfo, errors: Errors) -> None: if info.is_protocol: for type in info.bases: if not type.type.is_protocol and type.type.fullname != "builtins.object": - - def report(message: str, severity: str) -> None: - errors.report(info.line, info.column, message, severity=severity) - - report("All bases of a protocol must be protocols", "error") + errors.report( + info.line, + info.column, + "All bases of a protocol must be protocols", + severity="error", + ) def calculate_class_vars(info: TypeInfo) -> None: From 2e0b8687599fa033e78978dc78c42afa86934913 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 6 Jul 2024 21:05:46 +0100 Subject: [PATCH 13/36] Revert "Have namedtuple `__replace__` return `Self`" (#17496) Reverts python/mypy#17475 --- mypy/semanal_namedtuple.py | 4 +--- test-data/unit/check-namedtuple.test | 16 +--------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index bf526a1ee990..768dd265b338 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -57,7 +57,6 @@ TYPED_NAMEDTUPLE_NAMES, AnyType, CallableType, - Instance, LiteralType, TupleType, Type, @@ -632,10 +631,9 @@ def add_method( args=[Argument(var, var.type, EllipsisExpr(), ARG_NAMED_OPT) for var in vars], ) if self.options.python_version >= (3, 13): - type_vars = [tv for tv in info.defn.type_vars] add_method( "__replace__", - ret=Instance(info, type_vars), + ret=None, args=[Argument(var, var.type, EllipsisExpr(), ARG_NAMED_OPT) for var in vars], ) diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index f10217b9aa5f..e9d156754d9c 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1407,23 +1407,9 @@ from typing import NamedTuple class A(NamedTuple): x: int -replaced = A(x=0).__replace__(x=1) -reveal_type(replaced) # N: Revealed type is "__main__.A" - +A(x=0).__replace__(x=1) A(x=0).__replace__(x="asdf") # E: Argument "x" to "__replace__" of "A" has incompatible type "str"; expected "int" A(x=0).__replace__(y=1) # E: Unexpected keyword argument "y" for "__replace__" of "A" - -from typing import TypeVar, Generic - -T = TypeVar("T") - -class GenericA(NamedTuple, Generic[T]): - x: T - -replaced_2 = GenericA(x=0).__replace__(x=1) -reveal_type(replaced_2) # N: Revealed type is "__main__.GenericA" -GenericA(x=0).__replace__(x="abc") # E: Argument "x" to "__replace__" of "GenericA" has incompatible type "str"; expected "int" - [builtins fixtures/tuple.pyi] [typing fixtures/typing-namedtuple.pyi] From 1ea867624d392f90755c7ccd3b410690ad87e891 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sat, 6 Jul 2024 13:44:04 -0700 Subject: [PATCH 14/36] Run Python 3.13 tests in CI (with failures allowed) (#17484) https://github.com/python/mypy/issues/17264 --- .github/workflows/docs.yml | 2 +- .github/workflows/test.yml | 18 ++++++++++++++++++ test-requirements.in | 2 -- test-requirements.txt | 34 ++++------------------------------ 4 files changed, 23 insertions(+), 33 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index f13a3de1f2e3..8bded1d380aa 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -38,7 +38,7 @@ jobs: with: python-version: '3.8' - name: Install tox - run: pip install --upgrade 'setuptools!=50' tox==4.11.0 + run: pip install tox==4.11.0 - name: Setup tox environment run: tox run -e ${{ env.TOXENV }} --notest - name: Test diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 98a737a78b3b..01d5876635b0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -147,6 +147,24 @@ jobs: - name: Test run: tox run -e ${{ matrix.toxenv }} --skip-pkg-install -- ${{ matrix.tox_extra_args }} + python-nightly: + runs-on: ubuntu-latest + name: Test suite with Python nightly + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.13-dev' + - name: Install tox + run: pip install setuptools==68.2.2 tox==4.11.0 + - name: Setup tox environment + run: tox run -e py --notest + - name: Test + run: tox run -e py --skip-pkg-install -- "-n 4" + continue-on-error: true + - name: Mark as a success + run: exit 0 + python_32bits: runs-on: ubuntu-latest name: Test mypyc suite with 32-bit Python diff --git a/test-requirements.in b/test-requirements.in index 637f5b948055..5a888811bfcd 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -8,8 +8,6 @@ black==24.3.0 # must match version in .pre-commit-config.yaml filelock>=3.3.0 # lxml 4.9.3 switched to manylinux_2_28, the wheel builder still uses manylinux2014 lxml>=4.9.1,<4.9.3; (python_version<'3.11' or sys_platform!='win32') and python_version<'3.12' -pre-commit -pre-commit-hooks==4.5.0 psutil>=4.0 pytest>=8.1.0 pytest-xdist>=1.34.0 diff --git a/test-requirements.txt b/test-requirements.txt index 9005daab2876..75a970c5bf0e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,22 +8,14 @@ attrs==23.1.0 # via -r test-requirements.in black==24.3.0 # via -r test-requirements.in -cfgv==3.4.0 - # via pre-commit click==8.1.7 # via black coverage==7.3.2 # via pytest-cov -distlib==0.3.7 - # via virtualenv execnet==2.0.2 # via pytest-xdist filelock==3.12.4 - # via - # -r test-requirements.in - # virtualenv -identify==2.5.30 - # via pre-commit + # via -r test-requirements.in iniconfig==2.0.0 # via pytest lxml==4.9.2 ; (python_version < "3.11" or sys_platform != "win32") and python_version < "3.12" @@ -32,8 +24,6 @@ mypy-extensions==1.0.0 # via # -r mypy-requirements.txt # black -nodeenv==1.8.0 - # via pre-commit packaging==23.2 # via # black @@ -41,15 +31,9 @@ packaging==23.2 pathspec==0.11.2 # via black platformdirs==3.11.0 - # via - # black - # virtualenv + # via black pluggy==1.4.0 # via pytest -pre-commit==3.5.0 - # via -r test-requirements.in -pre-commit-hooks==4.5.0 - # via -r test-requirements.in psutil==5.9.6 # via -r test-requirements.in pytest==8.1.1 @@ -61,12 +45,6 @@ pytest-cov==4.1.0 # via -r test-requirements.in pytest-xdist==3.3.1 # via -r test-requirements.in -pyyaml==6.0.1 - # via pre-commit -ruamel-yaml==0.17.40 - # via pre-commit-hooks -ruamel-yaml-clib==0.2.8 - # via ruamel-yaml ruff==0.2.0 # via -r test-requirements.in tomli==2.0.1 @@ -75,13 +53,9 @@ types-psutil==5.9.5.17 # via -r build-requirements.txt types-setuptools==68.2.0.0 # via -r build-requirements.txt -typing-extensions==4.8.0 +typing-extensions==4.12.2 # via -r mypy-requirements.txt -virtualenv==20.24.5 - # via pre-commit # The following packages are considered to be unsafe in a requirements file: setuptools==68.2.2 - # via - # -r test-requirements.in - # nodeenv + # via -r test-requirements.in From 4ccf216492392ef673f4847ed8779fb54e826bd7 Mon Sep 17 00:00:00 2001 From: InSync Date: Sun, 7 Jul 2024 03:49:05 +0700 Subject: [PATCH 15/36] Fix cross-variable type-narrowing example (#17488) From [Type narrowing § Limitations](https://github.com/python/mypy/blob/606971807fad1de26ebc575d327d4c1c33f71c0e/docs/source/type_narrowing.rst#limitations): ```python def f(a: str | None, b: str | None) -> str: if a is not None or b is not None: return a or b # Incompatible return value type (got "str | None", expected "str") return 'spam' ``` A trivial counter-example is `f('', None)`, which returns `None`. Ironically, this somewhat makes Mypy's diagnostic "correct". I propose that `str` be replaced with a custom class `C` whose `__bool__()` is not defined (does it have to be `@final` too?): ```python class C: pass def f(a: C | None, b: C | None) -> C: if a is not None or b is not None: return a or b # Incompatible return value type (got "C | None", expected "C") return C() ``` --- docs/source/type_narrowing.rst | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/source/type_narrowing.rst b/docs/source/type_narrowing.rst index 4c5c2851edd0..d698f35c44bc 100644 --- a/docs/source/type_narrowing.rst +++ b/docs/source/type_narrowing.rst @@ -368,14 +368,18 @@ Limitations Mypy's analysis is limited to individual symbols and it will not track relationships between symbols. For example, in the following code it's easy to deduce that if :code:`a` is None then :code:`b` must not be, -therefore :code:`a or b` will always be a string, but Mypy will not be able to tell that: +therefore :code:`a or b` will always be an instance of :code:`C`, +but Mypy will not be able to tell that: .. code-block:: python - def f(a: str | None, b: str | None) -> str: + class C: + pass + + def f(a: C | None, b: C | None) -> C: if a is not None or b is not None: - return a or b # Incompatible return value type (got "str | None", expected "str") - return 'spam' + return a or b # Incompatible return value type (got "C | None", expected "C") + return C() Tracking these sort of cross-variable conditions in a type checker would add significant complexity and performance overhead. @@ -385,9 +389,9 @@ or rewrite the function to be slightly more verbose: .. code-block:: python - def f(a: str | None, b: str | None) -> str: + def f(a: C | None, b: C | None) -> C: if a is not None: return a elif b is not None: return b - return 'spam' + return C() From e5b3b563ea3f6d1f83d0fe3552965c8b05ea4c3d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 6 Jul 2024 22:04:07 +0100 Subject: [PATCH 16/36] Fix daemon crash on invalid type in TypedDict (#17495) Fixes https://github.com/python/mypy/issues/10007 Fixes https://github.com/python/mypy/issues/17477 This fixes the crash as proposed in https://github.com/python/mypy/pull/13732, but also fixes some inconsistencies in `Any` types exposed by the fix. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- mypy/semanal.py | 21 +++++++++++++++++ mypy/semanal_typeddict.py | 6 +++-- mypy/stats.py | 4 ++++ mypy/types.py | 5 +++- test-data/unit/check-flags.test | 4 ++-- test-data/unit/check-semanal-error.test | 31 ++++++++++++++++++++++++- test-data/unit/check-typeddict.test | 20 ++++++++++++++++ test-data/unit/reports.test | 16 ++++++------- test-data/unit/semanal-typeddict.test | 2 +- 9 files changed, 94 insertions(+), 15 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index f857c3e73381..f36149076fe6 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -3935,6 +3935,9 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool: # When this type alias gets "inlined", the Any is not explicit anymore, # so we need to replace it with non-explicit Anys. res = make_any_non_explicit(res) + if self.options.disallow_any_unimported and has_any_from_unimported_type(res): + self.msg.unimported_type_becomes_any("Type alias target", res, s) + res = make_any_non_unimported(res) # Note: with the new (lazy) type alias representation we only need to set no_args to True # if the expected number of arguments is non-zero, so that aliases like `A = List` work # but not aliases like `A = TypeAliasType("A", List)` as these need explicit type params. @@ -5407,6 +5410,9 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None: # When this type alias gets "inlined", the Any is not explicit anymore, # so we need to replace it with non-explicit Anys. res = make_any_non_explicit(res) + if self.options.disallow_any_unimported and has_any_from_unimported_type(res): + self.msg.unimported_type_becomes_any("Type alias target", res, s) + res = make_any_non_unimported(res) eager = self.is_func_scope() if isinstance(res, ProperType) and isinstance(res, Instance) and not res.args: fix_instance(res, self.fail, self.note, disallow_any=False, options=self.options) @@ -7433,6 +7439,21 @@ def visit_type_alias_type(self, t: TypeAliasType) -> Type: return t.copy_modified(args=[a.accept(self) for a in t.args]) +def make_any_non_unimported(t: Type) -> Type: + """Replace all Any types that come from unimported types with special form Any.""" + return t.accept(MakeAnyNonUnimported()) + + +class MakeAnyNonUnimported(TrivialSyntheticTypeTranslator): + def visit_any(self, t: AnyType) -> Type: + if t.type_of_any == TypeOfAny.from_unimported_type: + return t.copy_modified(TypeOfAny.special_form, missing_import_name=None) + return t + + def visit_type_alias_type(self, t: TypeAliasType) -> Type: + return t.copy_modified(args=[a.accept(self) for a in t.args]) + + def apply_semantic_analyzer_patches(patches: list[tuple[int, Callable[[], None]]]) -> None: """Call patch callbacks in the right order. diff --git a/mypy/semanal_typeddict.py b/mypy/semanal_typeddict.py index eee98d4d20fa..7b8d874337a2 100644 --- a/mypy/semanal_typeddict.py +++ b/mypy/semanal_typeddict.py @@ -310,11 +310,11 @@ def analyze_typeddict_classdef_fields( # Append stmt, name, and type in this case... fields.append(name) statements.append(stmt) - if stmt.type is None: + if stmt.unanalyzed_type is None: types.append(AnyType(TypeOfAny.unannotated)) else: analyzed = self.api.anal_type( - stmt.type, + stmt.unanalyzed_type, allow_required=True, allow_placeholder=not self.api.is_func_scope(), prohibit_self_type="TypedDict item type", @@ -322,6 +322,8 @@ def analyze_typeddict_classdef_fields( if analyzed is None: return None, [], [], set() # Need to defer types.append(analyzed) + if not has_placeholder(analyzed): + stmt.type = analyzed # ...despite possible minor failures that allow further analysis. if stmt.type is None or hasattr(stmt, "new_syntax") and not stmt.new_syntax: self.fail(TPDICT_CLASS_ERROR, stmt) diff --git a/mypy/stats.py b/mypy/stats.py index b167a41b0e34..9c69a245741b 100644 --- a/mypy/stats.py +++ b/mypy/stats.py @@ -203,7 +203,11 @@ def visit_assignment_stmt(self, o: AssignmentStmt) -> None: # Type variable definition -- not a real assignment. return if o.type: + # If there is an explicit type, don't visit the l.h.s. as an expression + # to avoid double-counting and mishandling special forms. self.type(o.type) + o.rvalue.accept(self) + return elif self.inferred and not self.all_nodes: # if self.all_nodes is set, lvalues will be visited later for lvalue in o.lvalues: diff --git a/mypy/types.py b/mypy/types.py index 2e7cbfd4e733..89609e8d0546 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1120,15 +1120,18 @@ def copy_modified( # Mark with Bogus because _dummy is just an object (with type Any) type_of_any: int = _dummy_int, original_any: Bogus[AnyType | None] = _dummy, + missing_import_name: Bogus[str | None] = _dummy, ) -> AnyType: if type_of_any == _dummy_int: type_of_any = self.type_of_any if original_any is _dummy: original_any = self.source_any + if missing_import_name is _dummy: + missing_import_name = self.missing_import_name return AnyType( type_of_any=type_of_any, source_any=original_any, - missing_import_name=self.missing_import_name, + missing_import_name=missing_import_name, line=self.line, column=self.column, ) diff --git a/test-data/unit/check-flags.test b/test-data/unit/check-flags.test index 62711d5f0071..4f327a2f0edc 100644 --- a/test-data/unit/check-flags.test +++ b/test-data/unit/check-flags.test @@ -924,9 +924,9 @@ class A(List[Unchecked]): # E: Base type becomes "List[Any]" due to an unfollowe from missing import Unchecked from typing import List -X = List[Unchecked] +X = List[Unchecked] # E: Type alias target becomes "List[Any]" due to an unfollowed import -def f(x: X) -> None: # E: Argument 1 to "f" becomes "List[Any]" due to an unfollowed import +def f(x: X) -> None: pass [builtins fixtures/list.pyi] diff --git a/test-data/unit/check-semanal-error.test b/test-data/unit/check-semanal-error.test index c6cf45d96691..d7ab272aed6c 100644 --- a/test-data/unit/check-semanal-error.test +++ b/test-data/unit/check-semanal-error.test @@ -151,4 +151,33 @@ class C: x: P[int] = C() [builtins fixtures/tuple.pyi] -[out] + +[case testSemanalDoesNotLeakSyntheticTypes] +# flags: --cache-fine-grained +from typing import Generic, NamedTuple, TypedDict, TypeVar +from dataclasses import dataclass + +T = TypeVar('T') +class Wrap(Generic[T]): pass + +invalid_1: 1 + 2 # E: Invalid type comment or annotation +invalid_2: Wrap[1 + 2] # E: Invalid type comment or annotation + +class A: + invalid_1: 1 + 2 # E: Invalid type comment or annotation + invalid_2: Wrap[1 + 2] # E: Invalid type comment or annotation + +class B(NamedTuple): + invalid_1: 1 + 2 # E: Invalid type comment or annotation + invalid_2: Wrap[1 + 2] # E: Invalid type comment or annotation + +class C(TypedDict): + invalid_1: 1 + 2 # E: Invalid type comment or annotation + invalid_2: Wrap[1 + 2] # E: Invalid type comment or annotation + +@dataclass +class D: + invalid_1: 1 + 2 # E: Invalid type comment or annotation + invalid_2: Wrap[1 + 2] # E: Invalid type comment or annotation +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index fa77d98e4a34..d35ec8ddd80e 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -2362,6 +2362,26 @@ Foo = TypedDict('Foo', {'camelCaseKey': str}) value: Foo = {} # E: Missing key "camelCaseKey" for TypedDict "Foo" [builtins fixtures/dict.pyi] +[case testTypedDictWithDeferredFieldTypeEval] +from typing import Generic, TypeVar, TypedDict, NotRequired + +class Foo(TypedDict): + y: NotRequired[int] + x: Outer[Inner[ForceDeferredEval]] + +var: Foo +reveal_type(var) # N: Revealed type is "TypedDict('__main__.Foo', {'y'?: builtins.int, 'x': __main__.Outer[__main__.Inner[__main__.ForceDeferredEval]]})" + +T1 = TypeVar("T1") +class Outer(Generic[T1]): pass + +T2 = TypeVar("T2", bound="ForceDeferredEval") +class Inner(Generic[T2]): pass + +class ForceDeferredEval: pass +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + -- Required[] [case testDoesRecognizeRequiredInTypedDictWithClass] diff --git a/test-data/unit/reports.test b/test-data/unit/reports.test index 16061d9c32bf..81e24240af2d 100644 --- a/test-data/unit/reports.test +++ b/test-data/unit/reports.test @@ -81,19 +81,19 @@ def foo(a: int) -> MyDict: return {"a": a} md: MyDict = MyDict(**foo(42)) [outfile build/cobertura.xml] - + $PWD - + - + - + @@ -155,9 +155,9 @@ z: NestedGen[Any] [outfile report/types-of-anys.txt] Name Unannotated Explicit Unimported Omitted Generics Error Special Form Implementation Artifact ----------------------------------------------------------------------------------------------------------------- - n 0 4 0 8 0 0 0 + n 0 2 0 8 0 0 0 ----------------------------------------------------------------------------------------------------------------- -Total 0 4 0 8 0 0 0 +Total 0 2 0 8 0 0 0 [case testTypeVarTreatedAsEmptyLine] # cmd: mypy --html-report report n.py @@ -371,9 +371,9 @@ z = g.does_not_exist() # type: ignore # Error [outfile report/types-of-anys.txt] Name Unannotated Explicit Unimported Omitted Generics Error Special Form Implementation Artifact ----------------------------------------------------------------------------------------------------------------- - n 2 4 2 1 3 0 0 + n 2 3 1 1 3 0 0 ----------------------------------------------------------------------------------------------------------------- -Total 2 4 2 1 3 0 0 +Total 2 3 1 1 3 0 0 [case testAnyExpressionsReportUnqualifiedError] # cmd: mypy --any-exprs-report report n.py diff --git a/test-data/unit/semanal-typeddict.test b/test-data/unit/semanal-typeddict.test index b9eb6e0c2b13..9ce89155c308 100644 --- a/test-data/unit/semanal-typeddict.test +++ b/test-data/unit/semanal-typeddict.test @@ -42,4 +42,4 @@ MypyFile:1( NameExpr(x) TempNode:4( Any) - str?))) + builtins.str))) From 1acdfd073010a13a7e3b982097a9a1a7a5d9a544 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 6 Jul 2024 22:38:54 +0100 Subject: [PATCH 17/36] Fix crash on NamedTuple with method and error in function (#17498) Fixes https://github.com/python/mypy/issues/16814 This one is tricky and may expose some other bugs. But IMO this is strictly correct thing to do. --- mypy/semanal_main.py | 2 ++ test-data/unit/check-namedtuple.test | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/mypy/semanal_main.py b/mypy/semanal_main.py index 1185a3821553..09a1223be6aa 100644 --- a/mypy/semanal_main.py +++ b/mypy/semanal_main.py @@ -291,6 +291,8 @@ def process_top_level_function( deferred, incomplete, progress = semantic_analyze_target( target, module, state, node, active_type, final_iteration, patches ) + if not incomplete: + state.manager.incomplete_namespaces.discard(module) if final_iteration: assert not deferred, "Must not defer during final iteration" if not progress: diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index e9d156754d9c..147270dff72e 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1423,3 +1423,27 @@ class Foo(typing.NamedTuple): reveal_type(x) # N: Revealed type is "builtins.int" [builtins fixtures/tuple.pyi] [typing fixtures/typing-namedtuple.pyi] + +[case testNameErrorInNamedTupleNestedInFunction1] +from typing import NamedTuple + +def bar() -> None: + class MyNamedTuple(NamedTuple): + a: int + def foo(self) -> None: + ... + int_set: Set[int] # E: Name "Set" is not defined \ + # N: Did you forget to import it from "typing"? (Suggestion: "from typing import Set") +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testNameErrorInNamedTupleNestedInFunction2] +from typing import NamedTuple + +def bar() -> None: + class MyNamedTuple(NamedTuple): + a: int + def foo(self) -> None: + misspelled_var_name # E: Name "misspelled_var_name" is not defined +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] From 9c0a6f9ba355b307cd87c861619040e4b4691af4 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 6 Jul 2024 23:57:05 +0100 Subject: [PATCH 18/36] Fix crash on self-type in callable protocol (#17499) Fixes https://github.com/python/mypy/issues/16629 This is really ad-hoc, but a proper fix would be much more hard, and this currently we have a crash in a relatively common scenario. --- mypy/solve.py | 12 ++++++++++++ test-data/unit/check-selftype.test | 28 ++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/mypy/solve.py b/mypy/solve.py index bb87b6576ada..8a1495a9a246 100644 --- a/mypy/solve.py +++ b/mypy/solve.py @@ -553,6 +553,11 @@ def pre_validate_solutions( """ new_solutions: list[Type | None] = [] for t, s in zip(original_vars, solutions): + if is_callable_protocol(t.upper_bound): + # This is really ad-hoc, but a proper fix would be much more complex, + # and otherwise this may cause crash in a relatively common scenario. + new_solutions.append(s) + continue if s is not None and not is_subtype(s, t.upper_bound): bound_satisfies_all = True for c in constraints: @@ -567,3 +572,10 @@ def pre_validate_solutions( continue new_solutions.append(s) return new_solutions + + +def is_callable_protocol(t: Type) -> bool: + proper_t = get_proper_type(t) + if isinstance(proper_t, Instance) and proper_t.type.is_protocol: + return "__call__" in proper_t.type.protocol_members + return False diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index 1480c83b2272..9601852ef823 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -2132,3 +2132,31 @@ class D: x: int x: Union[C, D] reveal_type(x.x) # N: Revealed type is "Union[__main__.C, builtins.int]" + +[case testCallableProtocolTypingSelf] +from typing import Protocol, Self + +class MyProtocol(Protocol): + __name__: str + + def __call__( + self: Self, + ) -> None: ... + +def test() -> None: ... +value: MyProtocol = test + +[case testCallableProtocolOldSelf] +from typing import Protocol, TypeVar + +Self = TypeVar("Self", bound="MyProtocol") + +class MyProtocol(Protocol): + __name__: str + + def __call__( + self: Self, + ) -> None: ... + +def test() -> None: ... +value: MyProtocol = test From 6d45f3cb58ce4781b7e7f47469358af9441ea48b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 7 Jul 2024 10:59:20 +0100 Subject: [PATCH 19/36] Experimental: allow inline/anonymous TypedDicts (#17457) Fixes https://github.com/python/mypy/issues/9884 I was always a bit skeptical about this thing, since it feels more like TypeScript than Python, but it is second most upvoted issue. Also (this specific) implementation is like 60 lines of code plus tests, so why not. I know there is no PEP etc., but IMO this syntax is obvious and it just works. cc @JukkaL --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- docs/source/command_line.rst | 21 +++++++++++ docs/source/typed_dict.rst | 38 +++++++++++++++++++ mypy/checker.py | 6 ++- mypy/exprtotype.py | 31 ++++++++++++++- mypy/fastparse.py | 20 +++++++++- mypy/message_registry.py | 1 + mypy/nodes.py | 1 + mypy/options.py | 3 +- mypy/semanal_typeddict.py | 15 ++------ mypy/typeanal.py | 55 +++++++++++++++++++++++---- mypy/types.py | 4 +- test-data/unit/check-literal.test | 19 +++++----- test-data/unit/check-python312.test | 15 +++++++- test-data/unit/check-typeddict.test | 58 ++++++++++++++++++++++++++++- 14 files changed, 247 insertions(+), 40 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 50a6ef65f4d0..906231dc7e42 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -1055,6 +1055,27 @@ List of currently incomplete/experimental features: # Without PreciseTupleTypes: tuple[int, ...] # With PreciseTupleTypes: tuple[()] | tuple[int] | tuple[int, int] +* ``NewGenericSyntax``: this feature enables support for syntax defined + by :pep:`695`. For example: + + .. code-block:: python + + class Container[T]: # defines a generic class + content: T + + def first[T](items: list[T]) -> T: # defines a generic function + return items[0] + + type Items[T] = list[tuple[T, T]] # defines a generic type alias + +* ``InlineTypedDict``: this feature enables non-standard syntax for inline + :ref:`TypedDicts `, for example: + + .. code-block:: python + + def test_values() -> {"int": int, "str": str}: + return {"int": 42, "str": "test"} + Miscellaneous ************* diff --git a/docs/source/typed_dict.rst b/docs/source/typed_dict.rst index e5ce2927db4d..c379b5449eae 100644 --- a/docs/source/typed_dict.rst +++ b/docs/source/typed_dict.rst @@ -248,3 +248,41 @@ section of the docs has a full description with an example, but in short, you wi need to give each TypedDict the same key where each value has a unique :ref:`Literal type `. Then, check that key to distinguish between your TypedDicts. + +Inline TypedDict types +---------------------- + +.. note:: + + This is an experimental (non-standard) feature. Use + ``--enable-incomplete-feature=InlineTypedDict`` to enable. + +Sometimes you may want to define a complex nested JSON schema, or annotate +a one-off function that returns a TypedDict. In such cases it may be convenient +to use inline TypedDict syntax. For example: + +.. code-block:: python + + def test_values() -> {"int": int, "str": str}: + return {"int": 42, "str": "test"} + + class Response(TypedDict): + status: int + msg: str + # Using inline syntax here avoids defining two additional TypedDicts. + content: {"items": list[{"key": str, "value": str}]} + +Inline TypedDicts can also by used as targets of type aliases, but due to +ambiguity with a regular variables it is only allowed for (newer) explicit +type alias forms: + +.. code-block:: python + + from typing import TypeAlias + + X = {"a": int, "b": int} # creates a variable with type dict[str, type[int]] + Y: TypeAlias = {"a": int, "b": int} # creates a type alias + type Z = {"a": int, "b": int} # same as above (Python 3.12+ only) + +Also, due to incompatibility with runtime type-checking it is strongly recommended +to *not* use inline syntax in union types. diff --git a/mypy/checker.py b/mypy/checker.py index 2df74cf7be8d..0ae499916ec6 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2971,7 +2971,8 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: self.msg.annotation_in_unchecked_function(context=s) def check_type_alias_rvalue(self, s: AssignmentStmt) -> None: - alias_type = self.expr_checker.accept(s.rvalue) + with self.msg.filter_errors(): + alias_type = self.expr_checker.accept(s.rvalue) self.store_type(s.lvalues[-1], alias_type) def check_assignment( @@ -5311,7 +5312,8 @@ def remove_capture_conflicts(self, type_map: TypeMap, inferred_types: dict[Var, del type_map[expr] def visit_type_alias_stmt(self, o: TypeAliasStmt) -> None: - self.expr_checker.accept(o.value) + with self.msg.filter_errors(): + self.expr_checker.accept(o.value) def make_fake_typeinfo( self, diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index d9bdf2e2b20b..92316d11926d 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -4,9 +4,11 @@ from mypy.fastparse import parse_type_string from mypy.nodes import ( + MISSING_FALLBACK, BytesExpr, CallExpr, ComplexExpr, + DictExpr, EllipsisExpr, Expression, FloatExpr, @@ -29,9 +31,11 @@ AnyType, CallableArgument, EllipsisType, + Instance, ProperType, RawExpressionType, Type, + TypedDictType, TypeList, TypeOfAny, UnboundType, @@ -55,7 +59,7 @@ def _extract_argument_name(expr: Expression) -> str | None: def expr_to_unanalyzed_type( expr: Expression, - options: Options | None = None, + options: Options, allow_new_syntax: bool = False, _parent: Expression | None = None, allow_unpack: bool = False, @@ -67,6 +71,8 @@ def expr_to_unanalyzed_type( If allow_new_syntax is True, allow all type syntax independent of the target Python version (used in stubs). + + # TODO: a lot of code here is duplicated in fastparse.py, refactor this. """ # The `parent` parameter is used in recursive calls to provide context for # understanding whether an CallableArgument is ok. @@ -116,7 +122,7 @@ def expr_to_unanalyzed_type( elif ( isinstance(expr, OpExpr) and expr.op == "|" - and ((options and options.python_version >= (3, 10)) or allow_new_syntax) + and ((options.python_version >= (3, 10)) or allow_new_syntax) ): return UnionType( [ @@ -206,5 +212,26 @@ def expr_to_unanalyzed_type( return UnpackType( expr_to_unanalyzed_type(expr.expr, options, allow_new_syntax), from_star_syntax=True ) + elif isinstance(expr, DictExpr): + if not expr.items: + raise TypeTranslationError() + items: dict[str, Type] = {} + extra_items_from = [] + for item_name, value in expr.items: + if not isinstance(item_name, StrExpr): + if item_name is None: + extra_items_from.append( + expr_to_unanalyzed_type(value, options, allow_new_syntax, expr) + ) + continue + raise TypeTranslationError() + items[item_name.value] = expr_to_unanalyzed_type( + value, options, allow_new_syntax, expr + ) + result = TypedDictType( + items, set(), Instance(MISSING_FALLBACK, ()), expr.line, expr.column + ) + result.extra_items_from = extra_items_from + return result else: raise TypeTranslationError() diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 01f6ed4733ae..75c4bd46550c 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -17,6 +17,7 @@ ARG_POS, ARG_STAR, ARG_STAR2, + MISSING_FALLBACK, PARAM_SPEC_KIND, TYPE_VAR_KIND, TYPE_VAR_TUPLE_KIND, @@ -42,7 +43,6 @@ EllipsisExpr, Expression, ExpressionStmt, - FakeInfo, FloatExpr, ForStmt, FuncDef, @@ -116,6 +116,7 @@ RawExpressionType, TupleType, Type, + TypedDictType, TypeList, TypeOfAny, UnboundType, @@ -190,7 +191,6 @@ def ast3_parse( # There is no way to create reasonable fallbacks at this stage, # they must be patched later. -MISSING_FALLBACK: Final = FakeInfo("fallback can't be filled out until semanal") _dummy_fallback: Final = Instance(MISSING_FALLBACK, [], -1) TYPE_IGNORE_PATTERN: Final = re.compile(r"[^#]*#\s*type:\s*ignore\s*(.*)") @@ -2106,6 +2106,22 @@ def visit_Tuple(self, n: ast3.Tuple) -> Type: column=self.convert_column(n.col_offset), ) + def visit_Dict(self, n: ast3.Dict) -> Type: + if not n.keys: + return self.invalid_type(n) + items: dict[str, Type] = {} + extra_items_from = [] + for item_name, value in zip(n.keys, n.values): + if not isinstance(item_name, ast3.Constant) or not isinstance(item_name.value, str): + if item_name is None: + extra_items_from.append(self.visit(value)) + continue + return self.invalid_type(n) + items[item_name.value] = self.visit(value) + result = TypedDictType(items, set(), _dummy_fallback, n.lineno, n.col_offset) + result.extra_items_from = extra_items_from + return result + # Attribute(expr value, identifier attr, expr_context ctx) def visit_Attribute(self, n: Attribute) -> Type: before_dot = self.visit(n.value) diff --git a/mypy/message_registry.py b/mypy/message_registry.py index befacc9e6182..06199e70d6b4 100644 --- a/mypy/message_registry.py +++ b/mypy/message_registry.py @@ -138,6 +138,7 @@ def with_additional_msg(self, info: str) -> ErrorMessage: TYPEDDICT_KEY_MUST_BE_STRING_LITERAL: Final = ErrorMessage( "Expected TypedDict key to be string literal" ) +TYPEDDICT_OVERRIDE_MERGE: Final = 'Overwriting TypedDict field "{}" while merging' MALFORMED_ASSERT: Final = ErrorMessage("Assertion is always true, perhaps remove parentheses?") DUPLICATE_TYPE_SIGNATURES: Final = ErrorMessage("Function has duplicate type signatures") DESCRIPTOR_SET_NOT_CALLABLE: Final = ErrorMessage("{}.__set__ is not callable") diff --git a/mypy/nodes.py b/mypy/nodes.py index 2eb39d4baaf6..4a5c7240fa83 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -3480,6 +3480,7 @@ def __getattribute__(self, attr: str) -> type: VAR_NO_INFO: Final[TypeInfo] = FakeInfo("Var is lacking info") CLASSDEF_NO_INFO: Final[TypeInfo] = FakeInfo("ClassDef is lacking info") FUNC_NO_INFO: Final[TypeInfo] = FakeInfo("FuncBase for non-methods lack info") +MISSING_FALLBACK: Final = FakeInfo("fallback can't be filled out until semanal") class TypeAlias(SymbolNode): diff --git a/mypy/options.py b/mypy/options.py index 5ef6bc2a35e7..bff096d82c15 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -74,7 +74,8 @@ class BuildType: UNPACK: Final = "Unpack" PRECISE_TUPLE_TYPES: Final = "PreciseTupleTypes" NEW_GENERIC_SYNTAX: Final = "NewGenericSyntax" -INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES, NEW_GENERIC_SYNTAX)) +INLINE_TYPEDDICT: Final = "InlineTypedDict" +INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES, NEW_GENERIC_SYNTAX, INLINE_TYPEDDICT)) COMPLETE_FEATURES: Final = frozenset((TYPE_VAR_TUPLE, UNPACK)) diff --git a/mypy/semanal_typeddict.py b/mypy/semanal_typeddict.py index 7b8d874337a2..e639871364ce 100644 --- a/mypy/semanal_typeddict.py +++ b/mypy/semanal_typeddict.py @@ -8,6 +8,7 @@ from mypy.errorcodes import ErrorCode from mypy.expandtype import expand_type from mypy.exprtotype import TypeTranslationError, expr_to_unanalyzed_type +from mypy.message_registry import TYPEDDICT_OVERRIDE_MERGE from mypy.messages import MessageBuilder from mypy.nodes import ( ARG_NAMED, @@ -216,7 +217,7 @@ def add_keys_and_types_from_base( valid_items = self.map_items_to_base(valid_items, tvars, base_args) for key in base_items: if key in keys: - self.fail(f'Overwriting TypedDict field "{key}" while merging', ctx) + self.fail(TYPEDDICT_OVERRIDE_MERGE.format(key), ctx) keys.extend(valid_items.keys()) types.extend(valid_items.values()) required_keys.update(base_typed_dict.required_keys) @@ -507,17 +508,7 @@ def parse_typeddict_fields_with_types( field_type_expr, self.options, self.api.is_stub_file ) except TypeTranslationError: - if ( - isinstance(field_type_expr, CallExpr) - and isinstance(field_type_expr.callee, RefExpr) - and field_type_expr.callee.fullname in TPDICT_NAMES - ): - self.fail_typeddict_arg( - "Inline TypedDict types not supported; use assignment to define TypedDict", - field_type_expr, - ) - else: - self.fail_typeddict_arg("Invalid field type", field_type_expr) + self.fail_typeddict_arg("Use dict literal for nested TypedDict", field_type_expr) return [], [], False analyzed = self.api.anal_type( type, diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 6651af7dad4f..f63aef30a09a 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -10,7 +10,11 @@ from mypy import errorcodes as codes, message_registry, nodes from mypy.errorcodes import ErrorCode from mypy.expandtype import expand_type -from mypy.message_registry import INVALID_PARAM_SPEC_LOCATION, INVALID_PARAM_SPEC_LOCATION_NOTE +from mypy.message_registry import ( + INVALID_PARAM_SPEC_LOCATION, + INVALID_PARAM_SPEC_LOCATION_NOTE, + TYPEDDICT_OVERRIDE_MERGE, +) from mypy.messages import ( MessageBuilder, format_type, @@ -25,6 +29,7 @@ ARG_POS, ARG_STAR, ARG_STAR2, + MISSING_FALLBACK, SYMBOL_FUNCBASE_TYPES, ArgKind, Context, @@ -43,7 +48,7 @@ check_arg_names, get_nongen_builtins, ) -from mypy.options import Options +from mypy.options import INLINE_TYPEDDICT, Options from mypy.plugin import AnalyzeTypeContext, Plugin, TypeAnalyzerPluginInterface from mypy.semanal_shared import ( SemanticAnalyzerCoreInterface, @@ -1220,10 +1225,45 @@ def visit_tuple_type(self, t: TupleType) -> Type: return TupleType(self.anal_array(t.items, allow_unpack=True), fallback, t.line) def visit_typeddict_type(self, t: TypedDictType) -> Type: - items = { - item_name: self.anal_type(item_type) for (item_name, item_type) in t.items.items() - } - return TypedDictType(items, set(t.required_keys), t.fallback) + req_keys = set() + items = {} + for item_name, item_type in t.items.items(): + analyzed = self.anal_type(item_type, allow_required=True) + if isinstance(analyzed, RequiredType): + if analyzed.required: + req_keys.add(item_name) + analyzed = analyzed.item + else: + # Keys are required by default. + req_keys.add(item_name) + items[item_name] = analyzed + if t.fallback.type is MISSING_FALLBACK: # anonymous/inline TypedDict + if INLINE_TYPEDDICT not in self.options.enable_incomplete_feature: + self.fail( + "Inline TypedDict is experimental," + " must be enabled with --enable-incomplete-feature=InlineTypedDict", + t, + ) + required_keys = req_keys + fallback = self.named_type("typing._TypedDict") + for typ in t.extra_items_from: + analyzed = self.analyze_type(typ) + p_analyzed = get_proper_type(analyzed) + if not isinstance(p_analyzed, TypedDictType): + if not isinstance(p_analyzed, (AnyType, PlaceholderType)): + self.fail("Can only merge-in other TypedDict", t, code=codes.VALID_TYPE) + continue + for sub_item_name, sub_item_type in p_analyzed.items.items(): + if sub_item_name in items: + self.fail(TYPEDDICT_OVERRIDE_MERGE.format(sub_item_name), t) + continue + items[sub_item_name] = sub_item_type + if sub_item_name in p_analyzed.required_keys: + req_keys.add(sub_item_name) + else: + required_keys = t.required_keys + fallback = t.fallback + return TypedDictType(items, required_keys, fallback, t.line, t.column) def visit_raw_expression_type(self, t: RawExpressionType) -> Type: # We should never see a bare Literal. We synthesize these raw literals @@ -1761,11 +1801,12 @@ def anal_type( allow_param_spec: bool = False, allow_unpack: bool = False, allow_ellipsis: bool = False, + allow_required: bool = False, ) -> Type: if nested: self.nesting_level += 1 old_allow_required = self.allow_required - self.allow_required = False + self.allow_required = allow_required old_allow_ellipsis = self.allow_ellipsis self.allow_ellipsis = allow_ellipsis old_allow_unpack = self.allow_unpack diff --git a/mypy/types.py b/mypy/types.py index 89609e8d0546..91b40536f1cf 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -2521,11 +2521,12 @@ class TypedDictType(ProperType): TODO: The fallback structure is perhaps overly complicated. """ - __slots__ = ("items", "required_keys", "fallback") + __slots__ = ("items", "required_keys", "fallback", "extra_items_from") items: dict[str, Type] # item_name -> item_type required_keys: set[str] fallback: Instance + extra_items_from: list[ProperType] # only used during semantic analysis def __init__( self, @@ -2541,6 +2542,7 @@ def __init__( self.fallback = fallback self.can_be_true = len(self.items) > 0 self.can_be_false = len(self.required_keys) == 0 + self.extra_items_from = [] def accept(self, visitor: TypeVisitor[T]) -> T: return visitor.visit_typeddict_type(self) diff --git a/test-data/unit/check-literal.test b/test-data/unit/check-literal.test index 8f8aaf6a3982..6d76ce176aaf 100644 --- a/test-data/unit/check-literal.test +++ b/test-data/unit/check-literal.test @@ -608,36 +608,35 @@ e: Literal[dummy()] # E: Invalid type: Literal[...] cannot contain a [case testLiteralDisallowCollections] from typing_extensions import Literal -a: Literal[{"a": 1, "b": 2}] # E: Invalid type: Literal[...] cannot contain arbitrary expressions +a: Literal[{"a": 1, "b": 2}] # E: Parameter 1 of Literal[...] is invalid b: Literal[{1, 2, 3}] # E: Invalid type: Literal[...] cannot contain arbitrary expressions -c: {"a": 1, "b": 2} # E: Invalid type comment or annotation +c: {"a": 1, "b": 2} # E: Inline TypedDict is experimental, must be enabled with --enable-incomplete-feature=InlineTypedDict \ + # E: Invalid type: try using Literal[1] instead? \ + # E: Invalid type: try using Literal[2] instead? d: {1, 2, 3} # E: Invalid type comment or annotation [builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] [case testLiteralDisallowCollections2] - from typing_extensions import Literal a: (1, 2, 3) # E: Syntax error in type annotation \ # N: Suggestion: Use Tuple[T1, ..., Tn] instead of (T1, ..., Tn) b: Literal[[1, 2, 3]] # E: Parameter 1 of Literal[...] is invalid c: [1, 2, 3] # E: Bracketed expression "[...]" is not valid as a type [builtins fixtures/tuple.pyi] -[out] [case testLiteralDisallowCollectionsTypeAlias] - from typing_extensions import Literal -at = Literal[{"a": 1, "b": 2}] # E: Invalid type alias: expression is not a valid type +at = Literal[{"a": 1, "b": 2}] # E: Parameter 1 of Literal[...] is invalid bt = {"a": 1, "b": 2} -a: at # E: Variable "__main__.at" is not valid as a type \ - # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases +a: at +reveal_type(a) # N: Revealed type is "Any" b: bt # E: Variable "__main__.bt" is not valid as a type \ # N: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases [builtins fixtures/dict.pyi] -[out] +[typing fixtures/typing-typeddict.pyi] [case testLiteralDisallowCollectionsTypeAlias2] - from typing_extensions import Literal at = Literal[{1, 2, 3}] # E: Invalid type alias: expression is not a valid type bt = {1, 2, 3} diff --git a/test-data/unit/check-python312.test b/test-data/unit/check-python312.test index 5307f47d539a..073ef7f4bdec 100644 --- a/test-data/unit/check-python312.test +++ b/test-data/unit/check-python312.test @@ -585,8 +585,7 @@ reveal_type(a) # N: Revealed type is "Any" [case testPEP695TypeAliasInvalidType] # flags: --enable-incomplete-feature=NewGenericSyntax -type A = int | 1 # E: Invalid type: try using Literal[1] instead? \ - # E: Unsupported operand types for | ("Type[int]" and "int") +type A = int | 1 # E: Invalid type: try using Literal[1] instead? a: A reveal_type(a) # N: Revealed type is "Union[builtins.int, Any]" @@ -1656,3 +1655,15 @@ type I2 = C[Any] | None type I3 = None | C[TD] [builtins fixtures/type.pyi] [typing fixtures/typing-full.pyi] + +[case testTypedDictInlineYesNewStyleAlias] +# flags: --enable-incomplete-feature=NewGenericSyntax --enable-incomplete-feature=InlineTypedDict +type X[T] = {"item": T, "other": X[T] | None} +x: X[str] +reveal_type(x) # N: Revealed type is "TypedDict({'item': builtins.str, 'other': Union[..., None]})" +if x["other"] is not None: + reveal_type(x["other"]["item"]) # N: Revealed type is "builtins.str" + +type Y[T] = {"item": T, **Y[T]} # E: Overwriting TypedDict field "item" while merging +[builtins fixtures/dict.pyi] +[typing fixtures/typing-full.pyi] diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index d35ec8ddd80e..6a5120159c2d 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -78,7 +78,7 @@ p = Point(x='meaning_of_life', y=1337) # E: Incompatible types (expression has [case testCannotCreateTypedDictInstanceWithInlineTypedDict] from mypy_extensions import TypedDict D = TypedDict('D', { - 'x': TypedDict('E', { # E: Inline TypedDict types not supported; use assignment to define TypedDict + 'x': TypedDict('E', { # E: Use dict literal for nested TypedDict 'y': int }) }) @@ -3570,3 +3570,59 @@ class Test: run(test2, other="yes", **params) run(test2, other=0, **params) # E: Argument "other" to "run" has incompatible type "int"; expected "str" [builtins fixtures/tuple.pyi] + +[case testTypedDictInlineNoOldStyleAlias] +# flags: --enable-incomplete-feature=InlineTypedDict +X = {"int": int, "str": str} +reveal_type(X) # N: Revealed type is "builtins.dict[builtins.str, def () -> builtins.object]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictInlineYesMidStyleAlias] +# flags: --enable-incomplete-feature=InlineTypedDict +from typing_extensions import TypeAlias +X: TypeAlias = {"int": int, "str": str} +x: X +reveal_type(x) # N: # N: Revealed type is "TypedDict({'int': builtins.int, 'str': builtins.str})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictInlineNoEmpty] +# flags: --enable-incomplete-feature=InlineTypedDict +x: {} # E: Invalid type comment or annotation +reveal_type(x) # N: Revealed type is "Any" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictInlineNotRequired] +# flags: --enable-incomplete-feature=InlineTypedDict +from typing import NotRequired + +x: {"one": int, "other": NotRequired[int]} +x = {"one": 1} # OK +y: {"one": int, "other": int} +y = {"one": 1} # E: Expected TypedDict keys ("one", "other") but found only key "one" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictInlineNestedSchema] +# flags: --enable-incomplete-feature=InlineTypedDict +def nested() -> {"one": str, "other": {"a": int, "b": int}}: + if bool(): + return {"one": "yes", "other": {"a": 1, "b": 2}} # OK + else: + return {"one": "no", "other": {"a": 1, "b": "2"}} # E: Incompatible types (expression has type "str", TypedDict item "b" has type "int") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictInlineMergeAnother] +# flags: --enable-incomplete-feature=InlineTypedDict +from typing import TypeVar +from typing_extensions import TypeAlias + +T = TypeVar("T") +X: TypeAlias = {"item": T} +x: {"a": int, **X[str], "b": int} +reveal_type(x) # N: Revealed type is "TypedDict({'a': builtins.int, 'b': builtins.int, 'item': builtins.str})" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-full.pyi] From 4c54801afefa545bee6ee205578f4f53c6afeedb Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Sun, 7 Jul 2024 22:00:14 +0200 Subject: [PATCH 20/36] Fix `_PyObject_FastCall` for Python 3.13 (#17502) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_PyObject_FastCall` will be removed in 3.13. It can safely replaced by `PyObject_Vectorcall` (available since `3.9`) / `_PyObject_Vectorcall` (available since `3.8`). https://github.com/python/cpython/issues/106023#issuecomment-1613963489 https://peps.python.org/pep-0590/ Fixes ```cpp /home/runner/work/mypy/mypy/mypyc/lib-rt/pythonsupport.h: In function ‘update_bases’: (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/pythonsupport.h:62:20: error: implicit declaration of function ‘_PyObject_FastCall’; did you mean ‘PyObject_Call’? [-Werror=implicit-function-declaration] (diff) 62 | new_base = _PyObject_FastCall(meth, stack, 1); (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/pythonsupport.h:62:18: error: assignment to ‘PyObject *’ {aka ‘struct _object *’} from ‘int’ makes pointer from integer without a cast [-Werror=int-conversion] (diff) 62 | new_base = _PyObject_FastCall(meth, stack, 1); (diff) | ^ (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/pythonsupport.h: In function ‘init_subclass’: (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/pythonsupport.h:111:11: error: assignment to ‘PyObject *’ {aka ‘struct _object *’} from ‘int’ makes pointer from integer without a cast [-Werror=int-conversion] (diff) 111 | super = _PyObject_FastCall((PyObject *)&PySuper_Type, args, 2); (diff) | ^ (diff) ``` --- mypyc/lib-rt/pythonsupport.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypyc/lib-rt/pythonsupport.h b/mypyc/lib-rt/pythonsupport.h index 85f9ec64ac90..87e034f1fc51 100644 --- a/mypyc/lib-rt/pythonsupport.h +++ b/mypyc/lib-rt/pythonsupport.h @@ -59,7 +59,7 @@ update_bases(PyObject *bases) } continue; } - new_base = _PyObject_FastCall(meth, stack, 1); + new_base = _PyObject_Vectorcall(meth, stack, 1, NULL); Py_DECREF(meth); if (!new_base) { goto error; @@ -108,7 +108,7 @@ init_subclass(PyTypeObject *type, PyObject *kwds) PyObject *super, *func, *result; PyObject *args[2] = {(PyObject *)type, (PyObject *)type}; - super = _PyObject_FastCall((PyObject *)&PySuper_Type, args, 2); + super = _PyObject_Vectorcall((PyObject *)&PySuper_Type, args, 2, NULL); if (super == NULL) { return -1; } From d4f7e5cd67c83f08f1e2dab418725351ff15da1b Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Sun, 7 Jul 2024 22:06:02 +0200 Subject: [PATCH 21/36] Fix `gen_is_coroutine` for Python 3.13 (#17501) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `_PyInterpreterFrame` struct was changed in https://github.com/python/cpython/pull/105727 to store the code object in `f_executable` instead of `f_code`. Fixes ```cpp /home/runner/work/mypy/mypy/mypyc/lib-rt/pythonsupport.h: In function ‘_PyGen_GetCode’: (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/pythonsupport.h:403:17: error: ‘_PyInterpreterFrame’ has no member named ‘f_code’ (diff) 403 | return frame->f_code; (diff) | ^~ (diff) ``` --- mypyc/lib-rt/mypyc_util.h | 3 +++ mypyc/lib-rt/pythonsupport.h | 25 ++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/mypyc/lib-rt/mypyc_util.h b/mypyc/lib-rt/mypyc_util.h index 3c888a581a33..9967f0a13b4f 100644 --- a/mypyc/lib-rt/mypyc_util.h +++ b/mypyc/lib-rt/mypyc_util.h @@ -115,4 +115,7 @@ static inline void CPyLong_SetUnsignedSize(PyLongObject *o, Py_ssize_t n) { #endif +// Are we targeting Python 3.13 or newer? +#define CPY_3_13_FEATURES (PY_VERSION_HEX >= 0x030d0000) + #endif diff --git a/mypyc/lib-rt/pythonsupport.h b/mypyc/lib-rt/pythonsupport.h index 87e034f1fc51..c5423ab0fab5 100644 --- a/mypyc/lib-rt/pythonsupport.h +++ b/mypyc/lib-rt/pythonsupport.h @@ -392,7 +392,30 @@ _CPyObject_HasAttrId(PyObject *v, _Py_Identifier *name) { _PyObject_CallMethodIdObjArgs((self), (name), (arg), NULL) #endif -#if CPY_3_12_FEATURES +#if CPY_3_13_FEATURES + +// These are copied from genobject.c in Python 3.13 + +/* Returns a borrowed reference */ +static inline PyCodeObject * +_PyGen_GetCode(PyGenObject *gen) { + _PyInterpreterFrame *frame = (_PyInterpreterFrame *)(gen->gi_iframe); + return _PyFrame_GetCode(frame); +} + +static int +gen_is_coroutine(PyObject *o) +{ + if (PyGen_CheckExact(o)) { + PyCodeObject *code = _PyGen_GetCode((PyGenObject*)o); + if (code->co_flags & CO_ITERABLE_COROUTINE) { + return 1; + } + } + return 0; +} + +#elif CPY_3_12_FEATURES // These are copied from genobject.c in Python 3.12 From 9175ce59322759d3e02727fd804008ee2234bf65 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Sun, 7 Jul 2024 22:10:09 +0200 Subject: [PATCH 22/36] Fix `_PyList_Extend` for Python 3.13 (#17503) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace `_PyList_Extend` with `PyList_Extend` from `pythoncapi_compat.h`. https://github.com/python/cpython/issues/111138 https://docs.python.org/dev/c-api/list.html#c.PyList_Extend Fixes ```cpp /home/runner/work/mypy/mypy/mypyc/lib-rt/list_ops.c: In function ‘CPyList_Extend’: (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/list_ops.c:259:12: error: implicit declaration of function ‘_PyList_Extend’; did you mean ‘CPyList_Extend’? [-Werror=implicit-function-declaration] (diff) 259 | return _PyList_Extend((PyListObject *)o1, o2); (diff) | ^~~~~~~~~~~~~~ (diff) | CPyList_Extend (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/list_ops.c:259:12: error: returning ‘int’ from a function with return type ‘PyObject *’ {aka ‘struct _object *’} makes pointer from integer without a cast [-Werror=int-conversion] (diff) 259 | return _PyList_Extend((PyListObject *)o1, o2); (diff) | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/dict_ops.c: In function ‘CPyDict_Keys’: (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/dict_ops.c:233:21: error: initialization of ‘PyObject *’ {aka ‘struct _object *’} from ‘int’ makes pointer from integer without a cast [-Werror=int-conversion] (diff) 233 | PyObject *res = _PyList_Extend((PyListObject *)list, view); (diff) | ^~~~~~~~~~~~~~ (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/dict_ops.c: In function ‘CPyDict_Values’: (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/dict_ops.c:253:21: error: initialization of ‘PyObject *’ {aka ‘struct _object *’} from ‘int’ makes pointer from integer without a cast [-Werror=int-conversion] (diff) 253 | PyObject *res = _PyList_Extend((PyListObject *)list, view); (diff) | ^~~~~~~~~~~~~~ (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/dict_ops.c: In function ‘CPyDict_Items’: (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/dict_ops.c:273:21: error: initialization of ‘PyObject *’ {aka ‘struct _object *’} from ‘int’ makes pointer from integer without a cast [-Werror=int-conversion] (diff) 273 | PyObject *res = _PyList_Extend((PyListObject *)list, view); (diff) | ^~~~~~~~~~~~~~ (diff) ``` --- mypyc/lib-rt/dict_ops.c | 15 ++++++--------- mypyc/lib-rt/list_ops.c | 5 ++++- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/mypyc/lib-rt/dict_ops.c b/mypyc/lib-rt/dict_ops.c index c0cc8d5a7f87..031df8f63c49 100644 --- a/mypyc/lib-rt/dict_ops.c +++ b/mypyc/lib-rt/dict_ops.c @@ -230,12 +230,11 @@ PyObject *CPyDict_Keys(PyObject *dict) { if (view == NULL) { return NULL; } - PyObject *res = _PyList_Extend((PyListObject *)list, view); + int res = PyList_Extend(list, view); Py_DECREF(view); - if (res == NULL) { + if (res < 0) { return NULL; } - Py_DECREF(res); return list; } @@ -250,12 +249,11 @@ PyObject *CPyDict_Values(PyObject *dict) { if (view == NULL) { return NULL; } - PyObject *res = _PyList_Extend((PyListObject *)list, view); + int res = PyList_Extend(list, view); Py_DECREF(view); - if (res == NULL) { + if (res < 0) { return NULL; } - Py_DECREF(res); return list; } @@ -270,12 +268,11 @@ PyObject *CPyDict_Items(PyObject *dict) { if (view == NULL) { return NULL; } - PyObject *res = _PyList_Extend((PyListObject *)list, view); + int res = PyList_Extend(list, view); Py_DECREF(view); - if (res == NULL) { + if (res < 0) { return NULL; } - Py_DECREF(res); return list; } diff --git a/mypyc/lib-rt/list_ops.c b/mypyc/lib-rt/list_ops.c index df87228a0d10..d297ece8f417 100644 --- a/mypyc/lib-rt/list_ops.c +++ b/mypyc/lib-rt/list_ops.c @@ -256,7 +256,10 @@ int CPyList_Insert(PyObject *list, CPyTagged index, PyObject *value) } PyObject *CPyList_Extend(PyObject *o1, PyObject *o2) { - return _PyList_Extend((PyListObject *)o1, o2); + if (PyList_Extend(o1, o2) < 0) { + return NULL; + } + Py_RETURN_NONE; } // Return -2 or error, -1 if not found, or index of first match otherwise. From 7f67090b10694657bc72f39bdeaa3b189c7be7fc Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Sun, 7 Jul 2024 23:56:33 +0200 Subject: [PATCH 23/36] Fix `_PyObject_LookupAttrId` for Python 3.13 (#17505) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_PyObject_LookupAttrId` was removed / replaced with `PyObject_GetOptionalAttrString` in https://github.com/python/cpython/pull/106522. https://docs.python.org/dev/c-api/object.html#c.PyObject_GetOptionalAttrString Fixes ```cpp /home/runner/work/mypy/mypy/mypyc/lib-rt/pythonsupport.h: In function ‘update_bases’: (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/pythonsupport.h:51:13: error: implicit declaration of function ‘_PyObject_LookupAttrId’; did you mean ‘_PyObject_GetAttrId’? [-Werror=implicit-function-declaration] (diff) 51 | if (_PyObject_LookupAttrId(base, &PyId___mro_entries__, &meth) < 0) { (diff) | ^~~~~~~~~~~~~~~~~~~~~~ (diff) | _PyObject_GetAttrId (diff) ``` --- mypyc/lib-rt/pythonsupport.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypyc/lib-rt/pythonsupport.h b/mypyc/lib-rt/pythonsupport.h index c5423ab0fab5..2d18e19b2c56 100644 --- a/mypyc/lib-rt/pythonsupport.h +++ b/mypyc/lib-rt/pythonsupport.h @@ -48,7 +48,7 @@ update_bases(PyObject *bases) } continue; } - if (_PyObject_LookupAttrId(base, &PyId___mro_entries__, &meth) < 0) { + if (PyObject_GetOptionalAttrString(base, PyId___mro_entries__.string, &meth) < 0) { goto error; } if (!meth) { @@ -374,7 +374,7 @@ _CPyDictView_New(PyObject *dict, PyTypeObject *type) static int _CPyObject_HasAttrId(PyObject *v, _Py_Identifier *name) { PyObject *tmp = NULL; - int result = _PyObject_LookupAttrId(v, name, &tmp); + int result = PyObject_GetOptionalAttrString(v, name->string, &tmp); if (tmp) { Py_DECREF(tmp); } From 966d6d36595f9b029afb04271dc90fa5fd106005 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Mon, 8 Jul 2024 01:00:01 +0200 Subject: [PATCH 24/36] Fix `PyUnicode` functions for Python 3.13 (#17504) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace `_PyUnicode_EqualToASCIIString` with `PyUnicode_EqualToUTF8`. https://docs.python.org/dev/c-api/unicode.html#c.PyUnicode_EqualToUTF8 https://github.com/python/cpython/issues/110289 Fixes ```cpp /home/runner/work/mypy/mypy/mypyc/lib-rt/getargs.c: In function ‘vgetargskeywords’: (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/pythonsupport.h:310:45: error: implicit declaration of function ‘_PyUnicode_EqualToASCIIString’; did you mean ‘CPyUnicode_EqualToASCIIString’? [-Werror=implicit-function-declaration] (diff) 310 | #define CPyUnicode_EqualToASCIIString(x, y) _PyUnicode_EqualToASCIIString(x, y) (diff) | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~ (diff) /home/runner/work/mypy/mypy/mypyc/lib-rt/getargs.c:398:21: note: in expansion of macro ‘CPyUnicode_EqualToASCIIString’ (diff) 398 | if (CPyUnicode_EqualToASCIIString(key, kwlist[i])) { (diff) | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~ (diff) ``` --- mypyc/lib-rt/getargs.c | 2 +- mypyc/lib-rt/pythonsupport.h | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/mypyc/lib-rt/getargs.c b/mypyc/lib-rt/getargs.c index 3c8b528f8048..1bc2f5b02ba8 100644 --- a/mypyc/lib-rt/getargs.c +++ b/mypyc/lib-rt/getargs.c @@ -395,7 +395,7 @@ vgetargskeywords(PyObject *args, PyObject *kwargs, const char *format, goto latefail; } for (i = pos; i < len; i++) { - if (CPyUnicode_EqualToASCIIString(key, kwlist[i])) { + if (PyUnicode_EqualToUTF8(key, kwlist[i])) { match = 1; break; } diff --git a/mypyc/lib-rt/pythonsupport.h b/mypyc/lib-rt/pythonsupport.h index 2d18e19b2c56..85eb6348eb81 100644 --- a/mypyc/lib-rt/pythonsupport.h +++ b/mypyc/lib-rt/pythonsupport.h @@ -307,8 +307,6 @@ list_count(PyListObject *self, PyObject *value) return CPyTagged_ShortFromSsize_t(count); } -#define CPyUnicode_EqualToASCIIString(x, y) _PyUnicode_EqualToASCIIString(x, y) - // Adapted from genobject.c in Python 3.7.2 // Copied because it wasn't in 3.5.2 and it is undocumented anyways. /* From 4e3346ee1dee83868adc2411c4a0f4050cf2f95a Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Mon, 8 Jul 2024 01:00:31 +0200 Subject: [PATCH 25/36] Add additional includes for Python 3.13 (#17506) Define `Py_BUILD_CORE` required by `internal/...` header files. Include additional headers for moved private functions. ```cpp /opt/hostedtoolcache/Python/3.13.0-beta.3/x64/include/python3.13/internal/pycore_frame.h:8:4: error: #error "this header requires Py_BUILD_CORE define" (diff) 8 | # error "this header requires Py_BUILD_CORE define" (diff) ``` --- mypyc/lib-rt/pythonsupport.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/mypyc/lib-rt/pythonsupport.h b/mypyc/lib-rt/pythonsupport.h index 85eb6348eb81..69ff120dd40d 100644 --- a/mypyc/lib-rt/pythonsupport.h +++ b/mypyc/lib-rt/pythonsupport.h @@ -13,6 +13,18 @@ #include #include "mypyc_util.h" +#if CPY_3_13_FEATURES +#ifndef Py_BUILD_CORE +#define Py_BUILD_CORE +#endif +#include "internal/pycore_bytesobject.h" // _PyBytes_Join +#include "internal/pycore_call.h" // _PyObject_CallMethodIdNoArgs, _PyObject_CallMethodIdObjArgs, _PyObject_CallMethodIdOneArg +#include "internal/pycore_genobject.h" // _PyGen_FetchStopIterationValue +#include "internal/pycore_object.h" // _PyType_CalculateMetaclass +#include "internal/pycore_pyerrors.h" // _PyErr_FormatFromCause, _PyErr_SetKeyError +#include "internal/pycore_unicodeobject.h" // _PyUnicode_EQ, _PyUnicode_FastCopyCharacters +#endif + #if CPY_3_12_FEATURES #include "internal/pycore_frame.h" #endif From acc65b5a1ec6065ed06b5be5c2c55ef9ff90d610 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Mon, 8 Jul 2024 17:07:33 +0200 Subject: [PATCH 26/36] Update mypyc test output for Python 3.13 (#17508) --- mypyc/test-data/run-exceptions.test | 86 ++++++++++++++++++++++++++++ mypyc/test-data/run-loops.test | 35 +++++++++++ mypyc/test-data/run-misc.test | 5 +- mypyc/test-data/run-multimodule.test | 17 ++++++ 4 files changed, 142 insertions(+), 1 deletion(-) diff --git a/mypyc/test-data/run-exceptions.test b/mypyc/test-data/run-exceptions.test index c591fc1d8c15..1b180b933197 100644 --- a/mypyc/test-data/run-exceptions.test +++ b/mypyc/test-data/run-exceptions.test @@ -80,6 +80,43 @@ Traceback (most recent call last): File "native.py", line 23, in __init__ raise Exception Exception +[out version>=3.13] +Traceback (most recent call last): + File "driver.py", line 4, in + f([]) + ~^^^^ + File "native.py", line 3, in f + g(x) + File "native.py", line 6, in g + x[5] = 2 +IndexError: list assignment index out of range +Traceback (most recent call last): + File "driver.py", line 8, in + r1() + ~~^^ + File "native.py", line 10, in r1 + q1() + File "native.py", line 13, in q1 + raise Exception("test") +Exception: test +Traceback (most recent call last): + File "driver.py", line 12, in + r2() + ~~^^ + File "native.py", line 16, in r2 + q2() + File "native.py", line 19, in q2 + raise Exception +Exception +Traceback (most recent call last): + File "driver.py", line 16, in + hey() + ~~~^^ + File "native.py", line 26, in hey + A() + File "native.py", line 23, in __init__ + raise Exception +Exception [case testTryExcept] from typing import Any, Iterator @@ -264,6 +301,55 @@ attr! -- 'object' object has no attribute 'lol' out! == l == key! -- 0 +[out version>=3.13] +== i == + +Traceback (most recent call last): + File "driver.py", line 6, in + i() + ~^^ + File "native.py", line 44, in i + r(0) + File "native.py", line 15, in r + [0][1] +IndexError: list index out of range +== k == +Traceback (most recent call last): + File "native.py", line 59, in k + r(1) + File "native.py", line 17, in r + raise Exception('hi') +Exception: hi + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "driver.py", line 12, in + k() + ~^^ + File "native.py", line 61, in k + r(0) + File "native.py", line 15, in r + [0][1] +IndexError: list index out of range +== g == +caught! +caught! +== f == +hi +None +list index out of range +None +== h == +gonna break +None +== j == +lookup! +lookup! +attr! -- 'object' object has no attribute 'lol' +out! +== l == +key! -- 0 [case testTryFinally] from typing import Any diff --git a/mypyc/test-data/run-loops.test b/mypyc/test-data/run-loops.test index 6f7d79059a6d..95b79af1a411 100644 --- a/mypyc/test-data/run-loops.test +++ b/mypyc/test-data/run-loops.test @@ -381,6 +381,41 @@ RuntimeError: dictionary changed size during iteration 1 2 3 +[out version>=3.13] +Traceback (most recent call last): + File "driver.py", line 16, in + iterate_over_any(5) + ~~~~~~~~~~~~~~~~^^^ + File "native.py", line 6, in iterate_over_any + for element in a: +TypeError: 'int' object is not iterable +Traceback (most recent call last): + File "driver.py", line 20, in + iterate_over_iterable(broken_generator(5)) + ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ + File "native.py", line 10, in iterate_over_iterable + for element in iterable: + File "driver.py", line 8, in broken_generator + raise Exception('Exception Manually Raised') +Exception: Exception Manually Raised +Traceback (most recent call last): + File "driver.py", line 24, in + iterate_and_delete(d) + ~~~~~~~~~~~~~~~~~~^^^ + File "native.py", line 14, in iterate_and_delete + for key in d: +RuntimeError: dictionary changed size during iteration +15 +6 +3 +0 +1 +2 +3 +4 +1 +2 +3 [case testContinueFor] def f() -> None: diff --git a/mypyc/test-data/run-misc.test b/mypyc/test-data/run-misc.test index 14bb5be979ae..f07ac51dae6c 100644 --- a/mypyc/test-data/run-misc.test +++ b/mypyc/test-data/run-misc.test @@ -968,7 +968,10 @@ print(z) [case testCheckVersion] import sys -if sys.version_info[:2] == (3, 12): +if sys.version_info[:2] == (3, 13): + def version() -> int: + return 13 +elif sys.version_info[:2] == (3, 12): def version() -> int: return 12 elif sys.version_info[:2] == (3, 11): diff --git a/mypyc/test-data/run-multimodule.test b/mypyc/test-data/run-multimodule.test index 70c73dc2088b..5edd5688140e 100644 --- a/mypyc/test-data/run-multimodule.test +++ b/mypyc/test-data/run-multimodule.test @@ -291,6 +291,23 @@ Traceback (most recent call last): File "other.py", line 3, in fail2 x[2] = 2 IndexError: list assignment index out of range +[out version>=3.13] +Traceback (most recent call last): + File "driver.py", line 6, in + other.fail2() + ~~~~~~~~~~~^^ + File "other.py", line 3, in fail2 + x[2] = 2 +IndexError: list assignment index out of range +Traceback (most recent call last): + File "driver.py", line 12, in + native.fail() + ~~~~~~~~~~~^^ + File "native.py", line 4, in fail + fail2() + File "other.py", line 3, in fail2 + x[2] = 2 +IndexError: list assignment index out of range [case testMultiModuleCycle] if False: From 78d1dfe9db3e2385c66b80c5e88dfca1655ae0e3 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Mon, 8 Jul 2024 17:08:00 +0200 Subject: [PATCH 27/36] Fix ManagedDict functions for Python 3.13 (#17507) `PyObject_VisitManagedDict` and `PyObject_ClearManagedDict` were made public in https://github.com/python/cpython/pull/108763. Both are available from `pythoncapi_compat.h`. --- mypyc/codegen/emitclass.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 8dcf7212b694..ad95a1b0f323 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -734,7 +734,7 @@ def generate_traverse_for_class(cl: ClassIR, func_name: str, emitter: Emitter) - for attr, rtype in base.attributes.items(): emitter.emit_gc_visit(f"self->{emitter.attr(attr)}", rtype) if has_managed_dict(cl, emitter): - emitter.emit_line("_PyObject_VisitManagedDict((PyObject *)self, visit, arg);") + emitter.emit_line("PyObject_VisitManagedDict((PyObject *)self, visit, arg);") elif cl.has_dict: struct_name = cl.struct_name(emitter.names) # __dict__ lives right after the struct and __weakref__ lives right after that @@ -757,7 +757,7 @@ def generate_clear_for_class(cl: ClassIR, func_name: str, emitter: Emitter) -> N for attr, rtype in base.attributes.items(): emitter.emit_gc_clear(f"self->{emitter.attr(attr)}", rtype) if has_managed_dict(cl, emitter): - emitter.emit_line("_PyObject_ClearManagedDict((PyObject *)self);") + emitter.emit_line("PyObject_ClearManagedDict((PyObject *)self);") elif cl.has_dict: struct_name = cl.struct_name(emitter.names) # __dict__ lives right after the struct and __weakref__ lives right after that From 3c9f69487d3f01b7c24cf4dbda2460c99094d1f3 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Mon, 8 Jul 2024 17:08:22 +0200 Subject: [PATCH 28/36] Add another include for Python 3.13 (#17509) Include `internal/pycore_setobject.h` necessary for `_PySet_Update`. --- mypyc/lib-rt/pythonsupport.h | 1 + 1 file changed, 1 insertion(+) diff --git a/mypyc/lib-rt/pythonsupport.h b/mypyc/lib-rt/pythonsupport.h index 69ff120dd40d..8edc9abcf9f8 100644 --- a/mypyc/lib-rt/pythonsupport.h +++ b/mypyc/lib-rt/pythonsupport.h @@ -22,6 +22,7 @@ #include "internal/pycore_genobject.h" // _PyGen_FetchStopIterationValue #include "internal/pycore_object.h" // _PyType_CalculateMetaclass #include "internal/pycore_pyerrors.h" // _PyErr_FormatFromCause, _PyErr_SetKeyError +#include "internal/pycore_setobject.h" // _PySet_Update #include "internal/pycore_unicodeobject.h" // _PyUnicode_EQ, _PyUnicode_FastCopyCharacters #endif From 45bb91ed37f94140132742ab63a9a0d01f4f1b07 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 14 Jul 2024 13:02:31 +0100 Subject: [PATCH 29/36] [mypyc] Don't use _PyUnicode_EQ on 3.13, as it's no longer exported (#17523) Work on mypyc/mypyc#1056. --- mypyc/lib-rt/getargsfast.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mypyc/lib-rt/getargsfast.c b/mypyc/lib-rt/getargsfast.c index 387deed4399b..62d0dfed0a6d 100644 --- a/mypyc/lib-rt/getargsfast.c +++ b/mypyc/lib-rt/getargsfast.c @@ -271,9 +271,16 @@ find_keyword(PyObject *kwnames, PyObject *const *kwstack, PyObject *key) for (i = 0; i < nkwargs; i++) { PyObject *kwname = PyTuple_GET_ITEM(kwnames, i); assert(PyUnicode_Check(kwname)); +#if CPY_3_13_FEATURES + if (_PyUnicode_Equal(kwname, key)) { + return kwstack[i]; + } +#else if (_PyUnicode_EQ(kwname, key)) { return kwstack[i]; } +#endif + } return NULL; } From 42337a0ca0923ce9cf7492c6d847a945a69ad7a6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 14 Jul 2024 13:02:43 +0100 Subject: [PATCH 30/36] [mypyc] Don't use _PyUnicode_FastCopyCharacters on 3.13 (#17524) Work on mypyc/mypyc#1056. --- mypyc/lib-rt/str_ops.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mypyc/lib-rt/str_ops.c b/mypyc/lib-rt/str_ops.c index 90b19001f8f0..4ba181bcce85 100644 --- a/mypyc/lib-rt/str_ops.c +++ b/mypyc/lib-rt/str_ops.c @@ -117,7 +117,11 @@ PyObject *CPyStr_Build(Py_ssize_t len, ...) { PyObject *item = va_arg(args, PyObject *); Py_ssize_t itemlen = PyUnicode_GET_LENGTH(item); if (itemlen != 0) { +#if CPY_3_13_FEATURES + PyUnicode_CopyCharacters(res, res_offset, item, 0, itemlen); +#else _PyUnicode_FastCopyCharacters(res, res_offset, item, 0, itemlen); +#endif res_offset += itemlen; } } From 6a0657e5959ba1777c4d427f8f355d499035d145 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 14 Jul 2024 13:02:51 +0100 Subject: [PATCH 31/36] [mypyc] Don't rely on _PyType_CalculateMetaclass on 3.13 (#17525) Copy the implementation from CPython (with minor changes), as it's no longer exported. Work on mypyc/mypyc#1056. --- mypyc/lib-rt/CPy.h | 5 +---- mypyc/lib-rt/misc_ops.c | 48 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 2ec04e4c5b5c..833b1bd2e76a 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -846,10 +846,7 @@ static inline bool CPy_TypeCheck(PyObject *o, PyObject *type) { return PyObject_TypeCheck(o, (PyTypeObject *)type); } -static inline PyObject *CPy_CalculateMetaclass(PyObject *type, PyObject *o) { - return (PyObject *)_PyType_CalculateMetaclass((PyTypeObject *)type, o); -} - +PyObject *CPy_CalculateMetaclass(PyObject *type, PyObject *o); PyObject *CPy_GetCoro(PyObject *obj); PyObject *CPyIter_Send(PyObject *iter, PyObject *val); int CPy_YieldFromErrorHandle(PyObject *iter, PyObject **outp); diff --git a/mypyc/lib-rt/misc_ops.c b/mypyc/lib-rt/misc_ops.c index 803123d436a2..1572c4496e30 100644 --- a/mypyc/lib-rt/misc_ops.c +++ b/mypyc/lib-rt/misc_ops.c @@ -131,6 +131,52 @@ static bool _CPy_IsSafeMetaClass(PyTypeObject *metaclass) { return matches; } +#if CPY_3_13_FEATURES + +// Adapted from CPython 3.13.0b3 +/* Determine the most derived metatype. */ +PyObject *CPy_CalculateMetaclass(PyObject *metatype, PyObject *bases) +{ + Py_ssize_t i, nbases; + PyTypeObject *winner; + PyObject *tmp; + PyTypeObject *tmptype; + + /* Determine the proper metatype to deal with this, + and check for metatype conflicts while we're at it. + Note that if some other metatype wins to contract, + it's possible that its instances are not types. */ + + nbases = PyTuple_GET_SIZE(bases); + winner = (PyTypeObject *)metatype; + for (i = 0; i < nbases; i++) { + tmp = PyTuple_GET_ITEM(bases, i); + tmptype = Py_TYPE(tmp); + if (PyType_IsSubtype(winner, tmptype)) + continue; + if (PyType_IsSubtype(tmptype, winner)) { + winner = tmptype; + continue; + } + /* else: */ + PyErr_SetString(PyExc_TypeError, + "metaclass conflict: " + "the metaclass of a derived class " + "must be a (non-strict) subclass " + "of the metaclasses of all its bases"); + return NULL; + } + return (PyObject *)winner; +} + +#else + +PyObject *CPy_CalculateMetaclass(PyObject *metatype, PyObject *bases) { + return (PyObject *)_PyType_CalculateMetaclass((PyTypeObject *)metatype, bases); +} + +#endif + // Create a heap type based on a template non-heap type. // This is super hacky and maybe we should suck it up and use PyType_FromSpec instead. // We allow bases to be NULL to represent just inheriting from object. @@ -163,7 +209,7 @@ PyObject *CPyType_FromTemplate(PyObject *template, // Find the appropriate metaclass from our base classes. We // care about this because Generic uses a metaclass prior to // Python 3.7. - metaclass = _PyType_CalculateMetaclass(metaclass, bases); + metaclass = (PyTypeObject *)CPy_CalculateMetaclass((PyObject *)metaclass, bases); if (!metaclass) goto error; From e062793d17a67766e6a44fc4095eb72e55e50d7c Mon Sep 17 00:00:00 2001 From: Max Murin Date: Fri, 19 Jul 2024 05:07:41 -0700 Subject: [PATCH 32/36] CHANGELOG.md update for 1.11 (#17539) Add a changelog for the 1.11 release. --- CHANGELOG.md | 168 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d5919cafe33..196a75992c24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,174 @@ ## Next release +## Mypy 1.11 + +We’ve just uploaded mypy 1.11 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). Mypy is a static type checker for Python. This release includes new features, performance improvements and bug fixes. You can install it as follows: + + python3 -m pip install -U mypy + +You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). + +#### Additional support for PEP 695 + +Mypy now has experimental support for the new type parameter syntax introduced in Python 3.12 ([PEP 695](https://peps.python.org/pep-0695/)). +This feature is still experimental and must be enabled with the `--enable-incomplete-feature=NewGenericSyntax` flag. + +This example demonstrates the new syntax: +```python + +def f[T](x: T) -> T: ... + +reveal_type(f(1)) # Revealed type is 'int' +``` + +This feature was contributed by Jukka Lehtosalo (PR [17233](https://github.com/python/mypy/pull/17233)). + + +#### Support for `functools.partial` + +Mypy now typechecks uses of `functools.partial`, which previous mypy would always accept. +This example would previously pass: + +```python +from functools import partial + +def f(a: int, b: str) -> None: ... + +g = partial(f, 1) +g(1) # error: Argument 1 to "f" has incompatible type "int"; expected "str" [arg-type] +``` + +This feature was contributed by Shantanu (PR [16939](https://github.com/python/mypy/pull/16939)). + + +#### Changes to stubtest + * Stubtest: ignore `_ios_support` (Alex Waygood, PR [17270](https://github.com/python/mypy/pull/17270)) + * stubtest: changes for py313 (Shantanu, PR [17261](https://github.com/python/mypy/pull/17261)) + + +#### Changes to stubgen + * stubgen: Gracefully handle invalid `Optional` and recognize aliases to PEP 604 unions (Ali Hamdan, PR [17386](https://github.com/python/mypy/pull/17386)) + * Fix stubgen for Python 3.13 (Jelle Zijlstra, PR [17290](https://github.com/python/mypy/pull/17290)) + * stubgen: preserve enum value initialisers (Shantanu, PR [17125](https://github.com/python/mypy/pull/17125)) + + +#### Changes to mypyc + * [mypyc] Sync pythoncapi_compat.h (Jukka Lehtosalo, PR [17390](https://github.com/python/mypy/pull/17390)) + * [mypyc] Support Python 3.12 type alias syntax (PEP 695) (Jukka Lehtosalo, PR [17384](https://github.com/python/mypy/pull/17384)) + * [mypyc] Support new syntax for generic functions and classes (PEP 695) (Jukka Lehtosalo, PR [17357](https://github.com/python/mypy/pull/17357)) + * [mypyc] Fix ParamSpec (Shantanu, PR [17309](https://github.com/python/mypy/pull/17309)) + * [mypyc] Inline fast paths of integer unboxing operations (Jukka Lehtosalo, PR [17266](https://github.com/python/mypy/pull/17266)) + * [mypyc] Inline tagged integer arithmetic and bitwise operations (Jukka Lehtosalo, PR [17265](https://github.com/python/mypy/pull/17265)) + * [mypyc] Allow specifying primitives as pure (Jukka Lehtosalo, PR [17263](https://github.com/python/mypy/pull/17263)) + + +#### Changes to error reporting + * Do not report plugin-generated methods with `explicit-override` (sobolevn, PR [17433](https://github.com/python/mypy/pull/17433)) + * Fix explicit type for partial (Ivan Levkivskyi, PR [17424](https://github.com/python/mypy/pull/17424)) + * Re-work overload overlap logic (Ivan Levkivskyi, PR [17392](https://github.com/python/mypy/pull/17392)) + * Use namespaces for function type variables (Ivan Levkivskyi, PR [17311](https://github.com/python/mypy/pull/17311)) + * Fix false positive for Final local scope variable in Protocol (GiorgosPapoutsakis, PR [17308](https://github.com/python/mypy/pull/17308)) + * Use Never in more messages, use ambiguous in join (Shantanu, PR [17304](https://github.com/python/mypy/pull/17304)) + * Log full path to config file in verbose output (dexterkennedy, PR [17180](https://github.com/python/mypy/pull/17180)) + * Added [prop-decorator] code for unsupported property decorators (#14461) (Christopher Barber, PR [16571](https://github.com/python/mypy/pull/16571)) + * Suppress second error message with `:=` and `[truthy-bool]` (Nikita Sobolev, PR [15941](https://github.com/python/mypy/pull/15941)) + * Error for assignment of functional Enum to variable of different name (Shantanu, PR [16805](https://github.com/python/mypy/pull/16805)) + * Add Error format support, and JSON output option (Tushar Sadhwani, PR [11396](https://github.com/python/mypy/pull/11396)) + + +#### Fixes for crashes + * Fix daemon crash on invalid type in TypedDict (Ivan Levkivskyi, PR [17495](https://github.com/python/mypy/pull/17495)) + * Some cleanup in partial plugin (Ivan Levkivskyi, PR [17423](https://github.com/python/mypy/pull/17423)) + * Fix crash when overriding with unpacked TypedDict (Ivan Levkivskyi, PR [17359](https://github.com/python/mypy/pull/17359)) + * Fix crash on TypedDict unpacking for ParamSpec (Ivan Levkivskyi, PR [17358](https://github.com/python/mypy/pull/17358)) + * Fix crash involving recursive union of tuples (Ivan Levkivskyi, PR [17353](https://github.com/python/mypy/pull/17353)) + * Fix crash on invalid callable property override (Ivan Levkivskyi, PR [17352](https://github.com/python/mypy/pull/17352)) + * Fix crash on unpacking self in NamedTuple (Ivan Levkivskyi, PR [17351](https://github.com/python/mypy/pull/17351)) + * Fix crash on recursive alias with an optional type (Ivan Levkivskyi, PR [17350](https://github.com/python/mypy/pull/17350)) + * Fix type comments crash inside generic definitions (Bénédikt Tran, PR [16849](https://github.com/python/mypy/pull/16849)) + + +#### Changes to documentation + * Mention --enable-incomplete-feature=NewGenericSyntax (Shantanu, PR [17462](https://github.com/python/mypy/pull/17462)) + * Use inline config in the optional error codes docs (Shantanu, PR [17374](https://github.com/python/mypy/pull/17374)) + * docs: Use lower-case generics (Seo Sanghyeon, PR [17176](https://github.com/python/mypy/pull/17176)) + * Add documentation for show-error-code-links (GiorgosPapoutsakis, PR [17144](https://github.com/python/mypy/pull/17144)) + * Update CONTRIBUTING.md to include commands for Windows (GiorgosPapoutsakis, PR [17142](https://github.com/python/mypy/pull/17142)) + + +#### Other notable contributions + * Fix ParamSpec inference against TypeVarTuple (Ivan Levkivskyi, PR [17431](https://github.com/python/mypy/pull/17431)) + * Always allow lambda calls (Ivan Levkivskyi, PR [17430](https://github.com/python/mypy/pull/17430)) + * Fix error reporting on cached run after uninstallation of third party library (Shantanu, PR [17420](https://github.com/python/mypy/pull/17420)) + * Fix isinstance checks with PEP 604 unions containing None (Shantanu, PR [17415](https://github.com/python/mypy/pull/17415)) + * Use (simplified) unions instead of joins for tuple fallbacks (Ivan Levkivskyi, PR [17408](https://github.com/python/mypy/pull/17408)) + * Fix self-referential upper bound in new-style type variables (Ivan Levkivskyi, PR [17407](https://github.com/python/mypy/pull/17407)) + * Consider overlap between instances and callables (Ivan Levkivskyi, PR [17389](https://github.com/python/mypy/pull/17389)) + * Support `enum.member` for python3.11+ (Nikita Sobolev, PR [17382](https://github.com/python/mypy/pull/17382)) + * Allow new-style self-types in classmethods (Ivan Levkivskyi, PR [17381](https://github.com/python/mypy/pull/17381)) + * Support `enum.nonmember` for python3.11+ (Nikita Sobolev, PR [17376](https://github.com/python/mypy/pull/17376)) + * Fix isinstance with type aliases to PEP 604 unions (Shantanu, PR [17371](https://github.com/python/mypy/pull/17371)) + * Properly handle unpacks in overlap checks (Ivan Levkivskyi, PR [17356](https://github.com/python/mypy/pull/17356)) + * Fix type application for classes with generic constructors (Ivan Levkivskyi, PR [17354](https://github.com/python/mypy/pull/17354)) + * Use polymorphic inference in unification (Ivan Levkivskyi, PR [17348](https://github.com/python/mypy/pull/17348)) + * Update 'typing_extensions' to >=4.6.0 to fix python 3.12 error (Ben Brown, PR [17312](https://github.com/python/mypy/pull/17312)) + * Avoid does not return error in lambda (Shantanu, PR [17294](https://github.com/python/mypy/pull/17294)) + * Fix for bug with descriptors in non-strict-optional (Max Murin, PR [17293](https://github.com/python/mypy/pull/17293)) + * Don’t leak unreachability from lambda body to surrounding scope (Anders Kaseorg, PR [17287](https://github.com/python/mypy/pull/17287)) + * Validate more about overrides on untyped methods (Steven Troxler, PR [17276](https://github.com/python/mypy/pull/17276)) + * Fix case involving non-ASCII chars on Windows (Alexander Leopold Shon, PR [17275](https://github.com/python/mypy/pull/17275)) + * Support namedtuple.__replace__ in Python 3.13 (Shantanu, PR [17259](https://github.com/python/mypy/pull/17259)) + * Fix for type narrowing of negative integer literals (gilesgc, PR [17256](https://github.com/python/mypy/pull/17256)) + * Support rename=True in collections.namedtuple (Jelle Zijlstra, PR [17247](https://github.com/python/mypy/pull/17247)) + * [dmypy] sort list of files for update by extension (Valentin Stanciu, PR [17245](https://github.com/python/mypy/pull/17245)) + * fix #16935 fix type of tuple[X,Y] expression (urnest, PR [17235](https://github.com/python/mypy/pull/17235)) + * Do not forget that a `TypedDict` was wrapped in `Unpack` after a `name-defined` error occurred. (Christoph Tyralla, PR [17226](https://github.com/python/mypy/pull/17226)) + * fix: annotated argument's `var` node type is explicit, not inferred (bzoracler, PR [17217](https://github.com/python/mypy/pull/17217)) + * Enum private attributes are not enum members (Ali Hamdan, PR [17182](https://github.com/python/mypy/pull/17182)) + * Fix Literal strings containing pipe characters (Jelle Zijlstra, PR [17148](https://github.com/python/mypy/pull/17148)) + * Add support for __spec__ (Shantanu, PR [14739](https://github.com/python/mypy/pull/14739)) + + +#### Typeshed Updates + +Please see [git log](https://github.com/python/typeshed/commits/main?after=6dda799d8ad1d89e0f8aad7ac41d2d34bd838ace+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. + + +#### Acknowledgements +Thanks to all mypy contributors who contributed to this release: + +- Alex Waygood +- Alexander Leopold Shon +- Ali Hamdan +- Anders Kaseorg +- Ben Brown +- Bénédikt Tran +- bzoracler +- Christoph Tyralla +- Christopher Barber +- dexterkennedy +- gilesgc +- GiorgosPapoutsakis +- Ivan Levkivskyi +- Jelle Zijlstra +- Jukka Lehtosalo +- Marc Mueller +- Matthieu Devlin +- Michael R. Crusoe +- Nikita Sobolev +- Seo Sanghyeon +- Shantanu +- sobolevn +- Steven Troxler +- Tadeu Manoel +- Tamir Duberstein +- Tushar Sadhwani +- urnest +- Valentin Stanciu + +I’d also like to thank my employer, Dropbox, for supporting mypy development. + ## Mypy 1.10 From 0753e2a82dad35034e000609b6e8daa37238bfaa Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 19 Jul 2024 16:49:09 +0100 Subject: [PATCH 33/36] Update CHANGELOG for mypy 1.11 (#17540) Added additional sections for major features and did various other updates. --- CHANGELOG.md | 188 +++++++++++++++++++++++++++++++++++---------------- 1 file changed, 131 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 196a75992c24..b544e05ee573 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,26 +11,40 @@ We’ve just uploaded mypy 1.11 to the Python Package Index ([PyPI](https://pypi You can read the full documentation for this release on [Read the Docs](http://mypy.readthedocs.io). -#### Additional support for PEP 695 +#### Support Python 3.12 Syntax for Generics (PEP 695) -Mypy now has experimental support for the new type parameter syntax introduced in Python 3.12 ([PEP 695](https://peps.python.org/pep-0695/)). -This feature is still experimental and must be enabled with the `--enable-incomplete-feature=NewGenericSyntax` flag. +Mypy now supports the new type parameter syntax introduced in Python 3.12 ([PEP 695](https://peps.python.org/pep-0695/)). +This feature is still experimental and must be enabled with the `--enable-incomplete-feature=NewGenericSyntax` flag, or with `enable_incomplete_feature = NewGenericSyntax` in the mypy configuration file. +We plan to enable this by default in the next mypy feature release. This example demonstrates the new syntax: -```python +```python +# Generic function def f[T](x: T) -> T: ... reveal_type(f(1)) # Revealed type is 'int' + +# Generic class +class C[T]: + def __init__(self, x: T) -> None: + self.x = x + +c = C('a') +reveal_type(c.x) # Revealed type is 'str' + +# Type alias +type A[T] = C[list[T]] ``` -This feature was contributed by Jukka Lehtosalo (PR [17233](https://github.com/python/mypy/pull/17233)). +This feature was contributed by Jukka Lehtosalo. #### Support for `functools.partial` -Mypy now typechecks uses of `functools.partial`, which previous mypy would always accept. -This example would previously pass: +Mypy now type checks uses of `functools.partial`. Previously mypy would accept arbitrary arguments. + +This example will now produce an error: ```python from functools import partial @@ -38,98 +52,158 @@ from functools import partial def f(a: int, b: str) -> None: ... g = partial(f, 1) -g(1) # error: Argument 1 to "f" has incompatible type "int"; expected "str" [arg-type] + +# Argument has incompatible type "int"; expected "str" +g(11) ``` This feature was contributed by Shantanu (PR [16939](https://github.com/python/mypy/pull/16939)). -#### Changes to stubtest - * Stubtest: ignore `_ios_support` (Alex Waygood, PR [17270](https://github.com/python/mypy/pull/17270)) - * stubtest: changes for py313 (Shantanu, PR [17261](https://github.com/python/mypy/pull/17261)) +#### Stricter Checks for Untyped Overrides + +Past mypy versions didn't check if untyped methods were compatible with overridden methods. This would result in false negatives. Now mypy performs these checks when using `--check-untyped-defs`. + +For example, this now generates an error if using `--check-untyped-defs`: + +```python +class Base: + def f(self, x: int = 0) -> None: ... + +class Derived(Base): + # Signature incompatible with "Base" + def f(self): ... +``` + +This feature was contributed by Steven Troxler (PR [17276](https://github.com/python/mypy/pull/17276)). + + +#### Type Inference Improvements + +The new polymorphic inference algorithm introduced in mypy 1.5 is now used in more situations. This improves type inference involving generic higher-order functions, in particular. + +This feature was contributed by Ivan Levkivskyi (PR [17348](https://github.com/python/mypy/pull/17348)). + +Mypy now uses unions of tuple item types in certain contexts to enable more precise inferred types. Example: + +```python +for x in (1, 'x'): + # Previously inferred as 'object' + reveal_type(x) # Revealed type is 'int | str' +``` + +This was also contributed by Ivan Levkivskyi (PR [17408](https://github.com/python/mypy/pull/17408)). + +#### Improvements to Detection of Overlapping Overloads -#### Changes to stubgen - * stubgen: Gracefully handle invalid `Optional` and recognize aliases to PEP 604 unions (Ali Hamdan, PR [17386](https://github.com/python/mypy/pull/17386)) - * Fix stubgen for Python 3.13 (Jelle Zijlstra, PR [17290](https://github.com/python/mypy/pull/17290)) - * stubgen: preserve enum value initialisers (Shantanu, PR [17125](https://github.com/python/mypy/pull/17125)) +The details of how mypy checks if two `@overload` signatures are unsafely overlapping were overhauled. This both fixes some false positives, and allows mypy to detect additional unsafe signatures. +This feature was contributed by Ivan Levkivskyi (PR [17392](https://github.com/python/mypy/pull/17392)). -#### Changes to mypyc - * [mypyc] Sync pythoncapi_compat.h (Jukka Lehtosalo, PR [17390](https://github.com/python/mypy/pull/17390)) - * [mypyc] Support Python 3.12 type alias syntax (PEP 695) (Jukka Lehtosalo, PR [17384](https://github.com/python/mypy/pull/17384)) - * [mypyc] Support new syntax for generic functions and classes (PEP 695) (Jukka Lehtosalo, PR [17357](https://github.com/python/mypy/pull/17357)) - * [mypyc] Fix ParamSpec (Shantanu, PR [17309](https://github.com/python/mypy/pull/17309)) - * [mypyc] Inline fast paths of integer unboxing operations (Jukka Lehtosalo, PR [17266](https://github.com/python/mypy/pull/17266)) - * [mypyc] Inline tagged integer arithmetic and bitwise operations (Jukka Lehtosalo, PR [17265](https://github.com/python/mypy/pull/17265)) - * [mypyc] Allow specifying primitives as pure (Jukka Lehtosalo, PR [17263](https://github.com/python/mypy/pull/17263)) +#### Better Support for Type Hints in Expressions -#### Changes to error reporting +Mypy now allows more expressions that evaluate to valid type annotations in all expression contexts. The inferred types of these expressions are also sometimes more precise. Previously they were often `object`. + +This example uses a union type that includes a callable type as an expression, and it no longer generates an error: + +```python +from typing import Callable + +print(Callable[[], int] | None) # No error +``` + +This feature was contributed by Jukka Lehtosalo (PR [17404](https://github.com/python/mypy/pull/17404)). + + +#### Mypyc Improvements + +Mypyc now supports the new syntax for generics introduced in Python 3.12 (see above). Another notable improvement is signficantly faster basic operations on `int` values. + + * Support Python 3.12 syntax for generic functions and classes (Jukka Lehtosalo, PR [17357](https://github.com/python/mypy/pull/17357)) + * Support Python 3.12 type alias syntax (Jukka Lehtosalo, PR [17384](https://github.com/python/mypy/pull/17384)) + * Fix ParamSpec (Shantanu, PR [17309](https://github.com/python/mypy/pull/17309)) + * Inline fast paths of integer unboxing operations (Jukka Lehtosalo, PR [17266](https://github.com/python/mypy/pull/17266)) + * Inline tagged integer arithmetic and bitwise operations (Jukka Lehtosalo, PR [17265](https://github.com/python/mypy/pull/17265)) + * Allow specifying primitives as pure (Jukka Lehtosalo, PR [17263](https://github.com/python/mypy/pull/17263)) + + +#### Changes to Stubtest + * Ignore `_ios_support` (Alex Waygood, PR [17270](https://github.com/python/mypy/pull/17270)) + * Improve support for Python 3.13 (Shantanu, PR [17261](https://github.com/python/mypy/pull/17261)) + + +#### Changes to Stubgen + * Gracefully handle invalid `Optional` and recognize aliases to PEP 604 unions (Ali Hamdan, PR [17386](https://github.com/python/mypy/pull/17386)) + * Fix for Python 3.13 (Jelle Zijlstra, PR [17290](https://github.com/python/mypy/pull/17290)) + * Preserve enum value initialisers (Shantanu, PR [17125](https://github.com/python/mypy/pull/17125)) + + +#### Miscellaneous New Features + * Add error format support and JSON output option via `--output json` (Tushar Sadhwani, PR [11396](https://github.com/python/mypy/pull/11396)) + * Support `enum.member` in Python 3.11+ (Nikita Sobolev, PR [17382](https://github.com/python/mypy/pull/17382)) + * Support `enum.nonmember` in Python 3.11+ (Nikita Sobolev, PR [17376](https://github.com/python/mypy/pull/17376)) + * Support `namedtuple.__replace__` in Python 3.13 (Shantanu, PR [17259](https://github.com/python/mypy/pull/17259)) + * Support `rename=True` in collections.namedtuple (Jelle Zijlstra, PR [17247](https://github.com/python/mypy/pull/17247)) + * Add support for `__spec__` (Shantanu, PR [14739](https://github.com/python/mypy/pull/14739)) + + +#### Changes to Error Reporting + * Mention `--enable-incomplete-feature=NewGenericSyntax` in messages (Shantanu, PR [17462](https://github.com/python/mypy/pull/17462)) * Do not report plugin-generated methods with `explicit-override` (sobolevn, PR [17433](https://github.com/python/mypy/pull/17433)) - * Fix explicit type for partial (Ivan Levkivskyi, PR [17424](https://github.com/python/mypy/pull/17424)) - * Re-work overload overlap logic (Ivan Levkivskyi, PR [17392](https://github.com/python/mypy/pull/17392)) - * Use namespaces for function type variables (Ivan Levkivskyi, PR [17311](https://github.com/python/mypy/pull/17311)) + * Use and display namespaces for function type variables (Ivan Levkivskyi, PR [17311](https://github.com/python/mypy/pull/17311)) * Fix false positive for Final local scope variable in Protocol (GiorgosPapoutsakis, PR [17308](https://github.com/python/mypy/pull/17308)) * Use Never in more messages, use ambiguous in join (Shantanu, PR [17304](https://github.com/python/mypy/pull/17304)) * Log full path to config file in verbose output (dexterkennedy, PR [17180](https://github.com/python/mypy/pull/17180)) - * Added [prop-decorator] code for unsupported property decorators (#14461) (Christopher Barber, PR [16571](https://github.com/python/mypy/pull/16571)) + * Added `[prop-decorator]` code for unsupported property decorators (#14461) (Christopher Barber, PR [16571](https://github.com/python/mypy/pull/16571)) * Suppress second error message with `:=` and `[truthy-bool]` (Nikita Sobolev, PR [15941](https://github.com/python/mypy/pull/15941)) - * Error for assignment of functional Enum to variable of different name (Shantanu, PR [16805](https://github.com/python/mypy/pull/16805)) - * Add Error format support, and JSON output option (Tushar Sadhwani, PR [11396](https://github.com/python/mypy/pull/11396)) + * Generate error for assignment of functional Enum to variable of different name (Shantanu, PR [16805](https://github.com/python/mypy/pull/16805)) + * Fix error reporting on cached run after uninstallation of third party library (Shantanu, PR [17420](https://github.com/python/mypy/pull/17420)) -#### Fixes for crashes +#### Fixes for Crashes * Fix daemon crash on invalid type in TypedDict (Ivan Levkivskyi, PR [17495](https://github.com/python/mypy/pull/17495)) - * Some cleanup in partial plugin (Ivan Levkivskyi, PR [17423](https://github.com/python/mypy/pull/17423)) + * Fix crash and bugs related to `partial()` (Ivan Levkivskyi, PR [17423](https://github.com/python/mypy/pull/17423)) * Fix crash when overriding with unpacked TypedDict (Ivan Levkivskyi, PR [17359](https://github.com/python/mypy/pull/17359)) * Fix crash on TypedDict unpacking for ParamSpec (Ivan Levkivskyi, PR [17358](https://github.com/python/mypy/pull/17358)) * Fix crash involving recursive union of tuples (Ivan Levkivskyi, PR [17353](https://github.com/python/mypy/pull/17353)) * Fix crash on invalid callable property override (Ivan Levkivskyi, PR [17352](https://github.com/python/mypy/pull/17352)) * Fix crash on unpacking self in NamedTuple (Ivan Levkivskyi, PR [17351](https://github.com/python/mypy/pull/17351)) * Fix crash on recursive alias with an optional type (Ivan Levkivskyi, PR [17350](https://github.com/python/mypy/pull/17350)) - * Fix type comments crash inside generic definitions (Bénédikt Tran, PR [16849](https://github.com/python/mypy/pull/16849)) + * Fix crash on type comment inside generic definitions (Bénédikt Tran, PR [16849](https://github.com/python/mypy/pull/16849)) -#### Changes to documentation - * Mention --enable-incomplete-feature=NewGenericSyntax (Shantanu, PR [17462](https://github.com/python/mypy/pull/17462)) - * Use inline config in the optional error codes docs (Shantanu, PR [17374](https://github.com/python/mypy/pull/17374)) - * docs: Use lower-case generics (Seo Sanghyeon, PR [17176](https://github.com/python/mypy/pull/17176)) +#### Changes to Documentation + * Use inline config in documentation for optional error codes (Shantanu, PR [17374](https://github.com/python/mypy/pull/17374)) + * Use lower-case generics in documentation (Seo Sanghyeon, PR [17176](https://github.com/python/mypy/pull/17176)) * Add documentation for show-error-code-links (GiorgosPapoutsakis, PR [17144](https://github.com/python/mypy/pull/17144)) * Update CONTRIBUTING.md to include commands for Windows (GiorgosPapoutsakis, PR [17142](https://github.com/python/mypy/pull/17142)) -#### Other notable contributions +#### Other Notable Improvements and Fixes * Fix ParamSpec inference against TypeVarTuple (Ivan Levkivskyi, PR [17431](https://github.com/python/mypy/pull/17431)) + * Fix explicit type for `partial` (Ivan Levkivskyi, PR [17424](https://github.com/python/mypy/pull/17424)) * Always allow lambda calls (Ivan Levkivskyi, PR [17430](https://github.com/python/mypy/pull/17430)) - * Fix error reporting on cached run after uninstallation of third party library (Shantanu, PR [17420](https://github.com/python/mypy/pull/17420)) * Fix isinstance checks with PEP 604 unions containing None (Shantanu, PR [17415](https://github.com/python/mypy/pull/17415)) - * Use (simplified) unions instead of joins for tuple fallbacks (Ivan Levkivskyi, PR [17408](https://github.com/python/mypy/pull/17408)) * Fix self-referential upper bound in new-style type variables (Ivan Levkivskyi, PR [17407](https://github.com/python/mypy/pull/17407)) * Consider overlap between instances and callables (Ivan Levkivskyi, PR [17389](https://github.com/python/mypy/pull/17389)) - * Support `enum.member` for python3.11+ (Nikita Sobolev, PR [17382](https://github.com/python/mypy/pull/17382)) * Allow new-style self-types in classmethods (Ivan Levkivskyi, PR [17381](https://github.com/python/mypy/pull/17381)) - * Support `enum.nonmember` for python3.11+ (Nikita Sobolev, PR [17376](https://github.com/python/mypy/pull/17376)) * Fix isinstance with type aliases to PEP 604 unions (Shantanu, PR [17371](https://github.com/python/mypy/pull/17371)) * Properly handle unpacks in overlap checks (Ivan Levkivskyi, PR [17356](https://github.com/python/mypy/pull/17356)) * Fix type application for classes with generic constructors (Ivan Levkivskyi, PR [17354](https://github.com/python/mypy/pull/17354)) - * Use polymorphic inference in unification (Ivan Levkivskyi, PR [17348](https://github.com/python/mypy/pull/17348)) - * Update 'typing_extensions' to >=4.6.0 to fix python 3.12 error (Ben Brown, PR [17312](https://github.com/python/mypy/pull/17312)) - * Avoid does not return error in lambda (Shantanu, PR [17294](https://github.com/python/mypy/pull/17294)) - * Fix for bug with descriptors in non-strict-optional (Max Murin, PR [17293](https://github.com/python/mypy/pull/17293)) + * Update `typing_extensions` to >=4.6.0 to fix Python 3.12 error (Ben Brown, PR [17312](https://github.com/python/mypy/pull/17312)) + * Avoid "does not return" error in lambda (Shantanu, PR [17294](https://github.com/python/mypy/pull/17294)) + * Fix bug with descriptors in non-strict-optional mode (Max Murin, PR [17293](https://github.com/python/mypy/pull/17293)) * Don’t leak unreachability from lambda body to surrounding scope (Anders Kaseorg, PR [17287](https://github.com/python/mypy/pull/17287)) - * Validate more about overrides on untyped methods (Steven Troxler, PR [17276](https://github.com/python/mypy/pull/17276)) - * Fix case involving non-ASCII chars on Windows (Alexander Leopold Shon, PR [17275](https://github.com/python/mypy/pull/17275)) - * Support namedtuple.__replace__ in Python 3.13 (Shantanu, PR [17259](https://github.com/python/mypy/pull/17259)) + * Fix issues with non-ASCII characters on Windows (Alexander Leopold Shon, PR [17275](https://github.com/python/mypy/pull/17275)) * Fix for type narrowing of negative integer literals (gilesgc, PR [17256](https://github.com/python/mypy/pull/17256)) - * Support rename=True in collections.namedtuple (Jelle Zijlstra, PR [17247](https://github.com/python/mypy/pull/17247)) - * [dmypy] sort list of files for update by extension (Valentin Stanciu, PR [17245](https://github.com/python/mypy/pull/17245)) - * fix #16935 fix type of tuple[X,Y] expression (urnest, PR [17235](https://github.com/python/mypy/pull/17235)) - * Do not forget that a `TypedDict` was wrapped in `Unpack` after a `name-defined` error occurred. (Christoph Tyralla, PR [17226](https://github.com/python/mypy/pull/17226)) - * fix: annotated argument's `var` node type is explicit, not inferred (bzoracler, PR [17217](https://github.com/python/mypy/pull/17217)) - * Enum private attributes are not enum members (Ali Hamdan, PR [17182](https://github.com/python/mypy/pull/17182)) + * Fix confusion between .py and .pyi files in mypy daemon (Valentin Stanciu, PR [17245](https://github.com/python/mypy/pull/17245)) + * Fix type of `tuple[X, Y]` expression (urnest, PR [17235](https://github.com/python/mypy/pull/17235)) + * Don't forget that a `TypedDict` was wrapped in `Unpack` after a `name-defined` error occurred (Christoph Tyralla, PR [17226](https://github.com/python/mypy/pull/17226)) + * Mark annotated argument as having an explicit, not inferred type (bzoracler, PR [17217](https://github.com/python/mypy/pull/17217)) + * Don't consider Enum private attributes as enum members (Ali Hamdan, PR [17182](https://github.com/python/mypy/pull/17182)) * Fix Literal strings containing pipe characters (Jelle Zijlstra, PR [17148](https://github.com/python/mypy/pull/17148)) - * Add support for __spec__ (Shantanu, PR [14739](https://github.com/python/mypy/pull/14739)) #### Typeshed Updates @@ -331,7 +405,7 @@ This feature was contributed by Shantanu (PR [16756](https://github.com/python/m #### Typeshed Updates -Please see [git log](https://github.com/python/typeshed/commits/main?after=7c8e82fe483a40ec4cb0a2505cfdb0f3e7cc81d9+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. +Please see [git log](https://github.com/python/typeshed/commits/main?after=6dda799d8ad1d89e0f8aad7ac41d2d34bd838ace+0&branch=main&path=stdlib) for full list of standard library typeshed stub changes. #### Mypy 1.10.1 From 0a040dd25dc1d3847994574152932112369c0840 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Fri, 19 Jul 2024 23:22:53 -0400 Subject: [PATCH 34/36] Fix types.GenericAlias lookup crash (#17543) Fixes #17542 --- mypy/checkexpr.py | 2 +- test-data/unit/check-functions.test | 20 +++++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 3532e18b93b2..2e8e6e2db9d5 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -4341,7 +4341,7 @@ def visit_index_with_type( elif isinstance(left_type, FunctionLike) and left_type.is_type_obj(): if left_type.type_object().is_enum: return self.visit_enum_index_expr(left_type.type_object(), e.index, e) - elif left_type.type_object().type_vars: + elif left_type.type_object().type_vars and self.chk.options.python_version >= (3, 9): return self.named_type("types.GenericAlias") elif ( left_type.type_object().fullname == "builtins.type" diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index 6c895c86e899..ef5a66b6ecde 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -1779,6 +1779,7 @@ def Arg(x, y): pass F = Callable[[Arg(int, 'x')], int] # E: Invalid argument constructor "__main__.Arg" [case testCallableParsingFromExpr] +# flags: --python-version 3.9 from typing import Callable, List from mypy_extensions import Arg, VarArg, KwArg import mypy_extensions @@ -1799,10 +1800,23 @@ L = Callable[[Arg(name='x', type=int)], int] # ok # I have commented out the following test because I don't know how to expect the "defined here" note part of the error. # M = Callable[[Arg(gnome='x', type=int)], int] E: Invalid type alias: expression is not a valid type E: Unexpected keyword argument "gnome" for "Arg" N = Callable[[Arg(name=None, type=int)], int] # ok -O = Callable[[List[Arg(int)]], int] # E: Invalid type alias: expression is not a valid type # E: Value of type "int" is not indexable # E: Type expected within [...] +O = Callable[[List[Arg(int)]], int] # E: Invalid type alias: expression is not a valid type \ + # E: Value of type "int" is not indexable \ + # E: Type expected within [...] P = Callable[[mypy_extensions.VarArg(int)], int] # ok -Q = Callable[[Arg(int, type=int)], int] # E: Invalid type alias: expression is not a valid type # E: Value of type "int" is not indexable # E: "Arg" gets multiple values for keyword argument "type" -R = Callable[[Arg(int, 'x', name='y')], int] # E: Invalid type alias: expression is not a valid type # E: Value of type "int" is not indexable # E: "Arg" gets multiple values for keyword argument "name" +Q = Callable[[Arg(int, type=int)], int] # E: Invalid type alias: expression is not a valid type \ + # E: Value of type "int" is not indexable \ + # E: "Arg" gets multiple values for keyword argument "type" +R = Callable[[Arg(int, 'x', name='y')], int] # E: Invalid type alias: expression is not a valid type \ + # E: Value of type "int" is not indexable \ + # E: "Arg" gets multiple values for keyword argument "name" + + + + + + + [builtins fixtures/dict.pyi] [case testCallableParsing] From 0b09116baae650441e4038b9cd486c7583d47f2a Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sat, 20 Jul 2024 02:39:32 -0400 Subject: [PATCH 35/36] Indexing a type also produces a GenericAlias (#17546) Mentioned by Jelle in https://github.com/python/mypy/pull/17543#issuecomment-2240829583 --- mypy/checkexpr.py | 11 ++++------- test-data/unit/pythoneval.test | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 2e8e6e2db9d5..ad1b416a0227 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -4341,14 +4341,11 @@ def visit_index_with_type( elif isinstance(left_type, FunctionLike) and left_type.is_type_obj(): if left_type.type_object().is_enum: return self.visit_enum_index_expr(left_type.type_object(), e.index, e) - elif left_type.type_object().type_vars and self.chk.options.python_version >= (3, 9): - return self.named_type("types.GenericAlias") - elif ( - left_type.type_object().fullname == "builtins.type" - and self.chk.options.python_version >= (3, 9) + elif self.chk.options.python_version >= (3, 9) and ( + left_type.type_object().type_vars + or left_type.type_object().fullname == "builtins.type" ): - # builtins.type is special: it's not generic in stubs, but it supports indexing - return self.named_type("typing._SpecialForm") + return self.named_type("types.GenericAlias") if isinstance(left_type, TypeVarType) and not self.has_member( left_type.upper_bound, "__getitem__" diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 222430c3ef55..832e55f333de 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -1781,7 +1781,7 @@ C = str | int D: TypeAlias = str | int [out] _testTypeAliasNotSupportedWithNewStyleUnion.py:3: error: Invalid type alias: expression is not a valid type -_testTypeAliasNotSupportedWithNewStyleUnion.py:3: error: Unsupported left operand type for | ("") +_testTypeAliasNotSupportedWithNewStyleUnion.py:3: error: Unsupported left operand type for | ("GenericAlias") _testTypeAliasNotSupportedWithNewStyleUnion.py:4: error: Invalid type alias: expression is not a valid type _testTypeAliasNotSupportedWithNewStyleUnion.py:4: error: Unsupported left operand type for | ("Type[str]") _testTypeAliasNotSupportedWithNewStyleUnion.py:5: error: Invalid type alias: expression is not a valid type From 6aa46f097cf30f3143c5ee4bae7f0d2e7ce914bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 20 Jul 2024 15:00:33 -0700 Subject: [PATCH 36/36] Bump setuptools from 68.2.2 to 70.0.0 (#17533) --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 75a970c5bf0e..f4fb4a20cce7 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -57,5 +57,5 @@ typing-extensions==4.12.2 # via -r mypy-requirements.txt # The following packages are considered to be unsafe in a requirements file: -setuptools==68.2.2 +setuptools==70.0.0 # via -r test-requirements.in