From 46ebacae0ca5b464a7d422ac1e3370cae32c135a Mon Sep 17 00:00:00 2001 From: Ali Hamdan Date: Sun, 18 Feb 2024 22:32:19 +0100 Subject: [PATCH] stubgen: Replace obsolete typing aliases with builtin containers (#16780) Addresses part of #16737 This only replaces typing symbols that have equivalents in the `builtins` module. Replacing other symbols, like those from the `collections.abc` module, are a bit more complicated so I suggest we handle them separately. I also changed the default `TypedDict` module from `typing_extensions` to `typing` as typeshed dropped support for Python 3.7. --- mypy/stubgen.py | 51 ++++++---- mypy/stubutil.py | 33 ++++++- .../pybind11_fixtures/__init__.pyi | 6 +- .../pybind11_fixtures/demo.pyi | 4 +- .../pybind11_fixtures/__init__.pyi | 6 +- .../pybind11_fixtures/demo.pyi | 4 +- test-data/unit/stubgen.test | 94 +++++++++++++++---- 7 files changed, 148 insertions(+), 50 deletions(-) diff --git a/mypy/stubgen.py b/mypy/stubgen.py index 279f0569174a..7721366f5c0c 100755 --- a/mypy/stubgen.py +++ b/mypy/stubgen.py @@ -47,7 +47,7 @@ import os.path import sys import traceback -from typing import Final, Iterable +from typing import Final, Iterable, Iterator import mypy.build import mypy.mixedtraverser @@ -114,6 +114,7 @@ from mypy.stubdoc import ArgSig, FunctionSig from mypy.stubgenc import InspectionStubGenerator, generate_stub_for_c_module from mypy.stubutil import ( + TYPING_BUILTIN_REPLACEMENTS, BaseStubGenerator, CantImport, ClassInfo, @@ -289,20 +290,19 @@ def visit_call_expr(self, node: CallExpr) -> str: raise ValueError(f"Unknown argument kind {kind} in call") return f"{callee}({', '.join(args)})" + def _visit_ref_expr(self, node: NameExpr | MemberExpr) -> str: + fullname = self.stubgen.get_fullname(node) + if fullname in TYPING_BUILTIN_REPLACEMENTS: + return self.stubgen.add_name(TYPING_BUILTIN_REPLACEMENTS[fullname], require=False) + qualname = get_qualified_name(node) + self.stubgen.import_tracker.require_name(qualname) + return qualname + def visit_name_expr(self, node: NameExpr) -> str: - self.stubgen.import_tracker.require_name(node.name) - return node.name + return self._visit_ref_expr(node) def visit_member_expr(self, o: MemberExpr) -> str: - node: Expression = o - trailer = "" - while isinstance(node, MemberExpr): - trailer = "." + node.name + trailer - node = node.expr - if not isinstance(node, NameExpr): - return ERROR_MARKER - self.stubgen.import_tracker.require_name(node.name) - return node.name + trailer + return self._visit_ref_expr(o) def visit_str_expr(self, node: StrExpr) -> str: return repr(node.value) @@ -351,11 +351,17 @@ def find_defined_names(file: MypyFile) -> set[str]: return finder.names +def get_assigned_names(lvalues: Iterable[Expression]) -> Iterator[str]: + for lvalue in lvalues: + if isinstance(lvalue, NameExpr): + yield lvalue.name + elif isinstance(lvalue, TupleExpr): + yield from get_assigned_names(lvalue.items) + + class DefinitionFinder(mypy.traverser.TraverserVisitor): """Find names of things defined at the top level of a module.""" - # TODO: Assignment statements etc. - def __init__(self) -> None: # Short names of things defined at the top level. self.names: set[str] = set() @@ -368,6 +374,10 @@ def visit_func_def(self, o: FuncDef) -> None: # Don't recurse, as we only keep track of top-level definitions. self.names.add(o.name) + def visit_assignment_stmt(self, o: AssignmentStmt) -> None: + for name in get_assigned_names(o.lvalues): + self.names.add(name) + def find_referenced_names(file: MypyFile) -> set[str]: finder = ReferenceFinder() @@ -1023,10 +1033,15 @@ def is_alias_expression(self, expr: Expression, top_level: bool = True) -> bool: and isinstance(expr.node, (FuncDef, Decorator, MypyFile)) or isinstance(expr.node, TypeInfo) ) and not self.is_private_member(expr.node.fullname) - elif ( - isinstance(expr, IndexExpr) - and isinstance(expr.base, NameExpr) - and not self.is_private_name(expr.base.name) + elif isinstance(expr, IndexExpr) and ( + (isinstance(expr.base, NameExpr) and not self.is_private_name(expr.base.name)) + or ( # Also some known aliases that could be member expression + isinstance(expr.base, MemberExpr) + and not self.is_private_member(get_qualified_name(expr.base)) + and self.get_fullname(expr.base).startswith( + ("builtins.", "typing.", "typing_extensions.", "collections.abc.") + ) + ) ): if isinstance(expr.index, TupleExpr): indices = expr.index.items diff --git a/mypy/stubutil.py b/mypy/stubutil.py index 69af643efab2..410672f89d09 100644 --- a/mypy/stubutil.py +++ b/mypy/stubutil.py @@ -22,6 +22,26 @@ # Modules that may fail when imported, or that may have side effects (fully qualified). NOT_IMPORTABLE_MODULES = () +# Typing constructs to be replaced by their builtin equivalents. +TYPING_BUILTIN_REPLACEMENTS: Final = { + # From typing + "typing.Text": "builtins.str", + "typing.Tuple": "builtins.tuple", + "typing.List": "builtins.list", + "typing.Dict": "builtins.dict", + "typing.Set": "builtins.set", + "typing.FrozenSet": "builtins.frozenset", + "typing.Type": "builtins.type", + # From typing_extensions + "typing_extensions.Text": "builtins.str", + "typing_extensions.Tuple": "builtins.tuple", + "typing_extensions.List": "builtins.list", + "typing_extensions.Dict": "builtins.dict", + "typing_extensions.Set": "builtins.set", + "typing_extensions.FrozenSet": "builtins.frozenset", + "typing_extensions.Type": "builtins.type", +} + class CantImport(Exception): def __init__(self, module: str, message: str) -> None: @@ -229,6 +249,8 @@ def visit_unbound_type(self, t: UnboundType) -> str: return " | ".join([item.accept(self) for item in t.args]) if fullname == "typing.Optional": return f"{t.args[0].accept(self)} | None" + if fullname in TYPING_BUILTIN_REPLACEMENTS: + s = self.stubgen.add_name(TYPING_BUILTIN_REPLACEMENTS[fullname], require=True) if self.known_modules is not None and "." in s: # see if this object is from any of the modules that we're currently processing. # reverse sort so that subpackages come before parents: e.g. "foo.bar" before "foo". @@ -476,7 +498,7 @@ def reexport(self, name: str) -> None: def import_lines(self) -> list[str]: """The list of required import lines (as strings with python code). - In order for a module be included in this output, an indentifier must be both + In order for a module be included in this output, an identifier must be both 'required' via require_name() and 'imported' via add_import_from() or add_import() """ @@ -585,9 +607,9 @@ def __init__( # a corresponding import statement. self.known_imports = { "_typeshed": ["Incomplete"], - "typing": ["Any", "TypeVar", "NamedTuple"], + "typing": ["Any", "TypeVar", "NamedTuple", "TypedDict"], "collections.abc": ["Generator"], - "typing_extensions": ["TypedDict", "ParamSpec", "TypeVarTuple"], + "typing_extensions": ["ParamSpec", "TypeVarTuple"], } def get_sig_generators(self) -> list[SignatureGenerator]: @@ -613,7 +635,10 @@ def add_name(self, fullname: str, require: bool = True) -> str: """ module, name = fullname.rsplit(".", 1) alias = "_" + name if name in self.defined_names else None - self.import_tracker.add_import_from(module, [(name, alias)], require=require) + while alias in self.defined_names: + alias = "_" + alias + if module != "builtins" or alias: # don't import from builtins unless needed + self.import_tracker.add_import_from(module, [(name, alias)], require=require) return alias or name def add_import_line(self, line: str) -> None: diff --git a/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/__init__.pyi b/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/__init__.pyi index bb939aa5a5e7..90afb46d6d94 100644 --- a/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/__init__.pyi +++ b/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/__init__.pyi @@ -1,6 +1,6 @@ import os from . import demo as demo -from typing import List, Tuple, overload +from typing import overload class StaticMethods: def __init__(self, *args, **kwargs) -> None: ... @@ -22,6 +22,6 @@ class TestStruct: def func_incomplete_signature(*args, **kwargs): ... def func_returning_optional() -> int | None: ... -def func_returning_pair() -> Tuple[int, float]: ... +def func_returning_pair() -> tuple[int, float]: ... def func_returning_path() -> os.PathLike: ... -def func_returning_vector() -> List[float]: ... +def func_returning_vector() -> list[float]: ... diff --git a/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/demo.pyi b/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/demo.pyi index 6f164a03edcc..87b8ec0e4ad6 100644 --- a/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/demo.pyi +++ b/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/demo.pyi @@ -1,4 +1,4 @@ -from typing import ClassVar, List, overload +from typing import ClassVar, overload PI: float __version__: str @@ -47,7 +47,7 @@ class Point: def __init__(self) -> None: ... @overload def __init__(self, x: float, y: float) -> None: ... - def as_list(self) -> List[float]: ... + def as_list(self) -> list[float]: ... @overload def distance_to(self, x: float, y: float) -> float: ... @overload diff --git a/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi b/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi index 622e5881a147..db04bccab028 100644 --- a/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi +++ b/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi @@ -1,6 +1,6 @@ import os from . import demo as demo -from typing import List, Tuple, overload +from typing import overload class StaticMethods: def __init__(self, *args, **kwargs) -> None: @@ -44,9 +44,9 @@ def func_incomplete_signature(*args, **kwargs): """func_incomplete_signature() -> dummy_sub_namespace::HasNoBinding""" def func_returning_optional() -> int | None: """func_returning_optional() -> Optional[int]""" -def func_returning_pair() -> Tuple[int, float]: +def func_returning_pair() -> tuple[int, float]: """func_returning_pair() -> Tuple[int, float]""" def func_returning_path() -> os.PathLike: """func_returning_path() -> os.PathLike""" -def func_returning_vector() -> List[float]: +def func_returning_vector() -> list[float]: """func_returning_vector() -> List[float]""" diff --git a/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/demo.pyi b/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/demo.pyi index 1527225ed009..1be0bc905a43 100644 --- a/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/demo.pyi +++ b/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/demo.pyi @@ -1,4 +1,4 @@ -from typing import ClassVar, List, overload +from typing import ClassVar, overload PI: float __version__: str @@ -73,7 +73,7 @@ class Point: 2. __init__(self: pybind11_fixtures.demo.Point, x: float, y: float) -> None """ - def as_list(self) -> List[float]: + def as_list(self) -> list[float]: """as_list(self: pybind11_fixtures.demo.Point) -> List[float]""" @overload def distance_to(self, x: float, y: float) -> float: diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test index 3503fd4ad808..53baa2c0ca06 100644 --- a/test-data/unit/stubgen.test +++ b/test-data/unit/stubgen.test @@ -1376,9 +1376,8 @@ x: List[collections.defaultdict] [out] import collections -from typing import List -x: List[collections.defaultdict] +x: list[collections.defaultdict] [case testAnnotationFwRefs] @@ -2216,9 +2215,9 @@ funcs: Dict[Any, Any] f = funcs[a.f] [out] from _typeshed import Incomplete -from typing import Any, Dict +from typing import Any -funcs: Dict[Any, Any] +funcs: dict[Any, Any] f: Incomplete [case testAbstractMethodNameExpr] @@ -3290,18 +3289,18 @@ def f(*args: Union[int, Tuple[int, int]]) -> int: [out] -from typing import Tuple, overload +from typing import overload class A: @overload def f(self, x: int, y: int) -> int: ... @overload - def f(self, x: Tuple[int, int]) -> int: ... + def f(self, x: tuple[int, int]) -> int: ... @overload def f(x: int, y: int) -> int: ... @overload -def f(x: Tuple[int, int]) -> int: ... +def f(x: tuple[int, int]) -> int: ... [case testOverload_fromTypingExtensionsImport] from typing import Tuple, Union @@ -3332,19 +3331,18 @@ def f(*args: Union[int, Tuple[int, int]]) -> int: [out] -from typing import Tuple from typing_extensions import overload class A: @overload def f(self, x: int, y: int) -> int: ... @overload - def f(self, x: Tuple[int, int]) -> int: ... + def f(self, x: tuple[int, int]) -> int: ... @overload def f(x: int, y: int) -> int: ... @overload -def f(x: Tuple[int, int]) -> int: ... +def f(x: tuple[int, int]) -> int: ... [case testOverload_importTyping] import typing @@ -3407,22 +3405,22 @@ class A: @typing.overload def f(self, x: int, y: int) -> int: ... @typing.overload - def f(self, x: typing.Tuple[int, int]) -> int: ... + def f(self, x: tuple[int, int]) -> int: ... @typing.overload @classmethod def g(cls, x: int, y: int) -> int: ... @typing.overload @classmethod - def g(cls, x: typing.Tuple[int, int]) -> int: ... + def g(cls, x: tuple[int, int]) -> int: ... @typing.overload def f(x: int, y: int) -> int: ... @typing.overload -def f(x: typing.Tuple[int, int]) -> int: ... +def f(x: tuple[int, int]) -> int: ... @typing_extensions.overload def g(x: int, y: int) -> int: ... @typing_extensions.overload -def g(x: typing.Tuple[int, int]) -> int: ... +def g(x: tuple[int, int]) -> int: ... [case testOverload_importTypingAs] import typing as t @@ -3485,22 +3483,22 @@ class A: @t.overload def f(self, x: int, y: int) -> int: ... @t.overload - def f(self, x: t.Tuple[int, int]) -> int: ... + def f(self, x: tuple[int, int]) -> int: ... @t.overload @classmethod def g(cls, x: int, y: int) -> int: ... @t.overload @classmethod - def g(cls, x: t.Tuple[int, int]) -> int: ... + def g(cls, x: tuple[int, int]) -> int: ... @t.overload def f(x: int, y: int) -> int: ... @t.overload -def f(x: t.Tuple[int, int]) -> int: ... +def f(x: tuple[int, int]) -> int: ... @te.overload def g(x: int, y: int) -> int: ... @te.overload -def g(x: t.Tuple[int, int]) -> int: ... +def g(x: tuple[int, int]) -> int: ... [case testOverloadFromImportAlias] from typing import overload as t_overload @@ -4249,6 +4247,66 @@ o = int | None def f1(a: int | tuple[int, int | None] | None) -> int: ... def f2(a: int | x.Union[int, int] | float | None) -> int: ... +[case testTypingBuiltinReplacements] +import typing +import typing as t +from typing import Tuple +import typing_extensions +import typing_extensions as te +from typing_extensions import List, Type + +# builtins are not builtins +tuple = int +[list,] = float +dict, set, frozenset = str, float, int + +x: Tuple[t.Text, t.FrozenSet[typing.Type[float]]] +y: typing.List[int] +z: t.Dict[str, float] +v: typing.Set[int] +w: List[typing_extensions.Dict[te.FrozenSet[Type[int]], te.Tuple[te.Set[te.Text], ...]]] + +x_alias = Tuple[str, ...] +y_alias = typing.List[int] +z_alias = t.Dict[str, float] +v_alias = typing.Set[int] +w_alias = List[typing_extensions.Dict[str, te.Tuple[int, ...]]] + +[out] +from _typeshed import Incomplete +from builtins import dict as _dict, frozenset as _frozenset, list as _list, set as _set, tuple as _tuple + +tuple = int +list: Incomplete +dict: Incomplete +set: Incomplete +frozenset: Incomplete +x: _tuple[str, _frozenset[type[float]]] +y: _list[int] +z: _dict[str, float] +v: _set[int] +w: _list[_dict[_frozenset[type[int]], _tuple[_set[str], ...]]] +x_alias = _tuple[str, ...] +y_alias = _list[int] +z_alias = _dict[str, float] +v_alias = _set[int] +w_alias = _list[_dict[str, _tuple[int, ...]]] + +[case testHandlingNameCollisions] +# flags: --include-private +from typing import Tuple +tuple = int +_tuple = range +__tuple = map +x: Tuple[int, str] +[out] +from builtins import tuple as ___tuple + +tuple = int +_tuple = range +__tuple = map +x: ___tuple[int, str] + [case testPEP570PosOnlyParams] def f(x=0, /): ... def f1(x: int, /): ...