Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support rename=True in collections.namedtuple #17247

Merged
merged 5 commits into from
May 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 48 additions & 12 deletions mypy/semanal_namedtuple.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,11 @@

from __future__ import annotations

import keyword
from contextlib import contextmanager
from typing import Final, Iterator, List, Mapping, cast
from typing import Container, Final, Iterator, List, Mapping, cast

from mypy.errorcodes import ARG_TYPE, ErrorCode
from mypy.exprtotype import TypeTranslationError, expr_to_unanalyzed_type
from mypy.messages import MessageBuilder
from mypy.nodes import (
Expand Down Expand Up @@ -352,6 +354,7 @@ def parse_namedtuple_args(
self.fail(f'Too few arguments for "{type_name}()"', call)
return None
defaults: list[Expression] = []
rename = False
if len(args) > 2:
# Typed namedtuple doesn't support additional arguments.
if fullname in TYPED_NAMEDTUPLE_NAMES:
Expand All @@ -370,7 +373,17 @@ def parse_namedtuple_args(
"{}()".format(type_name),
arg,
)
break
elif arg_name == "rename":
arg = args[i]
if isinstance(arg, NameExpr) and arg.name in ("True", "False"):
rename = arg.name == "True"
else:
self.fail(
'Boolean literal expected as the "rename" argument to '
f"{type_name}()",
arg,
code=ARG_TYPE,
)
if call.arg_kinds[:2] != [ARG_POS, ARG_POS]:
self.fail(f'Unexpected arguments to "{type_name}()"', call)
return None
Expand Down Expand Up @@ -417,17 +430,28 @@ def parse_namedtuple_args(
return [], [], [], typename, [], False
if not types:
types = [AnyType(TypeOfAny.unannotated) for _ in items]
underscore = [item for item in items if item.startswith("_")]
if underscore:
self.fail(
f'"{type_name}()" field names cannot start with an underscore: '
+ ", ".join(underscore),
call,
)
processed_items = []
seen_names: set[str] = set()
for i, item in enumerate(items):
problem = self.check_namedtuple_field_name(item, seen_names)
if problem is None:
processed_items.append(item)
seen_names.add(item)
else:
if not rename:
self.fail(f'"{type_name}()" {problem}', call)
# Even if rename=False, we pretend that it is True.
# At runtime namedtuple creation would throw an error;
# applying the rename logic means we create a more sensible
# namedtuple.
new_name = f"_{i}"
processed_items.append(new_name)
seen_names.add(new_name)

if len(defaults) > len(items):
self.fail(f'Too many defaults given in call to "{type_name}()"', call)
defaults = defaults[: len(items)]
return items, types, defaults, typename, tvar_defs, True
return processed_items, types, defaults, typename, tvar_defs, True

def parse_namedtuple_fields_with_types(
self, nodes: list[Expression], context: Context
Expand Down Expand Up @@ -666,5 +690,17 @@ def save_namedtuple_body(self, named_tuple_info: TypeInfo) -> Iterator[None]:

# Helpers

def fail(self, msg: str, ctx: Context) -> None:
self.api.fail(msg, ctx)
def check_namedtuple_field_name(self, field: str, seen_names: Container[str]) -> str | None:
"""Return None for valid fields, a string description for invalid ones."""
if field in seen_names:
return f'has duplicate field name "{field}"'
elif not field.isidentifier():
return f'field name "{field}" is not a valid identifier'
elif field.startswith("_"):
return f'field name "{field}" starts with an underscore'
elif keyword.iskeyword(field):
return f'field name "{field}" is a keyword'
return None

def fail(self, msg: str, ctx: Context, code: ErrorCode | None = None) -> None:
self.api.fail(msg, ctx, code=code)
26 changes: 24 additions & 2 deletions test-data/unit/check-namedtuple.test
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,13 @@ a, b, c = x # E: Need more than 2 values to unpack (3 expected)
x[2] # E: Tuple index out of range
[builtins fixtures/tuple.pyi]

[case testNamedTupleNoUnderscoreFields]
[case testNamedTupleInvalidFields]
from collections import namedtuple

X = namedtuple('X', 'x, _y, _z') # E: "namedtuple()" field names cannot start with an underscore: _y, _z
X = namedtuple('X', 'x, _y') # E: "namedtuple()" field name "_y" starts with an underscore
Y = namedtuple('Y', ['x', '1']) # E: "namedtuple()" field name "1" is not a valid identifier
Z = namedtuple('Z', ['x', 'def']) # E: "namedtuple()" field name "def" is a keyword
A = namedtuple('A', ['x', 'x']) # E: "namedtuple()" has duplicate field name "x"
[builtins fixtures/tuple.pyi]

[case testNamedTupleAccessingAttributes]
Expand Down Expand Up @@ -125,6 +128,8 @@ E = namedtuple('E', 'a b', 0)
[builtins fixtures/bool.pyi]

[out]
main:4: error: Boolean literal expected as the "rename" argument to namedtuple()
main:5: error: Boolean literal expected as the "rename" argument to namedtuple()
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A little unfortunate that we give two errors for the same line, but the same happens with T = TypeVar("T", covariant="True"); I don't think it's worth the effort to suppress one of the two.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think its fine to keep both errors but it would be better if both errors have the same code so that they can be disabled with a single type ignore comment. Currently the errors are:

nt.py:5: error: Boolean literal expected as the "rename" argument to namedtuple()  [misc]
nt.py:5: error: Argument "rename" to "namedtuple" has incompatible type "str"; expected "bool"  [arg-type]

How about giving the first error the code arg-type as well?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good idea, done.

main:5: error: Argument "rename" to "namedtuple" has incompatible type "str"; expected "int"
main:6: error: Unexpected keyword argument "unrecognized_arg" for "namedtuple"
<ROOT>/test-data/unit/lib-stub/collections.pyi:3: note: "namedtuple" defined here
Expand All @@ -145,6 +150,23 @@ Z = namedtuple('Z', ['x', 'y'], defaults='not a tuple') # E: List or tuple lite

[builtins fixtures/list.pyi]

[case testNamedTupleRename]
from collections import namedtuple

X = namedtuple('X', ['abc', 'def'], rename=False) # E: "namedtuple()" field name "def" is a keyword
Y = namedtuple('Y', ['x', 'x', 'def', '42', '_x'], rename=True)
y = Y(x=0, _1=1, _2=2, _3=3, _4=4)
reveal_type(y.x) # N: Revealed type is "Any"
reveal_type(y._1) # N: Revealed type is "Any"
reveal_type(y._2) # N: Revealed type is "Any"
reveal_type(y._3) # N: Revealed type is "Any"
reveal_type(y._4) # N: Revealed type is "Any"
y._0 # E: "Y" has no attribute "_0"
y._5 # E: "Y" has no attribute "_5"
y._x # E: "Y" has no attribute "_x"

[builtins fixtures/list.pyi]

[case testNamedTupleWithItemTypes]
from typing import NamedTuple
N = NamedTuple('N', [('a', int),
Expand Down
4 changes: 2 additions & 2 deletions test-data/unit/semanal-namedtuple.test
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ N = namedtuple('N', ['x', 1]) # E: String literal expected as "namedtuple()" ite

[case testNamedTupleWithUnderscoreItemName]
from collections import namedtuple
N = namedtuple('N', ['_fallback']) # E: "namedtuple()" field names cannot start with an underscore: _fallback
N = namedtuple('N', ['_fallback']) # E: "namedtuple()" field name "_fallback" starts with an underscore
[builtins fixtures/tuple.pyi]

-- NOTE: The following code works at runtime but is not yet supported by mypy.
Expand Down Expand Up @@ -197,7 +197,7 @@ N = NamedTuple('N', 1) # E: List or tuple literal expected as the second argumen

[case testTypingNamedTupleWithUnderscoreItemName]
from typing import NamedTuple
N = NamedTuple('N', [('_fallback', int)]) # E: "NamedTuple()" field names cannot start with an underscore: _fallback
N = NamedTuple('N', [('_fallback', int)]) # E: "NamedTuple()" field name "_fallback" starts with an underscore
[builtins fixtures/tuple.pyi]

[case testTypingNamedTupleWithUnexpectedNames]
Expand Down
Loading