Conformance tests: Fix automated scoring for pyright dataclasses tests (

python#1699)

Part of python#1692

conformance/results/mypy/dataclasses_slots.toml

@@ -3,13 +3,12 @@ notes = """
 Does not reject write to instance variable that is not defined in __slots__.
 """
 output = """
-dataclasses_slots.py:12: error: "DC1" both defines "__slots__" and is used with "slots=True"  [misc]
-dataclasses_slots.py:67: error: "type[DC6]" has no attribute "__slots__"  [attr-defined]
-dataclasses_slots.py:70: error: "DC6" has no attribute "__slots__"  [attr-defined]
+dataclasses_slots.py:11: error: "DC1" both defines "__slots__" and is used with "slots=True"  [misc]
+dataclasses_slots.py:66: error: "type[DC6]" has no attribute "__slots__"  [attr-defined]
+dataclasses_slots.py:69: error: "DC6" has no attribute "__slots__"  [attr-defined]
 """
 conformance_automated = "Fail"
 errors_diff = """
-Line 12: Unexpected errors ['dataclasses_slots.py:12: error: "DC1" both defines "__slots__" and is used with "slots=True"  [misc]']
-Line 67: Unexpected errors ['dataclasses_slots.py:67: error: "type[DC6]" has no attribute "__slots__"  [attr-defined]']
-Line 70: Unexpected errors ['dataclasses_slots.py:70: error: "DC6" has no attribute "__slots__"  [attr-defined]']
+Line 25: Expected 1 errors
+Line 38: Expected 1 errors
 """

conformance/results/mypy/dataclasses_transform_field.toml

@@ -10,7 +10,5 @@ dataclasses_transform_field.py:77: error: Missing named argument "id" for "Custo
 """
 conformance_automated = "Fail"
 errors_diff = """
-Line 64: Unexpected errors ['dataclasses_transform_field.py:64: error: Unexpected keyword argument "id" for "CustomerModel1"  [call-arg]']
-Line 75: Unexpected errors ['dataclasses_transform_field.py:75: error: Too many positional arguments for "CustomerModel2"  [misc]', 'dataclasses_transform_field.py:75: error: Missing named argument "name" for "CustomerModel2"  [call-arg]']
 Line 77: Unexpected errors ['dataclasses_transform_field.py:77: error: Missing named argument "id" for "CustomerModel2"  [call-arg]']
 """

conformance/results/mypy/dataclasses_transform_func.toml

@@ -13,10 +13,6 @@ dataclasses_transform_func.py:97: error: Property "id" defined in "Customer3" is
 """
 conformance_automated = "Fail"
 errors_diff = """
+Line 61: Expected 1 errors
 Line 53: Unexpected errors ['dataclasses_transform_func.py:53: error: Too many positional arguments for "Customer1"  [misc]']
-Line 57: Unexpected errors ['dataclasses_transform_func.py:57: error: Incompatible types in assignment (expression has type "int", variable has type "str")  [assignment]']
-Line 65: Unexpected errors ['dataclasses_transform_func.py:65: error: Unexpected keyword argument "salary" for "Customer1"  [call-arg]']
-Line 71: Unexpected errors ['dataclasses_transform_func.py:71: error: Too many positional arguments for "Customer2"  [misc]']
-Line 90: Unexpected errors ['dataclasses_transform_func.py:90: error: Cannot inherit non-frozen dataclass from a frozen one  [misc]']
-Line 97: Unexpected errors ['dataclasses_transform_func.py:97: error: Property "id" defined in "Customer3" is read-only  [misc]']
 """

conformance/results/pyre/dataclasses_frozen.toml

@@ -9,6 +9,6 @@ dataclasses_frozen.py:17:0 Invalid assignment [41]: Cannot reassign final attrib
 """
 conformance_automated = "Fail"
 errors_diff = """
-Line 23: Expected 1 errors
-Line 33: Expected 1 errors
+Lines 22, 23: Expected error (tag 'DC2')
+Lines 32, 33: Expected error (tag 'DC4')
 """

conformance/results/pyre/dataclasses_slots.toml

@@ -5,9 +5,12 @@ Does not reject write to instance variable that is not defined in __slots__.
 Does not reject access to `__slots__` from dataclass instance when `slots=False`.
 """
 output = """
-dataclasses_slots.py:67:0 Undefined attribute [16]: `DC6` has no attribute `__slots__`.
+dataclasses_slots.py:66:0 Undefined attribute [16]: `DC6` has no attribute `__slots__`.
 """
 conformance_automated = "Fail"
 errors_diff = """
-Line 67: Unexpected errors ['dataclasses_slots.py:67:0 Undefined attribute [16]: `DC6` has no attribute `__slots__`.']
+Line 25: Expected 1 errors
+Line 38: Expected 1 errors
+Line 69: Expected 1 errors
+Lines 10, 11: Expected error (tag 'DC1')
 """

conformance/results/pyre/dataclasses_transform_field.toml

@@ -13,7 +13,5 @@ conformance_automated = "Fail"
 errors_diff = """
 Line 49: Unexpected errors ["dataclasses_transform_field.py:49:42 Invalid type variable [34]: The type variable `Variable[T]` isn't present in the function's parameters."]
 Line 60: Unexpected errors ['dataclasses_transform_field.py:60:0 Unexpected keyword [28]: Unexpected keyword argument `name` to call `object.__init__`.']
-Line 64: Unexpected errors ['dataclasses_transform_field.py:64:0 Unexpected keyword [28]: Unexpected keyword argument `id` to call `object.__init__`.']
-Line 75: Unexpected errors ['dataclasses_transform_field.py:75:0 Too many arguments [19]: Call `object.__init__` expects 0 positional arguments, 1 was provided.']
 Line 77: Unexpected errors ['dataclasses_transform_field.py:77:0 Unexpected keyword [28]: Unexpected keyword argument `name` to call `object.__init__`.']
 """

conformance/results/pyre/dataclasses_transform_func.toml

@@ -26,6 +26,7 @@ dataclasses_transform_func.py:94:7 Unexpected keyword [28]: Unexpected keyword a
 """
 conformance_automated = "Fail"
 errors_diff = """
+Line 97: Expected 1 errors
 Line 20: Unexpected errors ['dataclasses_transform_func.py:20:0 Incompatible overload [43]: The implementation of `create_model` does not accept all possible arguments of overload defined on line `20`.']
 Line 25: Unexpected errors ["dataclasses_transform_func.py:25:5 Invalid type variable [34]: The type variable `Variable[T]` isn't present in the function's parameters."]
 Line 29: Unexpected errors ['dataclasses_transform_func.py:29:0 Incompatible overload [43]: This definition does not have the same decorators as the preceding overload(s).']
@@ -34,13 +35,8 @@ Line 40: Unexpected errors ['dataclasses_transform_func.py:40:0 Uninitialized at
 Line 46: Unexpected errors ['dataclasses_transform_func.py:46:0 Uninitialized attribute [13]: Attribute `salary` is declared in class `Customer2Subclass` to have type `float` but is never initialized.']
 Line 50: Unexpected errors ['dataclasses_transform_func.py:50:7 Unexpected keyword [28]: Unexpected keyword argument `id` to call `object.__init__`.']
 Line 53: Unexpected errors ['dataclasses_transform_func.py:53:7 Too many arguments [19]: Call `object.__init__` expects 0 positional arguments, 2 were provided.']
-Line 57: Unexpected errors ['dataclasses_transform_func.py:57:0 Incompatible attribute type [8]: Attribute `name` declared in class `Customer1` has type `str` but is used as type `int`.']
-Line 61: Unexpected errors ['dataclasses_transform_func.py:61:5 Unsupported operand [58]: `<` is not supported for operand types `Customer1` and `Customer1`.']
-Line 65: Unexpected errors ['dataclasses_transform_func.py:65:7 Unexpected keyword [28]: Unexpected keyword argument `id` to call `object.__init__`.']
 Line 67: Unexpected errors ['dataclasses_transform_func.py:67:7 Unexpected keyword [28]: Unexpected keyword argument `id` to call `object.__init__`.']
-Line 71: Unexpected errors ['dataclasses_transform_func.py:71:7 Too many arguments [19]: Call `object.__init__` expects 0 positional arguments, 2 were provided.']
 Line 73: Unexpected errors ['dataclasses_transform_func.py:73:5 Unsupported operand [58]: `<` is not supported for operand types `Customer2` and `Customer2`.']
 Line 82: Unexpected errors ['dataclasses_transform_func.py:82:0 Uninitialized attribute [13]: Attribute `id` is declared in class `Customer3` to have type `int` but is never initialized.', 'dataclasses_transform_func.py:82:0 Uninitialized attribute [13]: Attribute `name` is declared in class `Customer3` to have type `str` but is never initialized.']
-Line 90: Unexpected errors ['dataclasses_transform_func.py:90:0 Uninitialized attribute [13]: Attribute `age` is declared in class `Customer3Subclass` to have type `int` but is never initialized.']
 Line 94: Unexpected errors ['dataclasses_transform_func.py:94:7 Unexpected keyword [28]: Unexpected keyword argument `id` to call `object.__init__`.']
 """

conformance/results/pyright/dataclasses_frozen.toml

@@ -9,10 +9,6 @@ dataclasses_frozen.py:17:5 - error: Cannot assign member "b" for type "DC1"
 dataclasses_frozen.py:22:1 - error: A non-frozen class cannot inherit from a class that is frozen (reportGeneralTypeIssues)
 dataclasses_frozen.py:32:12 - error: A frozen class cannot inherit from a class that is not frozen (reportGeneralTypeIssues)
 """
-conformance_automated = "Fail"
+conformance_automated = "Pass"
 errors_diff = """
-Line 23: Expected 1 errors
-Line 33: Expected 1 errors
-Line 22: Unexpected errors ['dataclasses_frozen.py:22:1 - error: A non-frozen class cannot inherit from a class that is frozen (reportGeneralTypeIssues)']
-Line 32: Unexpected errors ['dataclasses_frozen.py:32:12 - error: A frozen class cannot inherit from a class that is not frozen (reportGeneralTypeIssues)']
 """

conformance/results/pyright/dataclasses_slots.toml

@@ -1,18 +1,13 @@
 conformant = "Pass"
 output = """
-dataclasses_slots.py:11:12 - error: __slots__ is already defined in class (reportGeneralTypeIssues)
-dataclasses_slots.py:26:14 - error: "y" is not specified in __slots__ (reportGeneralTypeIssues)
-dataclasses_slots.py:39:14 - error: "y" is not specified in __slots__ (reportGeneralTypeIssues)
-dataclasses_slots.py:67:5 - error: Cannot access member "__slots__" for type "type[DC6]"
+dataclasses_slots.py:10:12 - error: __slots__ is already defined in class (reportGeneralTypeIssues)
+dataclasses_slots.py:25:14 - error: "y" is not specified in __slots__ (reportGeneralTypeIssues)
+dataclasses_slots.py:38:14 - error: "y" is not specified in __slots__ (reportGeneralTypeIssues)
+dataclasses_slots.py:66:5 - error: Cannot access member "__slots__" for type "type[DC6]"
   Member "__slots__" is unknown (reportAttributeAccessIssue)
-dataclasses_slots.py:70:8 - error: Cannot access member "__slots__" for type "DC6"
+dataclasses_slots.py:69:8 - error: Cannot access member "__slots__" for type "DC6"
   Member "__slots__" is unknown (reportAttributeAccessIssue)
 """
-conformance_automated = "Fail"
+conformance_automated = "Pass"
 errors_diff = """
-Line 11: Unexpected errors ['dataclasses_slots.py:11:12 - error: __slots__ is already defined in class (reportGeneralTypeIssues)']
-Line 26: Unexpected errors ['dataclasses_slots.py:26:14 - error: "y" is not specified in __slots__ (reportGeneralTypeIssues)']
-Line 39: Unexpected errors ['dataclasses_slots.py:39:14 - error: "y" is not specified in __slots__ (reportGeneralTypeIssues)']
-Line 67: Unexpected errors ['dataclasses_slots.py:67:5 - error: Cannot access member "__slots__" for type "type[DC6]"']
-Line 70: Unexpected errors ['dataclasses_slots.py:70:8 - error: Cannot access member "__slots__" for type "DC6"']
 """

conformance/results/pyright/dataclasses_transform_field.toml

@@ -3,8 +3,6 @@ output = """
 dataclasses_transform_field.py:64:16 - error: No parameter named "id" (reportCallIssue)
 dataclasses_transform_field.py:75:16 - error: Expected 0 positional arguments (reportCallIssue)
 """
-conformance_automated = "Fail"
+conformance_automated = "Pass"
 errors_diff = """
-Line 64: Unexpected errors ['dataclasses_transform_field.py:64:16 - error: No parameter named "id" (reportCallIssue)']
-Line 75: Unexpected errors ['dataclasses_transform_field.py:75:16 - error: Expected 0 positional arguments (reportCallIssue)']
 """

conformance/results/pyright/dataclasses_transform_func.toml

@@ -10,12 +10,6 @@ dataclasses_transform_func.py:97:6 - error: Cannot assign member "id" for type "
   "Customer3" is frozen
     Member "__set__" is unknown (reportAttributeAccessIssue)
 """
-conformance_automated = "Fail"
+conformance_automated = "Pass"
 errors_diff = """
-Line 57: Unexpected errors ['dataclasses_transform_func.py:57:13 - error: Cannot assign member "name" for type "Customer1"']
-Line 61: Unexpected errors ['dataclasses_transform_func.py:61:6 - error: Operator "<" not supported for types "Customer1" and "Customer1" (reportOperatorIssue)']
-Line 65: Unexpected errors ['dataclasses_transform_func.py:65:36 - error: No parameter named "salary" (reportCallIssue)']
-Line 71: Unexpected errors ['dataclasses_transform_func.py:71:18 - error: Expected 0 positional arguments (reportCallIssue)']
-Line 89: Unexpected errors ['dataclasses_transform_func.py:89:1 - error: A non-frozen class cannot inherit from a class that is frozen (reportGeneralTypeIssues)']
-Line 97: Unexpected errors ['dataclasses_transform_func.py:97:6 - error: Cannot assign member "id" for type "Customer3"']
 """

conformance/results/pytype/dataclasses_frozen.toml

@@ -11,6 +11,6 @@ conformance_automated = "Fail"
 errors_diff = """
 Line 16: Expected 1 errors
 Line 17: Expected 1 errors
-Line 23: Expected 1 errors
-Line 33: Expected 1 errors
+Lines 22, 23: Expected error (tag 'DC2')
+Lines 32, 33: Expected error (tag 'DC4')
 """

conformance/results/pytype/dataclasses_slots.toml

@@ -5,15 +5,16 @@ Does not reject write to instance variable that is not defined in __slots__.
 Incorrectly reports error when accessing `__slots__` when `slots=True`.
 """
 output = """
-File "dataclasses_slots.py", line 57, in <module>: No attribute '__slots__' on Type[DC5] [attribute-error]
-File "dataclasses_slots.py", line 58, in <module>: No attribute '__slots__' on DC5 [attribute-error]
-File "dataclasses_slots.py", line 67, in <module>: No attribute '__slots__' on Type[DC6] [attribute-error]
-File "dataclasses_slots.py", line 70, in <module>: No attribute '__slots__' on DC6 [attribute-error]
+File "dataclasses_slots.py", line 56, in <module>: No attribute '__slots__' on Type[DC5] [attribute-error]
+File "dataclasses_slots.py", line 57, in <module>: No attribute '__slots__' on DC5 [attribute-error]
+File "dataclasses_slots.py", line 66, in <module>: No attribute '__slots__' on Type[DC6] [attribute-error]
+File "dataclasses_slots.py", line 69, in <module>: No attribute '__slots__' on DC6 [attribute-error]
 """
 conformance_automated = "Fail"
 errors_diff = """
-Line 57: Unexpected errors ['File "dataclasses_slots.py", line 57, in <module>: No attribute \\'__slots__\\' on Type[DC5] [attribute-error]']
-Line 58: Unexpected errors ['File "dataclasses_slots.py", line 58, in <module>: No attribute \\'__slots__\\' on DC5 [attribute-error]']
-Line 67: Unexpected errors ['File "dataclasses_slots.py", line 67, in <module>: No attribute \\'__slots__\\' on Type[DC6] [attribute-error]']
-Line 70: Unexpected errors ['File "dataclasses_slots.py", line 70, in <module>: No attribute \\'__slots__\\' on DC6 [attribute-error]']
+Line 25: Expected 1 errors
+Line 38: Expected 1 errors
+Lines 10, 11: Expected error (tag 'DC1')
+Line 56: Unexpected errors ['File "dataclasses_slots.py", line 56, in <module>: No attribute \\'__slots__\\' on Type[DC5] [attribute-error]']
+Line 57: Unexpected errors ['File "dataclasses_slots.py", line 57, in <module>: No attribute \\'__slots__\\' on DC5 [attribute-error]']
 """

conformance/results/pytype/dataclasses_transform_field.toml

@@ -8,6 +8,8 @@ File "dataclasses_transform_field.py", line 50, in create_model: bad return type
 """
 conformance_automated = "Fail"
 errors_diff = """
+Line 64: Expected 1 errors
+Line 75: Expected 1 errors
 Line 48: Unexpected errors ['File "dataclasses_transform_field.py", line 48, in <module>: Arguments to dataclass_transform not supported yet [not-supported-yet]']
 Line 50: Unexpected errors ['File "dataclasses_transform_field.py", line 50, in create_model: bad return type [bad-return-type]']
 """

conformance/results/pytype/dataclasses_transform_func.toml

@@ -19,14 +19,14 @@ File "dataclasses_transform_func.py", line 76, in <module>: Arguments to datacla
 """
 conformance_automated = "Fail"
 errors_diff = """
+Line 61: Expected 1 errors
+Line 97: Expected 1 errors
+Lines 89, 90: Expected error (tag 'Customer3Subclass')
 Line 13: Unexpected errors ['File "dataclasses_transform_func.py", line 13, in <module>: Arguments to dataclass_transform not supported yet [not-supported-yet]']
 Line 19: Unexpected errors ['File "dataclasses_transform_func.py", line 19, in <module>: Arguments to dataclass_transform not supported yet [not-supported-yet]']
 Line 30: Unexpected errors ['File "dataclasses_transform_func.py", line 30, in create_model: bad return type [bad-return-type]', 'File "dataclasses_transform_func.py", line 30, in create_model: bad return type [bad-return-type]']
 Line 50: Unexpected errors ['File "dataclasses_transform_func.py", line 50, in <module>: Invalid keyword arguments (id, name) to function Customer1.__init__ [wrong-keyword-args]']
 Line 53: Unexpected errors ['File "dataclasses_transform_func.py", line 53, in <module>: Function Customer1.__init__ expects 1 arg(s), got 3 [wrong-arg-count]']
-Line 57: Unexpected errors ['File "dataclasses_transform_func.py", line 57, in <module>: Type annotation for name does not match type of assignment [annotation-type-mismatch]']
-Line 65: Unexpected errors ['File "dataclasses_transform_func.py", line 65, in <module>: Invalid keyword arguments (id, name, salary) to function Customer1.__init__ [wrong-keyword-args]']
 Line 67: Unexpected errors ['File "dataclasses_transform_func.py", line 67, in <module>: Invalid keyword arguments (id, name) to function Customer2.__init__ [wrong-keyword-args]']
-Line 71: Unexpected errors ['File "dataclasses_transform_func.py", line 71, in <module>: Function Customer2.__init__ expects 1 arg(s), got 3 [wrong-arg-count]']
 Line 76: Unexpected errors ['File "dataclasses_transform_func.py", line 76, in <module>: Arguments to dataclass_transform not supported yet [not-supported-yet]']
 """

conformance/tests/dataclasses_frozen.py

@@ -19,8 +19,8 @@ class DC1:
 
 # This should generate an error because a non-frozen dataclass
 # cannot inherit from a frozen dataclass.
-@dataclass
-class DC2(DC1):  # E
+@dataclass  # E[DC2]
+class DC2(DC1):  # E[DC2]
     pass
 
 @dataclass
@@ -29,8 +29,8 @@ class DC3:
 
 # This should generate an error because a frozen dataclass
 # cannot inherit from a non-frozen dataclass.
-@dataclass(frozen=True)
-class DC4(DC3):  # E
+@dataclass(frozen=True)  # E[DC4]
+class DC4(DC3):  # E[DC4]
     pass
 
 

conformance/tests/dataclasses_slots.py

@@ -5,11 +5,10 @@
 # Specification: https://docs.python.org/3/library/dataclasses.html#module-contents
 
 from dataclasses import dataclass
-from typing import Iterable, assert_type
 
 # This should generate an error because __slots__ is already defined.
-@dataclass(slots=True)
-class DC1:
+@dataclass(slots=True)  # E[DC1]
+class DC1:  # E[DC1]
     x: int
 
     __slots__ = ()
@@ -23,7 +22,7 @@ def __init__(self):
         self.x = 3
 
         # This should generate an error because "y" is not in slots.
-        self.y = 3
+        self.y = 3  # E
 
 
 @dataclass(slots=False)
@@ -36,7 +35,7 @@ def __init__(self):
         self.x = 3
 
         # This should generate an error because "y" is not in slots.
-        self.y = 3
+        self.y = 3  # E
 
 
 @dataclass
@@ -64,7 +63,7 @@ class DC6:
 
 
 # This should generate an error because __slots__ is not defined.
-DC6.__slots__
+DC6.__slots__  # E
 
 # This should generate an error because __slots__ is not defined.
-DC6(1).__slots__
+DC6(1).__slots__  # E

conformance/tests/dataclasses_transform_field.py

@@ -61,7 +61,7 @@ class CustomerModel1:
 
 # This should generate an error because "id" is not
 # supposed to be part of the init function.
-CustomerModel1(id=1, name="hi")
+CustomerModel1(id=1, name="hi")  # E
 
 
 @create_model()
@@ -72,6 +72,6 @@ class CustomerModel2:
 
 # This should generate an error because kw_only is True
 # by default for field2.
-CustomerModel2(1)
+CustomerModel2(1)  # E
 
 CustomerModel2(name="Fred")

conformance/tests/dataclasses_transform_func.py

@@ -54,21 +54,21 @@ class Customer2Subclass(Customer2):
 c1_2.name = "Susan"
 
 # This should generate an error because of a type mismatch.
-c1_2.name = 3
+c1_2.name = 3  # E
 
 # This should generate an error because comparison methods are
 # not synthesized.
-v1 = c1_1 < c1_2
+v1 = c1_1 < c1_2   # E
 
 # This should generate an error because salary is not
 # a defined field.
-c1_3 = Customer1(id=3, name="Sue", salary=40000)
+c1_3 = Customer1(id=3, name="Sue", salary=40000)  # E
 
 c2_1 = Customer2(id=0, name="John")
 
 # This should generate an error because Customer2 supports
 # keyword-only parameters for its constructor.
-c2_2 = Customer2(0, "John")
+c2_2 = Customer2(0, "John")  # E
 
 v2 = c2_1 < c2_2
 
@@ -86,12 +86,12 @@ class Customer3:
 
 # This should generate an error because a non-frozen class
 # cannot inherit from a frozen class.
-@create_model
-class Customer3Subclass(Customer3):
+@create_model  # E[Customer3Subclass]
+class Customer3Subclass(Customer3):  # E[Customer3Subclass]
     age: int
 
 
 c3_1 = Customer3(id=2, name="hi")
 
 # This should generate an error because Customer3 is frozen.
-c3_1.id = 4
+c3_1.id = 4  # E