Update conformance tests for recent generics spec change

conformance/results/mypy/generics_base_class.toml

@@ -1,4 +1,7 @@
-conformant = "Pass"
+conformant = "Partial"
+notes = """
+Does not detect inconsistent type variable ordering.
+"""
 output = """
 generics_base_class.py:26: error: Argument 1 to "takes_dict_incorrect" has incompatible type "SymbolTable"; expected "dict[str, list[object]]"  [arg-type]
 generics_base_class.py:29: error: Variable "typing.Generic" is not valid as a type  [valid-type]
@@ -9,6 +12,7 @@ generics_base_class.py:49: error: "LinkedList" expects 1 type argument, but 2 gi
 generics_base_class.py:61: error: "MyDict" expects 1 type argument, but 2 given  [type-arg]
 generics_base_class.py:68: error: Duplicate type variables in Generic[...] or Protocol[...]  [misc]
 """
-conformance_automated = "Pass"
+conformance_automated = "Fail"
 errors_diff = """
+Line 98: Expected 1 errors
 """

conformance/results/mypy/generics_basic.toml

@@ -10,10 +10,14 @@ generics_basic.py:69: error: Value of type variable "AnyStr" of "concat" cannot
 generics_basic.py:121: error: Duplicate type variables in Generic[...] or Protocol[...]  [misc]
 generics_basic.py:157: error: Invalid index type "int" for "MyMap1[str, int]"; expected type "str"  [index]
 generics_basic.py:158: error: Invalid index type "int" for "MyMap2[int, str]"; expected type "str"  [index]
-generics_basic.py:191: error: Dynamic metaclass not supported for "GenericMetaInstance"  [misc]
-generics_basic.py:191: error: Type variable "generics_basic.T" is unbound  [valid-type]
-generics_basic.py:191: note: (Hint: Use "Generic[T]" or "Protocol[T]" base class to bind "T" inside a class)
-generics_basic.py:191: note: (Hint: Use "T" in function signature to bind "T" inside a function)
+generics_basic.py:162: error: Free type variable expected in Generic[...]  [misc]
+generics_basic.py:163: error: Free type variable expected in Protocol[...]  [misc]
+generics_basic.py:171: error: If Generic[...] or Protocol[...] is present it should list all type variables  [misc]
+generics_basic.py:172: error: If Generic[...] or Protocol[...] is present it should list all type variables  [misc]
+generics_basic.py:208: error: Dynamic metaclass not supported for "GenericMetaInstance"  [misc]
+generics_basic.py:208: error: Type variable "generics_basic.T" is unbound  [valid-type]
+generics_basic.py:208: note: (Hint: Use "Generic[T]" or "Protocol[T]" base class to bind "T" inside a class)
+generics_basic.py:208: note: (Hint: Use "T" in function signature to bind "T" inside a function)
 """
 conformance_automated = "Fail"
 errors_diff = """

conformance/results/mypy/version.toml

@@ -1,2 +1,2 @@
 version = "mypy 1.13.0"
-test_duration = 2.7
+test_duration = 2.5

conformance/results/pyre/generics_base_class.toml

@@ -2,6 +2,7 @@ conformant = "Partial"
 notes = """
 Does not reject illegal use of Generic.
 Does not allow using generic in assert_type expression.
+Does not detect inconsistent type variable ordering.
 """
 output = """
 generics_base_class.py:26:25 Incompatible parameter type [6]: In call `takes_dict_incorrect`, for 1st positional argument, expected `Dict[str, List[object]]` but got `SymbolTable`.
@@ -13,4 +14,5 @@ conformance_automated = "Fail"
 errors_diff = """
 Line 29: Expected 1 errors
 Line 30: Expected 1 errors
+Line 98: Expected 1 errors
 """

conformance/results/pyre/generics_basic.toml

@@ -4,6 +4,7 @@ False positives in examples using constrained type variables.
 False negative for constraint parameterized by a type variable.
 False negative in custom map example.
 False positive using `iter`.
+False negative for bad type arguments to Generic/Protocol.
 False negative for generic metaclass.
 """
 output = """
@@ -16,10 +17,14 @@ generics_basic.py:69:14 Incompatible parameter type [6]: In call `concat`, for 2
 generics_basic.py:121:0 Duplicate type variables [59]: Duplicate type variable `T` in Generic[...].
 generics_basic.py:157:7 Incompatible parameter type [6]: In call `typing.Mapping.__getitem__`, for 1st positional argument, expected `str` but got `int`.
 generics_basic.py:158:7 Incompatible parameter type [6]: In call `typing.Mapping.__getitem__`, for 1st positional argument, expected `str` but got `int`.
+generics_basic.py:171:11 Invalid type variable [34]: The current class isn't generic with respect to the type variable `Variable[T_co]`. To reference the type variable, you can modify the class to inherit from `typing.Generic[T_co]`.
+generics_basic.py:172:11 Invalid type variable [34]: The current class isn't generic with respect to the type variable `Variable[T_co]`. To reference the type variable, you can modify the class to inherit from `typing.Generic[T_co]`.
 """
 conformance_automated = "Fail"
 errors_diff = """
 Line 55: Expected 1 errors
-Line 191: Expected 1 errors
+Line 162: Expected 1 errors
+Line 163: Expected 1 errors
+Line 208: Expected 1 errors
 Line 34: Unexpected errors ['generics_basic.py:34:4 Incompatible return type [7]: Expected `Variable[AnyStr <: [str, bytes]]` but got `str`.', 'generics_basic.py:34:15 Incompatible parameter type [6]: In call `str.__add__`, for 1st positional argument, expected `str` but got `Variable[AnyStr <: [str, bytes]]`.']
 """

conformance/results/pyre/version.toml

@@ -1,3 +1,2 @@
 version = "pyre 0.9.23"
-test_duration = 9.1
-
+test_duration = 4.3

conformance/results/pyright/dataclasses_transform_converter.toml

@@ -40,5 +40,5 @@ dataclasses_transform_converter.py:130:67 - error: Argument of type "Literal[1]"
 dataclasses_transform_converter.py:133:75 - error: Argument of type "type[int]" cannot be assigned to parameter "default_factory" of type "(() -> S@model_field) | None" in function "model_field"
   Type "type[int]" is not assignable to type "(() -> str) | None"
     No overloaded function matches type "() -> str"
-    "type[type]" is not assignable to "type[None]" (reportArgumentType)
+    Type is not assignable to "None" (reportArgumentType)
 """

conformance/results/pyright/generics_base_class.toml

@@ -10,6 +10,8 @@ generics_base_class.py:30:8 - error: "Generic" is not valid in this context (rep
 generics_base_class.py:49:38 - error: Too many type arguments provided for "LinkedList"; expected 1 but received 2 (reportInvalidTypeArguments)
 generics_base_class.py:61:30 - error: Too many type arguments provided for "MyDict"; expected 1 but received 2 (reportInvalidTypeArguments)
 generics_base_class.py:68:28 - error: Type arguments for "Generic" must be unique (reportInvalidTypeForm)
+generics_base_class.py:98:7 - error: Base classes of BadChild are mutually incompatible
+  Base class "Grandparent[T2@BadChild, T1@BadChild]" derives from "Grandparent[T2@BadChild, T1@BadChild]" which is incompatible with type "Grandparent[T1@BadChild, T2@BadChild]" (reportGeneralTypeIssues)
 """
 conformance_automated = "Pass"
 errors_diff = """

conformance/results/pyright/generics_basic.toml

@@ -13,7 +13,13 @@ generics_basic.py:157:5 - error: Argument of type "Literal[0]" cannot be assigne
   "Literal[0]" is not assignable to "str" (reportArgumentType)
 generics_basic.py:158:5 - error: Argument of type "Literal[0]" cannot be assigned to parameter "key" of type "str" in function "__getitem__"
   "Literal[0]" is not assignable to "str" (reportArgumentType)
-generics_basic.py:191:37 - error: Metaclass cannot be generic (reportGeneralTypeIssues)
+generics_basic.py:162:20 - error: Type argument for "Generic" must be a type variable (reportInvalidTypeForm)
+generics_basic.py:163:21 - error: Type argument for "Protocol" must be a type parameter (reportInvalidTypeForm)
+generics_basic.py:171:7 - error: Generic[] or Protocol[] must include all type variables
+  Missing type variables: "T_co" (reportGeneralTypeIssues)
+generics_basic.py:172:7 - error: Generic[] or Protocol[] must include all type variables
+  Missing type variables: "T_co" (reportGeneralTypeIssues)
+generics_basic.py:208:37 - error: Metaclass cannot be generic (reportGeneralTypeIssues)
 """
 conformance_automated = "Pass"
 errors_diff = """

conformance/results/pyright/overloads_basic.toml

@@ -1,8 +1,8 @@
 conformant = "Pass"
 output = """
 overloads_basic.py:37:1 - error: No overloads for "__getitem__" match the provided arguments (reportCallIssue)
-overloads_basic.py:37:1 - error: Argument of type "Literal['']" cannot be assigned to parameter "__s" of type "slice" in function "__getitem__"
-  "Literal['']" is not assignable to "slice" (reportArgumentType)
+overloads_basic.py:37:1 - error: Argument of type "Literal['']" cannot be assigned to parameter "__s" of type "slice[Any, Any, Any]" in function "__getitem__"
+  "Literal['']" is not assignable to "slice[Any, Any, Any]" (reportArgumentType)
 overloads_basic.py:63:5 - error: "func1" is marked as overload, but additional overloads are missing (reportInconsistentOverload)
 overloads_basic.py:75:5 - error: "func2" is marked as overload, but no implementation is provided (reportNoOverloadImplementation)
 """

conformance/results/pyright/protocols_class_objects.toml

@@ -5,8 +5,8 @@ protocols_class_objects.py:29:5 - error: Argument of type "type[Proto]" cannot b
 protocols_class_objects.py:34:7 - error: Type "type[Proto]" is not assignable to declared type "type[Proto]"
   "Proto" is not a concrete class type and cannot be assigned to type "type[Proto]" (reportAssignmentType)
 protocols_class_objects.py:58:16 - error: Type "type[ConcreteA]" is not assignable to declared type "ProtoA1"
-  "type[type]" is not assignable to "type[ConcreteA]"
-  "type[type]" is not assignable to "type[ConcreteA]"
+  Type "type[ConcreteA]" is not assignable to type "ConcreteA"
+  Type "type[ConcreteA]" is not assignable to type "ConcreteA"
   "method1" is an incompatible type
     Type "(self: ConcreteA, x: int) -> int" is not assignable to type "(x: int) -> int"
       Parameter name mismatch: "x" versus "self"

conformance/results/pyright/specialtypes_none.toml

@@ -1,7 +1,7 @@
 conformant = "Pass"
 output = """
 specialtypes_none.py:21:7 - error: Argument of type "type[None]" cannot be assigned to parameter "val1" of type "None" in function "func1"
-  "type[type]" is not assignable to "type[None]" (reportArgumentType)
+  Type is not assignable to "None" (reportArgumentType)
 specialtypes_none.py:27:19 - error: Type "None" is not assignable to declared type "Iterable[Unknown]"
   "None" is incompatible with protocol "Iterable[Unknown]"
     "__iter__" is not present (reportAssignmentType)

conformance/results/pyright/version.toml

@@ -1,3 +1,2 @@
-version = "pyright 1.1.386"
-test_duration = 2.0
-
+version = "pyright 1.1.389"
+test_duration = 2.5

conformance/results/pytype/generics_base_class.toml

@@ -2,6 +2,7 @@ conformant = "Partial"
 notes = """
 False negative on passing SymbolTable to dict[str, list[object]].
 Does not reject illegal use of Generic.
+Wrong type variable order when inheriting from multiple generic base classes.
 """
 output = """
 generics_base_class.py:29:1: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in <module>: Invalid type annotation 'T'  [invalid-annotation]
@@ -26,9 +27,26 @@ generics_base_class.py:68:17: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in <m
 class BadClass1(Generic[T, T]):  # E
                 \u001b[1m\u001b[31m~~~~~~~~~~~~~\u001b[39m\u001b[0m
 
+generics_base_class.py:90:1: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in <module>: Function takes_parent1 was called with the wrong arguments [wrong-arg-types]
+
+takes_parent1(child)  # OK
+\u001b[1m\u001b[31m~~~~~~~~~~~~~~~~~~~~\u001b[39m\u001b[0m
+
+generics_base_class.py:91:1: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in <module>: Function takes_parent2 was called with the wrong arguments [wrong-arg-types]
+
+takes_parent2(child)  # OK
+\u001b[1m\u001b[31m~~~~~~~~~~~~~~~~~~~~\u001b[39m\u001b[0m
+
+generics_base_class.py:98:1: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in <module>: Invalid type annotation 'BadChild'  [invalid-annotation]
+
+class BadChild(Parent[T1, T2], Grandparent[T2, T1]): ...  # E
+\u001b[1m\u001b[31m~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\u001b[39m\u001b[0m
+
 """
 conformance_automated = "Fail"
 errors_diff = """
 Line 26: Expected 1 errors
 Line 30: Expected 1 errors
+Line 90: Unexpected errors ['generics_base_class.py:90:1: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in <module>: Function takes_parent1 was called with the wrong arguments [wrong-arg-types]']
+Line 91: Unexpected errors ['generics_base_class.py:91:1: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in <module>: Function takes_parent2 was called with the wrong arguments [wrong-arg-types]']
 """

conformance/results/pytype/generics_basic.toml

@@ -1,6 +1,8 @@
 conformant = "Partial"
 notes = """
 False positives in examples using constrained type variables.
+Argument 'covariant' to TypeVar not supported.
+False negative for type variables missing from Generic/Protocol base class.
 False negative for generic metaclass.
 """
 output = """
@@ -66,17 +68,41 @@ generics_basic.py:158:5: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in test_my
     m2[0]  # E
     \u001b[1m\u001b[31m~~~~~\u001b[39m\u001b[0m
 
-generics_basic.py:182:5: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in test_my_iterable_any: Iterator[nothing] [assert-type]
+generics_basic.py:162:12: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in <module>: Invalid type annotation 'Generic'  [invalid-annotation]
+
+class Bad1(Generic[int]): ...  # E
+           \u001b[1m\u001b[31m~~~~~~~~~~~~\u001b[39m\u001b[0m
+
+generics_basic.py:163:12: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in <module>: Invalid type annotation 'Protocol'  [invalid-annotation]
+
+class Bad2(Protocol[int]): ...  # E
+           \u001b[1m\u001b[31m~~~~~~~~~~~~~\u001b[39m\u001b[0m
+
+generics_basic.py:168:8: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in <module>: argument "covariant" to TypeVar not supported yet [not-supported-yet]
+
+T_co = TypeVar("T_co", covariant=True)
+       \u001b[1m\u001b[31m~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\u001b[39m\u001b[0m
+
+generics_basic.py:169:8: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in <module>: argument "covariant" to TypeVar not supported yet [not-supported-yet]
+
+S_co = TypeVar("S_co", covariant=True)
+       \u001b[1m\u001b[31m~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\u001b[39m\u001b[0m
+
+generics_basic.py:199:5: \u001b[1m\u001b[31merror\u001b[39m\u001b[0m: in test_my_iterable_any: Iterator[nothing] [assert-type]
 
     assert_type(iter(m), Iterator[Any])
     \u001b[1m\u001b[31m~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\u001b[39m\u001b[0m
 
 """
 conformance_automated = "Fail"
 errors_diff = """
-Line 191: Expected 1 errors
+Line 171: Expected 1 errors
+Line 172: Expected 1 errors
+Line 208: Expected 1 errors
 Line 34: Unexpected errors ['generics_basic.py:34:5: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in concat: bad return type [bad-return-type]']
 Line 67: Unexpected errors ['generics_basic.py:67:5: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in test_concat_subtype: MyStr [assert-type]']
 Line 68: Unexpected errors ['generics_basic.py:68:17: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in test_concat_subtype: Function concat was called with the wrong arguments [wrong-arg-types]', 'generics_basic.py:68:5: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in test_concat_subtype: Any [assert-type]']
-Line 182: Unexpected errors ['generics_basic.py:182:5: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in test_my_iterable_any: Iterator[nothing] [assert-type]']
+Line 168: Unexpected errors ['generics_basic.py:168:8: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in <module>: argument "covariant" to TypeVar not supported yet [not-supported-yet]']
+Line 169: Unexpected errors ['generics_basic.py:169:8: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in <module>: argument "covariant" to TypeVar not supported yet [not-supported-yet]']
+Line 199: Unexpected errors ['generics_basic.py:199:5: \\x1b[1m\\x1b[31merror\\x1b[39m\\x1b[0m: in test_my_iterable_any: Iterator[nothing] [assert-type]']
 """

conformance/results/pytype/version.toml

@@ -1,2 +1,2 @@
 version = "pytype 2024.10.11"
-test_duration = 50.2
+test_duration = 45.2

conformance/results/results.html

@@ -159,16 +159,16 @@ <h3>Python Type System Conformance Test Results</h3>
         <div class="table_container"><table><tbody>
 <tr><th class="col1">&nbsp;</th>
 <th class='tc-header'><div class='tc-name'>mypy 1.13.0</div>
-<div class='tc-time'>2.7sec</div>
+<div class='tc-time'>2.5sec</div>
 </th>
-<th class='tc-header'><div class='tc-name'>pyright 1.1.386</div>
-<div class='tc-time'>2.0sec</div>
+<th class='tc-header'><div class='tc-name'>pyright 1.1.389</div>
+<div class='tc-time'>2.5sec</div>
 </th>
 <th class='tc-header'><div class='tc-name'>pyre 0.9.23</div>
-<div class='tc-time'>9.1sec</div>
+<div class='tc-time'>4.3sec</div>
 </th>
 <th class='tc-header'><div class='tc-name'>pytype 2024.10.11</div>
-<div class='tc-time'>50.2sec</div>
+<div class='tc-time'>45.2sec</div>
 </th>
 </tr>
 <tr><th class="column" colspan="5">
@@ -241,16 +241,16 @@ <h3>Python Type System Conformance Test Results</h3>
 <a class="test_group" href="https://typing.readthedocs.io/en/latest/spec/generics.html">Generics</a>
 </th></tr>
 <tr><th class="column col1">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;generics_base_class</th>
+<th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Does not detect inconsistent type variable ordering.</p></span></div></th>
 <th class="column col2 conformant">Pass</th>
-<th class="column col2 conformant">Pass</th>
-<th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Does not reject illegal use of Generic.</p><p>Does not allow using generic in assert_type expression.</p></span></div></th>
-<th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>False negative on passing SymbolTable to dict[str, list[object]].</p><p>Does not reject illegal use of Generic.</p></span></div></th>
+<th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Does not reject illegal use of Generic.</p><p>Does not allow using generic in assert_type expression.</p><p>Does not detect inconsistent type variable ordering.</p></span></div></th>
+<th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>False negative on passing SymbolTable to dict[str, list[object]].</p><p>Does not reject illegal use of Generic.</p><p>Wrong type variable order when inheriting from multiple generic base classes.</p></span></div></th>
 </tr>
 <tr><th class="column col1">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;generics_basic</th>
 <th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Does not reject the use of a constraint parameterized by another type variable.</p></span></div></th>
 <th class="column col2 conformant">Pass</th>
-<th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>False positives in examples using constrained type variables.</p><p>False negative for constraint parameterized by a type variable.</p><p>False negative in custom map example.</p><p>False positive using `iter`.</p><p>False negative for generic metaclass.</p></span></div></th>
-<th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>False positives in examples using constrained type variables.</p><p>False negative for generic metaclass.</p></span></div></th>
+<th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>False positives in examples using constrained type variables.</p><p>False negative for constraint parameterized by a type variable.</p><p>False negative in custom map example.</p><p>False positive using `iter`.</p><p>False negative for bad type arguments to Generic/Protocol.</p><p>False negative for generic metaclass.</p></span></div></th>
+<th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>False positives in examples using constrained type variables.</p><p>Argument 'covariant' to TypeVar not supported.</p><p>False negative for type variables missing from Generic/Protocol base class.</p><p>False negative for generic metaclass.</p></span></div></th>
 </tr>
 <tr><th class="column col1">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;generics_defaults</th>
 <th class="column col2 not-conformant">Unsupported</th>