Literal unary annotation (#1580)

* Added command-line option "--report-only" that regenerates the report summary without re-running the tests.

* Updated conformance suite to reflect recent change to spec regarding the use of a unary `+` within a `Literal` type annotation.

conformance/results/mypy/literals_parameterizations.toml

@@ -1,22 +1,24 @@
 conformant = "Partial"
 notes = """
+Rejects integer Literal with unary `+` operator.
 Does not reject tuple within Literal.
 """
 output = """
-literals_parameterizations.py:40: error: Invalid type: Literal[...] cannot contain arbitrary expressions  [valid-type]
+literals_parameterizations.py:20: error: Invalid type: Literal[...] cannot contain arbitrary expressions  [valid-type]
 literals_parameterizations.py:41: error: Invalid type: Literal[...] cannot contain arbitrary expressions  [valid-type]
 literals_parameterizations.py:42: error: Invalid type: Literal[...] cannot contain arbitrary expressions  [valid-type]
 literals_parameterizations.py:43: error: Invalid type: Literal[...] cannot contain arbitrary expressions  [valid-type]
 literals_parameterizations.py:44: error: Invalid type: Literal[...] cannot contain arbitrary expressions  [valid-type]
-literals_parameterizations.py:46: error: Invalid type: Literal[...] cannot contain arbitrary expressions  [valid-type]
-literals_parameterizations.py:47: error: Parameter 1 of Literal[...] is invalid  [valid-type]
+literals_parameterizations.py:45: error: Invalid type: Literal[...] cannot contain arbitrary expressions  [valid-type]
+literals_parameterizations.py:47: error: Invalid type: Literal[...] cannot contain arbitrary expressions  [valid-type]
 literals_parameterizations.py:48: error: Parameter 1 of Literal[...] is invalid  [valid-type]
 literals_parameterizations.py:49: error: Parameter 1 of Literal[...] is invalid  [valid-type]
-literals_parameterizations.py:50: error: Parameter 1 of Literal[...] cannot be of type "float"  [valid-type]
-literals_parameterizations.py:51: error: Parameter 1 of Literal[...] cannot be of type "Any"  [valid-type]
-literals_parameterizations.py:52: error: Parameter 1 of Literal[...] is invalid  [valid-type]
-literals_parameterizations.py:55: error: Invalid type: Literal[...] cannot contain arbitrary expressions  [valid-type]
-literals_parameterizations.py:58: error: Literal[...] must have at least one parameter  [valid-type]
-literals_parameterizations.py:59: error: Parameter 1 of Literal[...] is invalid  [valid-type]
-literals_parameterizations.py:63: error: Incompatible types in assignment (expression has type "Literal[Color.RED]", variable has type "Literal['Color.RED']")  [assignment]
+literals_parameterizations.py:50: error: Parameter 1 of Literal[...] is invalid  [valid-type]
+literals_parameterizations.py:51: error: Parameter 1 of Literal[...] cannot be of type "float"  [valid-type]
+literals_parameterizations.py:52: error: Parameter 1 of Literal[...] cannot be of type "Any"  [valid-type]
+literals_parameterizations.py:53: error: Parameter 1 of Literal[...] is invalid  [valid-type]
+literals_parameterizations.py:56: error: Invalid type: Literal[...] cannot contain arbitrary expressions  [valid-type]
+literals_parameterizations.py:60: error: Literal[...] must have at least one parameter  [valid-type]
+literals_parameterizations.py:61: error: Parameter 1 of Literal[...] is invalid  [valid-type]
+literals_parameterizations.py:65: error: Incompatible types in assignment (expression has type "Literal[Color.RED]", variable has type "Literal['Color.RED']")  [assignment]
 """

conformance/results/pyre/literals_parameterizations.toml

@@ -7,24 +7,25 @@ Does not reject tuple in Literal type expression.
 Does not reject "bare" Literal in type expression.
 """
 output = """
-literals_parameterizations.py:32:0 Invalid type [31]: Expression `AppendMode` is not a literal value.
-literals_parameterizations.py:32:0 Invalid type [31]: Expression `ReadOnlyMode` is not a literal value.
-literals_parameterizations.py:32:0 Invalid type [31]: Expression `WriteAndTruncateMode` is not a literal value.
-literals_parameterizations.py:32:0 Invalid type [31]: Expression `WriteNoTruncateMode` is not a literal value.
-literals_parameterizations.py:32:0 Undefined or invalid type [11]: Annotation `` is not defined as a type.
-literals_parameterizations.py:34:8 Invalid type [31]: Expression `typing.Literal[(typing.Literal[(typing.Literal[(1, 2, 3)], "foo")], 5, None)]` is not a valid type.
-literals_parameterizations.py:40:6 Invalid type [31]: Expression `typing.Literal[3.__add__(4)]` is not a valid type.
-literals_parameterizations.py:41:6 Invalid type [31]: Expression `typing.Literal["foo".replace("o", "b")]` is not a valid type.
-literals_parameterizations.py:42:6 Invalid type [31]: Expression `typing.Literal[4.__add__(3.000000j)]` is not a valid type.
-literals_parameterizations.py:44:6 Invalid type [31]: Expression `typing.Literal[not False]` is not a valid type.
-literals_parameterizations.py:46:6 Invalid type [31]: Expression `typing.Literal[{ "a":"b","c":"d" }]` is not a valid type.
-literals_parameterizations.py:47:6 Invalid type [31]: Expression `typing.Literal[int]` is not a valid type.
-literals_parameterizations.py:48:6 Invalid type [31]: Expression `variable` is not a literal value.
-literals_parameterizations.py:49:7 Invalid type [31]: Expression `T` is not a literal value.
-literals_parameterizations.py:50:7 Invalid type [31]: Expression `typing.Literal[3.140000]` is not a valid type.
-literals_parameterizations.py:51:7 Invalid type [31]: Expression `Any` is not a literal value.
-literals_parameterizations.py:52:7 Invalid type [31]: Expression `typing.Literal[...]` is not a valid type.
-literals_parameterizations.py:55:19 Invalid type [31]: Expression `typing.Literal[1.__add__(2)]` is not a valid type.
-literals_parameterizations.py:59:3 Invalid type [31]: Expression `my_function` is not a literal value.
-literals_parameterizations.py:63:4 Incompatible variable type [9]: x1 is declared to have type `typing_extensions.Literal['Color.RED']` but is used as type `typing_extensions.Literal[Color.RED]`.
+literals_parameterizations.py:33:0 Invalid type [31]: Expression `AppendMode` is not a literal value.
+literals_parameterizations.py:33:0 Invalid type [31]: Expression `ReadOnlyMode` is not a literal value.
+literals_parameterizations.py:33:0 Invalid type [31]: Expression `WriteAndTruncateMode` is not a literal value.
+literals_parameterizations.py:33:0 Invalid type [31]: Expression `WriteNoTruncateMode` is not a literal value.
+literals_parameterizations.py:33:0 Undefined or invalid type [11]: Annotation `` is not defined as a type.
+literals_parameterizations.py:35:8 Invalid type [31]: Expression `typing.Literal[(typing.Literal[(typing.Literal[(1, 2, 3)], "foo")], 5, None)]` is not a valid type.
+literals_parameterizations.py:41:6 Invalid type [31]: Expression `typing.Literal[3.__add__(4)]` is not a valid type.
+literals_parameterizations.py:42:6 Invalid type [31]: Expression `typing.Literal["foo".replace("o", "b")]` is not a valid type.
+literals_parameterizations.py:43:6 Invalid type [31]: Expression `typing.Literal[4.__add__(3.000000j)]` is not a valid type.
+literals_parameterizations.py:44:6 Invalid type [31]: Expression `typing.Literal[~ 5]` is not a valid type.
+literals_parameterizations.py:45:6 Invalid type [31]: Expression `typing.Literal[not False]` is not a valid type.
+literals_parameterizations.py:47:6 Invalid type [31]: Expression `typing.Literal[{ "a":"b","c":"d" }]` is not a valid type.
+literals_parameterizations.py:48:6 Invalid type [31]: Expression `typing.Literal[int]` is not a valid type.
+literals_parameterizations.py:49:6 Invalid type [31]: Expression `variable` is not a literal value.
+literals_parameterizations.py:50:7 Invalid type [31]: Expression `T` is not a literal value.
+literals_parameterizations.py:51:7 Invalid type [31]: Expression `typing.Literal[3.140000]` is not a valid type.
+literals_parameterizations.py:52:7 Invalid type [31]: Expression `Any` is not a literal value.
+literals_parameterizations.py:53:7 Invalid type [31]: Expression `typing.Literal[...]` is not a valid type.
+literals_parameterizations.py:56:19 Invalid type [31]: Expression `typing.Literal[1.__add__(2)]` is not a valid type.
+literals_parameterizations.py:61:3 Invalid type [31]: Expression `my_function` is not a literal value.
+literals_parameterizations.py:65:4 Incompatible variable type [9]: x1 is declared to have type `typing_extensions.Literal['Color.RED']` but is used as type `typing_extensions.Literal[Color.RED]`.
 """

conformance/results/pyright/literals_parameterizations.toml

@@ -1,6 +1,9 @@
-conformant = "Pass"
+conformant = "Partial"
+notes = """
+Rejects integer Literal with unary `+` operator.
+"""
 output = """
-literals_parameterizations.py:40:15 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value
+literals_parameterizations.py:20:16 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value
 literals_parameterizations.py:41:15 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value
 literals_parameterizations.py:42:15 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value
 literals_parameterizations.py:43:15 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value
@@ -9,13 +12,14 @@ literals_parameterizations.py:45:15 - error: Type arguments for "Literal" must b
 literals_parameterizations.py:46:15 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value
 literals_parameterizations.py:47:15 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value
 literals_parameterizations.py:48:15 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value
-literals_parameterizations.py:49:16 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value
+literals_parameterizations.py:49:15 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value
 literals_parameterizations.py:50:16 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value
 literals_parameterizations.py:51:16 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value
 literals_parameterizations.py:52:16 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value
-literals_parameterizations.py:55:28 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value
-literals_parameterizations.py:58:4 - error: "Literal" cannot be used in this context without a type argument
-literals_parameterizations.py:59:12 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value
-literals_parameterizations.py:63:32 - error: Expression of type "Literal[Color.RED]" cannot be assigned to declared type "Literal['Color.RED']"
+literals_parameterizations.py:53:16 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value
+literals_parameterizations.py:56:28 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value
+literals_parameterizations.py:60:4 - error: "Literal" cannot be used in this context without a type argument
+literals_parameterizations.py:61:12 - error: Type arguments for "Literal" must be None, a literal value (int, bool, str, or bytes), or an enum value
+literals_parameterizations.py:65:32 - error: Expression of type "Literal[Color.RED]" cannot be assigned to declared type "Literal['Color.RED']"
   "Literal[Color.RED]" cannot be assigned to type "Literal['Color.RED']" (reportGeneralTypeIssues)
 """

conformance/results/pytype/literals_parameterizations.toml

@@ -9,33 +9,35 @@ File "literals_parameterizations.py", line 18, in <module>: Invalid type annotat
   Bad parameter 'int' at index 0
 File "literals_parameterizations.py", line 19, in <module>: Invalid type annotation 'Literal'  [invalid-annotation]
   Bad parameter 'int' at index 0
-File "literals_parameterizations.py", line 32, in <module>: Invalid type annotation 'Literal'  [invalid-annotation]
+File "literals_parameterizations.py", line 33, in <module>: Invalid type annotation 'Literal'  [invalid-annotation]
   Bad parameter 'Union' at index 0
   Bad parameter 'Union' at index 1
   Bad parameter 'Union' at index 2
   Bad parameter 'Union' at index 3
-File "literals_parameterizations.py", line 34, in <module>: Invalid type annotation 'Literal'  [invalid-annotation]
+File "literals_parameterizations.py", line 35, in <module>: Invalid type annotation 'Literal'  [invalid-annotation]
   Bad parameter 'Union' at index 0
-File "literals_parameterizations.py", line 41, in <module>: Invalid type annotation 'Literal'  [invalid-annotation]
-  Bad parameter 'str' at index 0
 File "literals_parameterizations.py", line 42, in <module>: Invalid type annotation 'Literal'  [invalid-annotation]
+  Bad parameter 'str' at index 0
+File "literals_parameterizations.py", line 43, in <module>: Invalid type annotation 'Literal'  [invalid-annotation]
   Bad parameter 'complex' at index 0
-File "literals_parameterizations.py", line 46, in <module>: Invalid type annotation 'Literal'  [invalid-annotation]
-  Bad parameter 'dict' at index 0
+File "literals_parameterizations.py", line 44, in <module>: Invalid type annotation 'Literal'  [invalid-annotation]
+  Bad parameter 'int' at index 0
 File "literals_parameterizations.py", line 47, in <module>: Invalid type annotation 'Literal'  [invalid-annotation]
+  Bad parameter 'dict' at index 0
+File "literals_parameterizations.py", line 48, in <module>: Invalid type annotation 'Literal'  [invalid-annotation]
   Bad parameter 'int' at index 0
-File "literals_parameterizations.py", line 49, in <module>: Invalid type annotation 'Literal'  [invalid-annotation]
-  Bad parameter 'T' at index 0
 File "literals_parameterizations.py", line 50, in <module>: Invalid type annotation 'Literal'  [invalid-annotation]
+  Bad parameter 'T' at index 0
+File "literals_parameterizations.py", line 51, in <module>: Invalid type annotation 'Literal'  [invalid-annotation]
   Bad parameter 'float' at index 0
-File "literals_parameterizations.py", line 52, in <module>: Invalid type annotation 'Literal'  [invalid-annotation]
+File "literals_parameterizations.py", line 53, in <module>: Invalid type annotation 'Literal'  [invalid-annotation]
   Bad parameter '...' at index 0
-File "literals_parameterizations.py", line 59, in <module>: Invalid type annotation 'Literal[my_function]'  [invalid-annotation]
+File "literals_parameterizations.py", line 61, in <module>: Invalid type annotation 'Literal[my_function]'  [invalid-annotation]
   Invalid type annotation 'Literal' 
   Bad parameter 'my_function' at index 0
-File "literals_parameterizations.py", line 59, in <module>: Invalid type annotation 'Literal'  [invalid-annotation]
+File "literals_parameterizations.py", line 61, in <module>: Invalid type annotation 'Literal'  [invalid-annotation]
   Bad parameter 'my_function' at index 0
-File "literals_parameterizations.py", line 63, in func2: Type annotation for x1 does not match type of assignment [annotation-type-mismatch]
+File "literals_parameterizations.py", line 65, in func2: Type annotation for x1 does not match type of assignment [annotation-type-mismatch]
   Annotation: Literal['Color.RED']
   Assignment: Color
 """

conformance/src/main.py

@@ -11,6 +11,7 @@
 import tomli
 import tomlkit
 
+from options import parse_options
 from reporting import generate_summary
 from test_groups import get_test_cases, get_test_groups
 from type_checker import TYPE_CHECKERS, TypeChecker
@@ -111,23 +112,26 @@ def main():
     # latest version of Python (3.12), so we need this version.
     assert sys.version_info >= (3, 12)
 
+    options = parse_options(sys.argv[1:])
+
     root_dir = Path(__file__).resolve().parent.parent
 
-    tests_dir = root_dir / "tests"
-    assert tests_dir.is_dir()
+    if not options.report_only:
+        tests_dir = root_dir / "tests"
+        assert tests_dir.is_dir()
 
-    test_groups = get_test_groups(root_dir)
-    test_cases = get_test_cases(test_groups, tests_dir)
+        test_groups = get_test_groups(root_dir)
+        test_cases = get_test_cases(test_groups, tests_dir)
 
-    # Switch to the tests directory.
-    os.chdir(tests_dir)
+        # Switch to the tests directory.
+        os.chdir(tests_dir)
 
-    # Run each test case with each type checker.
-    for type_checker in TYPE_CHECKERS:
-        if not type_checker.install():
-            print(f"Skipping tests for {type_checker.name}")
-        else:
-            run_tests(root_dir, type_checker, test_cases)
+        # Run each test case with each type checker.
+        for type_checker in TYPE_CHECKERS:
+            if not type_checker.install():
+                print(f"Skipping tests for {type_checker.name}")
+            else:
+                run_tests(root_dir, type_checker, test_cases)
 
     # Generate a summary report.
     generate_summary(root_dir)

conformance/src/options.py

@@ -0,0 +1,23 @@
+"""
+Command-line options for the test tool.
+"""
+
+import argparse
+from dataclasses import dataclass
+
+
+@dataclass
+class _Options:
+    report_only: bool | None
+
+
+def parse_options(argv: list[str]) -> _Options:
+    parser = argparse.ArgumentParser()
+    reporting_group = parser.add_argument_group("reporting")
+    reporting_group.add_argument(
+        "--report-only",
+        action="store_true",
+        help=("regenerates the test suite report from past results"),
+    )
+    ret = _Options(**vars(parser.parse_args(argv)))
+    return ret

conformance/tests/literals_parameterizations.py

@@ -17,12 +17,13 @@ class Color(Enum):
 good1: Literal[26]
 good2: Literal[0x1A]
 good3: Literal[-4]
-good4: Literal["hello world"]
-good5: Literal[b"hello world"]
-good6: Literal["hello world"]
-good7: Literal[True]
-good8: Literal[Color.RED]
-good9: Literal[None]
+good4: Literal[+5]
+good5: Literal["hello world"]
+good6: Literal[b"hello world"]
+good7: Literal["hello world"]
+good8: Literal[True]
+good9: Literal[Color.RED]
+good10: Literal[None]
 
 ReadOnlyMode = Literal["r", "r+"]
 WriteAndTruncateMode = Literal["w", "w+", "wt", "w+t"]
@@ -31,7 +32,7 @@ class Color(Enum):
 
 AllModes = Literal[ReadOnlyMode, WriteAndTruncateMode, WriteNoTruncateMode, AppendMode]
 
-good10: Literal[Literal[Literal[1, 2, 3], "foo"], 5, None]
+good11: Literal[Literal[Literal[1, 2, 3], "foo"], 5, None]
 
 variable = 3
 T = TypeVar("T")
@@ -40,7 +41,7 @@ class Color(Enum):
 bad1: Literal[3 + 4]  # Type error
 bad2: Literal["foo".replace("o", "b")]  # Type error
 bad3: Literal[4 + 3j]  # Type error
-bad4: Literal[+5]  # Type error
+bad4: Literal[~5]  # Type error
 bad5: Literal[not False]  # Type error
 bad6: Literal[(1, "foo", "bar")]  # Type error
 bad7: Literal[{"a": "b", "c": "d"}]  # Type error
@@ -55,6 +56,7 @@ class Color(Enum):
 def my_function(x: Literal[1 + 2]) -> int:  # Type error
     return x * 3
 
+
 x: Literal  # Type error
 y: Literal[my_function] = my_function  # Type error
 
@@ -63,5 +65,3 @@ def func2(a: Literal[Color.RED]):
     x1: Literal["Color.RED"] = a  # Type error
 
     x2: "Literal[Color.RED]" = a  # OK
-
-