Make pytype conformance test a little faster. (#1559)

This change gets the pytype test down to ~30s. It's still considerably
slower than the others, but any further improvements will likely require
untangling the sad state of affairs that is
google/pytype#1501, which will take some time.

* Runs pytype's 'check' option through pytype.io.check_py to skip the
  costly type inference that it does by default.
* Adds a progress bar so that it doesn't look like pytype is hanging.

This generates a few changes to results/pytype/ due to error messages
being formatted and parsed in a different way.

conformance/requirements.txt

@@ -1,5 +1,6 @@
 tomli
 tomlkit
+tqdm
 pyright
 mypy
 pyre-check

conformance/results/pytype/aliases_type_statement.toml

@@ -3,5 +3,4 @@ notes = """
 Does not support `type` statement.
 """
 output = """
-File "aliases_type_statement.py", line 8: Type statement is only supported in Python 3.12 and greater [python-compiler-error]
-"""
+SyntaxError: Type statement is only supported in Python 3.12 and greater (<unknown>, line 8)"""

conformance/results/pytype/literals_literalstring.toml

@@ -3,4 +3,29 @@ notes = """
 Does not understand `LiteralString` special form.
 """
 output = """
+File "literals_literalstring.py", line 8, in <module>: typing.LiteralString not supported yet [not-supported-yet]
+File "literals_literalstring.py", line 24, in my_function: bad return type [bad-return-type]
+           Expected: str
+  Actually returned: None
+File "literals_literalstring.py", line 36, in <module>: Invalid type annotation 'Literal'  [invalid-annotation]
+  Bad parameter 'str' at index 1
+File "literals_literalstring.py", line 37, in <module>: Invalid type annotation 'Literal'  [invalid-annotation]
+  Bad parameter 'str' at index 0
+File "literals_literalstring.py", line 43, in func1: Type annotation for x2 does not match type of assignment [annotation-type-mismatch]
+  Annotation: Literal['']
+  Assignment: Literal['two']
+File "literals_literalstring.py", line 74, in func2: Type annotation for x3 does not match type of assignment [annotation-type-mismatch]
+  Annotation: str
+  Assignment: int
+File "literals_literalstring.py", line 75, in func2: Type annotation for x4 does not match type of assignment [annotation-type-mismatch]
+  Annotation: str
+  Assignment: bytes
+File "literals_literalstring.py", line 157, in func8: bad return type [bad-return-type]
+           Expected: int
+  Actually returned: None
+Called from (traceback):
+  line 160, in current file
+File "literals_literalstring.py", line 162, in <module>: bool [assert-type]
+  Expected: str
+    Actual: bool
 """

conformance/results/pytype/typeddicts_operations.toml

@@ -9,15 +9,24 @@ output = """
 File "typeddicts_operations.py", line 28, in <module>: Type annotation for movie does not match type of assignment [annotation-type-mismatch]
   Annotation: Movie(TypedDict)
   Assignment: Dict[str, str]
+  
+  TypedDict missing keys: year
 File "typeddicts_operations.py", line 29, in <module>: Type annotation for movie does not match type of assignment [annotation-type-mismatch]
   Annotation: Movie(TypedDict)
   Assignment: Dict[str, Union[float, str]]
+  
+  TypedDict type errors: 
+    {'year': ...}: expected int, got float
 File "typeddicts_operations.py", line 32, in <module>: Type annotation for movie does not match type of assignment [annotation-type-mismatch]
   Annotation: Movie(TypedDict)
   Assignment: Dict[str, Union[int, str]]
+  
+  TypedDict extra keys: other
 File "typeddicts_operations.py", line 37, in func1: Type annotation for movie does not match type of assignment [annotation-type-mismatch]
   Annotation: Movie(TypedDict)
   Assignment: Dict[str, Union[int, str]]
+  
+  TypedDict missing keys: name
 File "typeddicts_operations.py", line 60, in <module>: str [assert-type]
   Expected: Optional[str]
     Actual: str

conformance/results/pytype/typeddicts_required.toml

@@ -5,4 +5,4 @@ Does not reject use of `Required` in function parameter annotation.
 Does not reject nested use of `Required` in type annotation.
 """
 output = """
-"""
+RecursionError: maximum recursion depth exceeded"""

conformance/results/pytype/typeddicts_type_consistency.toml

@@ -8,13 +8,22 @@ output = """
 File "typeddicts_type_consistency.py", line 62, in <module>: Type annotation for a3 does not match type of assignment [annotation-type-mismatch]
   Annotation: A3(TypedDict)
   Assignment: B3
+  
+  TypedDict extra keys: y
 File "typeddicts_type_consistency.py", line 65, in <module>: Type annotation for b3 does not match type of assignment [annotation-type-mismatch]
   Annotation: B3(TypedDict)
   Assignment: Dict[str, int]
+  
+  TypedDict missing keys: y
 File "typeddicts_type_consistency.py", line 69, in <module>: Type annotation for a3_1 does not match type of assignment [annotation-type-mismatch]
   Annotation: A3(TypedDict)
   Assignment: Dict[str, int]
+  
+  TypedDict extra keys: y
 File "typeddicts_type_consistency.py", line 124, in <module>: Type annotation for o2 does not match type of assignment [annotation-type-mismatch]
   Annotation: Outer1(TypedDict)
   Assignment: Dict[str, Dict[str, Dict[str, int]]]
+  
+  TypedDict type errors: 
+    {'outer_key': ...}: expected Inner2, got Dict[str, Dict[str, int]]
 """

conformance/results/pytype/typeddicts_usage.toml

@@ -11,4 +11,7 @@ File "typeddicts_usage.py", line 24, in <module>: Type annotation for key year i
 File "typeddicts_usage.py", line 28, in <module>: Type annotation for movie2 does not match type of assignment [annotation-type-mismatch]
   Annotation: Movie(TypedDict)
   Assignment: Dict[str, Union[int, str]]
+  
+  TypedDict missing keys: name
+  TypedDict extra keys: title
 """

conformance/results/pytype/version.toml

@@ -1,2 +1,2 @@
 version = "pytype 2023.12.18"
-test_duration = 51.5667941570282
+test_duration = 28.1415696144104

conformance/results/results.html

@@ -340,7 +340,7 @@ <h3>Python Type System Conformance Test Results</h3>
 <tr><th>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</th><th class="column col1">narrowing_typeguard</th><th class="column col2 partially-conformant">Partial</th><th class="column col3">Does not support `tuple` in `assert_type` call.<br>Does not reject TypeGuard method with too few parameters.<br></th></tr>
 <tr><th class="column spacer" colspan="4"></th></tr>
 </table></div>
-<div class='tc-header'><span class='tc-name'>pytype 2023.12.18<span class='tc-time'>(51.57sec)</span>
+<div class='tc-header'><span class='tc-name'>pytype 2023.12.18<span class='tc-time'>(28.14sec)</span>
 </div>
 <div class="table_container"><table>
 <tr><th class="column spacer" colspan="4"></th></tr>

conformance/src/type_checker.py

@@ -5,9 +5,14 @@
 from abc import ABC, abstractmethod
 import json
 from pathlib import Path
+import os
+from pytype import config as pytype_config
+from pytype import io as pytype_io
+from pytype import load_pytd as pytype_loader
 import re
 from subprocess import PIPE, run
 import sys
+from tqdm import tqdm
 from typing import Sequence
 
 
@@ -192,47 +197,27 @@ def run_tests(self, test_files: Sequence[str]) -> dict[str, str]:
         # Specify 3.11 for now to work around the fact that pytype
         # currently doesn't support 3.12 and emits an error when
         # running on 3.12.
-        command = f"{sys.executable} -m pytype -V 3.11 -k *.py"
-        proc = run(command, stdout=PIPE, text=True, shell=True)
-        lines = proc.stdout.split("\n")
+        options = pytype_config.Options.create(
+            python_version=(3, 11),
+            quick=True)
+        loader = pytype_loader.create_loader(options)
 
         # Add results to a dictionary keyed by the file name.
         results_dict: dict[str, str] = {}
-        accumulated_lines: list[str] = []
-        file_name: str | None = None
 
-        def log_accumulated():
-            if file_name is not None:
-                results_dict[file_name] = (
-                    results_dict.get(file_name, "") + "".join(accumulated_lines) + "\n"
-                )
-
-        for line in lines:
-            match = re.search(r'File "(.*?)",', line)
-
-            if not match or match.start() != 0:
-                # An empty line precedes the summary for the file. Ignore
-                # everything after that line until we see diagnostics for
-                # the next file.
-                if line.strip() == "":
-                    log_accumulated()
-                    file_name = None
-                    accumulated_lines = []
-                elif file_name is not None:
-                    accumulated_lines.append("\n" + line)
+        for fi in tqdm(os.listdir('.')):
+            if not fi.endswith('.py'):
+                continue
+            options.tweak(input=fi)
+            with open(fi, 'r') as test_file:
+                src = test_file.read()
+            try:
+                errorlog = pytype_io.check_py(
+                    src, options=options, loader=loader)
+            except Exception as e:
+                results_dict[fi] = f'{e.__class__.__name__}: {e}'
             else:
-                log_accumulated()
-
-                file_path = Path(match.group(1))
-                file_name = file_path.name
-
-                # Replace the full file path with the file name.
-                line = f'File "{file_name}",{line[match.end():]}'
-                accumulated_lines = [line]
-
-        # Log the final accumulated lines.
-        log_accumulated()
-
+                results_dict[fi] = str(errorlog)
         return results_dict