Added code to skip tests for a type checker if it cannot be installed…

… for some reason.

conformance/README.md

@@ -46,6 +46,8 @@ To run the conformance test suite:
 * Switch to the `conformance` subdirectory and install all dependencies (`pip install -r requirements.txt`).
 * Switch to the `src` subdirectory and run `python main.py`.
 
+Note that some type checkers may not run on some platforms. For example, pytype cannot be installed on Windows. If a type checker fails to install, tests will be skipped for that type checker.
+
 ## Reporting Conformance Results
 
 Different type checkers report errors in different ways (with different wording in error messages and different line numbers or character ranges for errors). This variation makes it difficult to fully automate test validation given that tests will want to check for both false positive and false negative type errors. Some level of manual inspection will therefore be needed to determine whether a type checker is fully conformant with all tests in any given test file. This "scoring" process is required only when the output of a test changes — e.g. when a new version of that type checker is released and the tests are rerun. We assume that the output of a type checker will be the same from one run to the next unless/until a new version is released that fixes or introduces a bug. In this case, the output will need to be manually inspected and the conformance results re-scored for those tests whose output has changed.

conformance/requirements.txt

@@ -3,4 +3,5 @@ tomlkit
 pyright
 mypy
 pyre-check
-pytype
+pytype; platform_system != "Windows"
+

conformance/src/main.py

@@ -116,8 +116,10 @@ def main():
 
     # Run each test case with each type checker.
     for type_checker in TYPE_CHECKERS:
-        type_checker.install()
-        run_tests(root_dir, type_checker, test_cases)
+        if not type_checker.install():
+            print(f'Skipping tests for {type_checker.name}')
+        else:
+            run_tests(root_dir, type_checker, test_cases)
 
     # Generate a summary report.
     generate_summary(root_dir)

conformance/src/type_checker.py

@@ -21,9 +21,10 @@ def name(self) -> str:
         raise NotImplementedError
 
     @abstractmethod
-    def install(self) -> None:
+    def install(self) -> bool:
         """
         Ensures that the latest version of the type checker is installed.
+        Returns False if installation fails.
         """
         raise NotImplementedError
 
@@ -48,8 +49,13 @@ class MypyTypeChecker(TypeChecker):
     def name(self) -> str:
         return "mypy"
 
-    def install(self) -> None:
-        run(f"{sys.executable} -m pip install mypy --upgrade", shell=True)
+    def install(self) -> bool:
+        try:
+            run(f"{sys.executable} -m pip install mypy --upgrade", check=True, shell=True)
+            return True
+        except:
+            print('Unable to install mypy')
+            return False
 
     def get_version(self) -> str:
         proc = run(
@@ -80,13 +86,18 @@ class PyrightTypeChecker(TypeChecker):
     def name(self) -> str:
         return "pyright"
 
-    def install(self) -> None:
-        # Install the Python wrapper if it's not installed.
-        run(f"{sys.executable} -m pip install pyright --upgrade", shell=True)
+    def install(self) -> bool:
+        try:
+            # Install the Python wrapper if it's not installed.
+            run(f"{sys.executable} -m pip install pyright --upgrade", check=True, shell=True)
 
-        # Force the Python wrapper to install node if needed
-        # and download the latest version of pyright.
-        self.get_version()
+            # Force the Python wrapper to install node if needed
+            # and download the latest version of pyright.
+            self.get_version()
+            return True
+        except:
+            print('Unable to install pyright')
+            return False
 
     def get_version(self) -> str:
         proc = run(
@@ -122,15 +133,21 @@ class PyreTypeChecker(TypeChecker):
     def name(self) -> str:
         return "pyre"
 
-    def install(self) -> None:
-        run(f"{sys.executable} -m pip install pyre-check --upgrade", shell=True)
+    def install(self) -> bool:
+        try:
+            run(f"{sys.executable} -m pip install pyre-check --upgrade", check=True, shell=True)
 
-        # Generate a default config file.
-        pyre_config = (
-            '{"site_package_search_strategy": "pep561", "source_directories": ["."]}\n'
-        )
-        with open(".pyre_configuration", "w") as f:
-            f.write(pyre_config)
+            # Generate a default config file.
+            pyre_config = (
+                '{"site_package_search_strategy": "pep561", "source_directories": ["."]}\n'
+            )
+            with open(".pyre_configuration", "w") as f:
+                f.write(pyre_config)
+
+            return True
+        except:
+            print('Unable to install pyre')
+            return False
 
     def get_version(self) -> str:
         proc = run("pyre --version", stdout=PIPE, text=True, shell=True)
@@ -156,8 +173,13 @@ class PytypeTypeChecker(TypeChecker):
     def name(self) -> str:
         return "pytype"
 
-    def install(self) -> None:
-        run(f"{sys.executable} -m pip install pytype --upgrade", shell=True)
+    def install(self) -> bool:
+        try:
+            run(f"{sys.executable} -m pip install pytype --upgrade", check=True, shell=True)
+            return True
+        except:
+            print('Unable to install pytype on this platform')
+            return False
 
     def get_version(self) -> str:
         proc = run(