diff --git a/conformance/README.md b/conformance/README.md index 757bc6d62..ebc39986c 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -46,6 +46,8 @@ To run the conformance test suite: * Switch to the `conformance` subdirectory and install all dependencies (`pip install -r requirements.txt`). * Switch to the `src` subdirectory and run `python main.py`. +Note that some type checkers may not run on some platforms. For example, pytype cannot be installed on Windows. If a type checker fails to install, tests will be skipped for that type checker. + ## Reporting Conformance Results Different type checkers report errors in different ways (with different wording in error messages and different line numbers or character ranges for errors). This variation makes it difficult to fully automate test validation given that tests will want to check for both false positive and false negative type errors. Some level of manual inspection will therefore be needed to determine whether a type checker is fully conformant with all tests in any given test file. This "scoring" process is required only when the output of a test changes — e.g. when a new version of that type checker is released and the tests are rerun. We assume that the output of a type checker will be the same from one run to the next unless/until a new version is released that fixes or introduces a bug. In this case, the output will need to be manually inspected and the conformance results re-scored for those tests whose output has changed. diff --git a/conformance/requirements.txt b/conformance/requirements.txt index b60d217d6..3f3979428 100644 --- a/conformance/requirements.txt +++ b/conformance/requirements.txt @@ -3,4 +3,5 @@ tomlkit pyright mypy pyre-check -pytype +pytype; platform_system != "Windows" + diff --git a/conformance/src/main.py b/conformance/src/main.py index fd7c1aadf..be474bac0 100644 --- a/conformance/src/main.py +++ b/conformance/src/main.py @@ -116,8 +116,10 @@ def main(): # Run each test case with each type checker. for type_checker in TYPE_CHECKERS: - type_checker.install() - run_tests(root_dir, type_checker, test_cases) + if not type_checker.install(): + print(f'Skipping tests for {type_checker.name}') + else: + run_tests(root_dir, type_checker, test_cases) # Generate a summary report. generate_summary(root_dir) diff --git a/conformance/src/type_checker.py b/conformance/src/type_checker.py index a6d1e76b0..905d56296 100644 --- a/conformance/src/type_checker.py +++ b/conformance/src/type_checker.py @@ -21,9 +21,10 @@ def name(self) -> str: raise NotImplementedError @abstractmethod - def install(self) -> None: + def install(self) -> bool: """ Ensures that the latest version of the type checker is installed. + Returns False if installation fails. """ raise NotImplementedError @@ -48,8 +49,13 @@ class MypyTypeChecker(TypeChecker): def name(self) -> str: return "mypy" - def install(self) -> None: - run(f"{sys.executable} -m pip install mypy --upgrade", shell=True) + def install(self) -> bool: + try: + run(f"{sys.executable} -m pip install mypy --upgrade", check=True, shell=True) + return True + except: + print('Unable to install mypy') + return False def get_version(self) -> str: proc = run( @@ -80,13 +86,18 @@ class PyrightTypeChecker(TypeChecker): def name(self) -> str: return "pyright" - def install(self) -> None: - # Install the Python wrapper if it's not installed. - run(f"{sys.executable} -m pip install pyright --upgrade", shell=True) + def install(self) -> bool: + try: + # Install the Python wrapper if it's not installed. + run(f"{sys.executable} -m pip install pyright --upgrade", check=True, shell=True) - # Force the Python wrapper to install node if needed - # and download the latest version of pyright. - self.get_version() + # Force the Python wrapper to install node if needed + # and download the latest version of pyright. + self.get_version() + return True + except: + print('Unable to install pyright') + return False def get_version(self) -> str: proc = run( @@ -122,15 +133,21 @@ class PyreTypeChecker(TypeChecker): def name(self) -> str: return "pyre" - def install(self) -> None: - run(f"{sys.executable} -m pip install pyre-check --upgrade", shell=True) + def install(self) -> bool: + try: + run(f"{sys.executable} -m pip install pyre-check --upgrade", check=True, shell=True) - # Generate a default config file. - pyre_config = ( - '{"site_package_search_strategy": "pep561", "source_directories": ["."]}\n' - ) - with open(".pyre_configuration", "w") as f: - f.write(pyre_config) + # Generate a default config file. + pyre_config = ( + '{"site_package_search_strategy": "pep561", "source_directories": ["."]}\n' + ) + with open(".pyre_configuration", "w") as f: + f.write(pyre_config) + + return True + except: + print('Unable to install pyre') + return False def get_version(self) -> str: proc = run("pyre --version", stdout=PIPE, text=True, shell=True) @@ -156,8 +173,13 @@ class PytypeTypeChecker(TypeChecker): def name(self) -> str: return "pytype" - def install(self) -> None: - run(f"{sys.executable} -m pip install pytype --upgrade", shell=True) + def install(self) -> bool: + try: + run(f"{sys.executable} -m pip install pytype --upgrade", check=True, shell=True) + return True + except: + print('Unable to install pytype on this platform') + return False def get_version(self) -> str: proc = run(