From 448fffed2ea58b14863e03576d887078ea97967c Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Tue, 26 Dec 2023 13:58:27 -0700 Subject: [PATCH] Improved report visuals. Added timing stats. --- conformance/results/mypy/version.toml | 1 + conformance/results/pyre/version.toml | 1 + conformance/results/pyright/version.toml | 1 + conformance/results/pytype/version.toml | 1 + conformance/results/results.html | 83 +++++++++++++++--------- conformance/src/main.py | 10 ++- conformance/src/reporting.py | 11 +++- conformance/src/results_template.html | 27 ++++++-- 8 files changed, 96 insertions(+), 39 deletions(-) diff --git a/conformance/results/mypy/version.toml b/conformance/results/mypy/version.toml index f0d528a44..533bde034 100644 --- a/conformance/results/mypy/version.toml +++ b/conformance/results/mypy/version.toml @@ -1 +1,2 @@ version = "mypy 1.8.0" +test_duration = 0.31742215156555176 diff --git a/conformance/results/pyre/version.toml b/conformance/results/pyre/version.toml index abda16e02..9a9efb11d 100644 --- a/conformance/results/pyre/version.toml +++ b/conformance/results/pyre/version.toml @@ -1 +1,2 @@ version = "pyre 0.9.19" +test_duration = 1.2770380973815918 diff --git a/conformance/results/pyright/version.toml b/conformance/results/pyright/version.toml index 493948174..0f3d3b9d2 100644 --- a/conformance/results/pyright/version.toml +++ b/conformance/results/pyright/version.toml @@ -1 +1,2 @@ version = "pyright 1.1.343" +test_duration = 0.7825243473052979 diff --git a/conformance/results/pytype/version.toml b/conformance/results/pytype/version.toml index 1c85d9c11..014bd5b04 100644 --- a/conformance/results/pytype/version.toml +++ b/conformance/results/pytype/version.toml @@ -1 +1,2 @@ version = "pytype 2023.12.18" +test_duration = 18.681317806243896 diff --git a/conformance/results/results.html b/conformance/results/results.html index b8b465954..dcbaea08d 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -44,10 +44,6 @@ font-weight: normal; } - th { - background-color: #f8f8f8; - } - .content_container { margin: 20px; } @@ -57,7 +53,6 @@ } .column { - background-color: #f8f8f8; padding-left: 8px; padding-right: 8px; padding-top: 4px; @@ -71,6 +66,12 @@ border: 0px solid #000; } + .spacer { + border-color: #f0f0f0; + border-style: solid; + border-bottom-width: 1px; + } + .col1 { width: 25%; vertical-align: top; @@ -86,6 +87,22 @@ vertical-align: top; } + .tc-header { + margin-top: 20px; + margin-bottom: 8px; + } + + .tc-name { + font-size: 18px; + font-weight: bold; + } + + .tc-time { + margin-left: 8px; + font-size: 12px; + font-weight: normal; + } + .test_group { font-size: 10pt; font-weight: bold; @@ -110,12 +127,14 @@

Python Type System Conformance Test Results

-

mypy 1.8.0

+
mypy 1.8.0(0.32sec) +
+ - + @@ -123,14 +142,14 @@

Python Type System Conformance Test Results

- + - + @@ -141,18 +160,20 @@

Python Type System Conformance Test Results

- + - +
Type annotations
     annotations_typeexprYes
Generics
     generics_self_advancedPartialDoes not infer the type of an unannotated `self` parameter to be type `Self`.
Does not retain `Self` when calling method that returns `Self`.
Does not infer the type of an unannotated `cls` parameter to be type `type[Self]`.
Does not retain `Self` when accessing attribute through `type[Self]`.
     generics_self_basicPartialDoes not properly handle constructor call through `cls` parameter.
     generics_self_protocolsYes
     generics_self_usageYes
Literals
     literals_interactionsPartialDoes not narrow type of `x` with `x in Literal` type guard pattern.
     literals_literalstringUnsupportedSupport for `LiteralString` has not been implemented in mypy.
     literals_parameterizationsPartialDoes not reject tuple within Literal.
     literals_semanticsYes
Typed dictionaries
     typeddicts_alt_syntaxPartialDoes not support keyword-argument form of alternative syntax (deprecated in 3.11).
     typeddicts_requiredPartialDoes not support nesting of `Annotated` and `Required` or `NotRequired`.
     typeddicts_type_consistencyYes
     typeddicts_usageYes
Type narrowing
     narrowing_typeguardYes
-

pyright 1.1.343

+
pyright 1.1.343(0.78sec) +
+ - + @@ -160,14 +181,14 @@

Python Type System Conformance Test Results

- + - + @@ -178,18 +199,20 @@

Python Type System Conformance Test Results

- + - +
Type annotations
     annotations_typeexprYes
Generics
     generics_self_advancedYes
     generics_self_basicYes
     generics_self_protocolsYes
     generics_self_usageYes
Literals
     literals_interactionsYes
     literals_literalstringYes
     literals_parameterizationsYes
     literals_semanticsYes
Typed dictionaries
     typeddicts_alt_syntaxYes
     typeddicts_requiredYes
     typeddicts_type_consistencyYes
     typeddicts_usageYes
Type narrowing
     narrowing_typeguardYes
-

pyre 0.9.19

+
pyre 0.9.19(1.28sec) +
+ - + @@ -197,14 +220,14 @@

Python Type System Conformance Test Results

- + - + @@ -215,18 +238,20 @@

Python Type System Conformance Test Results

- + - +
Type annotations
     annotations_typeexprYes
Generics
     generics_self_advancedUnsupportedDoes not understand `Self` type.
     generics_self_basicUnsupportedDoes not understand `Self` type.
     generics_self_protocolsPartialDoes not reject protocol compatibility due to method `Self` return type.
     generics_self_usageUnsupportedDoes not understand `Self` type.
Literals
     literals_interactionsPartialDoes not detect out-of-bound tuple literal index.
Does not narrow type of `x` with `x in Literal` type guard pattern.
Does not narrow type of `x` with `x == Literal` type guard pattern.
     literals_literalstringYes
     literals_parameterizationsPartialDoes not support type aliases in Literal type expression.
Does not support nested Literal type expression.
Does not reject unary + operator in Literal type expression.
Does not reject tuple in Literal type expression.
Does not reject "bare" Literal in type expression.
     literals_semanticsPartialDoes not reject augmented operation that modifies literal value.
Typed dictionaries
     typeddicts_alt_syntaxPartialDoes not report when name of TypedDict doesn't match assigned identifier name.
Does not support keyword-argument form of alternative syntax (deprecated in 3.11).
     typeddicts_requiredPartialDoes not reject use of `Required` in function parameter annotation.
Does not reject nested use of `Required` in type annotation.
Does not support recursive TypedDict definitions.
     typeddicts_type_consistencyPartialDoes not reject assignment of TypedDict with missing key.
Does not return non-Optional value from `get` method for required key.
Does not properly handle nested TypedDicts.
     typeddicts_usagePartialDoes not report errant use of TypedDict in `isinstance` call.
Does not reject use of TypedDict as TypeVar bound.
Type narrowing
     narrowing_typeguardPartialDoes not support `tuple` in `assert_type` call.
Does not reject TypeGuard method with too few parameters.
-

pytype 2023.12.18

+
pytype 2023.12.18(18.68sec) +
+ - + @@ -234,14 +259,14 @@

Python Type System Conformance Test Results

- + - + @@ -252,11 +277,11 @@

Python Type System Conformance Test Results

- + - +
Type annotations
     annotations_typeexprPartialDoes not reject call expressions in type annotation.
Does not reject call lambda expression in type annotation.
Does not reject list expression in type annotation.
Does not reject ternary expression in type annotation.
Does not reject f-string in type annotation.
Generics
     generics_self_advancedUnsupportedDoes not understand `Self` type.
     generics_self_basicUnsupportedDoes not understand `Self` type.
     generics_self_protocolsUnknown
     generics_self_usageUnsupportedDoes not understand `Self` type.
Literals
     literals_interactionsPartialIncorrectly rejects some legal Literal annotations.
Does not reject some illegal Literal annotations.
Does not use Literal to distinguish overloads.
Does not narrow based on `x is Literal` type guard pattern.
Does not narrow based on `x == Literal` type guard pattern.
     literals_literalstringUnsupportedDoes not understand `LiteralString` special form.
     literals_parameterizationsUnsupportedDoes not understand `Literal` type annotation.
     literals_semanticsUnsupportedDoes not understand `Literal` type annotation.
Typed dictionaries
     typeddicts_alt_syntaxPartialDoes not reject use of variable as second argument to `TypedDict` call.
Does not report when name of TypedDict doesn't match assigned identifier name.
Does not support keyword-argument form of alternative syntax (deprecated in 3.11).
     typeddicts_requiredPartialDoes not reject use of `Required` in non-TypedDict class.
Does not reject use of `Required` in function parameter annotation.
Does not reject nested use of `Required` in type annotation.
     typeddicts_type_consistencyPartialDoes not report some type violations for TypedDict type compatibility.
Incorrectly reports type violation in cases where there is none.
Does not report type incompatibility between TypedDict and `dict[str, Any]`.
     typeddicts_usagePartialDoes not report errant use of TypedDict in `isinstance` call.
Does not reject use of TypedDict as TypeVar bound.
Type narrowing
     narrowing_typeguardPartialDoes not reject TypeGuard method with too few parameters.