Fix update-allow-list logic, update test262harness (#228)

sebastienros · Feb 20, 2022 · 620798e · 620798e
1 parent 19b3b8b
commit 620798e
Show file tree

Hide file tree

Showing 8 changed files with 40 additions and 17 deletions.
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -46,7 +46,7 @@ jobs:
 
     - name: Generate test cases
       working-directory: ./test/Esprima.Tests.Test262
-      run: dotnet tool restore && dotnet test262 generate
+      run: dotnet tool restore && dotnet test262 generate && dotnet run -c Release -- --update-allow-list
 
     - name: Test
       run: dotnet test --configuration Release
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -26,7 +26,7 @@ jobs:
 
     - name: Generate test cases
       working-directory: ./test/Esprima.Tests.Test262
-      run: dotnet tool restore && dotnet test262 generate
+      run: dotnet tool restore && dotnet test262 generate && dotnet run -c Release -- --update-allow-list
 
     - name: Test
       run: dotnet test --configuration Release

diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml
@@ -29,7 +29,7 @@ jobs:
 
     - name: Generate test cases
       working-directory: ./test/Esprima.Tests.Test262
-      run: dotnet tool restore && dotnet test262 generate
+      run: dotnet tool restore && dotnet test262 generate && dotnet run -c Release -- --update-allow-list
 
     - name: Test
       run: dotnet test --configuration Release

diff --git a/test/Esprima.Tests.Test262/.config/dotnet-tools.json b/test/Esprima.Tests.Test262/.config/dotnet-tools.json
@@ -3,7 +3,7 @@
   "isRoot": true,
   "tools": {
     "test262harness.console": {
-      "version": "0.0.9",
+      "version": "0.0.11",
       "commands": [
         "test262"
       ]

diff --git a/test/Esprima.Tests.Test262/Esprima.Tests.Test262.csproj b/test/Esprima.Tests.Test262/Esprima.Tests.Test262.csproj
@@ -3,14 +3,15 @@
     <PropertyGroup>
         <TargetFramework>net6.0</TargetFramework>
         <Nullable>enable</Nullable>
+        <GenerateProgramFile>false</GenerateProgramFile>
     </PropertyGroup>
 
     <ItemGroup>
       <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
       <PackageReference Include="NUnit" Version="3.13.3-beta4" PrivateAssets="all" />
       <PackageReference Include="NUnit3TestAdapter" Version="4.2.1" />
       <PackageReference Include="Spectre.Console" Version="0.43.0" />
-      <PackageReference Include="Test262Harness" Version="0.0.9" />
+      <PackageReference Include="Test262Harness" Version="0.0.11" />
       <PackageReference Include="YamlDotNet" Version="11.2.1" />
     </ItemGroup>
 

diff --git a/test/Esprima.Tests.Test262/Program.cs b/test/Esprima.Tests.Test262/Program.cs
@@ -11,7 +11,7 @@ namespace Esprima.Tests.Test262;
 
 public static class Program
 {
-    public static async Task<int> MainNotUsed(string[] args)
+    public static async Task<int> Main(string[] args)
     {
         var rootDirectory = Path.GetDirectoryName(Assembly.GetEntryAssembly()?.Location) ?? string.Empty;
         var projectRoot = Path.Combine(rootDirectory, "../../..");
@@ -44,13 +44,8 @@ public static async Task<int> MainNotUsed(string[] args)
                 var readTask = ctx.AddTask("Loading tests", maxValue: 90_000);
                 readTask.StartTask();
 
-                Parallel.ForEach(stream.GetTestFiles(), file =>
-                {
-                    test262Files.Add(file);
-                    readTask.Increment(1);
-                });
-
-                readTask.MaxValue = test262Files.Count;
+                test262Files = new ConcurrentBag<Test262File>(stream.GetTestFiles());
+                readTask.Value = 100;
                 readTask.StopTask();
 
                 AnsiConsole.WriteLine();
@@ -79,7 +74,7 @@ public static async Task<int> MainNotUsed(string[] args)
                 };
 
                 var executor = new Test262Runner(options);
-                summary = executor.Run(stream.GetTestFiles());
+                summary = executor.Run(test262Files);
                 testTask.StopTask();
             });
 
@@ -168,16 +163,16 @@ private static void UpdateAllowList(
         List<string> knownFailing)
     {
         // make sure we don't keep new successful ones in list
-        var successs = new HashSet<string>(
+        var success = new HashSet<string>(
             testExecutionSummary.Allowed.Failure.Concat(testExecutionSummary.Allowed.Success).Select(x => x.ToString())
         );
 
         var failing = testExecutionSummary.Disallowed.FalseNegative
             .Concat(testExecutionSummary.Disallowed.FalsePositive)
             .Concat(testExecutionSummary.Disallowed.Failure)
             .Select(x => x.ToString())
+            .Where(x => !success.Contains(x))
             .Concat(knownFailing)
-            .Where(x => !successs.Contains(x))
             .Distinct()
             .OrderBy(x => x)
             .ToList();

diff --git a/test/Esprima.Tests.Test262/Test262Test.cs b/test/Esprima.Tests.Test262/Test262Test.cs
@@ -11,6 +11,13 @@ private JavaScriptParser BuildTestExecutor(Test262File file)
 
     private static void ExecuteTest(JavaScriptParser parser, Test262File file)
     {
-        parser.ParseScript(file.Strict);
+        if (file.Type == ProgramType.Script)
+        {
+            parser.ParseScript(file.Strict);
+        }
+        else
+        {
+            parser.ParseModule();
+        }
     }
 }
diff --git a/test/Esprima.Tests.Test262/allow-list.txt b/test/Esprima.Tests.Test262/allow-list.txt
@@ -69,6 +69,14 @@ test/built-ins/RegExp/named-groups/unicode-property-names-valid.js(default)
 test/built-ins/RegExp/named-groups/unicode-property-names-valid.js(strict mode)
 test/built-ins/RegExp/named-groups/unicode-property-names.js(default)
 test/built-ins/RegExp/named-groups/unicode-property-names.js(strict mode)
+test/built-ins/RegExp/property-escapes/character-class-range-end.js(default)
+test/built-ins/RegExp/property-escapes/character-class-range-end.js(strict mode)
+test/built-ins/RegExp/property-escapes/character-class-range-no-dash-end.js(default)
+test/built-ins/RegExp/property-escapes/character-class-range-no-dash-end.js(strict mode)
+test/built-ins/RegExp/property-escapes/character-class-range-no-dash-start.js(default)
+test/built-ins/RegExp/property-escapes/character-class-range-no-dash-start.js(strict mode)
+test/built-ins/RegExp/property-escapes/character-class-range-start.js(default)
+test/built-ins/RegExp/property-escapes/character-class-range-start.js(strict mode)
 test/built-ins/RegExp/property-escapes/generated/Alphabetic.js(default)
 test/built-ins/RegExp/property-escapes/generated/Alphabetic.js(strict mode)
 test/built-ins/RegExp/property-escapes/generated/Any.js(default)
@@ -895,6 +903,14 @@ test/built-ins/RegExp/property-escapes/generated/XID_Continue.js(default)
 test/built-ins/RegExp/property-escapes/generated/XID_Continue.js(strict mode)
 test/built-ins/RegExp/property-escapes/generated/XID_Start.js(default)
 test/built-ins/RegExp/property-escapes/generated/XID_Start.js(strict mode)
+test/built-ins/RegExp/property-escapes/grammar-extension-empty-negated.js(default)
+test/built-ins/RegExp/property-escapes/grammar-extension-empty-negated.js(strict mode)
+test/built-ins/RegExp/property-escapes/grammar-extension-empty.js(default)
+test/built-ins/RegExp/property-escapes/grammar-extension-empty.js(strict mode)
+test/built-ins/RegExp/property-escapes/grammar-extension-invalid-negated.js(default)
+test/built-ins/RegExp/property-escapes/grammar-extension-invalid-negated.js(strict mode)
+test/built-ins/RegExp/property-escapes/grammar-extension-invalid.js(default)
+test/built-ins/RegExp/property-escapes/grammar-extension-invalid.js(strict mode)
 test/built-ins/RegExp/property-escapes/non-existent-property-value-General_Category-negated.js(default)
 test/built-ins/RegExp/property-escapes/non-existent-property-value-General_Category-negated.js(strict mode)
 test/built-ins/RegExp/property-escapes/non-existent-property-value-general-category.js(default)
@@ -4058,6 +4074,10 @@ test/language/literals/regexp/u-invalid-non-empty-class-ranges-no-dash-a.js(defa
 test/language/literals/regexp/u-invalid-non-empty-class-ranges-no-dash-a.js(strict mode)
 test/language/literals/regexp/u-invalid-non-empty-class-ranges-no-dash-ab.js(default)
 test/language/literals/regexp/u-invalid-non-empty-class-ranges-no-dash-ab.js(strict mode)
+test/language/literals/regexp/u-invalid-non-empty-class-ranges-no-dash-b.js(default)
+test/language/literals/regexp/u-invalid-non-empty-class-ranges-no-dash-b.js(strict mode)
+test/language/literals/regexp/u-invalid-non-empty-class-ranges.js(default)
+test/language/literals/regexp/u-invalid-non-empty-class-ranges.js(strict mode)
 test/language/literals/regexp/u-invalid-optional-lookahead.js(default)
 test/language/literals/regexp/u-invalid-optional-lookahead.js(strict mode)
 test/language/literals/regexp/u-invalid-optional-lookbehind.js(default)