OSOE-935: Upgrade to xUnit 3 and handling test cancellations gracefully

Lombiq.Tests.UI.Samples/FrontendUITestBase.cs

@@ -3,7 +3,7 @@
 using System;
 using System.Diagnostics.CodeAnalysis;
 using System.Threading.Tasks;
-using Xunit.Abstractions;
+using Xunit;
 
 namespace Lombiq.Tests.UI.Samples;
 

Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj

@@ -3,6 +3,7 @@
   <PropertyGroup>
     <TargetFramework>net8.0</TargetFramework>
     <IsPackable>false</IsPackable>
+    <OutputType>Exe</OutputType>
   </PropertyGroup>
 
   <ItemGroup>
@@ -25,8 +26,9 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.7.2" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.12.0" />
+    <PackageReference Include="xunit.v3" Version="1.0.0" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="3.0.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

Lombiq.Tests.UI.Samples/Tests/AccessibilityTest.cs

@@ -2,7 +2,6 @@
 using Lombiq.Tests.UI.Services;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 

Lombiq.Tests.UI.Samples/Tests/AzureBlobStorageTests.cs

@@ -4,7 +4,6 @@
 using System.Linq;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 

Lombiq.Tests.UI.Samples/Tests/BasicOrchardFeaturesTests.cs

@@ -2,7 +2,6 @@
 using Lombiq.Tests.UI.Samples.Constants;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 

Lombiq.Tests.UI.Samples/Tests/BasicTests.cs

@@ -6,7 +6,6 @@
 using System.Linq;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 

Lombiq.Tests.UI.Samples/Tests/BasicVisualVerificationTests.cs

@@ -5,7 +5,6 @@
 using OpenQA.Selenium;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 

Lombiq.Tests.UI.Samples/Tests/DatabaseSnapshotTests.cs

@@ -4,7 +4,6 @@
 using System.IO;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 

Lombiq.Tests.UI.Samples/Tests/EmailTests.cs

@@ -2,7 +2,6 @@
 using Lombiq.Tests.UI.Helpers;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 

Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs

@@ -7,7 +7,6 @@
 using System.Linq;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 
@@ -38,7 +37,7 @@ public Task ServerSideErrorOnLoadedPageShouldHaltTest() =>
                 catch (PageChangeAssertionException)
                 {
                     // Remove all logs to have a clean slate.
-                    await context.ClearLogsAsync();
+                    await context.ClearLogsAsync(context.Configuration.TestCancellationToken);
                 }
             });
 

Lombiq.Tests.UI.Samples/Tests/FrontendTests.cs

@@ -2,7 +2,6 @@
 using OpenQA.Selenium;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 

Lombiq.Tests.UI.Samples/Tests/InteractiveModeTests.cs

@@ -1,10 +1,11 @@
 using Lombiq.Tests.UI.Extensions;
+using Lombiq.Tests.UI.Helpers;
 using OpenQA.Selenium;
 using Shouldly;
+using System;
 using System.Diagnostics.CodeAnalysis;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 
@@ -55,14 +56,18 @@ public Task EnteringInteractiveModeShouldWait() =>
 
                 await Task.WhenAll(
                     context.SwitchToInteractiveAsync(),
-                    Task.Run(async () =>
-                    {
-                        // Ensure that the interactive mode polls for status at least once, so the arbitrary waiting
-                        // actually works in a real testing scenario.
-                        await Task.Delay(5000);
+                    Task.Run(
+                        async () =>
+                        {
+                            ReliabilityHelper.DoWithRetriesOrFail(
+                                () => context.Driver.WindowHandles.Count > 1,
+                                TimeSpan.FromSeconds(5));
 
-                        await context.ClickReliablyOnAsync(By.ClassName("interactive__continue"));
-                    }));
+                            context.SwitchToLastWindow();
+
+                            await context.ClickReliablyOnAsync(By.ClassName("interactive__continue"));
+                        },
+                        context.Configuration.TestCancellationToken));
 
                 // Ensure that the info tab is closed and the control handed back to the last tab.
                 context.Driver.Url.ShouldBe(currentUrl);

Lombiq.Tests.UI.Samples/Tests/JavaScriptTests.cs

@@ -2,7 +2,6 @@
 using System.IO;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 

Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs

@@ -9,7 +9,6 @@
 using System.Linq;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 using LogLevel = OpenQA.Selenium.LogLevel;
 
 namespace Lombiq.Tests.UI.Samples.Tests;

Lombiq.Tests.UI.Samples/Tests/MultiBrowserTests.cs

@@ -5,7 +5,6 @@
 using Shouldly;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 

Lombiq.Tests.UI.Samples/Tests/RemoteTests.cs

@@ -4,7 +4,6 @@
 using System;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 
@@ -13,13 +12,26 @@ namespace Lombiq.Tests.UI.Samples.Tests;
 // to the internet.
 
 // However, sometimes in addition to this, you also want to test remote apps available online, like running rudimentary
-// smoke tests on your production app (e.g.: Can people still log in? Are payments still working?). The UI Testing
+// smoke tests on your production app (e.g.: Can people still log in? Are payments still working?). You can also run
+// such tests in your pre-production environment (like staging) before rolling it out the production. The UI Testing
 // Toolbox also supports this. Check out the example below!
 
 // Note how the test derives from RemoteUITestBase this time, not UITestBase. This is a generic base class for any
 // remote test. However, if you're testing an app behind Cloudflare, you might see requests rejected with an HTTP 403
 // due to Cloudflare's Bot Fight Mode; in that case, derive from CloudflareRemoteUITestBase instead after checking out
 // its documentation on how to set it up.
+
+// While there may be some common code between local and remote tests, they cannot be the same, and the common code must
+// be in a project independent of the web app (which is not the case here for the sake of simplicity):
+// - In local tests, you have a much lower-level access to the app: E.g., you can access the Orchard Core log, use
+//   shortcuts with all kinds of backdoors, and you can communicate directly with the Orchard Core shell (not just via #spell-check-ignore-line
+//   web APIs). You can also save a snapshot of the site's database and media. None of these are available (for a good
+//   reason) when the app is deployed to a server: There, you interact with it just as any ordinary user.
+// - For local tests, you need to build the web app and all its dependencies. For remote tests, you only need the test
+//   project and the UI Testing Toolbox.
+// - Some things are inherently different in a staging/production app, also on the UI. E.g., the app will load static
+//   resources from a CDN, or the content will be more up-to-date.
+
 public class RemoteTests : RemoteUITestBase
 {
     public RemoteTests(ITestOutputHelper testOutputHelper)

Lombiq.Tests.UI.Samples/Tests/SecurityScanningTests.cs

@@ -7,7 +7,6 @@
 using System;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 using YamlDotNet.RepresentationModel;
 
 namespace Lombiq.Tests.UI.Samples.Tests;

Lombiq.Tests.UI.Samples/Tests/ShiftTimeTests.cs

@@ -6,7 +6,6 @@
 using System.Globalization;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 

Lombiq.Tests.UI.Samples/Tests/SqlServerTests.cs

@@ -4,7 +4,6 @@
 using System.Linq;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 

Lombiq.Tests.UI.Samples/Tests/TenantTests.cs

@@ -5,7 +5,6 @@
 using Shouldly;
 using System.Threading.Tasks;
 using Xunit;
-using Xunit.Abstractions;
 
 namespace Lombiq.Tests.UI.Samples.Tests;
 

Lombiq.Tests.UI.Samples/UITestBase.cs

@@ -5,7 +5,7 @@
 using Microsoft.Extensions.Logging;
 using System;
 using System.Threading.Tasks;
-using Xunit.Abstractions;
+using Xunit;
 
 namespace Lombiq.Tests.UI.Samples;
 
@@ -114,11 +114,12 @@ protected override Task ExecuteTestAsync(
                 // JavaScript errors show up) are checked and if there are any errors, the test will fail. You can also
                 // enable the checking of accessibility rules as we'll see later. Maybe not all of the default checks
                 // are suitable for you. Then it's simple to override them; here we change which log entries cause the
-                // tests to fail, and allow certain log entries.
+                // tests to fail, and allow certain log entries: info-level log entries for ShellHost are allowed, see
+                // above.
                 configuration.AssertAppLogsAsync = webApplicationInstance =>
-                    webApplicationInstance.LogsShouldNotContainAsync(logEntry =>
-                    // Allowing info-level log entries for ShellHost, see above.
-                    logEntry.Message != "My permitted message." && logEntry.Level != LogLevel.Information);
+                    webApplicationInstance.LogsShouldNotContainAsync(
+                        logEntry => logEntry.Message != "My permitted message." && logEntry.Level != LogLevel.Information,
+                        configuration.TestCancellationToken);
 
                 // Strictly speaking this is not necessary here, because we always use the same static method for setup.
                 // However, if you used a dynamic setup operation (e.g. `context => SetupHelpers.RunSetupAsync(context,

Lombiq.Tests.UI.Tests.UI/TestCases/SecurityShortcutsTestCases.cs

@@ -48,7 +48,7 @@ public static Task AddUserToFakeRoleShouldThrowAsync(
                 await context.CreateUserAsync(UserUserName, DefaultUser.Password, UserEmail);
                 await context.AddUserToRoleAsync(UserUserName, FakeRole).ShouldThrowAsync<RoleNotFoundException>();
 
-                await context.ClearLogsAsync();
+                await context.ClearLogsAsync(context.Configuration.TestCancellationToken);
             },
             browser,
             ConfigurationHelper.DisableHtmlValidation);
@@ -61,7 +61,7 @@ public static Task AllowFakePermissionToRoleShouldThrowAsync(
                 await context.AddPermissionToRoleAsync(FakePermission, AuthorRole)
                     .ShouldThrowAsync<PermissionNotFoundException>();
 
-                await context.ClearLogsAsync();
+                await context.ClearLogsAsync(context.Configuration.TestCancellationToken);
             },
             browser,
             ConfigurationHelper.DisableHtmlValidation);

Lombiq.Tests.UI.Tests.UI/TestCases/TimeoutTestCases.cs

@@ -10,7 +10,7 @@ public static Task TestRunTimeoutShouldThrowAsync(
         Browser browser = Browser.None) =>
         Should.ThrowAsync(
             async () => await executeTestAfterSetupAsync(
-                context => Task.Delay(TimeSpan.FromSeconds(1)),
+                context => Task.Delay(TimeSpan.FromSeconds(1), context.Configuration.TestCancellationToken),
                 browser,
                 configuration =>
                 {

Lombiq.Tests.UI/Attributes/AllBrowsersAttribute.cs

@@ -2,19 +2,28 @@
 using System;
 using System.Collections.Generic;
 using System.Reflection;
+using System.Threading.Tasks;
+using Xunit;
 using Xunit.Sdk;
+using Xunit.v3;
 
 namespace Lombiq.Tests.UI.Attributes;
 
 [AttributeUsage(AttributeTargets.Method, AllowMultiple = false, Inherited = true)]
 public sealed class AllBrowsersAttribute : DataAttribute
 {
-    public override IEnumerable<object[]> GetData(MethodInfo testMethod)
+    public override ValueTask<IReadOnlyCollection<ITheoryDataRow>> GetData(MethodInfo testMethod, DisposalTracker disposalTracker)
     {
         var browsers = (IEnumerable<Browser>)Enum.GetValues(typeof(Browser));
+        var dataRows = new List<ITheoryDataRow>();
+
         foreach (var browser in browsers)
         {
-            yield return new[] { browser as object };
+            dataRows.Add(new TheoryDataRow(browser));
         }
+
+        return new ValueTask<IReadOnlyCollection<ITheoryDataRow>>(dataRows.AsReadOnly());
     }
+
+    public override bool SupportsDiscoveryEnumeration() => true;
 }

Lombiq.Tests.UI/Attributes/BrowserAttributeBase.cs

@@ -2,7 +2,10 @@
 using System;
 using System.Collections.Generic;
 using System.Reflection;
+using System.Threading.Tasks;
+using Xunit;
 using Xunit.Sdk;
+using Xunit.v3;
 
 namespace Lombiq.Tests.UI.Attributes;
 
@@ -11,8 +14,8 @@ public abstract class BrowserAttributeBase : DataAttribute
 {
     protected abstract Browser Browser { get; }
 
-    public override IEnumerable<object[]> GetData(MethodInfo testMethod)
-    {
-        yield return new[] { Browser as object };
-    }
+    public override ValueTask<IReadOnlyCollection<ITheoryDataRow>> GetData(MethodInfo testMethod, DisposalTracker disposalTracker) =>
+        new(new[] { new TheoryDataRow(Browser) }.AsReadOnly());
+
+    public override bool SupportsDiscoveryEnumeration() => true;
 }

Lombiq.Tests.UI/BasicOrchardFeaturesTesting/WorkflowsFeatureTestingUITestContextExtensions.cs

@@ -48,6 +48,8 @@ public static Task TestWorkflowsAsync(this UITestContext context) =>
                     By.XPath("//div[@class = 'jtk-endpoint jtk-endpoint-anchor jtk-draggable jtk-droppable']"), // #spell-check-ignore-line
                     By.XPath(taskXPath));
 
+                context.WaitElementToNotChange(By.ClassName("jtk-connector")); // #spell-check-ignore-line
+
                 // We need to save the workflow early, because sometimes the editor, thus the startup task button can be
                 // buggy during UI testing (it won't be clicked, even if we check for its existence). This way it's
                 // always clicked.

Lombiq.Tests.UI/CloudflareRemoteUITestBase.cs

@@ -2,7 +2,7 @@
 using Lombiq.Tests.UI.Services;
 using System;
 using System.Threading.Tasks;
-using Xunit.Abstractions;
+using Xunit;
 
 namespace Lombiq.Tests.UI;