diff --git a/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj b/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj index 83a572ce6..50f904f2b 100644 --- a/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj +++ b/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj @@ -3,9 +3,6 @@ net8.0 false - - false @@ -16,7 +13,7 @@ - Always + PreserveNewest PreserveNewest @@ -28,7 +25,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Lombiq.Tests.UI.Samples/Tests/InteractiveModeTests.cs b/Lombiq.Tests.UI.Samples/Tests/InteractiveModeTests.cs index b0438cd18..0d7302a1a 100644 --- a/Lombiq.Tests.UI.Samples/Tests/InteractiveModeTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/InteractiveModeTests.cs @@ -44,8 +44,8 @@ public Task SampleTest() => }); // This test checks if interactive mode works by opening it in one thread, and then clicking it away in a different - // thread. This ensures that the new tab correctly appears with the clickable "Continue Test" button, and then - // disappears once it's clicked. + // thread. Two threads are necessary because interactive mode stops test execution on its current thread, so we + // wouldn't be able to end it from within a test. [Fact] public Task EnteringInteractiveModeShouldWait() => ExecuteTestAfterSetupAsync( @@ -59,7 +59,7 @@ await Task.WhenAll( { // Ensure that the interactive mode polls for status at least once, so the arbitrary waiting // actually works in a real testing scenario. - await Task.Delay(1000); + await Task.Delay(5000); await context.ClickReliablyOnAsync(By.ClassName("interactive__continue")); })); diff --git a/Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs b/Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs index 363c31c2a..86bbcffc2 100644 --- a/Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs @@ -90,7 +90,7 @@ public Task TestAdminBackgroundTasksAsMonkeyRecursivelyShouldWorkWithAdminUser() // You could also configure the same thing with regex: ////_monkeyTestingOptions.UrlFilters.Add(new MatchesRegexMonkeyTestingUrlFilter(@"\/Admin\/BackgroundTasks")); - await context.SignInDirectlyAndGoToRelativeUrlAsync("/Admin/BackgroundTasks"); + await context.SignInDirectlyAndGoToAdminRelativeUrlAsync("/BackgroundTasks"); await context.TestCurrentPageAsMonkeyRecursivelyAsync(monkeyTestingOptions); }, configuration => configuration.AssertBrowserLog = (logEntries) => logEntries diff --git a/Lombiq.Tests.UI.Samples/UITestBase.cs b/Lombiq.Tests.UI.Samples/UITestBase.cs index 3fe59f397..774d2ccdb 100644 --- a/Lombiq.Tests.UI.Samples/UITestBase.cs +++ b/Lombiq.Tests.UI.Samples/UITestBase.cs @@ -79,7 +79,7 @@ protected override Task ExecuteTestAsync( // disable it with the below config). With this, you can make sure that the HTML markup the app // generates (also from content items) is valid. While the default settings for HTML validation are most // possibly suitable for your projects, check out the HtmlValidationConfiguration class for what else - // you can configure. We've also added a .htmlvalidate.json file (note the Content Build + // you can configure. We've also added a custom .htmlvalidate.json file (note the Content Build // Action) to further configure it. ////configuration.HtmlValidationConfiguration.RunHtmlValidationAssertionOnAllPageChanges = false; diff --git a/Lombiq.Tests.UI/Extensions/ElementRetrievalUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/ElementRetrievalUITestContextExtensions.cs index c202be07e..c7e7935f8 100644 --- a/Lombiq.Tests.UI/Extensions/ElementRetrievalUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/ElementRetrievalUITestContextExtensions.cs @@ -11,7 +11,7 @@ namespace Lombiq.Tests.UI.Extensions; /// -/// Extension methods to retrieve elements using Atata helpers. See the Atata docs ( ) for more information on what you can do /// with these. /// @@ -146,5 +146,5 @@ private static ExtendedSearchContext CreateSearchContext(this UITest new( context.Driver, context.Configuration.TimeoutConfiguration.RetryTimeout, - context.Configuration.TimeoutConfiguration.RetryTimeout); + context.Configuration.TimeoutConfiguration.RetryInterval); } diff --git a/Lombiq.Tests.UI/Extensions/ExtendedLoggingExtensions.cs b/Lombiq.Tests.UI/Extensions/ExtendedLoggingExtensions.cs index cb1a29ef5..68495d363 100644 --- a/Lombiq.Tests.UI/Extensions/ExtendedLoggingExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/ExtendedLoggingExtensions.cs @@ -148,15 +148,15 @@ private static async Task ExecuteSectionAsync( var notLast = i < StabilityRetryCount - 1; try { - // This is somewhat risky. ILogManager is not thread-safe and uses as stack to keep track of sections, so if - // multiple sections are started in concurrent threads, the result will be incorrect. This shouldn't be too much - // of an issue for now though since tests, while async, are single-threaded. - context.Scope.AtataContext.Log.Start(section); - context.Scope.AtataContext.Log.Info("Log section {0} started.", section.Message); - var result = await functionAsync(); - context.Scope.AtataContext.Log.Info("Log section {0} ended.", section.Message); - context.Scope.AtataContext.Log.EndSection(); - return result; + return await context.Scope.AtataContext.Log.ExecuteSectionAsync( + section, + async () => + { + context.Scope.AtataContext.Log.Info($"Log section {section.Message} started."); + var result = await functionAsync(); + context.Scope.AtataContext.Log.Info($"Log section {section.Message} ended."); + return result; + }); } catch (StaleElementReferenceException) when (notLast) { @@ -171,7 +171,5 @@ private static async Task ExecuteSectionAsync( private static void LogStaleElementReferenceExceptionRetry(UITestContext context, int tryIndex) => context.Scope.AtataContext.Log.Info( "The operation in the log section failed with StaleElementReferenceException but will be retried. This " + - "is try number {0} out of {1}.", - tryIndex + 1, - StabilityRetryCount); + $"is try number {(tryIndex + 1).ToTechnicalString()} out of {StabilityRetryCount}."); } diff --git a/Lombiq.Tests.UI/Extensions/FormUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/FormUITestContextExtensions.cs index 2bdf1c4d7..92fa7c3d0 100644 --- a/Lombiq.Tests.UI/Extensions/FormUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FormUITestContextExtensions.cs @@ -108,6 +108,8 @@ public static void FillInMonacoEditor( string editorId, string text) { + WaitForMonacoEditor(context, editorId); + var script = $@" monaco.editor.getEditors().find((element) => element.getContainerDomNode().id == {JsonConvert.SerializeObject(editorId)}).setValue({JsonConvert.SerializeObject(text)});"; @@ -122,6 +124,8 @@ public static string GetMonacoEditorText( this UITestContext context, string editorId) { + WaitForMonacoEditor(context, editorId); + var script = $@" return monaco.editor.getEditors().find((element) => element.getContainerDomNode().id == {JsonConvert.SerializeObject(editorId)}).getValue();"; @@ -375,4 +379,7 @@ private static IWebElement TryFillElement(UITestContext context, By by, string t return context.Driver.TryFillElement(element, text); } + + private static void WaitForMonacoEditor(UITestContext context, string editorId) => + context.Get(By.CssSelector($"#{editorId} .monaco-editor")); } diff --git a/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs b/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs index d6efabc9d..489d32f0c 100644 --- a/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs @@ -34,7 +34,8 @@ public static async Task> GetErrorsAsync(this HtmlValidation public static string GetParsedErrorMessageString(IEnumerable errors) => string.Join( - '\n', errors.Select(error => + '\n', + errors.Select(error => $"{error.Line.ToString(CultureInfo.InvariantCulture)}:{error.Column.ToString(CultureInfo.InvariantCulture)} - " + $"{error.Message} - " + $"{error.RuleId}")); diff --git a/Lombiq.Tests.UI/Extensions/NavigationUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/NavigationUITestContextExtensions.cs index 373a7a0cf..c0f411849 100644 --- a/Lombiq.Tests.UI/Extensions/NavigationUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/NavigationUITestContextExtensions.cs @@ -35,6 +35,17 @@ public static Task GoToAdminRelativeUrlAsync( return context.GoToAbsoluteUrlAsync(context.GetAbsoluteAdminUri(urlWithoutAdminPrefix), onlyIfNotAlreadyThere); } + public static async Task SignInDirectlyAndGoToAdminRelativeUrlAsync( + this UITestContext context, + string urlWithoutAdminPrefix = null, + bool onlyIfNotAlreadyThere = true, + string email = DefaultUser.UserName) + { + await context.SignInDirectlyAsync(email); + + await GoToAdminRelativeUrlAsync(context, urlWithoutAdminPrefix, onlyIfNotAlreadyThere); + } + public static Task GoToAbsoluteUrlAsync(this UITestContext context, Uri absoluteUri, bool onlyIfNotAlreadyThere = true) => context.ExecuteLoggedAsync( nameof(GoToAbsoluteUrlAsync), diff --git a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj index ba2500bf7..b86167642 100644 --- a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj +++ b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj @@ -59,31 +59,30 @@ - - - - + + + + - + - + - - + + - - + + - - - + + diff --git a/Lombiq.Tests.UI/MonkeyTesting/MonkeyTester.cs b/Lombiq.Tests.UI/MonkeyTesting/MonkeyTester.cs index 55addfc47..9b7821573 100644 --- a/Lombiq.Tests.UI/MonkeyTesting/MonkeyTester.cs +++ b/Lombiq.Tests.UI/MonkeyTesting/MonkeyTester.cs @@ -26,61 +26,49 @@ internal MonkeyTester(UITestContext context, MonkeyTestingOptions options = null _randomizer = new NonSecurityRandomizer(_options.BaseRandomSeed); } - internal async Task TestOnePageAsync(int? randomSeed = null) - { - Log.Start(new LogSection("Execute monkey testing against one page")); - - try - { - WriteOptionsToLog(); - - var pageTestInfo = GetCurrentPageTestInfo(); - - if (randomSeed is null) await TestCurrentPageAsync(pageTestInfo); - else await TestCurrentPageWithRandomSeedAsync(pageTestInfo, randomSeed.Value); - } - finally - { - Log.EndSection(); - } - } + internal Task TestOnePageAsync(int? randomSeed = null) => + Log.ExecuteSectionAsync( + new LogSection("Execute monkey testing against one page"), + async () => + { + WriteOptionsToLog(); - internal async Task TestRecursivelyAsync() - { - Log.Start(new LogSection($"Execute monkey testing recursively")); + var pageTestInfo = GetCurrentPageTestInfo(); - try - { - WriteOptionsToLog(); - - var pageTestInfo = GetCurrentPageTestInfo(); - await TestCurrentPageAsync(pageTestInfo); + if (randomSeed is null) await TestCurrentPageAsync(pageTestInfo); + else await TestCurrentPageWithRandomSeedAsync(pageTestInfo, randomSeed.Value); + }); - while (true) + internal Task TestRecursivelyAsync() => + Log.ExecuteSectionAsync( + new LogSection($"Execute monkey testing recursively"), + async () => { - pageTestInfo = GetCurrentPageTestInfo(); + WriteOptionsToLog(); - if (CanTestPage(pageTestInfo)) - { - await TestCurrentPageAsync(pageTestInfo); - } - else if (TryGetAvailablePageToTest(out var availablePageToTest)) - { - await _context.GoToAbsoluteUrlAsync(availablePageToTest.Url); + var pageTestInfo = GetCurrentPageTestInfo(); + await TestCurrentPageAsync(pageTestInfo); - await TestCurrentPageAsync(availablePageToTest); - } - else + while (true) { - return; + pageTestInfo = GetCurrentPageTestInfo(); + + if (CanTestPage(pageTestInfo)) + { + await TestCurrentPageAsync(pageTestInfo); + } + else if (TryGetAvailablePageToTest(out var availablePageToTest)) + { + await _context.GoToAbsoluteUrlAsync(availablePageToTest.Url); + + await TestCurrentPageAsync(availablePageToTest); + } + else + { + return; + } } - } - } - finally - { - Log.EndSection(); - } - } + }); private void WriteOptionsToLog() => Log.Trace(@$"Monkey testing options: diff --git a/Lombiq.Tests.UI/Pages/OrchardCoreAdminPage.cs b/Lombiq.Tests.UI/Pages/OrchardCoreAdminPage.cs index fd12b11d7..8c7e7ba2b 100644 --- a/Lombiq.Tests.UI/Pages/OrchardCoreAdminPage.cs +++ b/Lombiq.Tests.UI/Pages/OrchardCoreAdminPage.cs @@ -12,9 +12,9 @@ public abstract class OrchardCoreAdminPage : Page public ControlList, TOwner> AlertMessages { get; private set; } - public TOwner ShouldStayOnAdminPage() => AdminMenu.Should.Exist(); + public TOwner ShouldStayOnAdminPage() => AdminMenu.Should.BePresent(); - public TOwner ShouldLeaveAdminPage() => AdminMenu.Should.Not.Exist(); + public TOwner ShouldLeaveAdminPage() => AdminMenu.Should.Not.BePresent(); protected override void OnVerify() { diff --git a/Lombiq.Tests.UI/Services/AtataFactory.cs b/Lombiq.Tests.UI/Services/AtataFactory.cs index 641ea0be0..01e430efa 100644 --- a/Lombiq.Tests.UI/Services/AtataFactory.cs +++ b/Lombiq.Tests.UI/Services/AtataFactory.cs @@ -17,11 +17,13 @@ public class AtataConfiguration public static class AtataFactory { public static async Task StartAtataScopeAsync( + string contextId, ITestOutputHelper testOutputHelper, Uri baseUri, OrchardCoreUITestExecutorConfiguration configuration) { - AtataContext.ModeOfCurrent = AtataContextModeOfCurrent.AsyncLocal; + AtataContext.GlobalProperties.ModeOfCurrent = AtataContextModeOfCurrent.AsyncLocal; + AtataContext.GlobalProperties.UseUtcTimeZone(); // Since Atata 2.0 the default visibility option is Visibility.Any, these lines restore it to the 1.x behavior. AtataContext.GlobalConfiguration.UseDefaultControlVisibility(Visibility.Visible); @@ -37,8 +39,8 @@ public static async Task StartAtataScopeAsync( .UseTestName(configuration.AtataConfiguration.TestName) .UseBaseRetryTimeout(timeoutConfiguration.RetryTimeout) .UseBaseRetryInterval(timeoutConfiguration.RetryInterval) - .UseUtcTimeZone() - .PageSnapshots.UseCdpOrPageSourceStrategy(); // #spell-check-ignore-line + .PageSnapshots.UseCdpOrPageSourceStrategy() // #spell-check-ignore-line + .UseArtifactsPathTemplate(contextId); // Necessary to prevent long paths, an issue under Windows. builder.LogConsumers.AddDebug(); builder.LogConsumers.Add(new TestOutputLogConsumer(testOutputHelper)); diff --git a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs index e41493974..44578e3c7 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs @@ -180,7 +180,8 @@ await _configuration.BeforeAppStart private async Task StopOrchardAppAsync() { - _reverseProxy.DetachConnectionProvider(); + _reverseProxy?.DetachConnectionProvider(); + if (_orchardApplication == null) return; _testOutputHelper.WriteLineTimestampedAndDebug("Attempting to stop the Orchard Core instance."); diff --git a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs index 2149641a7..f23f35074 100644 --- a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs +++ b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs @@ -656,7 +656,7 @@ Task UITestingBeforeAppStartHandlerAsync(string contentRootPath, InstanceCommand _configuration.Events.AfterPageChange += TakeScreenshotIfEnabledAsync; } - var atataScope = await AtataFactory.StartAtataScopeAsync(_testOutputHelper, uri, _configuration); + var atataScope = await AtataFactory.StartAtataScopeAsync(contextId, _testOutputHelper, uri, _configuration); return new UITestContext( contextId, diff --git a/Lombiq.Tests.UI/Services/UITestExecutor.cs b/Lombiq.Tests.UI/Services/UITestExecutor.cs index a364d11ab..1c8b8782a 100644 --- a/Lombiq.Tests.UI/Services/UITestExecutor.cs +++ b/Lombiq.Tests.UI/Services/UITestExecutor.cs @@ -125,11 +125,26 @@ private static string PrepareDumpFolder( { var dumpConfiguration = configuration.FailureDumpConfiguration; var dumpFolderNameBase = testManifest.Name; - if (dumpConfiguration.UseShortNames && dumpFolderNameBase.Contains('(', StringComparison.Ordinal)) + if (dumpConfiguration.UseShortNames) { - var dumpFolderNameBeginningIndex = - dumpFolderNameBase[..dumpFolderNameBase.IndexOf('(', StringComparison.Ordinal)].LastIndexOf('.') + 1; - dumpFolderNameBase = dumpFolderNameBase[dumpFolderNameBeginningIndex..]; + if (dumpFolderNameBase.Contains('(', StringComparison.Ordinal)) + { + // The test uses parameters and is thus in the + // "Lombiq.Tests.UI.Samples.Tests.BasicTests.AnonymousHomePageShouldExist(browser: Chrome)" format. + var dumpFolderNameBeginningIndex = + dumpFolderNameBase[..dumpFolderNameBase.IndexOf('(', StringComparison.Ordinal)].LastIndexOf('.') + 1; + dumpFolderNameBase = dumpFolderNameBase[dumpFolderNameBeginningIndex..]; + } + else + { + // The test doesn't use parameters and is thus in the + // "Lombiq.Tests.UI.Samples.Tests.BasicTests.AnonymousHomePageShouldExist" format. + var dumpFolderNameBeginningIndex = dumpFolderNameBase.LastIndexOf('.') + 1; + dumpFolderNameBase = dumpFolderNameBase[dumpFolderNameBeginningIndex..]; + } + + // Can't use string.GetHasCode() because that varies between executions. + dumpFolderNameBase += "-" + Sha256Helper.ComputeHash(testManifest.Name); } dumpFolderNameBase = dumpFolderNameBase.MakeFileSystemFriendly(); @@ -159,9 +174,12 @@ private static string PrepareDumpFolder( var openingBracketIndex = dumpFolderNameBase.IndexOf('(', StringComparison.Ordinal); var closingBracketIndex = dumpFolderNameBase.LastIndexOf(')'); + // Only adding a hash of the parameters if the hash of the test's full name is not already there due to + // path shortening above. // Can't use string.GetHasCode() because that varies between executions. - var hashedParameters = Sha256Helper - .ComputeHash(dumpFolderNameBase[(openingBracketIndex + 1)..(closingBracketIndex + 1)]); + var hashedParameters = dumpConfiguration.UseShortNames + ? string.Empty + : Sha256Helper.ComputeHash(dumpFolderNameBase[(openingBracketIndex + 1)..(closingBracketIndex + 1)]); dumpFolderNameBase = dumpFolderNameBase[0..(openingBracketIndex + 1)] + diff --git a/Lombiq.Tests.UI/Services/UITestExecutorFailureDumpConfiguration.cs b/Lombiq.Tests.UI/Services/UITestExecutorFailureDumpConfiguration.cs index 1efcd1b6f..1c77d7f62 100644 --- a/Lombiq.Tests.UI/Services/UITestExecutorFailureDumpConfiguration.cs +++ b/Lombiq.Tests.UI/Services/UITestExecutorFailureDumpConfiguration.cs @@ -6,8 +6,9 @@ public class UITestExecutorFailureDumpConfiguration { /// /// Gets or sets a value indicating whether the subfolder of each test's dumps will use a shortened name, only - /// containing the name of the test method, without the name of the test class and its namespace. This is to - /// overcome the 260 character path length limitations on Windows. Defaults to on Windows. + /// containing the name of the test method suffixed with the test name's hash to make it unique, without the name of + /// the test class and its namespace. This is to overcome the 260 character path length limitations on Windows. + /// Defaults to on Windows. /// public bool UseShortNames { get; set; } = OperatingSystem.IsWindows(); diff --git a/Lombiq.Tests.UI/Services/WebDriverFactory.cs b/Lombiq.Tests.UI/Services/WebDriverFactory.cs index 630fef7cb..1128997f6 100644 --- a/Lombiq.Tests.UI/Services/WebDriverFactory.cs +++ b/Lombiq.Tests.UI/Services/WebDriverFactory.cs @@ -40,7 +40,7 @@ Task CreateDriverInnerAsync(ChromeDriverService service) chromeConfig.Service = service ?? ChromeDriverService.CreateDefaultService(); chromeConfig.Service.SuppressInitialDiagnosticInformation = true; // By default localhost is only allowed in IPv4. - chromeConfig.Service.WhitelistedIPAddresses += "::ffff:127.0.0.1"; + chromeConfig.Service.AllowedIPAddresses += "::ffff:127.0.0.1"; // Helps with misconfigured hosts. if (chromeConfig.Service.HostName == "localhost") chromeConfig.Service.HostName = "127.0.0.1";