From a334b80367e11f3d48233a1104875d4d28ff60e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Tue, 23 Apr 2024 20:16:34 +0200 Subject: [PATCH 01/36] Updating Atata packages to latest before v3, in preparation for v3 migration --- Lombiq.Tests.UI/Lombiq.Tests.UI.csproj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj index a7bea2578..c51c95c5e 100644 --- a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj +++ b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj @@ -59,10 +59,10 @@ - - - - + + + + From 83881f0d7ffde483ecdf70434540b012434b6272 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Tue, 23 Apr 2024 20:16:50 +0200 Subject: [PATCH 02/36] Code styling --- Lombiq.Tests.UI/Extensions/ExtendedLoggingExtensions.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lombiq.Tests.UI/Extensions/ExtendedLoggingExtensions.cs b/Lombiq.Tests.UI/Extensions/ExtendedLoggingExtensions.cs index cb1a29ef5..3e9fc5bfe 100644 --- a/Lombiq.Tests.UI/Extensions/ExtendedLoggingExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/ExtendedLoggingExtensions.cs @@ -148,9 +148,9 @@ private static async Task ExecuteSectionAsync( var notLast = i < StabilityRetryCount - 1; try { - // This is somewhat risky. ILogManager is not thread-safe and uses as stack to keep track of sections, so if - // multiple sections are started in concurrent threads, the result will be incorrect. This shouldn't be too much - // of an issue for now though since tests, while async, are single-threaded. + // This is somewhat risky. ILogManager is not thread-safe and uses as stack to keep track of sections, + // so if multiple sections are started in concurrent threads, the result will be incorrect. This + // shouldn't be too much of an issue for now though since tests, while async, are single-threaded. context.Scope.AtataContext.Log.Start(section); context.Scope.AtataContext.Log.Info("Log section {0} started.", section.Message); var result = await functionAsync(); From 03181d14a523eaa9bc0993ff7a0bce6b6114fb22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Tue, 23 Apr 2024 20:25:07 +0200 Subject: [PATCH 03/36] Fixing Atata Exist() deprecation --- Lombiq.Tests.UI/Pages/OrchardCoreAdminPage.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lombiq.Tests.UI/Pages/OrchardCoreAdminPage.cs b/Lombiq.Tests.UI/Pages/OrchardCoreAdminPage.cs index fd12b11d7..8c7e7ba2b 100644 --- a/Lombiq.Tests.UI/Pages/OrchardCoreAdminPage.cs +++ b/Lombiq.Tests.UI/Pages/OrchardCoreAdminPage.cs @@ -12,9 +12,9 @@ public abstract class OrchardCoreAdminPage : Page public ControlList, TOwner> AlertMessages { get; private set; } - public TOwner ShouldStayOnAdminPage() => AdminMenu.Should.Exist(); + public TOwner ShouldStayOnAdminPage() => AdminMenu.Should.BePresent(); - public TOwner ShouldLeaveAdminPage() => AdminMenu.Should.Not.Exist(); + public TOwner ShouldLeaveAdminPage() => AdminMenu.Should.Not.BePresent(); protected override void OnVerify() { From 6836be9b93a4eb88e272ba854522974345e32e76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Wed, 1 May 2024 00:21:56 +0200 Subject: [PATCH 04/36] Migrating away from deprecated Atata string.Format()-like log methods --- Lombiq.Tests.UI/Extensions/ExtendedLoggingExtensions.cs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Lombiq.Tests.UI/Extensions/ExtendedLoggingExtensions.cs b/Lombiq.Tests.UI/Extensions/ExtendedLoggingExtensions.cs index 3e9fc5bfe..71044f639 100644 --- a/Lombiq.Tests.UI/Extensions/ExtendedLoggingExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/ExtendedLoggingExtensions.cs @@ -152,9 +152,9 @@ private static async Task ExecuteSectionAsync( // so if multiple sections are started in concurrent threads, the result will be incorrect. This // shouldn't be too much of an issue for now though since tests, while async, are single-threaded. context.Scope.AtataContext.Log.Start(section); - context.Scope.AtataContext.Log.Info("Log section {0} started.", section.Message); + context.Scope.AtataContext.Log.Info($"Log section {section.Message} started."); var result = await functionAsync(); - context.Scope.AtataContext.Log.Info("Log section {0} ended.", section.Message); + context.Scope.AtataContext.Log.Info($"Log section {section.Message} ended."); context.Scope.AtataContext.Log.EndSection(); return result; } @@ -171,7 +171,5 @@ private static async Task ExecuteSectionAsync( private static void LogStaleElementReferenceExceptionRetry(UITestContext context, int tryIndex) => context.Scope.AtataContext.Log.Info( "The operation in the log section failed with StaleElementReferenceException but will be retried. This " + - "is try number {0} out of {1}.", - tryIndex + 1, - StabilityRetryCount); + $"is try number {(tryIndex + 1).ToTechnicalString()} out of {StabilityRetryCount}."); } From 90c92970e042cd2aa106c345eb687df413454b1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Wed, 1 May 2024 00:38:04 +0200 Subject: [PATCH 05/36] Updating to latest Atata packages --- .../Extensions/ExtendedLoggingExtensions.cs | 18 ++-- Lombiq.Tests.UI/Lombiq.Tests.UI.csproj | 8 +- Lombiq.Tests.UI/MonkeyTesting/MonkeyTester.cs | 82 ++++++++----------- Lombiq.Tests.UI/Services/AtataFactory.cs | 4 +- Lombiq.Tests.UI/Services/WebDriverFactory.cs | 2 +- 5 files changed, 51 insertions(+), 63 deletions(-) diff --git a/Lombiq.Tests.UI/Extensions/ExtendedLoggingExtensions.cs b/Lombiq.Tests.UI/Extensions/ExtendedLoggingExtensions.cs index 71044f639..68495d363 100644 --- a/Lombiq.Tests.UI/Extensions/ExtendedLoggingExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/ExtendedLoggingExtensions.cs @@ -148,15 +148,15 @@ private static async Task ExecuteSectionAsync( var notLast = i < StabilityRetryCount - 1; try { - // This is somewhat risky. ILogManager is not thread-safe and uses as stack to keep track of sections, - // so if multiple sections are started in concurrent threads, the result will be incorrect. This - // shouldn't be too much of an issue for now though since tests, while async, are single-threaded. - context.Scope.AtataContext.Log.Start(section); - context.Scope.AtataContext.Log.Info($"Log section {section.Message} started."); - var result = await functionAsync(); - context.Scope.AtataContext.Log.Info($"Log section {section.Message} ended."); - context.Scope.AtataContext.Log.EndSection(); - return result; + return await context.Scope.AtataContext.Log.ExecuteSectionAsync( + section, + async () => + { + context.Scope.AtataContext.Log.Info($"Log section {section.Message} started."); + var result = await functionAsync(); + context.Scope.AtataContext.Log.Info($"Log section {section.Message} ended."); + return result; + }); } catch (StaleElementReferenceException) when (notLast) { diff --git a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj index c51c95c5e..9891bdde3 100644 --- a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj +++ b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj @@ -59,10 +59,10 @@ - - - - + + + + diff --git a/Lombiq.Tests.UI/MonkeyTesting/MonkeyTester.cs b/Lombiq.Tests.UI/MonkeyTesting/MonkeyTester.cs index 55addfc47..9b7821573 100644 --- a/Lombiq.Tests.UI/MonkeyTesting/MonkeyTester.cs +++ b/Lombiq.Tests.UI/MonkeyTesting/MonkeyTester.cs @@ -26,61 +26,49 @@ internal MonkeyTester(UITestContext context, MonkeyTestingOptions options = null _randomizer = new NonSecurityRandomizer(_options.BaseRandomSeed); } - internal async Task TestOnePageAsync(int? randomSeed = null) - { - Log.Start(new LogSection("Execute monkey testing against one page")); - - try - { - WriteOptionsToLog(); - - var pageTestInfo = GetCurrentPageTestInfo(); - - if (randomSeed is null) await TestCurrentPageAsync(pageTestInfo); - else await TestCurrentPageWithRandomSeedAsync(pageTestInfo, randomSeed.Value); - } - finally - { - Log.EndSection(); - } - } + internal Task TestOnePageAsync(int? randomSeed = null) => + Log.ExecuteSectionAsync( + new LogSection("Execute monkey testing against one page"), + async () => + { + WriteOptionsToLog(); - internal async Task TestRecursivelyAsync() - { - Log.Start(new LogSection($"Execute monkey testing recursively")); + var pageTestInfo = GetCurrentPageTestInfo(); - try - { - WriteOptionsToLog(); - - var pageTestInfo = GetCurrentPageTestInfo(); - await TestCurrentPageAsync(pageTestInfo); + if (randomSeed is null) await TestCurrentPageAsync(pageTestInfo); + else await TestCurrentPageWithRandomSeedAsync(pageTestInfo, randomSeed.Value); + }); - while (true) + internal Task TestRecursivelyAsync() => + Log.ExecuteSectionAsync( + new LogSection($"Execute monkey testing recursively"), + async () => { - pageTestInfo = GetCurrentPageTestInfo(); + WriteOptionsToLog(); - if (CanTestPage(pageTestInfo)) - { - await TestCurrentPageAsync(pageTestInfo); - } - else if (TryGetAvailablePageToTest(out var availablePageToTest)) - { - await _context.GoToAbsoluteUrlAsync(availablePageToTest.Url); + var pageTestInfo = GetCurrentPageTestInfo(); + await TestCurrentPageAsync(pageTestInfo); - await TestCurrentPageAsync(availablePageToTest); - } - else + while (true) { - return; + pageTestInfo = GetCurrentPageTestInfo(); + + if (CanTestPage(pageTestInfo)) + { + await TestCurrentPageAsync(pageTestInfo); + } + else if (TryGetAvailablePageToTest(out var availablePageToTest)) + { + await _context.GoToAbsoluteUrlAsync(availablePageToTest.Url); + + await TestCurrentPageAsync(availablePageToTest); + } + else + { + return; + } } - } - } - finally - { - Log.EndSection(); - } - } + }); private void WriteOptionsToLog() => Log.Trace(@$"Monkey testing options: diff --git a/Lombiq.Tests.UI/Services/AtataFactory.cs b/Lombiq.Tests.UI/Services/AtataFactory.cs index 641ea0be0..f520befb9 100644 --- a/Lombiq.Tests.UI/Services/AtataFactory.cs +++ b/Lombiq.Tests.UI/Services/AtataFactory.cs @@ -21,7 +21,8 @@ public static async Task StartAtataScopeAsync( Uri baseUri, OrchardCoreUITestExecutorConfiguration configuration) { - AtataContext.ModeOfCurrent = AtataContextModeOfCurrent.AsyncLocal; + AtataContext.GlobalProperties.ModeOfCurrent = AtataContextModeOfCurrent.AsyncLocal; + AtataContext.GlobalProperties.UseUtcTimeZone(); // Since Atata 2.0 the default visibility option is Visibility.Any, these lines restore it to the 1.x behavior. AtataContext.GlobalConfiguration.UseDefaultControlVisibility(Visibility.Visible); @@ -37,7 +38,6 @@ public static async Task StartAtataScopeAsync( .UseTestName(configuration.AtataConfiguration.TestName) .UseBaseRetryTimeout(timeoutConfiguration.RetryTimeout) .UseBaseRetryInterval(timeoutConfiguration.RetryInterval) - .UseUtcTimeZone() .PageSnapshots.UseCdpOrPageSourceStrategy(); // #spell-check-ignore-line builder.LogConsumers.AddDebug(); diff --git a/Lombiq.Tests.UI/Services/WebDriverFactory.cs b/Lombiq.Tests.UI/Services/WebDriverFactory.cs index 630fef7cb..1128997f6 100644 --- a/Lombiq.Tests.UI/Services/WebDriverFactory.cs +++ b/Lombiq.Tests.UI/Services/WebDriverFactory.cs @@ -40,7 +40,7 @@ Task CreateDriverInnerAsync(ChromeDriverService service) chromeConfig.Service = service ?? ChromeDriverService.CreateDefaultService(); chromeConfig.Service.SuppressInitialDiagnosticInformation = true; // By default localhost is only allowed in IPv4. - chromeConfig.Service.WhitelistedIPAddresses += "::ffff:127.0.0.1"; + chromeConfig.Service.AllowedIPAddresses += "::ffff:127.0.0.1"; // Helps with misconfigured hosts. if (chromeConfig.Service.HostName == "localhost") chromeConfig.Service.HostName = "127.0.0.1"; From 0d96164e67617ed5baa599d8541b98e4fa98321b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Wed, 1 May 2024 01:48:57 +0200 Subject: [PATCH 06/36] Removing leftover WebDriverManager dependency --- Lombiq.Tests.UI/Lombiq.Tests.UI.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj index 9891bdde3..9c754d26a 100644 --- a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj +++ b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj @@ -81,7 +81,6 @@ - From 19e9ba7c0336dfa2d342f559606b4b99a939e101 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Wed, 1 May 2024 01:49:55 +0200 Subject: [PATCH 07/36] Fixing occasional HTML validation error in the samples due to the config file not being applied properly --- Lombiq.Tests.UI.Samples/.htmlvalidate.json | 19 ------------------- .../Lombiq.Tests.UI.Samples.csproj | 6 +----- Lombiq.Tests.UI.Samples/UITestBase.cs | 11 ++++++++++- .../samples.htmlvalidate.json | 11 +++++++++++ 4 files changed, 22 insertions(+), 25 deletions(-) delete mode 100644 Lombiq.Tests.UI.Samples/.htmlvalidate.json create mode 100644 Lombiq.Tests.UI.Samples/samples.htmlvalidate.json diff --git a/Lombiq.Tests.UI.Samples/.htmlvalidate.json b/Lombiq.Tests.UI.Samples/.htmlvalidate.json deleted file mode 100644 index 1b8ade5cf..000000000 --- a/Lombiq.Tests.UI.Samples/.htmlvalidate.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "extends": [ - "html-validate:recommended" - ], - - "rules": { - "attribute-boolean-style": "off", - "long-title": "off", - "no-trailing-whitespace": "off", - "no-inline-style": "off", - "no-dup-class": "off", - "wcag/h30": "off", - "wcag/h32": "off", - "wcag/h36": "off", - "wcag/h37": "off", - "wcag/h67": "off", - "wcag/h71": "off" - } -} diff --git a/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj b/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj index 83a572ce6..a107a6c52 100644 --- a/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj +++ b/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj @@ -3,19 +3,15 @@ net8.0 false - - false - - + Always diff --git a/Lombiq.Tests.UI.Samples/UITestBase.cs b/Lombiq.Tests.UI.Samples/UITestBase.cs index 3fe59f397..de1e23f7c 100644 --- a/Lombiq.Tests.UI.Samples/UITestBase.cs +++ b/Lombiq.Tests.UI.Samples/UITestBase.cs @@ -4,6 +4,7 @@ using Lombiq.Tests.UI.Services; using Shouldly; using System; +using System.IO; using System.Threading.Tasks; using Xunit.Abstractions; @@ -79,10 +80,18 @@ protected override Task ExecuteTestAsync( // disable it with the below config). With this, you can make sure that the HTML markup the app // generates (also from content items) is valid. While the default settings for HTML validation are most // possibly suitable for your projects, check out the HtmlValidationConfiguration class for what else - // you can configure. We've also added a .htmlvalidate.json file (note the Content Build + // you can configure. We've also added a custom .htmlvalidate.json file (note the Content Build // Action) to further configure it. ////configuration.HtmlValidationConfiguration.RunHtmlValidationAssertionOnAllPageChanges = false; + + // This is to make sure that the HTML validation config file in this project takes effect. Only needed + // until we fix https://github.com/Lombiq/UI-Testing-Toolbox/issues/359. + configuration.HtmlValidationConfiguration.HtmlValidationOptions = + configuration.HtmlValidationConfiguration.HtmlValidationOptions + .CloneWith(validationOptions => validationOptions.ConfigPath = + Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "samples.htmlvalidate.json")); // #spell-check-ignore-line + // The UI Testing Toolbox can run several checks for the app even if you don't add explicit // assertions: By default, the Orchard logs and the browser logs (where e.g. JavaScript errors show // up) are checked and if there are any errors, the test will fail. You can also enable the checking of diff --git a/Lombiq.Tests.UI.Samples/samples.htmlvalidate.json b/Lombiq.Tests.UI.Samples/samples.htmlvalidate.json new file mode 100644 index 000000000..cccd9b28f --- /dev/null +++ b/Lombiq.Tests.UI.Samples/samples.htmlvalidate.json @@ -0,0 +1,11 @@ +{ + "extends": [ + "./.htmlvalidate.json" + ], + + "rules": { + "long-title": "off", + "no-dup-class": "off" + }, + "root": true +} From 223b002bf4b012cd97422e1bf1fd749a9b6fdada Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Wed, 1 May 2024 01:56:19 +0200 Subject: [PATCH 08/36] Code styling --- Lombiq.Tests.UI.Samples/UITestBase.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/Lombiq.Tests.UI.Samples/UITestBase.cs b/Lombiq.Tests.UI.Samples/UITestBase.cs index de1e23f7c..de52f8d53 100644 --- a/Lombiq.Tests.UI.Samples/UITestBase.cs +++ b/Lombiq.Tests.UI.Samples/UITestBase.cs @@ -84,7 +84,6 @@ protected override Task ExecuteTestAsync( // Action) to further configure it. ////configuration.HtmlValidationConfiguration.RunHtmlValidationAssertionOnAllPageChanges = false; - // This is to make sure that the HTML validation config file in this project takes effect. Only needed // until we fix https://github.com/Lombiq/UI-Testing-Toolbox/issues/359. configuration.HtmlValidationConfiguration.HtmlValidationOptions = From d7aa43a479133277edbbb37362f8655832dd2baa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Wed, 1 May 2024 02:01:40 +0200 Subject: [PATCH 09/36] Updating basic Microsoft libraries --- Lombiq.Tests.UI/Lombiq.Tests.UI.csproj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj index 9c754d26a..c50b24835 100644 --- a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj +++ b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj @@ -64,20 +64,20 @@ - + - - + + - + From 66efc38d55a320db3e2106c9e516c42baa420234 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Wed, 1 May 2024 02:29:54 +0200 Subject: [PATCH 10/36] Simpler configuration to point to a custom HTML-validate config file --- Lombiq.Tests.UI/Docs/Configuration.md | 21 +++++-------------- .../HtmlValidationOptionsExtensions.cs | 15 +++++++++++++ 2 files changed, 20 insertions(+), 16 deletions(-) create mode 100644 Lombiq.Tests.UI/Extensions/HtmlValidationOptionsExtensions.cs diff --git a/Lombiq.Tests.UI/Docs/Configuration.md b/Lombiq.Tests.UI/Docs/Configuration.md index b97ec21b0..5f46547a4 100644 --- a/Lombiq.Tests.UI/Docs/Configuration.md +++ b/Lombiq.Tests.UI/Docs/Configuration.md @@ -59,26 +59,17 @@ Recommendations and notes for such configuration: ### HTML validation configuration -If you want to change some HTML validation rules from only a few specific tests, you can create a custom _.htmlvalidate.json_ file (e.g. _TestName.htmlvalidate.json_). For example: +If you want to change some HTML validation rules from only a few specific tests, you can create a custom _.htmlvalidate.json_ file (e.g. _TestName.htmlvalidate.json_). It can also extend the default one, only overriding the configuration necessary. For example: ```json { "extends": [ - "html-validate:recommended" + "./.htmlvalidate.json" ], "rules": { - "attribute-boolean-style": "off", "element-required-attributes": "off", - "no-trailing-whitespace": "off", - "no-inline-style": "off", - "no-implicit-button-type": "off", - "wcag/h30": "off", - "wcag/h32": "off", - "wcag/h36": "off", - "wcag/h37": "off", - "wcag/h67": "off", - "wcag/h71": "off" + "no-implicit-button-type": "off" }, "root": true @@ -88,10 +79,8 @@ If you want to change some HTML validation rules from only a few specific tests, Then you can change the configuration to use that: ```cs - changeConfiguration: configuration => configuration.HtmlValidationConfiguration.HtmlValidationOptions = - configuration.HtmlValidationConfiguration.HtmlValidationOptions - .CloneWith(validationOptions => validationOptions.ConfigPath = - Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "TestName.htmlvalidate.json"))); + changeConfiguration: configuration => + configuration.HtmlValidationConfiguration.HtmlValidationOptions.SetLocalConfigFile("TestName.htmlvalidate.json"); ``` Make sure to also include the `root` attribute and set it to `true` inside the custom _.htmlvalidate.json_ file and include it in the test project like this: diff --git a/Lombiq.Tests.UI/Extensions/HtmlValidationOptionsExtensions.cs b/Lombiq.Tests.UI/Extensions/HtmlValidationOptionsExtensions.cs new file mode 100644 index 000000000..450e9bb34 --- /dev/null +++ b/Lombiq.Tests.UI/Extensions/HtmlValidationOptionsExtensions.cs @@ -0,0 +1,15 @@ +using System; +using System.IO; + +namespace Atata.HtmlValidation; + +public static class HtmlValidationOptionsExtensions +{ + /// + /// Sets the path of an HTML-validate config file local to the current app as . + /// + /// The relative path to the HTML-validate config file. + public static void SetLocalConfigFile(this HtmlValidationOptions htmlValidationOptions, string configFileRelativePath) => + htmlValidationOptions.ConfigPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, configFileRelativePath); +} From dc0c1a2428acbc4d0573756500c6f9d044200e9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Wed, 1 May 2024 14:54:16 +0200 Subject: [PATCH 11/36] Shorter Atata artifact path --- Lombiq.Tests.UI/Services/AtataFactory.cs | 4 +++- Lombiq.Tests.UI/Services/UITestExecutionSession.cs | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Lombiq.Tests.UI/Services/AtataFactory.cs b/Lombiq.Tests.UI/Services/AtataFactory.cs index f520befb9..01e430efa 100644 --- a/Lombiq.Tests.UI/Services/AtataFactory.cs +++ b/Lombiq.Tests.UI/Services/AtataFactory.cs @@ -17,6 +17,7 @@ public class AtataConfiguration public static class AtataFactory { public static async Task StartAtataScopeAsync( + string contextId, ITestOutputHelper testOutputHelper, Uri baseUri, OrchardCoreUITestExecutorConfiguration configuration) @@ -38,7 +39,8 @@ public static async Task StartAtataScopeAsync( .UseTestName(configuration.AtataConfiguration.TestName) .UseBaseRetryTimeout(timeoutConfiguration.RetryTimeout) .UseBaseRetryInterval(timeoutConfiguration.RetryInterval) - .PageSnapshots.UseCdpOrPageSourceStrategy(); // #spell-check-ignore-line + .PageSnapshots.UseCdpOrPageSourceStrategy() // #spell-check-ignore-line + .UseArtifactsPathTemplate(contextId); // Necessary to prevent long paths, an issue under Windows. builder.LogConsumers.AddDebug(); builder.LogConsumers.Add(new TestOutputLogConsumer(testOutputHelper)); diff --git a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs index 2149641a7..f23f35074 100644 --- a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs +++ b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs @@ -656,7 +656,7 @@ Task UITestingBeforeAppStartHandlerAsync(string contentRootPath, InstanceCommand _configuration.Events.AfterPageChange += TakeScreenshotIfEnabledAsync; } - var atataScope = await AtataFactory.StartAtataScopeAsync(_testOutputHelper, uri, _configuration); + var atataScope = await AtataFactory.StartAtataScopeAsync(contextId, _testOutputHelper, uri, _configuration); return new UITestContext( contextId, From 07619ce7c7c486978052d6fa1e5fde054c7b1873 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Wed, 1 May 2024 15:23:53 +0200 Subject: [PATCH 12/36] Fixing short dump folder name generation --- Lombiq.Tests.UI/Services/UITestExecutor.cs | 26 ++++++++++++++++--- .../UITestExecutorFailureDumpConfiguration.cs | 5 ++-- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/Lombiq.Tests.UI/Services/UITestExecutor.cs b/Lombiq.Tests.UI/Services/UITestExecutor.cs index a364d11ab..46c33f2a0 100644 --- a/Lombiq.Tests.UI/Services/UITestExecutor.cs +++ b/Lombiq.Tests.UI/Services/UITestExecutor.cs @@ -125,11 +125,29 @@ private static string PrepareDumpFolder( { var dumpConfiguration = configuration.FailureDumpConfiguration; var dumpFolderNameBase = testManifest.Name; - if (dumpConfiguration.UseShortNames && dumpFolderNameBase.Contains('(', StringComparison.Ordinal)) + if (dumpConfiguration.UseShortNames) { - var dumpFolderNameBeginningIndex = - dumpFolderNameBase[..dumpFolderNameBase.IndexOf('(', StringComparison.Ordinal)].LastIndexOf('.') + 1; - dumpFolderNameBase = dumpFolderNameBase[dumpFolderNameBeginningIndex..]; + if (dumpFolderNameBase.Contains('(', StringComparison.Ordinal)) + { + + // The test uses parameters and is thus in the + // "Lombiq.Tests.UI.Samples.Tests.BasicTests.AnonymousHomePageShouldExist(browser: Chrome)" format. + var dumpFolderNameBeginningIndex = + dumpFolderNameBase[..dumpFolderNameBase.IndexOf('(', StringComparison.Ordinal)].LastIndexOf('.') + 1; + dumpFolderNameBase = dumpFolderNameBase[dumpFolderNameBeginningIndex..]; + } + else + { + // The test doesn't use parameters and is thus in the + // "Lombiq.Tests.UI.Samples.Tests.BasicTests.AnonymousHomePageShouldExist" format. + var dumpFolderNameBeginningIndex = dumpFolderNameBase.LastIndexOf('.') + 1; + dumpFolderNameBase = dumpFolderNameBase[dumpFolderNameBeginningIndex..]; + } + + // Not using an SHA265 hash, which would always compute the same value, because it doesn't matter if + // different runs compute a different hash, and the chance of a hash collision for the name of a couple + // dozen tests is tiny. So, it's better to have the shorter numeric hash codes. + dumpFolderNameBase += " " + StringComparer.Ordinal.GetHashCode(testManifest.Name).ToTechnicalString(); } dumpFolderNameBase = dumpFolderNameBase.MakeFileSystemFriendly(); diff --git a/Lombiq.Tests.UI/Services/UITestExecutorFailureDumpConfiguration.cs b/Lombiq.Tests.UI/Services/UITestExecutorFailureDumpConfiguration.cs index 1efcd1b6f..1c77d7f62 100644 --- a/Lombiq.Tests.UI/Services/UITestExecutorFailureDumpConfiguration.cs +++ b/Lombiq.Tests.UI/Services/UITestExecutorFailureDumpConfiguration.cs @@ -6,8 +6,9 @@ public class UITestExecutorFailureDumpConfiguration { /// /// Gets or sets a value indicating whether the subfolder of each test's dumps will use a shortened name, only - /// containing the name of the test method, without the name of the test class and its namespace. This is to - /// overcome the 260 character path length limitations on Windows. Defaults to on Windows. + /// containing the name of the test method suffixed with the test name's hash to make it unique, without the name of + /// the test class and its namespace. This is to overcome the 260 character path length limitations on Windows. + /// Defaults to on Windows. /// public bool UseShortNames { get; set; } = OperatingSystem.IsWindows(); From 13ee6f74cfad2f88c41f134793174652199b1fd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Wed, 1 May 2024 15:28:44 +0200 Subject: [PATCH 13/36] Fixing dump folder name double-hashing --- Lombiq.Tests.UI/Services/UITestExecutor.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Lombiq.Tests.UI/Services/UITestExecutor.cs b/Lombiq.Tests.UI/Services/UITestExecutor.cs index 46c33f2a0..5691f92c1 100644 --- a/Lombiq.Tests.UI/Services/UITestExecutor.cs +++ b/Lombiq.Tests.UI/Services/UITestExecutor.cs @@ -129,7 +129,6 @@ private static string PrepareDumpFolder( { if (dumpFolderNameBase.Contains('(', StringComparison.Ordinal)) { - // The test uses parameters and is thus in the // "Lombiq.Tests.UI.Samples.Tests.BasicTests.AnonymousHomePageShouldExist(browser: Chrome)" format. var dumpFolderNameBeginningIndex = @@ -144,10 +143,8 @@ private static string PrepareDumpFolder( dumpFolderNameBase = dumpFolderNameBase[dumpFolderNameBeginningIndex..]; } - // Not using an SHA265 hash, which would always compute the same value, because it doesn't matter if - // different runs compute a different hash, and the chance of a hash collision for the name of a couple - // dozen tests is tiny. So, it's better to have the shorter numeric hash codes. - dumpFolderNameBase += " " + StringComparer.Ordinal.GetHashCode(testManifest.Name).ToTechnicalString(); + // Can't use string.GetHasCode() because that varies between executions. + dumpFolderNameBase += " " + Sha256Helper.ComputeHash(testManifest.Name); } dumpFolderNameBase = dumpFolderNameBase.MakeFileSystemFriendly(); @@ -177,9 +174,12 @@ private static string PrepareDumpFolder( var openingBracketIndex = dumpFolderNameBase.IndexOf('(', StringComparison.Ordinal); var closingBracketIndex = dumpFolderNameBase.LastIndexOf(')'); + // Only adding a hash of the parameters if the hash of the test's full name is not already there due to + // path shortening above. // Can't use string.GetHasCode() because that varies between executions. - var hashedParameters = Sha256Helper - .ComputeHash(dumpFolderNameBase[(openingBracketIndex + 1)..(closingBracketIndex + 1)]); + var hashedParameters = dumpConfiguration.UseShortNames + ? string.Empty + : Sha256Helper.ComputeHash(dumpFolderNameBase[(openingBracketIndex + 1)..(closingBracketIndex + 1)]); dumpFolderNameBase = dumpFolderNameBase[0..(openingBracketIndex + 1)] + From 6b5886c37683b5510853b4593c29c2f6dccf6a56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Wed, 1 May 2024 15:33:44 +0200 Subject: [PATCH 14/36] Better dump folder hash separator --- Lombiq.Tests.UI/Services/UITestExecutor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Services/UITestExecutor.cs b/Lombiq.Tests.UI/Services/UITestExecutor.cs index 5691f92c1..1c8b8782a 100644 --- a/Lombiq.Tests.UI/Services/UITestExecutor.cs +++ b/Lombiq.Tests.UI/Services/UITestExecutor.cs @@ -144,7 +144,7 @@ private static string PrepareDumpFolder( } // Can't use string.GetHasCode() because that varies between executions. - dumpFolderNameBase += " " + Sha256Helper.ComputeHash(testManifest.Name); + dumpFolderNameBase += "-" + Sha256Helper.ComputeHash(testManifest.Name); } dumpFolderNameBase = dumpFolderNameBase.MakeFileSystemFriendly(); From efa930b834494184c25cf84ef40988fe94c18f25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Wed, 1 May 2024 15:41:32 +0200 Subject: [PATCH 15/36] Updating YamlDotNet to latest --- Lombiq.Tests.UI/Lombiq.Tests.UI.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj index c50b24835..124988a91 100644 --- a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj +++ b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj @@ -82,7 +82,7 @@ - + From 05a7c86ce898e5d6860b0b3a09e0732bbde1c490 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Wed, 1 May 2024 15:44:15 +0200 Subject: [PATCH 16/36] Updating Selenium.Axe --- Lombiq.Tests.UI/Lombiq.Tests.UI.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj index 124988a91..50db463c8 100644 --- a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj +++ b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj @@ -78,7 +78,7 @@ - + From 59e46e92459739834bd9dd5a0aa395ecb752c55c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Wed, 1 May 2024 16:21:27 +0200 Subject: [PATCH 17/36] Trying to make InteractiveModeTests safer --- Lombiq.Tests.UI.Samples/Tests/InteractiveModeTests.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lombiq.Tests.UI.Samples/Tests/InteractiveModeTests.cs b/Lombiq.Tests.UI.Samples/Tests/InteractiveModeTests.cs index b0438cd18..0d7302a1a 100644 --- a/Lombiq.Tests.UI.Samples/Tests/InteractiveModeTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/InteractiveModeTests.cs @@ -44,8 +44,8 @@ public Task SampleTest() => }); // This test checks if interactive mode works by opening it in one thread, and then clicking it away in a different - // thread. This ensures that the new tab correctly appears with the clickable "Continue Test" button, and then - // disappears once it's clicked. + // thread. Two threads are necessary because interactive mode stops test execution on its current thread, so we + // wouldn't be able to end it from within a test. [Fact] public Task EnteringInteractiveModeShouldWait() => ExecuteTestAfterSetupAsync( @@ -59,7 +59,7 @@ await Task.WhenAll( { // Ensure that the interactive mode polls for status at least once, so the arbitrary waiting // actually works in a real testing scenario. - await Task.Delay(1000); + await Task.Delay(5000); await context.ClickReliablyOnAsync(By.ClassName("interactive__continue")); })); From d9d63d2ef6556a98f924ab0e7bfbf41d6c540c1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Wed, 1 May 2024 17:16:18 +0200 Subject: [PATCH 18/36] Making filling Monaco editors safer --- Lombiq.Tests.UI/Extensions/FormUITestContextExtensions.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lombiq.Tests.UI/Extensions/FormUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/FormUITestContextExtensions.cs index 11e4bf1c7..ec81dc0d2 100644 --- a/Lombiq.Tests.UI/Extensions/FormUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FormUITestContextExtensions.cs @@ -108,6 +108,9 @@ public static void FillInMonacoEditor( string editorId, string text) { + // Waiting for the editor to load. + context.Get(By.CssSelector($"#{editorId} .monaco-editor")); + var script = $@" monaco.editor.getEditors().find((element) => element.getContainerDomNode().id == {JsonConvert.SerializeObject(editorId)}).setValue({JsonConvert.SerializeObject(text)});"; From 2b7d85e492623bdbf33b70071941840dc1212261 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Thu, 2 May 2024 18:44:29 +0200 Subject: [PATCH 19/36] Updating Codeuctivity.ImageSharpCompare to latest --- Lombiq.Tests.UI/Lombiq.Tests.UI.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj index 50db463c8..8ff7e10e7 100644 --- a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj +++ b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj @@ -66,7 +66,7 @@ - + From 14e176a356f7eb8c76aacd9f1090432a0f791c53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Thu, 2 May 2024 18:52:35 +0200 Subject: [PATCH 20/36] Updating xUnit dependencies to latest --- Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj | 2 +- Lombiq.Tests.UI/Lombiq.Tests.UI.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj b/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj index a107a6c52..c66c9437c 100644 --- a/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj +++ b/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj @@ -24,7 +24,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj index 8ff7e10e7..914b0c746 100644 --- a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj +++ b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj @@ -81,7 +81,7 @@ - + From 4e7ec2cb9f8bfb2adb73206a3c3fdc12379b99ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Mon, 6 May 2024 15:50:29 +0200 Subject: [PATCH 21/36] Fixing that RetryTimeout was passed instead of RetryInterval, making lookups slower --- .../Extensions/ElementRetrievalUITestContextExtensions.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lombiq.Tests.UI/Extensions/ElementRetrievalUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/ElementRetrievalUITestContextExtensions.cs index c202be07e..c7e7935f8 100644 --- a/Lombiq.Tests.UI/Extensions/ElementRetrievalUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/ElementRetrievalUITestContextExtensions.cs @@ -11,7 +11,7 @@ namespace Lombiq.Tests.UI.Extensions; /// -/// Extension methods to retrieve elements using Atata helpers. See the Atata docs ( ) for more information on what you can do /// with these. /// @@ -146,5 +146,5 @@ private static ExtendedSearchContext CreateSearchContext(this UITest new( context.Driver, context.Configuration.TimeoutConfiguration.RetryTimeout, - context.Configuration.TimeoutConfiguration.RetryTimeout); + context.Configuration.TimeoutConfiguration.RetryInterval); } From f6040ae27b1dcacfbd271d6b982e1be15569ad2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Mon, 6 May 2024 21:13:04 +0200 Subject: [PATCH 22/36] Fixing HTML validation config of the samples --- .../Lombiq.Tests.UI.Samples.csproj | 5 +++-- Lombiq.Tests.UI.Samples/UITestBase.cs | 8 -------- Lombiq.Tests.UI.Samples/samples.htmlvalidate.json | 11 ----------- .../Extensions/HtmlValidationOptionsExtensions.cs | 15 --------------- 4 files changed, 3 insertions(+), 36 deletions(-) delete mode 100644 Lombiq.Tests.UI.Samples/samples.htmlvalidate.json delete mode 100644 Lombiq.Tests.UI/Extensions/HtmlValidationOptionsExtensions.cs diff --git a/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj b/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj index c66c9437c..50f904f2b 100644 --- a/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj +++ b/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj @@ -6,13 +6,14 @@ + - - Always + + PreserveNewest PreserveNewest diff --git a/Lombiq.Tests.UI.Samples/UITestBase.cs b/Lombiq.Tests.UI.Samples/UITestBase.cs index de52f8d53..774d2ccdb 100644 --- a/Lombiq.Tests.UI.Samples/UITestBase.cs +++ b/Lombiq.Tests.UI.Samples/UITestBase.cs @@ -4,7 +4,6 @@ using Lombiq.Tests.UI.Services; using Shouldly; using System; -using System.IO; using System.Threading.Tasks; using Xunit.Abstractions; @@ -84,13 +83,6 @@ protected override Task ExecuteTestAsync( // Action) to further configure it. ////configuration.HtmlValidationConfiguration.RunHtmlValidationAssertionOnAllPageChanges = false; - // This is to make sure that the HTML validation config file in this project takes effect. Only needed - // until we fix https://github.com/Lombiq/UI-Testing-Toolbox/issues/359. - configuration.HtmlValidationConfiguration.HtmlValidationOptions = - configuration.HtmlValidationConfiguration.HtmlValidationOptions - .CloneWith(validationOptions => validationOptions.ConfigPath = - Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "samples.htmlvalidate.json")); // #spell-check-ignore-line - // The UI Testing Toolbox can run several checks for the app even if you don't add explicit // assertions: By default, the Orchard logs and the browser logs (where e.g. JavaScript errors show // up) are checked and if there are any errors, the test will fail. You can also enable the checking of diff --git a/Lombiq.Tests.UI.Samples/samples.htmlvalidate.json b/Lombiq.Tests.UI.Samples/samples.htmlvalidate.json deleted file mode 100644 index cccd9b28f..000000000 --- a/Lombiq.Tests.UI.Samples/samples.htmlvalidate.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "extends": [ - "./.htmlvalidate.json" - ], - - "rules": { - "long-title": "off", - "no-dup-class": "off" - }, - "root": true -} diff --git a/Lombiq.Tests.UI/Extensions/HtmlValidationOptionsExtensions.cs b/Lombiq.Tests.UI/Extensions/HtmlValidationOptionsExtensions.cs deleted file mode 100644 index 450e9bb34..000000000 --- a/Lombiq.Tests.UI/Extensions/HtmlValidationOptionsExtensions.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System; -using System.IO; - -namespace Atata.HtmlValidation; - -public static class HtmlValidationOptionsExtensions -{ - /// - /// Sets the path of an HTML-validate config file local to the current app as . - /// - /// The relative path to the HTML-validate config file. - public static void SetLocalConfigFile(this HtmlValidationOptions htmlValidationOptions, string configFileRelativePath) => - htmlValidationOptions.ConfigPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, configFileRelativePath); -} From c448f871a0302e799d1c992a80257650dbaabb14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Mon, 6 May 2024 21:39:20 +0200 Subject: [PATCH 23/36] Making Monaco editor reads also safer --- Lombiq.Tests.UI/Extensions/FormUITestContextExtensions.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Lombiq.Tests.UI/Extensions/FormUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/FormUITestContextExtensions.cs index ec81dc0d2..148ff181e 100644 --- a/Lombiq.Tests.UI/Extensions/FormUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FormUITestContextExtensions.cs @@ -108,8 +108,7 @@ public static void FillInMonacoEditor( string editorId, string text) { - // Waiting for the editor to load. - context.Get(By.CssSelector($"#{editorId} .monaco-editor")); + WaitForMonacoEditor(context, editorId); var script = $@" monaco.editor.getEditors().find((element) => @@ -125,6 +124,8 @@ public static string GetMonacoEditorText( this UITestContext context, string editorId) { + WaitForMonacoEditor(context, editorId); + var script = $@" return monaco.editor.getEditors().find((element) => element.getContainerDomNode().id == {JsonConvert.SerializeObject(editorId)}).getValue();"; @@ -378,4 +379,7 @@ private static IWebElement TryFillElement(UITestContext context, By by, string t return context.Driver.TryFillElement(element, text); } + + private static void WaitForMonacoEditor(UITestContext context, string editorId) => + context.Get(By.CssSelector($"#{editorId} .monaco-editor")); } From a6518ae64fe8e1e7aba4ab3f351e7943478d75bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Mon, 6 May 2024 23:26:16 +0200 Subject: [PATCH 24/36] Fixing potential NRE --- Lombiq.Tests.UI/Services/OrchardCoreInstance.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs index e41493974..44578e3c7 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs @@ -180,7 +180,8 @@ await _configuration.BeforeAppStart private async Task StopOrchardAppAsync() { - _reverseProxy.DetachConnectionProvider(); + _reverseProxy?.DetachConnectionProvider(); + if (_orchardApplication == null) return; _testOutputHelper.WriteLineTimestampedAndDebug("Attempting to stop the Orchard Core instance."); From 84423cc137bdbfacb11a628878a2f7c20a672fb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20P=C3=B3r?= Date: Fri, 10 May 2024 15:06:50 +0200 Subject: [PATCH 25/36] Adding SignInDirectlyAndGoToAdminRelativeUrlAsync() --- .../Extensions/NavigationUITestContextExtensions.cs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Lombiq.Tests.UI/Extensions/NavigationUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/NavigationUITestContextExtensions.cs index 0eebf4ba3..670a85c72 100644 --- a/Lombiq.Tests.UI/Extensions/NavigationUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/NavigationUITestContextExtensions.cs @@ -34,6 +34,17 @@ public static Task GoToAdminRelativeUrlAsync( return context.GoToAbsoluteUrlAsync(context.GetAbsoluteAdminUri(urlWithoutAdminPrefix), onlyIfNotAlreadyThere); } + public static async Task SignInDirectlyAndGoToAdminRelativeUrlAsync( + this UITestContext context, + string urlWithoutAdminPrefix = null, + bool onlyIfNotAlreadyThere = true, + string email = DefaultUser.UserName) + { + await context.SignInDirectlyAsync(email); + + await GoToAdminRelativeUrlAsync(context, urlWithoutAdminPrefix, onlyIfNotAlreadyThere); + } + public static Task GoToAbsoluteUrlAsync(this UITestContext context, Uri absoluteUri, bool onlyIfNotAlreadyThere = true) => context.ExecuteLoggedAsync( nameof(GoToAbsoluteUrlAsync), From 2fa9c3e24346c590b6c3562975dd8533c879b2b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Fri, 10 May 2024 15:26:30 +0200 Subject: [PATCH 26/36] Adding debug file logging to investigate hangs in GitHub Actions --- .../GitHubActionsGroupingTestOutputHelper.cs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Services/GitHub/GitHubActionsGroupingTestOutputHelper.cs b/Lombiq.Tests.UI/Services/GitHub/GitHubActionsGroupingTestOutputHelper.cs index 3fcaed90a..d31473355 100644 --- a/Lombiq.Tests.UI/Services/GitHub/GitHubActionsGroupingTestOutputHelper.cs +++ b/Lombiq.Tests.UI/Services/GitHub/GitHubActionsGroupingTestOutputHelper.cs @@ -1,5 +1,7 @@ using Lombiq.Tests.UI.Models; using System; +using System.Globalization; +using System.IO; using Xunit.Abstractions; namespace Lombiq.Tests.UI.Services.GitHub; @@ -7,6 +9,7 @@ namespace Lombiq.Tests.UI.Services.GitHub; internal sealed class GitHubActionsGroupingTestOutputHelper : ITestOutputHelperDecorator { private readonly string _groupName; + private FileStream _commongLog; private bool _isStarted; @@ -21,12 +24,14 @@ private GitHubActionsGroupingTestOutputHelper(ITestOutputHelper decorated, strin public void WriteLine(string message) { Start(); + _commongLog.Write(FormatMessage(message)); Decorated.WriteLine(message); } public void WriteLine(string format, params object[] args) { Start(); + _commongLog.Write(FormatMessage(string.Format(CultureInfo.InvariantCulture, format, args))); Decorated.WriteLine(format, args); } @@ -36,13 +41,20 @@ private void Start() Decorated.WriteLine($"::group::{_groupName}"); _isStarted = true; + _commongLog = File.Open("FailureDumps\\DebugLog.txt", FileMode.Append, FileAccess.Write, FileShare.Write); } private void EndGroup() { - if (_isStarted) Decorated.WriteLine("::endgroup::"); + if (_isStarted) + { + Decorated.WriteLine("::endgroup::"); + _commongLog.Dispose(); + } } + private byte[] FormatMessage(string message) => System.Text.Encoding.UTF8.GetBytes($"{_groupName} - {message}{Environment.NewLine}"); + public static (ITestOutputHelper DecoratedOutputHelper, Action AfterTest) CreateDecorator( ITestOutputHelper testOutputHelper, UITestManifest testManifest) From 7f1ef880983fface1fba363eb9dfd20d3513c273 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20P=C3=B3r?= Date: Fri, 10 May 2024 15:44:05 +0200 Subject: [PATCH 27/36] Using extension method --- Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs b/Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs index 363c31c2a..86bbcffc2 100644 --- a/Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs @@ -90,7 +90,7 @@ public Task TestAdminBackgroundTasksAsMonkeyRecursivelyShouldWorkWithAdminUser() // You could also configure the same thing with regex: ////_monkeyTestingOptions.UrlFilters.Add(new MatchesRegexMonkeyTestingUrlFilter(@"\/Admin\/BackgroundTasks")); - await context.SignInDirectlyAndGoToRelativeUrlAsync("/Admin/BackgroundTasks"); + await context.SignInDirectlyAndGoToAdminRelativeUrlAsync("/BackgroundTasks"); await context.TestCurrentPageAsMonkeyRecursivelyAsync(monkeyTestingOptions); }, configuration => configuration.AssertBrowserLog = (logEntries) => logEntries From ab5913be9d49f7d5765046457b7cf231d7e9b642 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Fri, 10 May 2024 15:53:23 +0200 Subject: [PATCH 28/36] Null check --- .../Services/GitHub/GitHubActionsGroupingTestOutputHelper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Services/GitHub/GitHubActionsGroupingTestOutputHelper.cs b/Lombiq.Tests.UI/Services/GitHub/GitHubActionsGroupingTestOutputHelper.cs index d31473355..53f98fdf4 100644 --- a/Lombiq.Tests.UI/Services/GitHub/GitHubActionsGroupingTestOutputHelper.cs +++ b/Lombiq.Tests.UI/Services/GitHub/GitHubActionsGroupingTestOutputHelper.cs @@ -49,7 +49,7 @@ private void EndGroup() if (_isStarted) { Decorated.WriteLine("::endgroup::"); - _commongLog.Dispose(); + _commongLog?.Dispose(); } } From 71aeafff3968590e69061233fa809fbd800e7218 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Fri, 10 May 2024 16:14:14 +0200 Subject: [PATCH 29/36] Need to create the dump folder and also use a subfolder for the debug log so it'll be uploaded as artifact --- .../Services/GitHub/GitHubActionsGroupingTestOutputHelper.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Services/GitHub/GitHubActionsGroupingTestOutputHelper.cs b/Lombiq.Tests.UI/Services/GitHub/GitHubActionsGroupingTestOutputHelper.cs index 53f98fdf4..7f84ae739 100644 --- a/Lombiq.Tests.UI/Services/GitHub/GitHubActionsGroupingTestOutputHelper.cs +++ b/Lombiq.Tests.UI/Services/GitHub/GitHubActionsGroupingTestOutputHelper.cs @@ -41,7 +41,8 @@ private void Start() Decorated.WriteLine($"::group::{_groupName}"); _isStarted = true; - _commongLog = File.Open("FailureDumps\\DebugLog.txt", FileMode.Append, FileAccess.Write, FileShare.Write); + Directory.CreateDirectory("FailureDumps\\DebugLog"); + _commongLog = File.Open("FailureDumps\\DebugLog\\Log.txt", FileMode.Append, FileAccess.Write, FileShare.Write); } private void EndGroup() From e3f395e2584a5ef473f789d5b37df2f13e97d742 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Fri, 10 May 2024 16:18:49 +0200 Subject: [PATCH 30/36] Safer dispose --- .../GitHubActionsGroupingTestOutputHelper.cs | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/Lombiq.Tests.UI/Services/GitHub/GitHubActionsGroupingTestOutputHelper.cs b/Lombiq.Tests.UI/Services/GitHub/GitHubActionsGroupingTestOutputHelper.cs index 7f84ae739..b6d5746a9 100644 --- a/Lombiq.Tests.UI/Services/GitHub/GitHubActionsGroupingTestOutputHelper.cs +++ b/Lombiq.Tests.UI/Services/GitHub/GitHubActionsGroupingTestOutputHelper.cs @@ -9,7 +9,7 @@ namespace Lombiq.Tests.UI.Services.GitHub; internal sealed class GitHubActionsGroupingTestOutputHelper : ITestOutputHelperDecorator { private readonly string _groupName; - private FileStream _commongLog; + private FileStream _logFileStream; private bool _isStarted; @@ -24,25 +24,29 @@ private GitHubActionsGroupingTestOutputHelper(ITestOutputHelper decorated, strin public void WriteLine(string message) { Start(); - _commongLog.Write(FormatMessage(message)); + _logFileStream.Write(FormatMessage(message)); Decorated.WriteLine(message); } public void WriteLine(string format, params object[] args) { Start(); - _commongLog.Write(FormatMessage(string.Format(CultureInfo.InvariantCulture, format, args))); + _logFileStream.Write(FormatMessage(string.Format(CultureInfo.InvariantCulture, format, args))); Decorated.WriteLine(format, args); } private void Start() { + if (_logFileStream == null) + { + if (!Directory.Exists("FailureDumps\\DebugLog")) Directory.CreateDirectory("FailureDumps\\DebugLog"); + _logFileStream = File.Open("FailureDumps\\DebugLog\\Log.txt", FileMode.Append, FileAccess.Write, FileShare.Write); + } + if (_isStarted) return; Decorated.WriteLine($"::group::{_groupName}"); _isStarted = true; - Directory.CreateDirectory("FailureDumps\\DebugLog"); - _commongLog = File.Open("FailureDumps\\DebugLog\\Log.txt", FileMode.Append, FileAccess.Write, FileShare.Write); } private void EndGroup() @@ -50,7 +54,12 @@ private void EndGroup() if (_isStarted) { Decorated.WriteLine("::endgroup::"); - _commongLog?.Dispose(); + } + + if (_logFileStream != null) + { + _logFileStream.Dispose(); + _logFileStream = null; } } From c86cc66e135717f9c4eaccda4176c12634be7594 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Sun, 12 May 2024 21:56:22 +0200 Subject: [PATCH 31/36] Formatting --- Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs b/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs index 2ccc67d25..99008752d 100644 --- a/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs @@ -34,7 +34,8 @@ public static async Task> GetErrorsAsync(this HtmlValidation public static string GetParsedErrorMessageString(IEnumerable errors) => string.Join( - '\n', errors.Select(error => + '\n', + errors.Select(error => $"{error.Line.ToString(CultureInfo.InvariantCulture)}:{error.Column.ToString(CultureInfo.InvariantCulture)} - " + $"{error.Message} - " + $"{error.RuleId}")); From 76e5ca3069021f3b9ccdc6ed650d65c855d3b5ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Mon, 13 May 2024 01:42:32 +0200 Subject: [PATCH 32/36] Disabling DebugLog --- .../GitHub/GitHubActionsGroupingTestOutputHelper.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lombiq.Tests.UI/Services/GitHub/GitHubActionsGroupingTestOutputHelper.cs b/Lombiq.Tests.UI/Services/GitHub/GitHubActionsGroupingTestOutputHelper.cs index b6d5746a9..6acfbf413 100644 --- a/Lombiq.Tests.UI/Services/GitHub/GitHubActionsGroupingTestOutputHelper.cs +++ b/Lombiq.Tests.UI/Services/GitHub/GitHubActionsGroupingTestOutputHelper.cs @@ -24,14 +24,14 @@ private GitHubActionsGroupingTestOutputHelper(ITestOutputHelper decorated, strin public void WriteLine(string message) { Start(); - _logFileStream.Write(FormatMessage(message)); + _logFileStream?.Write(FormatMessage(message)); Decorated.WriteLine(message); } public void WriteLine(string format, params object[] args) { Start(); - _logFileStream.Write(FormatMessage(string.Format(CultureInfo.InvariantCulture, format, args))); + _logFileStream?.Write(FormatMessage(string.Format(CultureInfo.InvariantCulture, format, args))); Decorated.WriteLine(format, args); } @@ -39,8 +39,8 @@ private void Start() { if (_logFileStream == null) { - if (!Directory.Exists("FailureDumps\\DebugLog")) Directory.CreateDirectory("FailureDumps\\DebugLog"); - _logFileStream = File.Open("FailureDumps\\DebugLog\\Log.txt", FileMode.Append, FileAccess.Write, FileShare.Write); + //if (!Directory.Exists("FailureDumps\\DebugLog")) Directory.CreateDirectory("FailureDumps\\DebugLog"); + //_logFileStream = File.Open("FailureDumps\\DebugLog\\Log.txt", FileMode.Append, FileAccess.Write, FileShare.Write); } if (_isStarted) return; From f1d0c9d86b64c92bb3aa102dd17849f523e5b18c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Mon, 13 May 2024 23:07:28 +0200 Subject: [PATCH 33/36] Reverting GitHubActionsGroupingTestOutputHelper --- .../GitHubActionsGroupingTestOutputHelper.cs | 24 +------------------ 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/Lombiq.Tests.UI/Services/GitHub/GitHubActionsGroupingTestOutputHelper.cs b/Lombiq.Tests.UI/Services/GitHub/GitHubActionsGroupingTestOutputHelper.cs index 6acfbf413..3fcaed90a 100644 --- a/Lombiq.Tests.UI/Services/GitHub/GitHubActionsGroupingTestOutputHelper.cs +++ b/Lombiq.Tests.UI/Services/GitHub/GitHubActionsGroupingTestOutputHelper.cs @@ -1,7 +1,5 @@ using Lombiq.Tests.UI.Models; using System; -using System.Globalization; -using System.IO; using Xunit.Abstractions; namespace Lombiq.Tests.UI.Services.GitHub; @@ -9,7 +7,6 @@ namespace Lombiq.Tests.UI.Services.GitHub; internal sealed class GitHubActionsGroupingTestOutputHelper : ITestOutputHelperDecorator { private readonly string _groupName; - private FileStream _logFileStream; private bool _isStarted; @@ -24,25 +21,17 @@ private GitHubActionsGroupingTestOutputHelper(ITestOutputHelper decorated, strin public void WriteLine(string message) { Start(); - _logFileStream?.Write(FormatMessage(message)); Decorated.WriteLine(message); } public void WriteLine(string format, params object[] args) { Start(); - _logFileStream?.Write(FormatMessage(string.Format(CultureInfo.InvariantCulture, format, args))); Decorated.WriteLine(format, args); } private void Start() { - if (_logFileStream == null) - { - //if (!Directory.Exists("FailureDumps\\DebugLog")) Directory.CreateDirectory("FailureDumps\\DebugLog"); - //_logFileStream = File.Open("FailureDumps\\DebugLog\\Log.txt", FileMode.Append, FileAccess.Write, FileShare.Write); - } - if (_isStarted) return; Decorated.WriteLine($"::group::{_groupName}"); @@ -51,20 +40,9 @@ private void Start() private void EndGroup() { - if (_isStarted) - { - Decorated.WriteLine("::endgroup::"); - } - - if (_logFileStream != null) - { - _logFileStream.Dispose(); - _logFileStream = null; - } + if (_isStarted) Decorated.WriteLine("::endgroup::"); } - private byte[] FormatMessage(string message) => System.Text.Encoding.UTF8.GetBytes($"{_groupName} - {message}{Environment.NewLine}"); - public static (ITestOutputHelper DecoratedOutputHelper, Action AfterTest) CreateDecorator( ITestOutputHelper testOutputHelper, UITestManifest testManifest) From 04565b792c040f06f7c539cd554a324b78eac903 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Mon, 13 May 2024 23:18:58 +0200 Subject: [PATCH 34/36] Udpating YamlDotNet to 15.1.4 --- Lombiq.Tests.UI/Lombiq.Tests.UI.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj index d66c75f38..b86167642 100644 --- a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj +++ b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj @@ -82,7 +82,7 @@ - + From 803e87f553dc16885995616958bd4a08ea3d8644 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Wed, 15 May 2024 19:36:57 +0200 Subject: [PATCH 35/36] Temporarily adding package validation suppression file --- Lombiq.Tests.UI/CompatibilitySuppressions.xml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 Lombiq.Tests.UI/CompatibilitySuppressions.xml diff --git a/Lombiq.Tests.UI/CompatibilitySuppressions.xml b/Lombiq.Tests.UI/CompatibilitySuppressions.xml new file mode 100644 index 000000000..7fb6e6629 --- /dev/null +++ b/Lombiq.Tests.UI/CompatibilitySuppressions.xml @@ -0,0 +1,18 @@ + + + + + CP0001 + T:Lombiq.Tests.UI.Extensions.BasicOrchardFeaturesTestingUITestContextExtensions + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + + CP0002 + M:Lombiq.Tests.UI.Services.AtataFactory.StartAtataScopeAsync(Xunit.Abstractions.ITestOutputHelper,System.Uri,Lombiq.Tests.UI.Services.OrchardCoreUITestExecutorConfiguration) + lib/net8.0/Lombiq.Tests.UI.dll + lib/net8.0/Lombiq.Tests.UI.dll + true + + \ No newline at end of file From 09dfc740ea86e0c22a662789d61d6c9d12bc621e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Wed, 15 May 2024 19:45:48 +0200 Subject: [PATCH 36/36] Revert "Temporarily adding package validation suppression file" This reverts commit 803e87f553dc16885995616958bd4a08ea3d8644. --- Lombiq.Tests.UI/CompatibilitySuppressions.xml | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 Lombiq.Tests.UI/CompatibilitySuppressions.xml diff --git a/Lombiq.Tests.UI/CompatibilitySuppressions.xml b/Lombiq.Tests.UI/CompatibilitySuppressions.xml deleted file mode 100644 index 7fb6e6629..000000000 --- a/Lombiq.Tests.UI/CompatibilitySuppressions.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - CP0001 - T:Lombiq.Tests.UI.Extensions.BasicOrchardFeaturesTestingUITestContextExtensions - lib/net8.0/Lombiq.Tests.UI.dll - lib/net8.0/Lombiq.Tests.UI.dll - true - - - CP0002 - M:Lombiq.Tests.UI.Services.AtataFactory.StartAtataScopeAsync(Xunit.Abstractions.ITestOutputHelper,System.Uri,Lombiq.Tests.UI.Services.OrchardCoreUITestExecutorConfiguration) - lib/net8.0/Lombiq.Tests.UI.dll - lib/net8.0/Lombiq.Tests.UI.dll - true - - \ No newline at end of file