From 1761d3e283243c0a0eb266894933ce9030bb80f8 Mon Sep 17 00:00:00 2001 From: Charlie Poole Date: Fri, 14 Feb 2025 10:58:17 -0800 Subject: [PATCH] Further work on 1613 --- build.cake | 47 +- choco/nunit-console-runner.nuspec | 7 + package-tests.cake | 757 ++++++++++-------- .../nunit3-console/ConsoleRunner.cs | 4 +- src/NUnitEngine/nunit-agent/Program.cs | 4 +- .../Services/DriverService.cs | 18 +- .../Services/ExtensionManager.cs | 32 +- .../nunit.engine/nunit.engine.addins | 1 - .../nunit.engine/nunit.engine.csproj | 6 - .../FakeExtensions/fake-extensions.addins | 3 + 10 files changed, 468 insertions(+), 411 deletions(-) delete mode 100644 src/NUnitEngine/nunit.engine/nunit.engine.addins create mode 100644 src/TestData/FakeExtensions/fake-extensions.addins diff --git a/build.cake b/build.cake index b10527b19..f56815e76 100644 --- a/build.cake +++ b/build.cake @@ -1,5 +1,5 @@ // Load the recipe -#load nuget:?package=NUnit.Cake.Recipe&version=1.3.1-alpha.1 +#load nuget:?package=NUnit.Cake.Recipe&version=1.4.0-alpha.1 // Comment out above line and uncomment below for local tests of recipe changes //#load ../NUnit.Cake.Recipe/recipe/*.cake @@ -74,7 +74,7 @@ BuildSettings.Packages.AddRange(new PackageDefinition[] { }, testRunner: new ConsoleRunnerSelfTester(BuildSettings.NuGetTestDirectory + $"NUnit.ConsoleRunner.{BuildSettings.PackageVersion}/tools/nunit3-console.exe"), - tests: StandardRunnerTests), + tests: PackageTests.StandardRunnerTests), // NOTE: Must follow ConsoleRunner, upon which it depends NUnitConsoleNuGetPackage = new NuGetPackage( @@ -91,9 +91,8 @@ BuildSettings.Packages.AddRange(new PackageDefinition[] { HasDirectory(".store/nunit.consolerunner.netcore/**/tools/net8.0/any") .WithFiles(ENGINE_FILES).AndFiles(ConsoleFiles).AndFile("Microsoft.Extensions.DependencyModel.dll") }, - testRunner: new ConsoleRunnerSelfTester(BuildSettings.NuGetTestDirectory - + $"NUnit.ConsoleRunner.NetCore.{BuildSettings.PackageVersion}/nunit.exe"), - tests: NetCoreRunnerTests), + testRunner: new ConsoleRunnerSelfTester(BuildSettings.NuGetTestDirectory + "nunit.exe"), + tests: PackageTests.NetCoreRunnerTests), NUnitConsoleRunnerChocolateyPackage = new ChocolateyPackage( id: "nunit-console-runner", @@ -106,7 +105,7 @@ BuildSettings.Packages.AddRange(new PackageDefinition[] { }, testRunner: new ConsoleRunnerSelfTester(BuildSettings.ChocolateyTestDirectory + $"nunit-console-runner.{BuildSettings.PackageVersion}/tools/nunit3-console.exe"), - tests: StandardRunnerTests), + tests: PackageTests.StandardRunnerTests), NUnitConsoleZipPackage = new ZipPackage( id: "NUnit.Console", @@ -126,7 +125,7 @@ BuildSettings.Packages.AddRange(new PackageDefinition[] { }, testRunner: new ConsoleRunnerSelfTester(BuildSettings.ZipTestDirectory + $"NUnit.Console.{BuildSettings.PackageVersion}/bin/net462/nunit3-console.exe"), - tests: StandardRunnerTests, + tests: PackageTests.ZipRunnerTests, bundledExtensions: new [] { KnownExtensions.VSProjectLoader.NuGetPackage, KnownExtensions.NUnitProjectLoader.NuGetPackage, @@ -168,36 +167,6 @@ BuildSettings.Packages.AddRange(new PackageDefinition[] { }) }); -Task("BuildZipPackage") - .Does(() => - { - NUnitConsoleZipPackage.BuildPackage(); - }); - -Task("InstallZipPackage") - .Does(() => - { - NUnitConsoleZipPackage.InstallPackage(); - }); - -Task("VerifyZipPackage") - .Does(() => - { - NUnitConsoleZipPackage.VerifyPackage(); - }); - -Task("TestZipPackage") - .Does(() => - { - NUnitConsoleZipPackage.RunPackageTests(); - }); - -Task("TestNetCorePackage") - .Does(() => - { - NUnitConsoleRunnerNetCorePackage.RunPackageTests(); - }); - ////////////////////////////////////////////////////////////////////// // TEST RUNNERS ////////////////////////////////////////////////////////////////////// @@ -212,10 +181,10 @@ public class ConsoleRunnerSelfTester : TestRunner, IPackageTestRunner _executablePath = executablePath; } - public int RunPackageTest(string arguments) + public int RunPackageTest(string arguments, bool redirectOutput = false) { Console.WriteLine("Running package test"); - return base.RunTest(_executablePath, arguments); + return base.RunPackageTest(_executablePath, new ProcessSettings { Arguments = arguments, RedirectStandardOutput = redirectOutput }); } } diff --git a/choco/nunit-console-runner.nuspec b/choco/nunit-console-runner.nuspec index 1b05a1446..2090d1ad0 100644 --- a/choco/nunit-console-runner.nuspec +++ b/choco/nunit-console-runner.nuspec @@ -28,6 +28,7 @@ + @@ -48,6 +49,8 @@ + + @@ -58,6 +61,8 @@ + + @@ -68,5 +73,7 @@ + + diff --git a/package-tests.cake b/package-tests.cake index c39888913..6dc4d8a1f 100644 --- a/package-tests.cake +++ b/package-tests.cake @@ -1,350 +1,455 @@ -// Tests run for all runner packages except NETCORE runner -var StandardRunnerTests = new List(); - -// Tests run for the NETCORE runner package -var NetCoreRunnerTests = new List(); - -// Method for adding to both lists -void AddToBothLists(PackageTest test) +public static class PackageTests { - StandardRunnerTests.Add(test); - NetCoreRunnerTests.Add(test); -} + // Tests run for the Standard runner packages (both nuget and chocolatey) + public static List StandardRunnerTests = new List(); -////////////////////////////////////////////////////////////////////// -// RUN MOCK-ASSEMBLY UNDER EACH RUNTIME -////////////////////////////////////////////////////////////////////// + // Tests run for the NETCORE runner package + public static List NetCoreRunnerTests = new List(); -class MockAssemblyExpectedResult : ExpectedResult -{ - public MockAssemblyExpectedResult(params string[] runtimes) : base("Failed") - { - int nCopies = runtimes.Length; - Total = 37 * nCopies; - Passed = 23 * nCopies; - Failed = 5 * nCopies; - Warnings = 1 * nCopies; - Inconclusive = 1 * nCopies; - Skipped = 7 * nCopies; - Assemblies = new ExpectedAssemblyResult[nCopies]; - for (int i = 0; i < nCopies; i++) - Assemblies[i] = new ExpectedAssemblyResult("mock-assembly.dll", runtimes[i]); - } -} + // Tests Run for the ZIP Package + public static List ZipRunnerTests = new List(); -StandardRunnerTests.Add(new PackageTest( - 1, "Net462Test", - "Run mock-assembly.dll under .NET 4.6.2", - "testdata/net462/mock-assembly.dll", - new MockAssemblyExpectedResult("net-4.6.2"))); - -AddToBothLists(new PackageTest( - 1, "Net80Test", - "Run mock-assembly.dll under .NET 8.0", - "testdata/net8.0/mock-assembly.dll", - new MockAssemblyExpectedResult("netcore-8.0"))); - -AddToBothLists(new PackageTest( - 1, "Net70Test", - "Run mock-assembly.dll under .NET 7.0", - "testdata/net7.0/mock-assembly.dll", - new MockAssemblyExpectedResult("netcore-7.0"))); - -AddToBothLists(new PackageTest( - 1, "Net60Test", - "Run mock-assembly.dll under .NET 6.0", - "testdata/net6.0/mock-assembly.dll", - new MockAssemblyExpectedResult("netcore-6.0"))); - -AddToBothLists(new PackageTest( - 1, "NetCore31Test", - "Run mock-assembly.dll under .NET Core 3.1", - "testdata/netcoreapp3.1/mock-assembly.dll", - new MockAssemblyExpectedResult("netcore-3.1"))); - -////////////////////////////////////////////////////////////////////// -// RUN MOCK-ASSEMBLY-X86 UNDER EACH RUNTIME -////////////////////////////////////////////////////////////////////// - -const string DOTNET_EXE_X86 = @"C:\Program Files (x86)\dotnet\dotnet.exe"; -// TODO: Remove the limitation to Windows -bool dotnetX86Available = IsRunningOnWindows() && System.IO.File.Exists(DOTNET_EXE_X86); - -class MockAssemblyX86ExpectedResult : MockAssemblyExpectedResult -{ - public MockAssemblyX86ExpectedResult(params string[] runtimes) : base(runtimes) + static PackageTests() { - for (int i = 0; i < runtimes.Length; i++) - Assemblies[i] = new ExpectedAssemblyResult("mock-assembly-x86.dll", runtimes[i]); - } -} + // Wrapper to add tests to all three Lists + var AllLists = new ListWrapper(StandardRunnerTests, NetCoreRunnerTests, ZipRunnerTests); -// X86 is always available for .NET Framework -StandardRunnerTests.Add(new PackageTest( - 1, "Net462X86Test", - "Run mock-assembly-x86.dll under .NET 4.6.2", - "testdata/net462/mock-assembly-x86.dll", - new MockAssemblyX86ExpectedResult("net-4.6.2"))); + // Wrapper to add tests to the standard and netcore lists + var StandardAndNetCoreLists = new ListWrapper(StandardRunnerTests, NetCoreRunnerTests); -if (dotnetX86Available) -{ - // TODO: Make tests run on all build platforms - bool onAppVeyor = BuildSystem.IsRunningOnAppVeyor; - bool onGitHubActions = BuildSystem.IsRunningOnGitHubActions; - - if (!onAppVeyor) - StandardRunnerTests.Add(new PackageTest( - 1, "Net80X86Test", - "Run mock-assembly-x86.dll under .NET 8.0", - "testdata/net8.0/mock-assembly-x86.dll", - new MockAssemblyX86ExpectedResult("netcore-8.0"))); - - if (!onAppVeyor && !onGitHubActions) - StandardRunnerTests.Add(new PackageTest( - 1, "Net70X86Test", - "Run mock-assembly-x86.dll under .NET 7.0", - "testdata/net7.0/mock-assembly-x86.dll", - new MockAssemblyX86ExpectedResult("netcore-7.0"))); - - StandardRunnerTests.Add(new PackageTest( - 1, "Net60X86Test", - "Run mock-assembly-x86.dll under .NET 6.0", - "testdata/net6.0/mock-assembly-x86.dll", - new MockAssemblyX86ExpectedResult("netcore-6.0"))); - - if (!onAppVeyor && !onGitHubActions) - StandardRunnerTests.Add(new PackageTest( - 1, "NetCore31X86Test", - "Run mock-assembly-x86.dll under .NET Core 3.1", - "testdata/netcoreapp3.1/mock-assembly-x86.dll", - new MockAssemblyX86ExpectedResult("netcore-3.1"))); -} + // Wrapper to add tests to the standard and zip Lists + var StandardAndZipLists = new ListWrapper(StandardRunnerTests, ZipRunnerTests); -////////////////////////////////////////////////////////////////////// -// RUN MULTIPLE COPIES OF MOCK-ASSEMBLY -////////////////////////////////////////////////////////////////////// - -StandardRunnerTests.Add(new PackageTest( - 1, "Net462PlusNet462Test", - "Run two copies of mock-assembly together", - "testdata/net462/mock-assembly.dll testdata/net462/mock-assembly.dll", - new MockAssemblyExpectedResult("net-4.6.2", "net-4.6.2"))); - -StandardRunnerTests.Add(new PackageTest( - 1, "Net60PlusNet80Test", - "Run mock-assembly under .NET6.0 and 8.0 together", - "testdata/net6.0/mock-assembly.dll testdata/net8.0/mock-assembly.dll", - new MockAssemblyExpectedResult("netcore-6.0", "netcore-8.0"))); - -StandardRunnerTests.Add(new PackageTest( - 1, "Net462PlusNet60Test", - "Run mock-assembly under .Net Framework 4.6.2 and .Net 6.0 together", - "testdata/net462/mock-assembly.dll testdata/net6.0/mock-assembly.dll", - new MockAssemblyExpectedResult("net-4.6.2", "netcore-6.0"))); - -////////////////////////////////////////////////////////////////////// -// ASP.NETCORE TESTS -////////////////////////////////////////////////////////////////////// - -AddToBothLists(new PackageTest( - 1, "Net60AspNetCoreTest", "Run test using AspNetCore targeting .NET 6.0", - "testdata/net6.0/aspnetcore-test.dll", new ExpectedResult("Passed") - { - Total = 3, Passed = 3, Failed = 0, Warnings = 0, Inconclusive = 0, Skipped = 0, - Assemblies = new ExpectedAssemblyResult[] { new ExpectedAssemblyResult("aspnetcore-test.dll", "netcore-6.0") } - })); + ////////////////////////////////////////////////////////////////////// + // RUN MOCK-ASSEMBLY UNDER EACH RUNTIME + ////////////////////////////////////////////////////////////////////// -AddToBothLists(new PackageTest( - 1, "Net80AspNetCoreTest", "Run test using AspNetCore targeting .NET 8.0", - "testdata/net8.0/aspnetcore-test.dll", new ExpectedResult("Passed") - { - Total = 3, Passed = 3, Failed = 0, Warnings = 0, Inconclusive = 0, Skipped = 0, - Assemblies = new ExpectedAssemblyResult[] { new ExpectedAssemblyResult("aspnetcore-test.dll", "netcore-8.0") } - })); + StandardAndZipLists.Add(new PackageTest(1, "Net462Test") + { + Description="Run mock-assembly.dll under .NET 4.6.2", + Arguments="testdata/net462/mock-assembly.dll", + ExpectedResult=new MockAssemblyExpectedResult("net-4.6.2") + }); -////////////////////////////////////////////////////////////////////// -// WINDOWS FORMS TESTS -////////////////////////////////////////////////////////////////////// + AllLists.Add(new PackageTest(1, "Net80Test") + { + Description = "Run mock-assembly.dll under .NET 8.0", + Arguments = "testdata/net8.0/mock-assembly.dll", + ExpectedResult = new MockAssemblyExpectedResult("netcore-8.0") + }); -AddToBothLists(new PackageTest( - 1, "Net60WindowsFormsTest", "Run test using windows forms under .NET 6.0", - "testdata/net6.0-windows/windows-forms-test.dll", new ExpectedResult("Passed") - { - Total = 2, Passed = 2, Failed = 0, Warnings = 0, Inconclusive = 0, Skipped = 0, - Assemblies = new ExpectedAssemblyResult[] { new ExpectedAssemblyResult("windows-forms-test.dll", "netcore-6.0") } - })); - -// Runs under Net80 runner but not NetCore -StandardRunnerTests.Add(new PackageTest( - 1, "Net80WindowsFormsTest", "Run test using windows forms under .NET 8.0", - "testdata/net8.0-windows/windows-forms-test.dll", new ExpectedResult("Passed") - { - Total = 2, Passed = 2, Failed = 0, Warnings = 0, Inconclusive = 0, Skipped = 0, - Assemblies = new ExpectedAssemblyResult[] { new ExpectedAssemblyResult("windows-forms-test.dll", "netcore-8.0") } - })); - -////////////////////////////////////////////////////////////////////// -// WPF TESTS -////////////////////////////////////////////////////////////////////// - -AddToBothLists(new PackageTest( - 1, "Net60WPFTest", "Run test using WPF under .NET 6.0", - "testdata/net6.0-windows/WpfTest.dll --trace=Debug", - new ExpectedResult("Passed") { Assemblies = new[] { new ExpectedAssemblyResult("WpfTest.dll", "netcore-6.0") } })); - -AddToBothLists(new PackageTest( - 1, "Net80WPFTest", "Run test using WPF under .NET 8.0", - "testdata/net8.0-windows/WpfTest.dll --trace=Debug", - new ExpectedResult("Passed") { Assemblies = new[] { new ExpectedAssemblyResult("WpfTest.dll", "netcore-8.0") } })); - -////////////////////////////////////////////////////////////////////// -// TESTS USING EACH OF OUR EXTENSIONS -////////////////////////////////////////////////////////////////////// - -// NUnit Project Loader Tests -StandardRunnerTests.Add(new PackageTest( - 1, "NUnitProjectTest", - "Run NUnit project with mock-assembly.dll built for .NET 4.6.2 and 6.0", - "../../MixedTests.nunit --config=Release", - new MockAssemblyExpectedResult("net-4.6.2", "net-6.0"), - KnownExtensions.NUnitProjectLoader)); - -NetCoreRunnerTests.Add(new PackageTest( - 1, "NUnitProjectTest", - "Run NUnit project with mock-assembly.dll built for .NET 6.0 and 8.0", - "../../NetCoreTests.nunit --config=Release", - new MockAssemblyExpectedResult("netcore-6.0", "netcore-8.0"), - KnownExtensions.NUnitProjectLoader)); - -// V2 Result Writer Test -AddToBothLists(new PackageTest( - 1, "V2ResultWriterTest", - "Run mock-assembly under .NET 6.0 and produce V2 output", - "testdata/net6.0/mock-assembly.dll --result=TestResult.xml --result=NUnit2TestResult.xml;format=nunit2", - new MockAssemblyExpectedResult("netcore-6.0"), - KnownExtensions.NUnitV2ResultWriter)); - -// VS Project Loader Tests -StandardRunnerTests.Add(new PackageTest( - 1, "VSProjectLoaderTest_Project", - "Run mock-assembly using the .csproj file", - "../../src/TestData/mock-assembly/mock-assembly.csproj --config=Release", - new MockAssemblyExpectedResult("net462", "netcore-3.1", "netcore-6.0", "netcore-7.0", "netcore-8.0"), - KnownExtensions.VSProjectLoader)); - -StandardRunnerTests.Add(new PackageTest( - 1, "VSProjectLoaderTest_Solution", - "Run mock-assembly using the .sln file", - "../../src/TestData/TestData.sln --config=Release --trace=Debug", - new ExpectedResult("Failed") - { - Total = 37 * 5, - Passed = 23 * 5, - Failed = 5 * 5, - Warnings = 1 * 5, - Inconclusive = 1 * 5, - Skipped = 7 * 5, - Assemblies = new ExpectedAssemblyResult[] - { - new ExpectedAssemblyResult("mock-assembly.dll", "net-4.6.2"), - new ExpectedAssemblyResult("mock-assembly.dll", "netcore-3.1"), - new ExpectedAssemblyResult("mock-assembly.dll", "netcore-6.0"), - new ExpectedAssemblyResult("mock-assembly.dll", "netcore-7.0"), - new ExpectedAssemblyResult("mock-assembly.dll", "netcore-8.0"), - new ExpectedAssemblyResult("notest-assembly.dll", "net-4.6.2"), - new ExpectedAssemblyResult("notest-assembly.dll", "netcore-3.1"), - new ExpectedAssemblyResult("notest-assembly.dll", "netstandard-2.0"), - new ExpectedAssemblyResult("WpfApp.exe") + AllLists.Add(new PackageTest(1, "Net70Test") + { + Description = "Run mock-assembly.dll under .NET 7.0", + Arguments = "testdata/net7.0/mock-assembly.dll", + ExpectedResult = new MockAssemblyExpectedResult("netcore-7.0") + }); + + AllLists.Add(new PackageTest(1, "Net60Test") + { + Description = "Run mock-assembly.dll under .NET 6.0", + Arguments = "testdata/net6.0/mock-assembly.dll", + ExpectedResult = new MockAssemblyExpectedResult("netcore-6.0") + }); + + AllLists.Add(new PackageTest(1, "NetCore31Test") + { + Description = "Run mock-assembly.dll under .NET Core 3.1", + Arguments = "testdata/netcoreapp3.1/mock-assembly.dll", + ExpectedResult = new MockAssemblyExpectedResult("netcore-3.1") + }); + + ////////////////////////////////////////////////////////////////////// + // RUN MOCK-ASSEMBLY-X86 UNDER EACH RUNTIME + ////////////////////////////////////////////////////////////////////// + + const string DOTNET_EXE_X86 = @"C:\Program Files (x86)\dotnet\dotnet.exe"; + // TODO: Remove the limitation to Windows + bool dotnetX86Available = BuildSettings.IsRunningOnWindows && System.IO.File.Exists(DOTNET_EXE_X86); + + // X86 is always available for .NET Framework + StandardAndZipLists.Add(new PackageTest(1, "Net462X86Test") + { + Description = "Run mock-assembly-x86.dll under .NET 4.6.2", + Arguments = "testdata/net462/mock-assembly-x86.dll", + ExpectedResult = new MockAssemblyX86ExpectedResult("net-4.6.2") + }); + + if (dotnetX86Available) + { + // TODO: Make tests run on all build platforms + bool onGitHubActions = BuildSettings.IsRunningOnGitHubActions; + + StandardAndZipLists.Add(new PackageTest(1, "Net80X86Test") + { + Description = "Run mock-assembly-x86.dll under .NET 8.0", + Arguments = "testdata/net8.0/mock-assembly-x86.dll", + ExpectedResult = new MockAssemblyX86ExpectedResult("netcore-8.0") + }); + + if (!onGitHubActions) + StandardAndZipLists.Add(new PackageTest(1, "Net70X86Test") + { + Description = "Run mock-assembly-x86.dll under .NET 7.0", + Arguments = "testdata/net7.0/mock-assembly-x86.dll", + ExpectedResult = new MockAssemblyX86ExpectedResult("netcore-7.0") + }); + + StandardAndZipLists.Add(new PackageTest(1, "Net60X86Test") + { + Description = "Run mock-assembly-x86.dll under .NET 6.0", + Arguments = "testdata/net6.0/mock-assembly-x86.dll", + ExpectedResult = new MockAssemblyX86ExpectedResult("netcore-6.0") + }); + + if (!onGitHubActions) + StandardAndZipLists.Add(new PackageTest(1, "NetCore31X86Test") + { + Description = "Run mock-assembly-x86.dll under .NET Core 3.1", + Arguments = "testdata/netcoreapp3.1/mock-assembly-x86.dll", + ExpectedResult = new MockAssemblyX86ExpectedResult("netcore-3.1") + }); } - }, - KnownExtensions.VSProjectLoader)); - -// TeamCity Event Listener Test -StandardRunnerTests.Add(new PackageTest( - 1, "TeamCityListenerTest", - "Run mock-assembly with --teamcity enabled", - "testdata/net462/mock-assembly.dll --teamcity", - new MockAssemblyExpectedResult("net-4.6.2"), - KnownExtensions.TeamCityEventListener)); - -// V2 Framework Driver Tests -StandardRunnerTests.Add(new PackageTest( - 1, "V2FrameworkDriverTest", - "Run mock-assembly-v2 using the V2 Driver in process", - "v2-tests/net462/mock-assembly-v2.dll", - new ExpectedResult("Failed") - { - Total = 28, - Passed = 18, - Failed = 5, - Warnings = 0, - Inconclusive = 1, - Skipped = 4, - Assemblies = new ExpectedAssemblyResult[] { new ExpectedAssemblyResult("mock-assembly-v2.dll", "net-4.6.2") } - }, - KnownExtensions.NUnitV2Driver)); - -StandardRunnerTests.Add(new PackageTest( - 1, "V2FrameworkDriverTest", - "Run mock-assembly-v2 using the V2 Driver out of process", - "v2-tests/net462/mock-assembly-v2.dll", - new ExpectedResult("Failed") - { - Total = 28, - Passed = 18, - Failed = 5, - Warnings = 0, - Inconclusive = 1, - Skipped = 4, - Assemblies = new ExpectedAssemblyResult[] { new ExpectedAssemblyResult("mock-assembly-v2.dll", "net-4.6.2") } - }, - KnownExtensions.NUnitV2Driver)); - -////////////////////////////////////////////////////////////////////// -// SPECIAL CASES -////////////////////////////////////////////////////////////////////// - -StandardRunnerTests.Add(new PackageTest( - 1, "InvalidTestNameTest_Net462", - "Ensure we handle invalid test names correctly under .NET 4.6.2", - "testdata/net462/InvalidTestNames.dll --trace:Debug", - new ExpectedResult("Passed") + + ////////////////////////////////////////////////////////////////////// + // RUN MULTIPLE COPIES OF MOCK-ASSEMBLY + ////////////////////////////////////////////////////////////////////// + + StandardAndZipLists.Add(new PackageTest(1, "Net462PlusNet462Test") + { + Description = "Run two copies of mock-assembly together", + Arguments = "testdata/net462/mock-assembly.dll testdata/net462/mock-assembly.dll", + ExpectedResult = new MockAssemblyExpectedResult("net-4.6.2", "net-4.6.2") + }); + + StandardAndZipLists.Add(new PackageTest(1, "Net60PlusNet80Test") + { + Description = "Run mock-assembly under .NET6.0 and 8.0 together", + Arguments = "testdata/net6.0/mock-assembly.dll testdata/net8.0/mock-assembly.dll", + ExpectedResult = new MockAssemblyExpectedResult("netcore-6.0", "netcore-8.0") + }); + + StandardAndZipLists.Add(new PackageTest(1, "Net462PlusNet60Test") + { + Description = "Run mock-assembly under .Net Framework 4.6.2 and .Net 6.0 together", + Arguments = "testdata/net462/mock-assembly.dll testdata/net6.0/mock-assembly.dll", + ExpectedResult = new MockAssemblyExpectedResult("net-4.6.2", "netcore-6.0") + }); + + ////////////////////////////////////////////////////////////////////// + // ASP.NETCORE TESTS + ////////////////////////////////////////////////////////////////////// + + AllLists.Add(new PackageTest(1, "Net60AspNetCoreTest") + { + Description = "Run test using AspNetCore targeting .NET 6.0", + Arguments = "testdata/net6.0/aspnetcore-test.dll", + ExpectedResult = new ExpectedResult("Passed") + { + Total = 3, + Passed = 3, + Failed = 0, + Warnings = 0, + Inconclusive = 0, + Skipped = 0, + Assemblies = new ExpectedAssemblyResult[] { new ExpectedAssemblyResult("aspnetcore-test.dll", "netcore-6.0") } + } + }); + + AllLists.Add(new PackageTest(1, "Net80AspNetCoreTest") + { + Description = "Run test using AspNetCore targeting .NET 8.0", + Arguments = "testdata/net8.0/aspnetcore-test.dll", + ExpectedResult = new ExpectedResult("Passed") + { + Total = 3, + Passed = 3, + Failed = 0, + Warnings = 0, + Inconclusive = 0, + Skipped = 0, + Assemblies = new ExpectedAssemblyResult[] { new ExpectedAssemblyResult("aspnetcore-test.dll", "netcore-8.0") } + } + }); + + ////////////////////////////////////////////////////////////////////// + // WINDOWS FORMS TESTS + ////////////////////////////////////////////////////////////////////// + + AllLists.Add(new PackageTest(1, "Net60WindowsFormsTest") + { + Description = "Run test using windows forms under .NET 6.0", + Arguments = "testdata/net6.0-windows/windows-forms-test.dll", + ExpectedResult = new ExpectedResult("Passed") + { + Total = 2, + Passed = 2, + Failed = 0, + Warnings = 0, + Inconclusive = 0, + Skipped = 0, + Assemblies = new ExpectedAssemblyResult[] { new ExpectedAssemblyResult("windows-forms-test.dll", "netcore-6.0") } + } + }); + + StandardAndZipLists.Add(new PackageTest(1, "Net80WindowsFormsTest") + { + Description = "Run test using windows forms under .NET 8.0", + Arguments = "testdata/net8.0-windows/windows-forms-test.dll", + ExpectedResult = new ExpectedResult("Passed") + { + Total = 2, + Passed = 2, + Failed = 0, + Warnings = 0, + Inconclusive = 0, + Skipped = 0, + Assemblies = new ExpectedAssemblyResult[] { new ExpectedAssemblyResult("windows-forms-test.dll", "netcore-8.0") } + } + }); + + ////////////////////////////////////////////////////////////////////// + // WPF TESTS + ////////////////////////////////////////////////////////////////////// + + AllLists.Add(new PackageTest(1, "Net60WPFTest") + { + Description = "Run test using WPF under .NET 6.0", + Arguments = "testdata/net6.0-windows/WpfTest.dll --trace=Debug", + ExpectedResult = new ExpectedResult("Passed") { Assemblies = new[] { new ExpectedAssemblyResult("WpfTest.dll", "netcore-6.0") } } + }); + + AllLists.Add(new PackageTest(1, "Net80WPFTest") + { + Description = "Run test using WPF under .NET 8.0", + Arguments = "testdata/net8.0-windows/WpfTest.dll --trace=Debug", + ExpectedResult = new ExpectedResult("Passed") { Assemblies = new[] { new ExpectedAssemblyResult("WpfTest.dll", "netcore-8.0") } } + }); + + ////////////////////////////////////////////////////////////////////// + // TESTS OF EXTENSION LISTING + ////////////////////////////////////////////////////////////////////// + + StandardAndNetCoreLists.Add(new PackageTest(1, "NoExtensionsInstalled") + { + Description = "List Extensions shows none installed", + Arguments = "--list-extensions", + OutputCheck = new OutputDoesNotContain("Extension:"), + }); + + StandardAndNetCoreLists.Add(new PackageTest(1, "ExtensionsInstalledFromAddedDirectory") + { + Description = "List Extensions shows extension from added directory", + Arguments = "--extensionDirectory ../../src/TestData/FakeExtensions --list-extensions", + OutputCheck = new OutputContains("Extension:", exactly: 5) + }); + + ZipRunnerTests.Add(new PackageTest(1, "BundledExtensionsInstalled") + { + Description = "List Extensions shows bundled extensions", + Arguments = "--list-extensions", + OutputCheck = new OutputContains("Extension:", exactly: 5) + }); + + ////////////////////////////////////////////////////////////////////// + // TESTS USING EACH OF OUR EXTENSIONS + ////////////////////////////////////////////////////////////////////// + + // NUnit Project Loader Tests + StandardAndZipLists.Add(new PackageTest(1, "NUnitProjectTest") + { + Description = "Run NUnit project with mock-assembly.dll built for .NET 4.6.2 and 6.0", + Arguments = "../../MixedTests.nunit --config=Release", + ExpectedResult = new MockAssemblyExpectedResult("net-4.6.2", "net-6.0"), + ExtensionsNeeded = new [] { KnownExtensions.NUnitProjectLoader } + }); + + NetCoreRunnerTests.Add(new PackageTest(1, "NUnitProjectTest") + { + Description="Run NUnit project with mock-assembly.dll built for .NET 6.0 and 8.0", + Arguments="../../NetCoreTests.nunit --config=Release", + ExpectedResult = new MockAssemblyExpectedResult("netcore-6.0", "netcore-8.0"), + ExtensionsNeeded = new [] { KnownExtensions.NUnitProjectLoader } + }); + + // V2 Result Writer Test + AllLists.Add(new PackageTest(1, "V2ResultWriterTest") + { + Description = "Run mock-assembly under .NET 6.0 and produce V2 output", + Arguments = "testdata/net6.0/mock-assembly.dll --result=TestResult.xml --result=NUnit2TestResult.xml;format=nunit2", + ExpectedResult = new MockAssemblyExpectedResult("netcore-6.0"), + ExtensionsNeeded = new[] { KnownExtensions.NUnitV2ResultWriter } + }); + + // VS Project Loader Tests + StandardAndZipLists.Add(new PackageTest(1, "VSProjectLoaderTest_Project") + { + Description = "Run mock-assembly using the .csproj file", + Arguments = "../../src/TestData/mock-assembly/mock-assembly.csproj --config=Release", + ExpectedResult = new MockAssemblyExpectedResult("net462", "netcore-3.1", "netcore-6.0", "netcore-7.0", "netcore-8.0"), + ExtensionsNeeded = new[] { KnownExtensions.VSProjectLoader } + }); + + StandardAndZipLists.Add(new PackageTest(1, "VSProjectLoaderTest_Solution") + { + Description = "Run mock-assembly using the .sln file", + Arguments = "../../src/TestData/TestData.sln --config=Release --trace=Debug", + ExpectedResult = new ExpectedResult("Failed") + { + Total = 37 * 5, + Passed = 23 * 5, + Failed = 5 * 5, + Warnings = 1 * 5, + Inconclusive = 1 * 5, + Skipped = 7 * 5, + Assemblies = new ExpectedAssemblyResult[] + { + new ExpectedAssemblyResult("mock-assembly.dll", "net-4.6.2"), + new ExpectedAssemblyResult("mock-assembly.dll", "netcore-3.1"), + new ExpectedAssemblyResult("mock-assembly.dll", "netcore-6.0"), + new ExpectedAssemblyResult("mock-assembly.dll", "netcore-7.0"), + new ExpectedAssemblyResult("mock-assembly.dll", "netcore-8.0"), + new ExpectedAssemblyResult("notest-assembly.dll", "net-4.6.2"), + new ExpectedAssemblyResult("notest-assembly.dll", "netcore-3.1"), + new ExpectedAssemblyResult("notest-assembly.dll", "netstandard-2.0"), + new ExpectedAssemblyResult("WpfApp.exe") + } + }, + ExtensionsNeeded = new[] { KnownExtensions.VSProjectLoader } + }); + + // TeamCity Event Listener Test + StandardAndZipLists.Add(new PackageTest(1, "TeamCityListenerTest") + { + Description = "Run mock-assembly with --teamcity enabled", + Arguments = "testdata/net462/mock-assembly.dll --teamcity", + ExpectedResult = new MockAssemblyExpectedResult("net-4.6.2"), + ExtensionsNeeded = new[] { KnownExtensions.TeamCityEventListener } + }); + + // V2 Framework Driver Tests + StandardAndZipLists.Add(new PackageTest(1, "V2FrameworkDriverTest") + { + Description = "Run mock-assembly-v2 using the V2 Driver in process", + Arguments = "v2-tests/net462/mock-assembly-v2.dll", + ExpectedResult = new ExpectedResult("Failed") + { + Total = 28, + Passed = 18, + Failed = 5, + Warnings = 0, + Inconclusive = 1, + Skipped = 4, + Assemblies = new ExpectedAssemblyResult[] { new ExpectedAssemblyResult("mock-assembly-v2.dll", "net-4.6.2") } + }, + ExtensionsNeeded = new[] { KnownExtensions.NUnitV2Driver } + }); + + StandardAndZipLists.Add(new PackageTest(1, "V2FrameworkDriverTest") + { + Description = "Run mock-assembly-v2 using the V2 Driver out of process", + Arguments = "v2-tests/net462/mock-assembly-v2.dll", + ExpectedResult = new ExpectedResult("Failed") + { + Total = 28, + Passed = 18, + Failed = 5, + Warnings = 0, + Inconclusive = 1, + Skipped = 4, + Assemblies = new ExpectedAssemblyResult[] { new ExpectedAssemblyResult("mock-assembly-v2.dll", "net-4.6.2") } + }, + ExtensionsNeeded = new[] { KnownExtensions.NUnitV2Driver } + }); + + ////////////////////////////////////////////////////////////////////// + // SPECIAL CASES + ////////////////////////////////////////////////////////////////////// + + StandardAndZipLists.Add(new PackageTest(1, "InvalidTestNameTest_Net462") + { + Description = "Ensure we handle invalid test names correctly under .NET 4.6.2", + Arguments = "testdata/net462/InvalidTestNames.dll --trace:Debug", + ExpectedResult = new ExpectedResult("Passed") + { + Assemblies = new ExpectedAssemblyResult[] { new ExpectedAssemblyResult("InvalidTestNames.dll", "net-4.6.2") } + } + }); + + AllLists.Add(new PackageTest(1, "InvalidTestNameTest_Net60") + { + Description = "Ensure we handle invalid test names correctly under .NET 6.0", + Arguments = "testdata/net6.0/InvalidTestNames.dll --trace:Debug", + ExpectedResult = new ExpectedResult("Passed") + { + Assemblies = new ExpectedAssemblyResult[] { new ExpectedAssemblyResult("InvalidTestNames.dll", "netcore-6.0") } + } + }); + + AllLists.Add(new PackageTest(1, "InvalidTestNameTest_Net80") + { + Description = "Ensure we handle invalid test names correctly under .NET 8.0", + Arguments = "testdata/net8.0/InvalidTestNames.dll --trace:Debug", + ExpectedResult = new ExpectedResult("Passed") + { + Assemblies = new ExpectedAssemblyResult[] { new ExpectedAssemblyResult("InvalidTestNames.dll", "netcore-8.0") } + } + }); + + StandardAndZipLists.Add(new PackageTest(1, "AppContextBaseDirectory_NET80") + { + Description = "Test Setting the BaseDirectory to match test assembly location under .NET 8.0", + Arguments = "testdata/net8.0/AppContextTest.dll", + ExpectedResult = new ExpectedResult("Passed") + { + Assemblies = new ExpectedAssemblyResult[] { new ExpectedAssemblyResult("AppContextTest.dll", "netcore-8.0") } + } + }); + } + + #region Nested Classes + + class ListWrapper { - Assemblies = new ExpectedAssemblyResult[] + private List[] _lists; + + public ListWrapper(params List[] lists) { - new ExpectedAssemblyResult("InvalidTestNames.dll", "net-4.6.2") + _lists = lists; } - })); -AddToBothLists(new PackageTest( - 1, "InvalidTestNameTest_Net60", - "Ensure we handle invalid test names correctly under .NET 6.0", - "testdata/net6.0/InvalidTestNames.dll --trace:Debug", - new ExpectedResult("Passed") - { - Assemblies = new ExpectedAssemblyResult[] + public void Add(PackageTest test) { - new ExpectedAssemblyResult("InvalidTestNames.dll", "netcore-6.0") + foreach (var list in _lists) + list.Add(test); } - })); + } -AddToBothLists(new PackageTest( - 1, "InvalidTestNameTest_Net80", - "Ensure we handle invalid test names correctly under .NET 8.0", - "testdata/net8.0/InvalidTestNames.dll --trace:Debug", - new ExpectedResult("Passed") + class MockAssemblyExpectedResult : ExpectedResult { - Assemblies = new ExpectedAssemblyResult[] + public MockAssemblyExpectedResult(params string[] runtimes) : base("Failed") { - new ExpectedAssemblyResult("InvalidTestNames.dll", "netcore-8.0") + int nCopies = runtimes.Length; + Total = 37 * nCopies; + Passed = 23 * nCopies; + Failed = 5 * nCopies; + Warnings = 1 * nCopies; + Inconclusive = 1 * nCopies; + Skipped = 7 * nCopies; + Assemblies = new ExpectedAssemblyResult[nCopies]; + for (int i = 0; i < nCopies; i++) + Assemblies[i] = new ExpectedAssemblyResult("mock-assembly.dll", runtimes[i]); } - })); + } -StandardRunnerTests.Add(new PackageTest( - 1, "AppContextBaseDirectory_NET80", - "Test Setting the BaseDirectory to match test assembly location under .NET 8.0", - "testdata/net8.0/AppContextTest.dll", - new ExpectedResult("Passed") + class MockAssemblyX86ExpectedResult : MockAssemblyExpectedResult { - Assemblies = new ExpectedAssemblyResult[] { new ExpectedAssemblyResult("AppContextTest.dll", "netcore-8.0") } - })); + public MockAssemblyX86ExpectedResult(params string[] runtimes) : base(runtimes) + { + for (int i = 0; i < runtimes.Length; i++) + Assemblies[i] = new ExpectedAssemblyResult("mock-assembly-x86.dll", runtimes[i]); + } + } + + #endregion +} diff --git a/src/NUnitConsole/nunit3-console/ConsoleRunner.cs b/src/NUnitConsole/nunit3-console/ConsoleRunner.cs index 366409715..606fec6d0 100644 --- a/src/NUnitConsole/nunit3-console/ConsoleRunner.cs +++ b/src/NUnitConsole/nunit3-console/ConsoleRunner.cs @@ -52,7 +52,7 @@ public class ConsoleRunner public ConsoleRunner(ITestEngine engine, ConsoleOptions options, ExtendedTextWriter writer) { - Guard.ArgumentNotNull(_engine = engine, nameof(engine)); + Guard.ArgumentNotNull(_engine = engine, nameof(engine)); Guard.ArgumentNotNull(_options = options, nameof(options)); Guard.ArgumentNotNull(_outWriter = writer, nameof(writer)); @@ -71,7 +71,7 @@ public ConsoleRunner(ITestEngine engine, ConsoleOptions options, ExtendedTextWri foreach (string extensionDirectory in extensionPath.Split(new[] { Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) _extensionService.FindExtensionAssemblies(extensionDirectory); - foreach (string extensionDirectory in _options.ExtensionDirectories) + foreach (string extensionDirectory in _options.ExtensionDirectories) _extensionService.FindExtensionAssemblies(extensionDirectory); _workDirectory = options.WorkDirectory; diff --git a/src/NUnitEngine/nunit-agent/Program.cs b/src/NUnitEngine/nunit-agent/Program.cs index 99bb8d0c2..0f182ae5e 100644 --- a/src/NUnitEngine/nunit-agent/Program.cs +++ b/src/NUnitEngine/nunit-agent/Program.cs @@ -62,8 +62,8 @@ public static void Main(string[] args) } } - // var logName = $"nunit-agent_{pid}.log"; - // InternalTrace.Initialize(Path.Combine(workDirectory, logName), traceLevel); // See PR 1214 https://github.com/nunit/nunit-console/pull/1214/files + var logName = $"nunit-agent_{pid}.log"; + InternalTrace.Initialize(Path.Combine(workDirectory, logName), traceLevel); // See PR 1214 https://github.com/nunit/nunit-console/pull/1214/files log = InternalTrace.GetLogger(typeof(NUnitTestAgent)); log.Info("Agent process {0} starting", pid); diff --git a/src/NUnitEngine/nunit.engine.core/Services/DriverService.cs b/src/NUnitEngine/nunit.engine.core/Services/DriverService.cs index 8220831b0..13764b8bf 100644 --- a/src/NUnitEngine/nunit.engine.core/Services/DriverService.cs +++ b/src/NUnitEngine/nunit.engine.core/Services/DriverService.cs @@ -21,7 +21,7 @@ public class DriverService : Service, IDriverService static ILogger log = InternalTrace.GetLogger("DriverService"); private ExtensionService _extensionService; - private IList _factories; + private List _factories; /// /// Get a driver suitable for use with a particular test assembly. @@ -76,7 +76,8 @@ public IFrameworkDriver GetDriver(AppDomain domain, string assemblyPath, string foreach (var factory in _factories) { - log.Debug($"Trying {factory.GetType().Name}"); + string factoryName = factory.GetType().Name; + log.Debug($"Trying {factoryName}"); foreach (var reference in references) { @@ -87,6 +88,8 @@ public IFrameworkDriver GetDriver(AppDomain domain, string assemblyPath, string return factory.GetDriver(reference); #endif } + + log.Debug($"No driver found using {factoryName}"); } } } @@ -98,8 +101,8 @@ public IFrameworkDriver GetDriver(AppDomain domain, string assemblyPath, string if (skipNonTestAssemblies) return new SkippedAssemblyFrameworkDriver(assemblyPath); else - return new InvalidAssemblyFrameworkDriver(assemblyPath, string.Format("No suitable tests found in '{0}'.\n" + - "Either assembly contains no tests or proper test driver has not been found.", assemblyPath)); + return new InvalidAssemblyFrameworkDriver(assemblyPath, string.Format( + $"No suitable tests found in '{assemblyPath}'.\r\nEither assembly contains no tests or proper test driver has not been found.")); } public override void StartService() @@ -123,10 +126,11 @@ private void InitializeDriverFactories() { _factories = new List(); - if (_extensionService != null) + if (_extensionService == null) + log.Debug("ExtensionService is not available, no driver extensions will be loaded"); + else { - foreach (IDriverFactory factory in _extensionService.GetExtensions()) - _factories.Add(factory); + _factories.AddRange(_extensionService.GetExtensions()); #if NETFRAMEWORK var node = _extensionService.GetExtensionNode("/NUnit/Engine/NUnitV2Driver"); diff --git a/src/NUnitEngine/nunit.engine.core/Services/ExtensionManager.cs b/src/NUnitEngine/nunit.engine.core/Services/ExtensionManager.cs index e50f0a0ac..9837ae245 100644 --- a/src/NUnitEngine/nunit.engine.core/Services/ExtensionManager.cs +++ b/src/NUnitEngine/nunit.engine.core/Services/ExtensionManager.cs @@ -10,7 +10,6 @@ using NUnit.Engine.Internal.FileSystemAccess; using NUnit.Engine.Internal.FileSystemAccess.Default; using System.Linq; -using System.Diagnostics; namespace NUnit.Engine.Services { @@ -21,7 +20,6 @@ public class ExtensionManager static readonly Logger log = InternalTrace.GetLogger(typeof(ExtensionManager)); private readonly IFileSystem _fileSystem; - //private readonly IAddinsFileReader _addinsReader; private readonly IDirectoryFinder _directoryFinder; // List of all ExtensionPoints discovered @@ -154,10 +152,7 @@ public void FindExtensionAssemblies(string startDir) log.Info($"FindExtensionAssemblies examining extension directory {startDir}"); - // Create the list of possible extension assemblies, - // eliminating duplicates, start in the provided directory. - // In this top level directory, we only look at .addins files. - ProcessAddinsFiles(_fileSystem.GetDirectory(startDir), false); + ProcessDirectory(_fileSystem.GetDirectory(startDir), false); } } @@ -242,7 +237,9 @@ public IEnumerable GetExtensionNodes(bool includeDisabled = fa EnsureExtensionsAreLoaded(); var ep = GetExtensionPoint(typeof(T)); - if (ep != null) + if (ep == null) + log.Debug("There is no extension point of type {typeof(T).Name}"); + else foreach (var node in ep.Extensions) if (includeDisabled || node.Enabled) yield return node; @@ -631,27 +628,6 @@ internal bool CanLoadTargetFramework(Assembly runnerAsm, ExtensionAssembly exten } } - //string extensionFrameworkName = AssemblyDefinition.ReadAssembly(extensionAsm.FilePath).GetFrameworkName(); - //string runnerFrameworkName = AssemblyDefinition.ReadAssembly(runnerAsm.Location).GetFrameworkName(); - //if (runnerFrameworkName?.StartsWith(".NETStandard") == true) - //{ - // throw new NUnitEngineException($"{runnerAsm.FullName} test runner must target .NET Core or .NET Framework, not .NET Standard"); - //} - //else if (runnerFrameworkName?.StartsWith(".NETCoreApp") == true) - //{ - // if (extensionFrameworkName?.StartsWith(".NETStandard") != true && extensionFrameworkName?.StartsWith(".NETCoreApp") != true) - // { - // log.Info($".NET Core runners require .NET Core or .NET Standard extension for {extensionAsm.FilePath}"); - // return false; - // } - //} - //else if (extensionFrameworkName?.StartsWith(".NETCoreApp") == true) - //{ - // log.Info($".NET Framework runners cannot load .NET Core extension {extensionAsm.FilePath}"); - // return false; - //} - - //return true; } private System.Runtime.Versioning.FrameworkName GetTargetRuntime(string filePath) diff --git a/src/NUnitEngine/nunit.engine/nunit.engine.addins b/src/NUnitEngine/nunit.engine/nunit.engine.addins deleted file mode 100644 index 4ddd66c2e..000000000 --- a/src/NUnitEngine/nunit.engine/nunit.engine.addins +++ /dev/null @@ -1 +0,0 @@ -../../../bundled-extensions/NUnit.Extension.*/tools/**/ \ No newline at end of file diff --git a/src/NUnitEngine/nunit.engine/nunit.engine.csproj b/src/NUnitEngine/nunit.engine/nunit.engine.csproj index c8a16bcdf..c9dd0007f 100644 --- a/src/NUnitEngine/nunit.engine/nunit.engine.csproj +++ b/src/NUnitEngine/nunit.engine/nunit.engine.csproj @@ -30,10 +30,4 @@ - - - PreserveNewest - - - \ No newline at end of file diff --git a/src/TestData/FakeExtensions/fake-extensions.addins b/src/TestData/FakeExtensions/fake-extensions.addins new file mode 100644 index 000000000..0e9627ea7 --- /dev/null +++ b/src/TestData/FakeExtensions/fake-extensions.addins @@ -0,0 +1,3 @@ +# This file isn't copied anywhere but is used by one of our +# package tests to exercise the --extensionDirectory option +../../../bin/Release/fakes/**/FakeExtensions.dll \ No newline at end of file