Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[149] - tests rerun in verbosity #158

Merged
merged 1 commit into from
Jun 30, 2024
Merged

[149] - tests rerun in verbosity #158

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 58 additions & 49 deletions test/dotnet-test-rerun.IntegrationTests/DotNetTestRerunTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ public async Task DotnetTestRerun_RunXUnitExample_Success()

// Assert
output.Should().Contain("Passed!", Exactly.Once());
output.Should().NotContainAny(new string[] {"Failed!", "Rerun attempt"});
output.Should().NotContainAny(new string[] { "Failed!", "Rerun attempt" });
Environment.ExitCode.Should().Be(0);
IsThereACoverageFile().Should().BeFalse();
}
Expand All @@ -46,27 +46,28 @@ public async Task DotnetTestRerun_RunMSTestExample_Success()
Environment.ExitCode = 0;

// Act
var output = await RunDotNetTestRerunAndCollectOutputMessage("MSTestExample", "--collect \"XPlat Code Coverage\" --configuration \"debug\" --verbosity \"minimal\"");
var output = await RunDotNetTestRerunAndCollectOutputMessage("MSTestExample",
"--collect \"XPlat Code Coverage\" --configuration \"debug\" --verbosity \"minimal\"");

// Assert
output.Should().Contain("Passed!", Exactly.Once());
output.Should().NotContainAny(new string[] {"Failed!", "Rerun attempt"});
output.Should().NotContainAny(new string[] { "Failed!", "Rerun attempt" });
output.Should().Contain("-c \"debug\" -v \"Minimal\"");
Environment.ExitCode.Should().Be(0);
IsThereACoverageFile().Should().BeTrue();
}

[Fact]
public async Task DotnetTestRerun_RunMSTestExample_RunningProcess_Success()
{
{
// Arrange
Environment.ExitCode = 0;

// Arrange
Process process = new Process();
process.StartInfo.FileName = "test-rerun";
process.StartInfo.Arguments = $"{_dir}\\MSTestExample --rerunMaxAttempts 3 --results-directory {_dir}";

// Act
process.Start();

Expand All @@ -88,7 +89,7 @@ public async Task DotnetTestRerun_FailingMSTest_Fails()
output.Should().NotContain("Passed!");
output.Should().Contain("Failed!", Exactly.Times(4));
output.Should().Contain("Rerun filter: FullyQualifiedName~MSTestExample.UnitTest1.SimpleNumberFailCompare",
Exactly.Thrice());
Exactly.Thrice());
output.Should().Contain("Failed: 2, Passed: 6",
Exactly.Once());
Environment.ExitCode.Should().Be(1);
Expand All @@ -99,13 +100,13 @@ public async Task DotnetTestRerun_RunNUnitExample_Success()
{
// Arrange
Environment.ExitCode = 0;

// Act
var output = await RunDotNetTestRerunAndCollectOutputMessage("NUnitTestExample");

// Assert
output.Should().Contain("Passed!", Exactly.Once());
output.Should().NotContainAny(new string[] {"Failed!", "Rerun attempt"});
output.Should().NotContainAny(new string[] { "Failed!", "Rerun attempt" });
Environment.ExitCode.Should().Be(0);
}

Expand All @@ -114,13 +115,15 @@ public async Task DotnetTestRerun_RunNUnitExample_WithPropertiesActive_Success()
{
// Arrange
Environment.ExitCode = 0;

// Act
var output = await RunDotNetTestRerunAndCollectOutputMessage("NUnitTestExample", extraArgs: $"/p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:CoverletOutput={_dir}\\TestResults\\Coverage\\UnitTests-TestResults.opencover.xml");
var output = await RunDotNetTestRerunAndCollectOutputMessage("NUnitTestExample",
extraArgs:
$"/p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:CoverletOutput={_dir}\\TestResults\\Coverage\\UnitTests-TestResults.opencover.xml");

// Assert
output.Should().Contain("Passed!", Exactly.Once());
output.Should().NotContainAny(new string[] {"Failed!", "Rerun attempt"});
output.Should().NotContainAny(new string[] { "Failed!", "Rerun attempt" });
Environment.ExitCode.Should().Be(0);
}

Expand All @@ -130,13 +133,13 @@ public async Task DotnetTestRerun_RunNUnitExample_WithDeleteFiles()
// Arrange
var testDir = TestUtilities.GetTmpDirectory();
TestUtilities.CopyFixture(string.Empty, new DirectoryInfo(testDir));

// Act
var output = await RunDotNetTestRerunAndCollectOutputMessage("NUnitTestExample", "--deleteReports", testDir);

// Assert
output.Should().Contain("Passed!", Exactly.Once());
output.Should().NotContainAny(new string[] {"Failed!", "Rerun attempt"});
output.Should().NotContainAny(new string[] { "Failed!", "Rerun attempt" });

var files = FileSystem.Directory.EnumerateFiles(testDir, "*trx");
files.Should().HaveCount(0);
Expand All @@ -147,20 +150,20 @@ public async Task DotnetTestRerun_FailingXUnit_Fails()
{
// Arrange
Environment.ExitCode = 0;

// Arrange
var testDir = TestUtilities.GetTmpDirectory();
TestUtilities.CopyFixture(string.Empty, new DirectoryInfo(testDir));
Environment.ExitCode = 0;

// Act
var output = await RunDotNetTestRerunAndCollectOutputMessage("FailingXUnitExample", dir: testDir);

// Assert
output.Should().NotContain("Passed!");
output.Should().Contain("Failed!", Exactly.Times(4));
output.Should().Contain("Rerun filter: FullyQualifiedName~FailingXUnitExample.SimpleTest.SimpleStringCompare",
Exactly.Thrice());
Exactly.Thrice());
output.Should().Contain("Passed: 4",
Exactly.Once());
var files = FileSystem.Directory.EnumerateFiles(testDir, "*trx");
Expand All @@ -172,27 +175,26 @@ public async Task DotnetTestRerun_FailingXUnit_Fails()
[InlineData("quiet")]
[InlineData("minimal")]
[InlineData("detailed")]
[InlineData("diagnostic")]
[Theory]
public async Task DotnetTestRerun_FailingXUnit_WithVerbosity_Fails(string verbosity)
{
// Arrange
Environment.ExitCode = 0;

// Arrange
var testDir = TestUtilities.GetTmpDirectory();
TestUtilities.CopyFixture(string.Empty, new DirectoryInfo(testDir));
Environment.ExitCode = 0;

// Act
var output = await RunDotNetTestRerunAndCollectOutputMessage("FailingXUnitExample", dir: testDir, extraArgs: $"--verbosity {verbosity}");
var output = await RunDotNetTestRerunAndCollectOutputMessage("FailingXUnitExample", dir: testDir,
extraArgs: $"--verbosity {verbosity}");

// Assert
output.Should().NotContain("Passed!");
output.Should().Contain("Failed:", Exactly.Times(4));
output.Should().Contain("Rerun filter: FullyQualifiedName~FailingXUnitExample.SimpleTest.SimpleStringCompare",
Exactly.Thrice());
output.Should().Contain("Passed: 4",
Exactly.Once());
Exactly.Thrice());
output.Should().ContainAny("Passed: 4", "Passed: 4");
var files = FileSystem.Directory.EnumerateFiles(testDir, "*trx");
files.Should().HaveCount(4);
Environment.ExitCode.Should().Be(1);
Expand All @@ -203,18 +205,20 @@ public async Task DotnetTestRerun_FailingXUnit_WithMultipleFrameworks_Fails()
{
// Arrange
Environment.ExitCode = 0;

// Arrange
var testDir = TestUtilities.GetTmpDirectory();
TestUtilities.CopyFixture(string.Empty, new DirectoryInfo(testDir));
Environment.ExitCode = 0;

// Act
var output = await RunDotNetTestRerunAndCollectOutputMessage("FailingXUnitWithMultipleFrameworksExample", dir: testDir);
var output =
await RunDotNetTestRerunAndCollectOutputMessage("FailingXUnitWithMultipleFrameworksExample", dir: testDir);

// Assert
output.Should().MatchRegex("Rerun filter: FullyQualifiedName~(FailingXUnitExample.)*SimpleTest.SimpleStringCompare",
Exactly.Thrice());
output.Should().MatchRegex(
"Rerun filter: FullyQualifiedName~(FailingXUnitExample.)*SimpleTest.SimpleStringCompare",
Exactly.Thrice());
output.Should().Contain("Passed: 1",
Exactly.Once());
output.Should().Contain("Failed!", Exactly.Times(4));
Expand All @@ -233,7 +237,7 @@ public async Task DotnetTestRerun_FailingXUnit_RunningProcess_Fails()
Process process = new Process();
process.StartInfo.FileName = "test-rerun";
process.StartInfo.Arguments = $"{_dir}\\FailingXUnitExample --rerunMaxAttempts 3 --results-directory {_dir}";

// Act
process.Start();

Expand All @@ -247,11 +251,11 @@ public async Task DotnetTestRerun_RunNonExistentXUnitProject_MissingArguments_Fa
{
// Arrange
Environment.ExitCode = 0;

Process process = new Process();
process.StartInfo.FileName = "test-rerun";
process.StartInfo.Arguments = $"{_dir}\\XUnitThatDoesNotExist --rerunMaxAttempts 3 --results-directory {_dir}";

// Act
process.Start();

Expand All @@ -272,8 +276,9 @@ public async Task DotnetTestRerun_FailingMultipleXUnit_Fails()
// Assert
output.Should().NotContain("Passed!");
output.Should().Contain("Failed!", Exactly.Times(4));
output.Should().Contain("Rerun filter: FullyQualifiedName~FailingXUnitExample.SimpleTest.SimpleFailedNumberCompare",
Exactly.Thrice());
output.Should().Contain(
"Rerun filter: FullyQualifiedName~FailingXUnitExample.SimpleTest.SimpleFailedNumberCompare",
Exactly.Thrice());
output.Should().Contain("Failed: 2, Passed: 5",
Exactly.Once());
Environment.ExitCode.Should().Be(1);
Expand All @@ -292,7 +297,7 @@ public async Task DotnetTestRerun_FailingNUnit_PassOnSecond()
output.Should().Contain("Passed!");
output.Should().Contain("Failed!", Exactly.Times(1));
output.Should().Contain("Rerun filter: FullyQualifiedName~NUnitTestExample.Tests.SecondSimpleNumberCompare",
Exactly.Once());
Exactly.Once());
output.Should().Contain("Failed: 1, Passed: 1",
Exactly.Once());
Environment.ExitCode.Should().Be(0);
Expand All @@ -311,35 +316,36 @@ public async Task DotnetTestRerun_FailingNUnit_PassOnSecond_WithCategory()
// Assert
output.Should().Contain("Passed!");
output.Should().Contain("Failed!", Exactly.Times(1));
output.Should().Contain("Rerun filter: (TestCategory=FirstCategory|TestCategory=SecondCategory)&(FullyQualifiedName~NUnitTestExample.Tests.SecondSimpleNumberCompare)",
Exactly.Once());
output.Should().Contain(
"Rerun filter: (TestCategory=FirstCategory|TestCategory=SecondCategory)&(FullyQualifiedName~NUnitTestExample.Tests.SecondSimpleNumberCompare)",
Exactly.Once());
output.Should().Contain("Failed: 1, Passed: 1",
Exactly.Once());
Environment.ExitCode.Should().Be(0);
}

[Fact]
public async Task DotnetTestRerun_FailingXUnit_WithDeleteFiles()
{
// Arrange
var testDir = TestUtilities.GetTmpDirectory();
TestUtilities.CopyFixture(string.Empty, new DirectoryInfo(testDir));

// Act
var output = await RunDotNetTestRerunAndCollectOutputMessage("FailingXUnitExample", "--deleteReports", testDir);

// Assert
output.Should().NotContain("Passed!");
output.Should().Contain("Failed!", Exactly.Times(4));
output.Should().Contain("Rerun filter: FullyQualifiedName~FailingXUnitExample.SimpleTest.SimpleStringCompare",
Exactly.Thrice());
Exactly.Thrice());
output.Should().Contain("Passed: 4",
Exactly.Once());

var files = FileSystem.Directory.EnumerateFiles(testDir, "*trx");
files.Should().HaveCount(0);
}

[Fact]
public async Task DotnetTestRerun_FailingNUnit_PassOnSecond_TwoFailingProjects()
{
Expand All @@ -355,9 +361,9 @@ public async Task DotnetTestRerun_FailingNUnit_PassOnSecond_TwoFailingProjects()
output.Should().Contain("Rerun filter:",
Exactly.Twice());
output.Should().Contain("Failed: 2, Passed: 1",
Exactly.Once());
Exactly.Once());
output.Should().Contain("Failed: 1, Passed: 1",
Exactly.Twice());
Exactly.Twice());
output.Should().Contain("Failed: 0, Passed: 1",
Exactly.Twice());
Environment.ExitCode.Should().Be(0);
Expand All @@ -375,16 +381,18 @@ public async Task DotnetTestRerun_FailingNUnit_PassOnSecond_WithFailingTestWithS
// Assert
output.Should().Contain("Passed!");
output.Should().Contain("Failed!", Exactly.Times(1));
output.Should().Contain("Rerun filter: FullyQualifiedName~NUnitTestExample.SimpleTest.SecondSimpleNumberCompare",
Exactly.Once());
output.Should().Contain(
"Rerun filter: FullyQualifiedName~NUnitTestExample.SimpleTest.SecondSimpleNumberCompare",
Exactly.Once());
output.Should().Contain("Failed: 1, Passed: 2",
Exactly.Once());
Exactly.Once());
output.Should().Contain("Failed: 0, Passed: 1",
Exactly.Once());
Environment.ExitCode.Should().Be(0);
}

private async Task<string> RunDotNetTestRerunAndCollectOutputMessage(string proj, string extraArgs = "", string? dir = null)
private async Task<string> RunDotNetTestRerunAndCollectOutputMessage(string proj, string extraArgs = "",
string? dir = null)
{
var testDir = dir ?? _dir;
var stringWriter = new StringWriter();
Expand All @@ -397,7 +405,8 @@ private async Task<string> RunDotNetTestRerunAndCollectOutputMessage(string proj
rerunCommandConfiguration.Set(command);

ParseResult result =
new Parser(command).Parse($"{testDir}\\{proj} --rerunMaxAttempts 3 --results-directory {testDir} {extraArgs}");
new Parser(command).Parse(
$"{testDir}\\{proj} --rerunMaxAttempts 3 --results-directory {testDir} {extraArgs}");
InvocationContext context = new(result);

rerunCommandConfiguration.GetValues(context);
Expand All @@ -415,5 +424,5 @@ private async Task<string> RunDotNetTestRerunAndCollectOutputMessage(string proj
}

private static bool IsThereACoverageFile()
=> Directory.GetFiles(_dir, "*cobertura.xml", SearchOption.AllDirectories).Any();
=> Directory.GetFiles(_dir, "*cobertura.xml", SearchOption.AllDirectories).Any();
}
Loading