From 36862639c2520b60864bffa3b34900053f09aaac Mon Sep 17 00:00:00 2001 From: Christian Sandfeld Date: Wed, 10 Apr 2024 23:47:54 +0200 Subject: [PATCH] Handle tests marked as inconclusive (#2405) --- src/Main.ps1 | 16 +-- src/Pester.RSpec.ps1 | 7 ++ src/Pester.Runtime.ps1 | 20 +++- src/csharp/Pester/Container.cs | 2 + src/csharp/Pester/Run.cs | 2 + src/csharp/Pester/Test.cs | 1 + src/csharp/Pester/ToStringConverter.cs | 1 + src/en-US/about_Pester.help.txt | 2 +- src/functions/It.ps1 | 2 +- src/functions/Output.ps1 | 27 +++-- src/functions/Set-ItResult.ps1 | 31 +++--- src/functions/TestResults.NUnit25.ps1 | 15 +-- src/functions/TestResults.NUnit3.ps1 | 80 ++++++++------- tst/Pester.RSpec.Output.ts.ps1 | 21 ++++ tst/Pester.RSpec.ResultObject.ts.ps1 | 15 ++- tst/Pester.RSpec.TestResults.NUnit25.ts.ps1 | 96 ++++++++++++++++++ tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 | 106 +++++++++++++++++++- 17 files changed, 367 insertions(+), 77 deletions(-) diff --git a/src/Main.ps1 b/src/Main.ps1 index 8bfb3568c..3b923fc66 100644 --- a/src/Main.ps1 +++ b/src/Main.ps1 @@ -747,12 +747,12 @@ function Invoke-Pester { } $plugins.Add(( - # decorator plugin needs to be added after output - # because on teardown they will run in opposite order - # and that way output can consume the fixed object that decorator - # decorated, not nice but works - Get-RSpecObjectDecoratorPlugin - )) + # decorator plugin needs to be added after output + # because on teardown they will run in opposite order + # and that way output can consume the fixed object that decorator + # decorated, not nice but works + Get-RSpecObjectDecoratorPlugin + )) if ($PesterPreference.TestDrive.Enabled.Value) { $plugins.Add((Get-TestDrivePlugin)) @@ -1458,11 +1458,13 @@ function ConvertTo-Pester4Result { "Skipped" { $legacyResult.SkippedCount++ } + "Inconclusive" { + $legacyResult.InconclusiveCount++ + } } } $legacyResult.TotalCount = $legacyResult.TestResult.Count $legacyResult.PendingCount = 0 - $legacyResult.InconclusiveCount = 0 $legacyResult.Time = $PesterResult.Duration $legacyResult diff --git a/src/Pester.RSpec.ps1 b/src/Pester.RSpec.ps1 index ee7729459..c9f8d5973 100644 --- a/src/Pester.RSpec.ps1 +++ b/src/Pester.RSpec.ps1 @@ -107,6 +107,9 @@ function Add-RSpecTestObjectProperties { $TestObject.Result = if ($TestObject.Skipped) { "Skipped" } + elseif ($TestObject.Inconclusive) { + "Inconclusive" + } elseif ($TestObject.Passed) { "Passed" } @@ -160,6 +163,9 @@ function PostProcess-RspecTestRun ($TestRun) { "Skipped" { $null = $TestRun.Skipped.Add($t) } + "Inconclusive" { + $null = $TestRun.Inconclusive.Add($t) + } default { throw "Result $($t.Result) is not supported." } } @@ -236,6 +242,7 @@ function PostProcess-RspecTestRun ($TestRun) { $TestRun.PassedCount = $TestRun.Passed.Count $TestRun.FailedCount = $TestRun.Failed.Count $TestRun.SkippedCount = $TestRun.Skipped.Count + $TestRun.InconclusiveCount = $TestRun.Inconclusive.Count $TestRun.NotRunCount = $TestRun.NotRun.Count $TestRun.TotalCount = $TestRun.Tests.Count diff --git a/src/Pester.Runtime.ps1 b/src/Pester.Runtime.ps1 index 72b4914d6..07c23d57b 100644 --- a/src/Pester.Runtime.ps1 +++ b/src/Pester.Runtime.ps1 @@ -687,7 +687,18 @@ function Invoke-TestItem { Write-PesterDebugMessage -Scope Skip "($path) Test is skipped." } $Test.Passed = $true - $Test.Skipped = $true + if ('PesterTestInconclusive' -eq $Result.ErrorRecord.FullyQualifiedErrorId) { + $Test.Inconclusive = $true + } + else { + $Test.Skipped = $true + + # Pending test is still considered a skipped, we don't have a special category for it. + # Mark the run to show deprecation message. + if ('PesterTestPending' -eq $Result.ErrorRecord.FullyQualifiedErrorId) { + $test.Block.Root.FrameworkData['ShowPendingDeprecation'] = $true + } + } } else { $Test.Passed = $result.Success @@ -2194,6 +2205,7 @@ function PostProcess-ExecutedBlock { $b.OwnFailedCount = 0 $b.OwnPassedCount = 0 $b.OwnSkippedCount = 0 + $b.OwnInconclusiveCount = 0 $b.OwnNotRunCount = 0 $testDuration = [TimeSpan]::Zero @@ -2205,6 +2217,9 @@ function PostProcess-ExecutedBlock { if (-not $t.ShouldRun) { $b.OwnNotRunCount++ } + elseif ($t.ShouldRun -and $t.Inconclusive) { + $b.OwnInconclusiveCount++ + } elseif ($t.ShouldRun -and $t.Skipped) { $b.OwnSkippedCount++ } @@ -2245,6 +2260,7 @@ function PostProcess-ExecutedBlock { $b.FailedCount = $b.OwnFailedCount $b.PassedCount = $b.OwnPassedCount $b.SkippedCount = $b.OwnSkippedCount + $b.InconclusiveCount = $b.OwnInconclusiveCount $b.NotRunCount = $b.OwnNotRunCount } else { @@ -2266,6 +2282,7 @@ function PostProcess-ExecutedBlock { $b.PassedCount += $child.PassedCount $b.FailedCount += $child.FailedCount $b.SkippedCount += $child.SkippedCount + $b.InconclusiveCount += $child.InconclusiveCount $b.NotRunCount += $child.NotRunCount } @@ -2274,6 +2291,7 @@ function PostProcess-ExecutedBlock { $b.PassedCount += $b.OwnPassedCount $b.FailedCount += $b.OwnFailedCount $b.SkippedCount += $b.OwnSkippedCount + $b.InconclusiveCount += $b.OwnInconclusiveCount $b.NotRunCount += $b.OwnNotRunCount $b.Passed = -not ($thisBlockFailed -or $anyTestFailed -or $anyChildBlockFailed) diff --git a/src/csharp/Pester/Container.cs b/src/csharp/Pester/Container.cs index 7c15e2c9b..76bd80c69 100644 --- a/src/csharp/Pester/Container.cs +++ b/src/csharp/Pester/Container.cs @@ -18,6 +18,7 @@ public static Container CreateFromBlock (Block block) { FailedCount = block.FailedCount, PassedCount = block.PassedCount, SkippedCount = block.SkippedCount, + InconclusiveCount = block.InconclusiveCount, NotRunCount = block.NotRunCount, TotalCount = block.TotalCount, ErrorRecord = block.ErrorRecord ?? new List(), @@ -56,6 +57,7 @@ public static Container CreateFromFile(FileInfo file) public int FailedCount { get; set; } public int PassedCount { get; set; } public int SkippedCount { get; set; } + public int InconclusiveCount { get; set; } public int NotRunCount { get; set; } public int TotalCount { get; set; } public List ErrorRecord { get; set; } = new List(); diff --git a/src/csharp/Pester/Run.cs b/src/csharp/Pester/Run.cs index 66b7d8212..40c1cda3e 100644 --- a/src/csharp/Pester/Run.cs +++ b/src/csharp/Pester/Run.cs @@ -20,6 +20,7 @@ public static Run Create() public int PassedCount { get; set; } public int SkippedCount { get; set; } + public int InconclusiveCount { get; set; } public int NotRunCount { get; set; } public int TotalCount { get; set; } @@ -47,6 +48,7 @@ public static Run Create() public List Passed { get; set; } = new List(); public List Skipped { get; set; } = new List(); + public List Inconclusive { get; set; } = new List(); public List NotRun { get; set; } = new List(); public List Tests { get; set; } = new List(); diff --git a/src/csharp/Pester/Test.cs b/src/csharp/Pester/Test.cs index 1b52fe94c..a44031ba8 100644 --- a/src/csharp/Pester/Test.cs +++ b/src/csharp/Pester/Test.cs @@ -63,6 +63,7 @@ public Test() public DateTime? ExecutedAt { get; set; } public bool Passed { get; set; } public bool Skipped { get; set; } + public bool Inconclusive { get; set; } public TimeSpan UserDuration { get; set; } public TimeSpan FrameworkDuration { get; set; } diff --git a/src/csharp/Pester/ToStringConverter.cs b/src/csharp/Pester/ToStringConverter.cs index 4fe3fd62b..cc6f77f45 100644 --- a/src/csharp/Pester/ToStringConverter.cs +++ b/src/csharp/Pester/ToStringConverter.cs @@ -12,6 +12,7 @@ static string ResultToString(string result) "Passed" => "[+]", "Failed" => "[-]", "Skipped" => "[!]", + "Inconclusive" => "[?]", "NotRun" => "[ ]", _ => "[ERR]", }; diff --git a/src/en-US/about_Pester.help.txt b/src/en-US/about_Pester.help.txt index a7f0ec7a9..a32ecf78b 100644 --- a/src/en-US/about_Pester.help.txt +++ b/src/en-US/about_Pester.help.txt @@ -255,7 +255,7 @@ LONG DESCRIPTION The output ends with a summary of the test results. Tests completed in 3.47s - Passed: 20 Failed: 1 Skipped: 0 Pending: 0 Inconclusive: 0 + Tests Passed: 20, Failed: 1, Skipped: 0, Inconclusive: 0, NotRun: 0 However, because Pester uses Write-Host, it does not write to the output stream (stdout), so there are no output objects to save in a variable or diff --git a/src/functions/It.ps1 b/src/functions/It.ps1 index a8d1012b4..227fc7c3a 100644 --- a/src/functions/It.ps1 +++ b/src/functions/It.ps1 @@ -15,7 +15,7 @@ In addition to using your own logic to test expectations and throw exceptions, you may also use Pester's Should command to perform assertions in plain language. - You can intentionally mark It block result as inconclusive by using Set-TestInconclusive + You can intentionally mark It block result as inconclusive by using Set-ItResult -Inconclusive command as the first tested statement in the It block. .PARAMETER Name diff --git a/src/functions/Output.ps1 b/src/functions/Output.ps1 index 264d5ea0b..53712bb26 100644 --- a/src/functions/Output.ps1 +++ b/src/functions/Output.ps1 @@ -26,7 +26,7 @@ $script:ReportStrings = DATA { TestsPassed = 'Tests Passed: {0}, ' TestsFailed = 'Failed: {0}, ' - TestsSkipped = 'Skipped: {0} ' + TestsSkipped = 'Skipped: {0}, ' TestsPending = 'Pending: {0}, ' TestsInconclusive = 'Inconclusive: {0}, ' TestsNotRun = 'NotRun: {0}' @@ -61,6 +61,7 @@ $script:ReportTheme = DATA { Discovery = 'Magenta' Container = 'Magenta' BlockFail = 'Red' + Warning = 'Yellow' } } @@ -345,12 +346,12 @@ function Write-PesterReport { # else { # $ReportTheme.Information # } - # $Inconclusive = if ($RunResult.InconclusiveCount -gt 0) { - # $ReportTheme.Inconclusive - # } - # else { - # $ReportTheme.Information - # } + $Inconclusive = if ($RunResult.InconclusiveCount -gt 0) { + $ReportTheme.Inconclusive + } + else { + $ReportTheme.Information + } # Try { # $PesterStatePassedScenariosCount = $PesterState.PassedScenarios.Count @@ -374,6 +375,7 @@ function Write-PesterReport { Write-PesterHostMessage ($ReportStrings.TestsPassed -f $RunResult.PassedCount) -Foreground $Success -NoNewLine Write-PesterHostMessage ($ReportStrings.TestsFailed -f $RunResult.FailedCount) -Foreground $Failure -NoNewLine Write-PesterHostMessage ($ReportStrings.TestsSkipped -f $RunResult.SkippedCount) -Foreground $Skipped -NoNewLine + Write-PesterHostMessage ($ReportStrings.TestsInconclusive -f $RunResult.InconclusiveCount) -Foreground $Inconclusive -NoNewLine Write-PesterHostMessage ($ReportStrings.TestsTotal -f $RunResult.TotalCount) -Foreground $Total -NoNewLine Write-PesterHostMessage ($ReportStrings.TestsNotRun -f $RunResult.NotRunCount) -Foreground $NotRun @@ -402,6 +404,15 @@ function Write-PesterReport { # & $SafeCommands['Write-Host'] ($ReportStrings.TestsPending -f $RunResult.PendingCount) -Foreground $Pending -NoNewLine # & $SafeCommands['Write-Host'] ($ReportStrings.TestsInconclusive -f $RunResult.InconclusiveCount) -Foreground $Inconclusive # } + + $rootFrameworkData = @($RunResult.Containers.Blocks.Root.FrameworkData) + foreach ($frameworkData in $rootFrameworkData) { + if ($null -ne $frameworkData -and $frameworkData['ShowPendingDeprecation']) { + Write-PesterHostMessage '**DEPRECATED**: The -Pending parameter of Set-ItResult is deprecated. The parameter will be removed in a future version of Pester.' -ForegroundColor $ReportTheme.Warning + # Show it only once. + break + } + } } function Write-CoverageReport { @@ -843,7 +854,7 @@ function Get-WriteScreenPlugin ($Verbosity) { if ($PesterPreference.Output.Verbosity.Value -in 'Detailed', 'Diagnostic') { $because = if ($_test.FailureMessage) { ", because $($_test.FailureMessage)" } else { $null } Write-PesterHostMessage -ForegroundColor $ReportTheme.Inconclusive "$margin[?] $out" -NoNewLine - Write-PesterHostMessage -ForegroundColor $ReportTheme.Inconclusive ", is inconclusive$because" -NoNewLine + Write-PesterHostMessage -ForegroundColor $ReportTheme.Inconclusive "$because" -NoNewLine Write-PesterHostMessage -ForegroundColor $ReportTheme.InconclusiveTime " $humanTime" } diff --git a/src/functions/Set-ItResult.ps1 b/src/functions/Set-ItResult.ps1 index 8698e1eac..1d3e97a04 100644 --- a/src/functions/Set-ItResult.ps1 +++ b/src/functions/Set-ItResult.ps1 @@ -14,7 +14,7 @@ backwards compatible .PARAMETER Inconclusive - **DEPRECATED** Sets the test result to inconclusive. Cannot be used at the same time as -Pending or -Skipped + Sets the test result to inconclusive. Cannot be used at the same time as -Pending or -Skipped .PARAMETER Pending **DEPRECATED** Sets the test result to pending. Cannot be used at the same time as -Inconclusive or -Skipped @@ -29,6 +29,9 @@ .EXAMPLE ```powershell Describe "Example" { + It "Inconclusive test" { + Set-ItResult -Inconclusive -Because "we want it to be inconclusive" + } It "Skipped test" { Set-ItResult -Skipped -Because "we want it to be skipped" } @@ -38,9 +41,11 @@ the output should be ``` - [!] Skipped test is skipped, because we want it to be skipped - Tests completed in 0ms - Tests Passed: 0, Failed: 0, Skipped: 0, Pending: 0, Inconclusive 1 + Describing Example + [?] Inconclusive test is inconclusive, because we want it to be inconclusive 35ms (32ms|3ms) + [!] Skipped test is skipped, because we want it to be skipped 3ms (2ms|1ms) + Tests completed in 78ms + Tests Passed: 0, Failed: 0, Skipped: 1, Inconclusive: 1, NotRun: 0 ``` .LINK @@ -58,14 +63,6 @@ $result = $PSCmdlet.ParameterSetName - [String]$Message = "is skipped" - if ($Result -ne 'Skipped') { - [String]$Because = if ($Because) { $Result.ToUpper(), $Because -join ': ' } else { $Result.ToUpper() } - } - if ($Because) { - [String]$Message += ", because $Because" - } - switch ($null) { $File { [String]$File = $MyInvocation.ScriptName @@ -81,15 +78,25 @@ switch ($result) { 'Inconclusive' { [String]$errorId = 'PesterTestInconclusive' + [String]$message = "is inconclusive" + break } 'Pending' { [String]$errorId = 'PesterTestPending' + [String]$message = "is pending" + break } 'Skipped' { [String]$errorId = 'PesterTestSkipped' + [String]$message = "is skipped" + break } } + if ($Because) { + [String]$message += ", because $(Format-Because $Because)" + } + throw [Pester.Factory]::CreateErrorRecord( $errorId, #string errorId $Message, #string message diff --git a/src/functions/TestResults.NUnit25.ps1 b/src/functions/TestResults.NUnit25.ps1 index 64cd6cffb..3d3cf6a41 100644 --- a/src/functions/TestResults.NUnit25.ps1 +++ b/src/functions/TestResults.NUnit25.ps1 @@ -13,7 +13,7 @@ } function Write-NUnitTestResultAttributes { - [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns','')] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns', '')] param($Result, [System.Xml.XmlWriter] $XmlWriter) $XmlWriter.WriteAttributeString('xmlns', 'xsi', $null, 'http://www.w3.org/2001/XMLSchema-instance') @@ -23,7 +23,7 @@ function Write-NUnitTestResultAttributes { $XmlWriter.WriteAttributeString('errors', '0') $XmlWriter.WriteAttributeString('failures', $Result.FailedCount) $XmlWriter.WriteAttributeString('not-run', $Result.NotRunCount) - $XmlWriter.WriteAttributeString('inconclusive', '0') # $Result.PendingCount + $Result.InconclusiveCount) #TODO: reflect inconclusive count once it is added + $XmlWriter.WriteAttributeString('inconclusive', $Result.InconclusiveCount) $XmlWriter.WriteAttributeString('ignored', '0') $XmlWriter.WriteAttributeString('skipped', $Result.SkippedCount) $XmlWriter.WriteAttributeString('invalid', '0') @@ -32,7 +32,7 @@ function Write-NUnitTestResultAttributes { } function Write-NUnitTestResultChildNodes { - [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns','')] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns', '')] param($Result, [System.Xml.XmlWriter] $XmlWriter) Write-NUnitEnvironmentInformation -Result $Result -XmlWriter $XmlWriter @@ -98,7 +98,7 @@ function Write-NUnitCultureInformation { } function Write-NUnitTestSuiteElements { - [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns','')] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns', '')] param($Node, [System.Xml.XmlWriter] $XmlWriter, [string] $Path) $suiteInfo = Get-TestSuiteInfo -TestSuite $Node -Path $Path @@ -249,7 +249,7 @@ function Get-TestSuiteInfo { } function Write-NUnitTestSuiteAttributes { - [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns','')] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns', '')] param($TestSuiteInfo, [string] $TestSuiteType = 'TestFixture', [System.Xml.XmlWriter] $XmlWriter, [string] $Path) $name = $TestSuiteInfo.Name @@ -279,7 +279,7 @@ function Write-NUnitTestCaseElement { } function Write-NUnitTestCaseAttributes { - [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns','')] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns', '')] param($TestResult, [System.Xml.XmlWriter] $XmlWriter, [string] $ParameterizedSuiteName) $testName = $TestResult.ExpandedPath @@ -399,5 +399,8 @@ function Get-GroupResult ($InputObject) { if ($InputObject.PendingCount -gt 0) { return 'Inconclusive' } + if ($InputObject.InconclusiveCount -gt 0) { + return 'Inconclusive' + } return 'Success' } diff --git a/src/functions/TestResults.NUnit3.ps1 b/src/functions/TestResults.NUnit3.ps1 index 2a47fbffe..5ac45d5be 100644 --- a/src/functions/TestResults.NUnit3.ps1 +++ b/src/functions/TestResults.NUnit3.ps1 @@ -19,7 +19,7 @@ function Write-NUnit3Report($Result, [System.Xml.XmlWriter] $XmlWriter) { } function Write-NUnit3TestRunAttributes { - [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns','')] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns', '')] param($Result, [System.Xml.XmlWriter] $XmlWriter) $XmlWriter.WriteAttributeString('id', '0') @@ -30,7 +30,7 @@ function Write-NUnit3TestRunAttributes { $XmlWriter.WriteAttributeString('total', ($Result.TotalCount - $Result.NotRunCount)) # testcasecount - filtered $XmlWriter.WriteAttributeString('passed', $Result.PassedCount) $XmlWriter.WriteAttributeString('failed', $Result.FailedCount) - $XmlWriter.WriteAttributeString('inconclusive', '0') # required attr. $Result.PendingCount + $Result.InconclusiveCount when/if implemented? + $XmlWriter.WriteAttributeString('inconclusive', $Result.InconclusiveCount) $XmlWriter.WriteAttributeString('skipped', $Result.SkippedCount) $XmlWriter.WriteAttributeString('warnings', '0') # required attr. $XmlWriter.WriteAttributeString('start-time', (Get-UTCTimeString $Result.ExecutedAt)) @@ -297,6 +297,7 @@ function Get-NUnit3TestSuiteInfo { passed = $TestSuite.PassedCount failed = $TestSuite.FailedCount skipped = $TestSuite.SkippedCount + inconclusive = $TestSuite.InconclusiveCount site = $site shouldrun = $TestSuite.ShouldRun } @@ -305,14 +306,14 @@ function Get-NUnit3TestSuiteInfo { } function Write-NUnit3TestSuiteAttributes { - [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns','')] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns', '')] param($TestSuiteInfo, [System.Xml.XmlWriter] $XmlWriter) $XmlWriter.WriteAttributeString('type', $TestSuiteInfo.type) $XmlWriter.WriteAttributeString('id', (Get-NUnit3NodeId)) $XmlWriter.WriteAttributeString('name', $TestSuiteInfo.name) $XmlWriter.WriteAttributeString('fullname', $TestSuiteInfo.fullname) - if ($TestSuiteInfo.type -in 'TestFixture','ParameterizedMethod') { + if ($TestSuiteInfo.type -in 'TestFixture', 'ParameterizedMethod') { $XmlWriter.WriteAttributeString('classname', $TestSuiteInfo.classname) } $XmlWriter.WriteAttributeString('runstate', $TestSuiteInfo.runstate) @@ -327,7 +328,7 @@ function Write-NUnit3TestSuiteAttributes { $XmlWriter.WriteAttributeString('total', $TestSuiteInfo.total) $XmlWriter.WriteAttributeString('passed', $TestSuiteInfo.passed) $XmlWriter.WriteAttributeString('failed', $TestSuiteInfo.failed) - $XmlWriter.WriteAttributeString('inconclusive', '0') # required attribute + $XmlWriter.WriteAttributeString('inconclusive', $TestSuiteInfo.inconclusive) # required attribute $XmlWriter.WriteAttributeString('warnings', '0') # required attribute $XmlWriter.WriteAttributeString('skipped', $TestSuiteInfo.skipped) $XmlWriter.WriteAttributeString('asserts', $TestSuiteInfo.testcasecount) # required attr. hardcode 1:1 per testcase @@ -344,9 +345,12 @@ function Get-NUnit3Result ($InputObject) { elseif ($InputObject.SkippedCount -gt 0) { 'Skipped' } - else { + elseif ($InputObject.PassedCount -gt 0) { 'Passed' } + else { + 'Inconclusive' + } } function Get-NUnit3Site ($Node) { @@ -391,20 +395,21 @@ function Get-NUnit3ParameterizedMethodSuiteInfo { $sampleTest = $TestSuiteGroup.Group[0] $node = [PSCustomObject] @{ - Name = $sampleTest.Name - ExpandedName = $sampleTest.Name - Path = $sampleTest.Block.Path # used for classname -> block path - Data = $null - TotalCount = 0 - Duration = [timespan]0 - ExecutedAt = [datetime]::MinValue - PassedCount = 0 - FailedCount = 0 - SkippedCount = 0 - NotRunCount = 0 - OwnTotalCount = 0 - ShouldRun = $true - Skip = $sampleTest.Skip + Name = $sampleTest.Name + ExpandedName = $sampleTest.Name + Path = $sampleTest.Block.Path # used for classname -> block path + Data = $null + TotalCount = 0 + Duration = [timespan]0 + ExecutedAt = [datetime]::MinValue + PassedCount = 0 + FailedCount = 0 + SkippedCount = 0 + InconclusiveCount = 0 + NotRunCount = 0 + OwnTotalCount = 0 + ShouldRun = $true + Skip = $sampleTest.Skip } foreach ($testCase in $TestSuiteGroup.Group) { @@ -417,6 +422,7 @@ function Get-NUnit3ParameterizedMethodSuiteInfo { Passed { $node.PassedCount++; break; } Failed { $node.FailedCount++; break; } Skipped { $node.SkippedCount++; break; } + Inconclusive { $node.InconclusiveCount++; break; } NotRun { $node.NotRunCount++; break; } } @@ -438,20 +444,21 @@ function Get-NUnit3ParameterizedFixtureSuiteInfo { $sampleBlock = $TestSuiteGroup.Group[0] $node = [PSCustomObject] @{ - Name = $sampleBlock.Name - ExpandedName = $sampleBlock.Name - Path = $sampleBlock.Path - Data = $null - TotalCount = 0 - Duration = [timespan]0 - ExecutedAt = [datetime]::MinValue - PassedCount = 0 - FailedCount = 0 - SkippedCount = 0 - NotRunCount = 0 - OwnTotalCount = 0 - ShouldRun = $true - Skip = $false # ParameterizedFixture are always Runnable, even with -Skip + Name = $sampleBlock.Name + ExpandedName = $sampleBlock.Name + Path = $sampleBlock.Path + Data = $null + TotalCount = 0 + Duration = [timespan]0 + ExecutedAt = [datetime]::MinValue + PassedCount = 0 + FailedCount = 0 + SkippedCount = 0 + InconclusiveCount = 0 + NotRunCount = 0 + OwnTotalCount = 0 + ShouldRun = $true + Skip = $false # ParameterizedFixture are always Runnable, even with -Skip } foreach ($block in $TestSuiteGroup.Group) { @@ -463,6 +470,7 @@ function Get-NUnit3ParameterizedFixtureSuiteInfo { $node.PassedCount += $block.PassedCount $node.FailedCount += $block.FailedCount $node.SkippedCount += $block.SkippedCount + $node.InconclusiveCount += $block.InconclusiveCount $node.NotRunCount += $block.NotRunCount $node.TotalCount += $block.TotalCount @@ -505,7 +513,7 @@ function Write-NUnit3TestCaseElement { } function Write-NUnit3TestCaseAttributes { - [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns','')] + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns', '')] param($TestResult, [string] $ParentPath, [System.Xml.XmlWriter] $XmlWriter) # add parameters to name for testcase with data when not using variables in name @@ -577,7 +585,7 @@ function Write-NUnit3FailureElement ($TestResult, [System.Xml.XmlWriter] $XmlWri $XmlWriter.WriteEndElement() # Close failure } -function Write-NUnitReasonElement ($TestResult,[System.Xml.XmlWriter] $XmlWriter) { +function Write-NUnitReasonElement ($TestResult, [System.Xml.XmlWriter] $XmlWriter) { # TODO: do not format the errors here, instead format them in the core using some unified function so we get the same thing on the screen and in nunit $result = Get-ErrorForXmlReport -TestResult $TestResult diff --git a/tst/Pester.RSpec.Output.ts.ps1 b/tst/Pester.RSpec.Output.ts.ps1 index f900f8d3a..92a72c106 100644 --- a/tst/Pester.RSpec.Output.ts.ps1 +++ b/tst/Pester.RSpec.Output.ts.ps1 @@ -295,4 +295,25 @@ i -PassThru:$PassThru { $normalOutput | Verify-Equal $writehostOutput } } + + b 'Pending is deprecated' { + t 'Shows deprecated message when -pending is used' { + $sb = { + $container = New-PesterContainer -ScriptBlock { + Describe 'd' { + It 'i' { + Set-ItResult -Pending + } + } + } + + Invoke-Pester -Container $container + } + + $output = Invoke-InNewProcess -ScriptBlock $sb + + $deprecated = $output | Select-String -Pattern '\*DEPRECATED\*' + @($deprecated).Count | Verify-Equal 1 + } + } } diff --git a/tst/Pester.RSpec.ResultObject.ts.ps1 b/tst/Pester.RSpec.ResultObject.ts.ps1 index cd518bbc7..69439ebd0 100644 --- a/tst/Pester.RSpec.ResultObject.ts.ps1 +++ b/tst/Pester.RSpec.ResultObject.ts.ps1 @@ -100,6 +100,14 @@ i -PassThru:$PassThru { It "not run" -Tag "Slow" { 1 | Should -Be 1 } + + It "Set-ItResult -Inconclusive" { + Set-ItResult -Inconclusive + } + + It "Set-ItResult -Skipped" { + Set-ItResult -Skipped + } } } $result = Invoke-Pester -Configuration @{ @@ -119,7 +127,7 @@ i -PassThru:$PassThru { $result | Verify-Property "Containers" $result.Containers.Count | Verify-Equal 2 - $result.TotalCount | Verify-Equal 4 + $result.TotalCount | Verify-Equal 6 $result.Tests | Verify-NotNull $result.PassedCount | Verify-Equal 1 @@ -128,12 +136,15 @@ i -PassThru:$PassThru { $result.FailedCount | Verify-Equal 1 $result.Failed | Verify-NotNull - $result.SkippedCount | Verify-Equal 1 + $result.SkippedCount | Verify-Equal 2 $result.Skipped | Verify-NotNull $result.NotRunCount | Verify-Equal 1 $result.NotRun | Verify-NotNull + $result.InconclusiveCount | Verify-Equal 1 + $result.Inconclusive | Verify-NotNull + $result.Duration | Verify-Equal ($result.Containers[0].Duration + $result.Containers[1].Duration) $result.UserDuration | Verify-Equal ($result.Containers[0].UserDuration + $result.Containers[1].UserDuration) $result.FrameworkDuration | Verify-Equal ($result.Containers[0].FrameworkDuration + $result.Containers[1].FrameworkDuration) diff --git a/tst/Pester.RSpec.TestResults.NUnit25.ts.ps1 b/tst/Pester.RSpec.TestResults.NUnit25.ts.ps1 index 3712bc6ed..996e1197b 100644 --- a/tst/Pester.RSpec.TestResults.NUnit25.ts.ps1 +++ b/tst/Pester.RSpec.TestResults.NUnit25.ts.ps1 @@ -144,6 +144,51 @@ i -PassThru:$PassThru { } + t "should write a skipped test result" { + $sb = { + Describe "Mocked Describe 1" { + It "Skipped testcase" -Skip { + } + } + Describe "Mocked Describe 2" { + It "Skipped testcase" { + Set-ItResult -Skipped + } + } + } + $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) + + $xmlResult = $r | ConvertTo-NUnitReport + $xmlTestSuite = $xmlResult.'test-results'.'test-suite'.'results'.'test-suite'.'results'.'test-suite' + $xmlTestCase1 = $xmlTestSuite.results.'test-case'[0] + $xmlTestCase2 = $xmlTestSuite.results.'test-case'[1] + + $xmlTestCase1.name | Verify-Equal "Mocked Describe 1.Skipped testcase" + $xmlTestCase1.result | Verify-Equal "Ignored" + $xmlTestCase1.time | Verify-XmlTime $r.Containers[0].Blocks[0].Tests[0].Duration + + $xmlTestCase2.name | Verify-Equal "Mocked Describe 2.Skipped testcase" + $xmlTestCase2.result | Verify-Equal "Ignored" + $xmlTestCase2.time | Verify-XmlTime $r.Containers[0].Blocks[1].Tests[0].Duration + } + + t "should write an inconclusive test result" { + $sb = { + Describe "Mocked Describe" { + It "Inconclusive testcase" { + Set-ItResult -Inconclusive + } + } + } + $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) + + $xmlResult = $r | ConvertTo-NUnitReport + $xmlTestCase = $xmlResult.'test-results'.'test-suite'.'results'.'test-suite'.'results'.'test-suite'.'results'.'test-case' + $xmlTestCase.name | Verify-Equal "Mocked Describe.Inconclusive testcase" + $xmlTestCase.result | Verify-Equal "Inconclusive" + $xmlTestCase.time | Verify-XmlTime $r.Containers[0].Blocks[0].Tests[0].Duration + } + t "should write the test summary" { $sb = { Describe "Mocked Describe" { @@ -162,6 +207,41 @@ i -PassThru:$PassThru { $xmlTestResult.time | Verify-Equal (Get-Date -Format "HH:mm:ss" $r.ExecutedAt) } + t "should write inconclusive count" { + $sb = { + Describe "Mocked Describe 1" { + It "Inconclusive testcase 1" { + Set-ItResult -Inconclusive + } + } + Describe "Mocked Describe 2" { + It "Inconclusive testcase 2" { + Set-ItResult -Inconclusive + } + } + } + $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) + $xmlResult = $r | ConvertTo-NUnitReport + $xmlResult.'test-results'.inconclusive | Verify-Equal 2 + } + + t "should write skipped count" { + $sb = { + Describe "Mocked Describe 1" { + It "Skipped testcase 1" -Skip { + } + } + Describe "Mocked Describe 2" { + It "Skippde testcase 2" { + Set-ItResult -Skipped + } + } + } + $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) + $xmlResult = $r | ConvertTo-NUnitReport + $xmlResult.'test-results'.skipped | Verify-Equal 2 + } + t "should write the test-suite information" { $sb = { Describe "Mocked Describe" { @@ -246,6 +326,22 @@ i -PassThru:$PassThru { $false | Should -Be $true 5 } } + + Describe "Describe #3" { + It "Skipped testcase #1" -Skip {} + } + + Describe "Describe #4" { + It "Skipped testcase #2" { + Set-ItResult -Skipped + } + } + + Describe "Describe #5" { + It "Inconclusive testcase" { + Set-ItResult -Inconclusive + } + } } $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) diff --git a/tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 b/tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 index e81f147f9..bc593ff59 100644 --- a/tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 +++ b/tst/Pester.RSpec.TestResults.NUnit3.ts.ps1 @@ -147,6 +147,51 @@ i -PassThru:$PassThru { } + t 'should write a skipped test result' { + $sb = { + Describe 'Describe 1' { + It 'Skipped testcase' -Skip { + } + } + Describe 'Describe 2' { + It 'Skipped testcase' { + Set-ItResult -Skipped + } + } + } + $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) + + $xmlResult = $r | ConvertTo-NUnitReport -Format NUnit3 + $xmlTestSuite = $xmlResult.'test-run'.'test-suite'.'test-suite' + $xmlTestCase1 = $xmlTestSuite.'test-case'[0] + $xmlTestCase2 = $xmlTestSuite.'test-case'[1] + + $xmlTestCase1.name | Verify-Equal 'Describe 1.Skipped testcase' + $xmlTestCase1.result | Verify-Equal 'Skipped' + $xmlTestCase1.duration | Verify-XmlTime $r.Containers[0].Blocks[0].Tests[0].Duration + + $xmlTestCase2.name | Verify-Equal 'Describe 2.Skipped testcase' + $xmlTestCase2.result | Verify-Equal 'Skipped' + $xmlTestCase2.duration | Verify-XmlTime $r.Containers[0].Blocks[1].Tests[0].Duration + } + + t 'should write an inconclusive test result' { + $sb = { + Describe 'Describe' { + It 'Inconclusive testcase' { + Set-ItResult -Inconclusive + } + } + } + $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) + + $xmlResult = $r | ConvertTo-NUnitReport -Format NUnit3 + $xmlTestCase = $xmlResult.'test-run'.'test-suite'.'test-suite'.'test-case' + $xmlTestCase.name | Verify-Equal 'Describe.Inconclusive testcase' + $xmlTestCase.result | Verify-Equal 'Inconclusive' + $xmlTestCase.duration | Verify-XmlTime $r.Containers[0].Blocks[0].Tests[0].Duration + } + t 'should write the test summary' { $sb = { Describe 'Describe' { @@ -167,6 +212,45 @@ i -PassThru:$PassThru { $xmlTestResult.'end-time' | Verify-Equal (($r.ExecutedAt + $r.Duration).ToUniversalTime().ToString('o')) } + t "should write inconclusive count" { + $sb = { + Describe "Mocked Describe 1" { + It "Inconclusive testcase 1" { + Set-ItResult -Inconclusive + } + } + Describe "Mocked Describe 2" { + It "Inconclusive testcase 2" { + Set-ItResult -Inconclusive + } + } + } + $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) + $xmlResult = $r | ConvertTo-NUnitReport -Format NUnit3 + $xmlResult.'test-run'.inconclusive | Verify-Equal 2 + $xmlResult.'test-run'.'test-suite'.'test-suite'[0].inconclusive | Verify-Equal 1 + $xmlResult.'test-run'.'test-suite'.'test-suite'[1].inconclusive | Verify-Equal 1 + } + + t "should write skipped count" { + $sb = { + Describe "Mocked Describe 1" { + It "Skipped testcase 1" -Skip { + } + } + Describe "Mocked Describe 2" { + It "Skippde testcase 2" { + Set-ItResult -Skipped + } + } + } + $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) + $xmlResult = $r | ConvertTo-NUnitReport -Format NUnit3 + $xmlResult.'test-run'.skipped | Verify-Equal 2 + $xmlResult.'test-run'.'test-suite'.'test-suite'[0].skipped | Verify-Equal 1 + $xmlResult.'test-run'.'test-suite'.'test-suite'[1].skipped | Verify-Equal 1 + } + t 'should write the test-suite information' { $sb = { Describe 'Describe' { @@ -258,6 +342,22 @@ i -PassThru:$PassThru { $false | Should -Be $true } } + + Describe "Describe #3" { + It "Skipped testcase #1" -Skip {} + } + + Describe "Describe #4" { + It "Skipped testcase #2" { + Set-ItResult -Skipped + } + } + + Describe "Describe #5" { + It "Inconclusive testcase" { + Set-ItResult -Inconclusive + } + } } $r = Invoke-Pester -Configuration ([PesterConfiguration]@{ Run = @{ ScriptBlock = $sb; PassThru = $true }; Output = @{ Verbosity = 'None' } }) @@ -317,7 +417,7 @@ i -PassThru:$PassThru { t 'should add tags as Category-properties to blocks and tests' { $sb = { Describe 'Describe' -Tag 'abc' { - It 'Successful testcase' -Tag 'hello','world' { + It 'Successful testcase' -Tag 'hello', 'world' { $true | Should -Be $true } } @@ -524,7 +624,7 @@ i -PassThru:$PassThru { # behavior based on NUnit3 runner $sb = { Describe 'Describe' { - It 'Testcase <_>' -Tag 'hello','world' -ForEach @(1,2) { + It 'Testcase <_>' -Tag 'hello', 'world' -ForEach @(1, 2) { $true | Should -Be $true } } @@ -681,7 +781,7 @@ i -PassThru:$PassThru { t 'should add tags as Category-properties on child test-suites only' { # behavior based on NUnit3 runner $sb = { - Describe 'Describe <_>' -Tag 'abc' -ForEach @(1,2) { + Describe 'Describe <_>' -Tag 'abc' -ForEach @(1, 2) { It 'Testcase' { $true | Should -Be $true }