Skip to content

Commit

Permalink
Ignore excluded/skipped tests and unnecessary setup/teardown using Sk…
Browse files Browse the repository at this point in the history 
…ipRemainingOnFailure (#2442)

* Refactor GetSkipRemainingOnFailure and ignore excluded and skipped tests

* Skip blocks to avoid running BeforeAll/AfterAll

* Fix tests

* Skip root-level BeforeAll/AfterAll

* Fix Skipped container result

Skipped-property only exists on Test

* Add tests for block-level skip

* Cleanup

* Cleanup test
  • Loading branch information
@fflaten
fflaten authored May 17, 2024
1 parent d324f29 commit c9c380d
Show file tree
Hide file tree
Showing 4 changed files with 230 additions and 75 deletions.
2 changes: 1 addition & 1 deletion src/Pester.RSpec.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ function PostProcess-RspecTestRun ($TestRun) {
## decorate

# here we add result
$b.result = if ($b.Skipped) {
$b.result = if ($b.Skip) {
"Skipped"
}
elseif ($b.Passed) {
Expand Down
2 changes: 1 addition & 1 deletion src/Pester.Utility.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -334,7 +334,7 @@ function Fold-Block {
foreach ($b in $Block) {
$Accumulator = & $OnBlock $Block $Accumulator
foreach ($test in $Block.Tests) {
$Accumulator = &$OnTest $test $Accumulator
$Accumulator = & $OnTest $test $Accumulator
}

foreach ($b in $Block.Blocks) {
Expand Down
128 changes: 55 additions & 73 deletions src/functions/Get-SkipRemainingOnFailurePlugin.ps1
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,46 @@
function New-SkippedTestMessage {
[OutputType([string])]
[CmdletBinding()]
param (
[Parameter(Mandatory)]
[Pester.Test]
$Test
)
"Skipped due to previous failure at '$($Test.ExpandedPath)' and Run.SkipRemainingOnFailure set to '$($PesterPreference.Run.SkipRemainingOnFailure.Value)'"
}

function Resolve-SkipRemainingOnFailureConfiguration {
$supportedValues = 'None', 'Block', 'Container', 'Run'
if ($PesterPreference.Run.SkipRemainingOnFailure.Value -notin $supportedValues) {
throw (Get-StringOptionErrorMessage -OptionPath 'Run.SkipRemainingOnFailure' -SupportedValues $supportedValues -Value $PesterPreference.Run.SkipRemainingOnFailure.Value)
}
}

function Set-RemainingAsSkipped {
param(
[Parameter(Mandatory)]
[Pester.Test]
$FailedTest,

[Parameter(Mandatory)]
[Pester.Block]
$Block
)

$errorRecord = [Pester.Factory]::CreateErrorRecord(
'PesterTestSkipped',
"Skipped due to previous failure at '$($FailedTest.ExpandedPath)' and Run.SkipRemainingOnFailure set to '$($PesterPreference.Run.SkipRemainingOnFailure.Value)'",
$null,
$null,
$null,
$false
)

Fold-Block -Block $Block -OnTest {
param ($test)
if ($test.ShouldRun -and -not $test.Skip -and -not $test.Executed) {
# Skipping and counting remaining unexecuted tests
$Context.Configuration.SkipRemainingOnFailureCount += 1
$test.Skip = $true
$test.ErrorRecord.Add($errorRecord)
}
} -OnBlock {
param($block)
if ($block.ShouldRun -and -not $block.Skip -and -not $block.Executed) {
# Marking remaining blocks as Skip to avoid executing BeforeAll/AfterAll
$block.Skip = $true
}
}
}

function Get-SkipRemainingOnFailurePlugin {
# Validate configuration
Expand All @@ -29,41 +54,18 @@ function Get-SkipRemainingOnFailurePlugin {
$p.Start = {
param ($Context)

# TODO: Use $Context.GlobalPluginData.SkipRemainingOnFailure.SkippedCount when exposed in $Context
$Context.Configuration.SkipRemainingOnFailureCount = 0
}

if ($PesterPreference.Run.SkipRemainingOnFailure.Value -eq 'Block') {
$p.EachTestTeardownEnd = {
param($Context)

# If test is not marked skipped and failed
# Go through block tests and child tests and mark unexecuted tests as skipped
# If test was not skipped and failed
if (-not $Context.Test.Skipped -and -not $Context.Test.Passed) {

$errorRecord = [Pester.Factory]::CreateErrorRecord(
'PesterTestSkipped',
(New-SkippedTestMessage -Test $Context.Test),
$null,
$null,
$null,
$false
)

foreach ($test in $Context.Block.Tests) {
if (-not $test.Executed) {
$Context.Configuration.SkipRemainingOnFailureCount += 1
$test.Skip = $true
$test.ErrorRecord.Add($errorRecord)
}
}

foreach ($test in ($Context.Block | View-Flat)) {
if (-not $test.Executed) {
$Context.Configuration.SkipRemainingOnFailureCount += 1
$test.Skip = $true
$test.ErrorRecord.Add($errorRecord)
}
}
# Skip all remaining tests in the block recursively
Set-RemainingAsSkipped -FailedTest $Context.Test -Block $Context.Block
}
}
}
Expand All @@ -72,57 +74,37 @@ function Get-SkipRemainingOnFailurePlugin {
$p.EachTestTeardownEnd = {
param($Context)

# If test is not marked skipped and failed
# Go through every test in container from block root and marked unexecuted tests as skipped
# If test was not skipped and failed
if (-not $Context.Test.Skipped -and -not $Context.Test.Passed) {

$errorRecord = [Pester.Factory]::CreateErrorRecord(
'PesterTestSkipped',
(New-SkippedTestMessage -Test $Context.Test),
$null,
$null,
$null,
$false
)

foreach ($test in ($Context.Block.Root | View-Flat)) {
if (-not $test.Executed) {
$Context.Configuration.SkipRemainingOnFailureCount += 1
$test.Skip = $true
$test.ErrorRecord.Add($errorRecord)
}
}
# Skip all remaining tests in the container recursively
Set-RemainingAsSkipped -FailedTest $Context.Test -Block $Context.Block.Root
}
}
}

elseif ($PesterPreference.Run.SkipRemainingOnFailure.Value -eq 'Run') {
$p.EachTestSetupStart = {
$p.ContainerRunStart = {
param($Context)

# If a test has failed at some point during the run
# Skip the test before it runs
# This handles skipping tests that failed from different containers in the same run
# If a test failed in a previous container, skip all tests
if ($Context.Configuration.SkipRemainingFailedTest) {
$Context.Configuration.SkipRemainingOnFailureCount += 1
$Context.Test.Skip = $true

$errorRecord = [Pester.Factory]::CreateErrorRecord(
'PesterTestSkipped',
(New-SkippedTestMessage -Test $Context.Configuration.SkipRemainingFailedTest),
$null,
$null,
$null,
$false
)
$Context.Test.ErrorRecord.Add($errorRecord)
# Skip container root block to avoid root-level BeforeAll/AfterAll from running. Only applicable in this mode
$Context.Block.Root.Skip = $true
# Skip all remaining tests in current container
Set-RemainingAsSkipped -FailedTest $Context.Configuration.SkipRemainingFailedTest -Block $Context.Block
}
}

$p.EachTestTeardownEnd = {
param($Context)

# If test was not skipped but failed
if (-not $Context.Test.Skipped -and -not $Context.Test.Passed) {
# Skip all remaining tests in current container
Set-RemainingAsSkipped -FailedTest $Context.Test -Block $Context.Block.Root

# Store failed test so we can skip remaining containers in ContainerRunStart-step
# TODO: Use $Context.GlobalPluginData.SkipRemainingOnFailure.FailedTest when exposed in $Context
$Context.Configuration.SkipRemainingFailedTest = $Context.Test
}
}
Expand Down
173 changes: 173 additions & 0 deletions tst/Pester.RSpec.ts.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2591,6 +2591,179 @@ i -PassThru:$PassThru {
$r.Containers[1].Blocks[0].Tests[0].ErrorRecord.FullyQualifiedErrorID | Verify-Equal 'PesterTestSkipped'
$r.Containers[1].Blocks[0].Tests[0].ErrorRecord.TargetObject.Message | Verify-Equal "Skipped due to previous failure at 'a.b' and Run.SkipRemainingOnFailure set to 'Run'"
}

foreach ($mode in 'Block', 'Container', 'Run') {
t "Ignore tests with -Skip or excluded by filter in mode '$mode'" {
$sb1 = {
Describe 'a' {
It 'Included - fails' -Tag 'Demo' {
$false | Should -BeTrue
}
It 'Excluded - ignore' {
$true | Should -BeTrue
}
Describe 'b' {
It 'Included - skip' -Tag 'Demo' {
$true | Should -BeTrue
}
It 'Included but skipped - ignore' -Tag 'Demo' -Skip {
$true | Should -BeTrue
}
}
}
Describe 'c' {
It 'Included - skip on Container and Run' -Tag 'Demo' {
$true | Should -BeTrue
}
}
}

$sb2 = {
Describe 'd' {
It 'Included - skip on Run' -Tag 'Demo' {
$true | Should -BeTrue
}
}
}

$c = [PesterConfiguration] @{
Filter = @{
Tag = 'Demo'
}
Run = @{
ScriptBlock = $sb1, $sb2
PassThru = $true
SkipRemainingOnFailure = $mode
}
Output = @{
CIFormat = 'None'
}
}

$r = Invoke-Pester -Configuration $c

$r.Tests[0].Skipped | Verify-False
$r.Tests[0].Result | Verify-Equal 'Failed'

# Should not mark excluded tests as Skipped
$r.Tests[1].Skipped | Verify-False
$r.Tests[1].Result | Verify-Equal 'NotRun'

# Should mark included test as Skipped
$r.Tests[2].Skipped | Verify-True
$r.Tests[2].Result | Verify-Equal 'Skipped'
$r.Tests[2].ErrorRecord.TargetObject.Message -match '^Skipped due to previous failure' | Verify-True

# Should not modify explicitly skipped tests
$r.Tests[3].Skipped | Verify-True
$r.Tests[3].Result | Verify-Equal 'Skipped'
$r.Tests[3].ErrorRecord | Verify-Null

switch ($mode) {
'Block' { $r.PluginConfiguration.SkipRemainingOnFailureCount | Verify-Equal 1 }
'Container' { $r.PluginConfiguration.SkipRemainingOnFailureCount | Verify-Equal 2 }
'Run' { $r.PluginConfiguration.SkipRemainingOnFailureCount | Verify-Equal 3 }
}
}
}

foreach ($mode in 'Block', 'Container', 'Run') {
t "Remaining blocks are skipped in mode '$mode'" {
$container = [ordered]@{
RootBeforeAll = 0
RootAfterAll = 0
BlockBeforeAll = 0
BlockAfterAll = 0
}

$sb1 = {
BeforeAll { $container.RootBeforeAll++ }
AfterAll { $container.RootAfterAll++ }

Describe 'd1' {
BeforeAll { $container.BlockBeforeAll++ }
AfterAll { $container.BlockAfterAll++ }

It 'Fails' { $false | Should -BeTrue }

Context 'c1' {
BeforeAll { $container.BlockBeforeAll++ }
AfterAll { $container.BlockAfterAll++ }
It 'Skipped' { $true | Should -BeTrue }
}
}
Describe 'd2' {
BeforeAll { $container.BlockBeforeAll++ }
AfterAll { $container.BlockAfterAll++ }

It 'Skipped' { $true | Should -BeTrue }
}
}

$sb2 = {
BeforeAll { $container.RootBeforeAll++ }
AfterAll { $container.RootAfterAll++ }

Describe 'd1' {
BeforeAll { $container.BlockBeforeAll++ }
AfterAll { $container.BlockAfterAll++ }

It 'Skipped' { $true | Should -BeTrue }
}
}

$c = [PesterConfiguration] @{
Run = @{
ScriptBlock = $sb1, $sb2
PassThru = $true
SkipRemainingOnFailure = $mode
}
Output = @{
CIFormat = 'None'
}
}

$r = Invoke-Pester -Configuration $c
$r.Containers[0].Result | Verify-Equal 'Failed'
$r.Containers[0].Blocks[0].Result | Verify-Equal 'Failed'
$r.Containers[0].Blocks[0].Blocks[0].Result | Verify-Equal 'Skipped'

# AfterAll should always execute for current and parent blocks of the failure
# BeforeAll and AfterAll should not be executed for remaining children or siblings
switch ($mode) {
'Block' {
$r.Containers[0].Blocks[1].Result | Verify-Equal 'Passed'
$r.Containers[1].Result | Verify-Equal 'Passed'
$r.Containers[1].Blocks[0].Result | Verify-Equal 'Passed'

$container.RootBeforeAll | Verify-Equal 2
$container.RootAfterAll | Verify-Equal 2
$container.BlockBeforeAll | Verify-Equal 3
$container.BlockAfterAll | Verify-Equal 3
}
'Container' {
$r.Containers[0].Blocks[1].Result | Verify-Equal 'Skipped'
$r.Containers[1].Result | Verify-Equal 'Passed'
$r.Containers[1].Blocks[0].Result | Verify-Equal 'Passed'

$container.RootBeforeAll | Verify-Equal 2
$container.RootAfterAll | Verify-Equal 2
$container.BlockBeforeAll | Verify-Equal 2
$container.BlockAfterAll | Verify-Equal 2
}
'Run' {
$r.Containers[0].Blocks[1].Result | Verify-Equal 'Skipped'
$r.Containers[1].Result | Verify-Equal 'Skipped'
$r.Containers[1].Blocks[0].Result | Verify-Equal 'Skipped'

$container.RootBeforeAll | Verify-Equal 1
$container.RootAfterAll | Verify-Equal 1
$container.BlockBeforeAll | Verify-Equal 1
$container.BlockAfterAll | Verify-Equal 1
}
}
}
}
}

b 'Changes to CWD are reverted on exit' {
Expand Down

0 comments on commit c9c380d

Please sign in to comment.