From 3d0b82a062127a17ee128093d17c6296cebe9a87 Mon Sep 17 00:00:00 2001 From: Daniel Bak Date: Wed, 5 Jun 2019 11:58:20 -0600 Subject: [PATCH 1/4] Added test displaying issue #1650 --- projects/SelfTest/UsageTests/Misc.tests.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/projects/SelfTest/UsageTests/Misc.tests.cpp b/projects/SelfTest/UsageTests/Misc.tests.cpp index 9a87169b93..858607f8f4 100644 --- a/projects/SelfTest/UsageTests/Misc.tests.cpp +++ b/projects/SelfTest/UsageTests/Misc.tests.cpp @@ -135,6 +135,18 @@ TEST_CASE( "even more nested SECTION tests", "[sections]" ) { } } +TEST_CASE( "#1650 - failing nested section tests should not interfere with names of other sections", + "[.][failing][sections]"){ + SECTION("Upper section"){ + SECTION("Lower section with success"){ + SUCCEED(); + } + SECTION("Lower section with failure"){ + FAIL(); + } + } +} + TEST_CASE( "looped SECTION tests", "[.][failing][sections]" ) { int a = 1; From 198548011196d04569edcdf39bd102de9542103b Mon Sep 17 00:00:00 2001 From: Daniel Bak Date: Wed, 5 Jun 2019 16:47:42 -0600 Subject: [PATCH 2/4] Added test to reveal bug #1650 Editied approval tests to expect correct behavior. --- .../Baselines/compact.sw.approved.txt | 5 +- .../Baselines/console.std.approved.txt | 30 ++- .../Baselines/console.sw.approved.txt | 41 +++- .../Baselines/console.swa4.approved.txt | 182 ++---------------- .../SelfTest/Baselines/junit.sw.approved.txt | 15 +- .../SelfTest/Baselines/xml.sw.approved.txt | 31 ++- projects/SelfTest/UsageTests/Misc.tests.cpp | 10 +- 7 files changed, 141 insertions(+), 173 deletions(-) diff --git a/projects/SelfTest/Baselines/compact.sw.approved.txt b/projects/SelfTest/Baselines/compact.sw.approved.txt index 89ebdfb1b5..7f3345b6c6 100644 --- a/projects/SelfTest/Baselines/compact.sw.approved.txt +++ b/projects/SelfTest/Baselines/compact.sw.approved.txt @@ -21,6 +21,9 @@ This would not be caught previously Nor would this Tricky.tests.cpp:: failed: explicitly with 1 message: '1514' Compilation.tests.cpp:: passed: std::is_same, TypeList>::value for: true +Misc.tests.cpp:: passed: +Misc.tests.cpp:: failed: explicitly with 1 message: 'Failure in lower section' +Misc.tests.cpp:: failed: explicitly with 1 message: 'Failure in second lower section' Exception.tests.cpp:: failed: unexpected exception with message: 'answer := 42' with 1 message: 'expected exception' Exception.tests.cpp:: failed: unexpected exception with message: 'answer := 42'; expression was: thisThrows() with 1 message: 'expected exception' Exception.tests.cpp:: passed: thisThrows() with 1 message: 'answer := 42' @@ -1629,5 +1632,5 @@ Misc.tests.cpp:: passed: v.size() == 5 for: 5 == 5 Misc.tests.cpp:: passed: v.capacity() >= 5 for: 5 >= 5 Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: -Failed 86 test cases, failed 148 assertions. +Failed 87 test cases, failed 150 assertions. diff --git a/projects/SelfTest/Baselines/console.std.approved.txt b/projects/SelfTest/Baselines/console.std.approved.txt index 5f8e036272..6685d18c0c 100644 --- a/projects/SelfTest/Baselines/console.std.approved.txt +++ b/projects/SelfTest/Baselines/console.std.approved.txt @@ -28,6 +28,32 @@ Tricky.tests.cpp:: FAILED: explicitly with message: 1514 +------------------------------------------------------------------------------- +#1650 failing nested section tests should not interfere with names of other +sections + Upper section + Lower section with failure +------------------------------------------------------------------------------- +Misc.tests.cpp: +............................................................................... + +Misc.tests.cpp:: FAILED: +explicitly with message: + Failure in lower section + +------------------------------------------------------------------------------- +#1650 failing nested section tests should not interfere with names of other +sections + Upper section + Second lower section with failure +------------------------------------------------------------------------------- +Misc.tests.cpp: +............................................................................... + +Misc.tests.cpp:: FAILED: +explicitly with message: + Failure in second lower section + ------------------------------------------------------------------------------- #748 - captures with unexpected exceptions outside assertions @@ -1380,6 +1406,6 @@ due to unexpected exception with message: Why would you throw a std::string? =============================================================================== -test cases: 289 | 215 passed | 70 failed | 4 failed as expected -assertions: 1539 | 1387 passed | 131 failed | 21 failed as expected +test cases: 290 | 215 passed | 71 failed | 4 failed as expected +assertions: 1542 | 1388 passed | 133 failed | 21 failed as expected diff --git a/projects/SelfTest/Baselines/console.sw.approved.txt b/projects/SelfTest/Baselines/console.sw.approved.txt index cd65bc1709..994926582e 100644 --- a/projects/SelfTest/Baselines/console.sw.approved.txt +++ b/projects/SelfTest/Baselines/console.sw.approved.txt @@ -172,6 +172,43 @@ Compilation.tests.cpp:: PASSED: with expansion: true +------------------------------------------------------------------------------- +#1650 failing nested section tests should not interfere with names of other +sections + Upper section + Lower section with success +------------------------------------------------------------------------------- +Misc.tests.cpp: +............................................................................... + +Misc.tests.cpp:: PASSED: + +------------------------------------------------------------------------------- +#1650 failing nested section tests should not interfere with names of other +sections + Upper section + Lower section with failure +------------------------------------------------------------------------------- +Misc.tests.cpp: +............................................................................... + +Misc.tests.cpp:: FAILED: +explicitly with message: + Failure in lower section + +------------------------------------------------------------------------------- +#1650 failing nested section tests should not interfere with names of other +sections + Upper section + Second lower section with failure +------------------------------------------------------------------------------- +Misc.tests.cpp: +............................................................................... + +Misc.tests.cpp:: FAILED: +explicitly with message: + Failure in second lower section + ------------------------------------------------------------------------------- #748 - captures with unexpected exceptions outside assertions @@ -12218,6 +12255,6 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 289 | 199 passed | 86 failed | 4 failed as expected -assertions: 1556 | 1387 passed | 148 failed | 21 failed as expected +test cases: 290 | 199 passed | 87 failed | 4 failed as expected +assertions: 1559 | 1388 passed | 150 failed | 21 failed as expected diff --git a/projects/SelfTest/Baselines/console.swa4.approved.txt b/projects/SelfTest/Baselines/console.swa4.approved.txt index bfa6675066..d6a30c8b37 100644 --- a/projects/SelfTest/Baselines/console.swa4.approved.txt +++ b/projects/SelfTest/Baselines/console.swa4.approved.txt @@ -173,185 +173,43 @@ with expansion: true ------------------------------------------------------------------------------- -#748 - captures with unexpected exceptions - outside assertions -------------------------------------------------------------------------------- -Exception.tests.cpp: -............................................................................... - -Exception.tests.cpp:: FAILED: -due to unexpected exception with messages: - answer := 42 - expected exception - -------------------------------------------------------------------------------- -#748 - captures with unexpected exceptions - inside REQUIRE_NOTHROW -------------------------------------------------------------------------------- -Exception.tests.cpp: -............................................................................... - -Exception.tests.cpp:: FAILED: - REQUIRE_NOTHROW( thisThrows() ) -due to unexpected exception with messages: - answer := 42 - expected exception - -------------------------------------------------------------------------------- -#748 - captures with unexpected exceptions - inside REQUIRE_THROWS -------------------------------------------------------------------------------- -Exception.tests.cpp: -............................................................................... - -Exception.tests.cpp:: PASSED: - REQUIRE_THROWS( thisThrows() ) -with message: - answer := 42 - -------------------------------------------------------------------------------- -#809 -------------------------------------------------------------------------------- -Compilation.tests.cpp: -............................................................................... - -Compilation.tests.cpp:: PASSED: - REQUIRE( 42 == f ) -with expansion: - 42 == {?} - -------------------------------------------------------------------------------- -#833 -------------------------------------------------------------------------------- -Compilation.tests.cpp: -............................................................................... - -Compilation.tests.cpp:: PASSED: - REQUIRE( a == t ) -with expansion: - 3 == 3 - -Compilation.tests.cpp:: PASSED: - CHECK( a == t ) -with expansion: - 3 == 3 - -Compilation.tests.cpp:: PASSED: - REQUIRE_THROWS( throws_int(true) ) - -Compilation.tests.cpp:: PASSED: - CHECK_THROWS_AS( throws_int(true), int ) - -Compilation.tests.cpp:: PASSED: - REQUIRE_NOTHROW( throws_int(false) ) - -Compilation.tests.cpp:: PASSED: - REQUIRE_THAT( "aaa", Catch::EndsWith("aaa") ) -with expansion: - "aaa" ends with: "aaa" - -Compilation.tests.cpp:: PASSED: - REQUIRE( templated_tests(3) ) -with expansion: - true - -------------------------------------------------------------------------------- -#835 -- errno should not be touched by Catch +#1650 failing nested section tests should not interfere with names of other +sections + Upper section + Lower section with success ------------------------------------------------------------------------------- Misc.tests.cpp: ............................................................................... -Misc.tests.cpp:: FAILED: - CHECK( f() == 0 ) -with expansion: - 1 == 0 - Misc.tests.cpp:: PASSED: - REQUIRE( errno == 1 ) -with expansion: - 1 == 1 ------------------------------------------------------------------------------- -#872 -------------------------------------------------------------------------------- -Compilation.tests.cpp: -............................................................................... - -Compilation.tests.cpp:: PASSED: - REQUIRE( x == 4 ) -with expansion: - {?} == 4 -with message: - dummy := 0 - -------------------------------------------------------------------------------- -#961 -- Dynamically created sections should all be reported - Looped section 0 +#1650 failing nested section tests should not interfere with names of other +sections + Upper section + Lower section with failure ------------------------------------------------------------------------------- Misc.tests.cpp: ............................................................................... -Misc.tests.cpp:: PASSED: -with message: - Everything is OK - -------------------------------------------------------------------------------- -#961 -- Dynamically created sections should all be reported - Looped section 1 -------------------------------------------------------------------------------- -Misc.tests.cpp: -............................................................................... - -Misc.tests.cpp:: PASSED: -with message: - Everything is OK - -------------------------------------------------------------------------------- -#961 -- Dynamically created sections should all be reported - Looped section 2 -------------------------------------------------------------------------------- -Misc.tests.cpp: -............................................................................... - -Misc.tests.cpp:: PASSED: -with message: - Everything is OK - -------------------------------------------------------------------------------- -#961 -- Dynamically created sections should all be reported - Looped section 3 -------------------------------------------------------------------------------- -Misc.tests.cpp: -............................................................................... - -Misc.tests.cpp:: PASSED: -with message: - Everything is OK +Misc.tests.cpp:: FAILED: +explicitly with message: + Failure in lower section ------------------------------------------------------------------------------- -#961 -- Dynamically created sections should all be reported - Looped section 4 +#1650 failing nested section tests should not interfere with names of other +sections + Upper section + Second lower section with failure ------------------------------------------------------------------------------- Misc.tests.cpp: ............................................................................... -Misc.tests.cpp:: PASSED: -with message: - Everything is OK - -------------------------------------------------------------------------------- -'Not' checks that should fail -------------------------------------------------------------------------------- -Condition.tests.cpp: -............................................................................... - -Condition.tests.cpp:: FAILED: - CHECK( false != false ) - -Condition.tests.cpp:: FAILED: - CHECK( true != true ) +Misc.tests.cpp:: FAILED: +explicitly with message: + Failure in second lower section =============================================================================== -test cases: 18 | 13 passed | 3 failed | 2 failed as expected -assertions: 40 | 33 passed | 4 failed | 3 failed as expected +test cases: 12 | 9 passed | 3 failed +assertions: 22 | 18 passed | 4 failed diff --git a/projects/SelfTest/Baselines/junit.sw.approved.txt b/projects/SelfTest/Baselines/junit.sw.approved.txt index 94f0138552..8461977119 100644 --- a/projects/SelfTest/Baselines/junit.sw.approved.txt +++ b/projects/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ - + @@ -28,6 +28,19 @@ Nor would this + + + +Failure in lower section +Misc.tests.cpp: + + + + +Failure in second lower section +Misc.tests.cpp: + + expected exception diff --git a/projects/SelfTest/Baselines/xml.sw.approved.txt b/projects/SelfTest/Baselines/xml.sw.approved.txt index dd2c3f4a96..a228d080ce 100644 --- a/projects/SelfTest/Baselines/xml.sw.approved.txt +++ b/projects/SelfTest/Baselines/xml.sw.approved.txt @@ -177,6 +177,33 @@ Nor would this + +
+
+ +
+ +
+
+
+ + Failure in lower section + + +
+ +
+
+
+ + Failure in second lower section + + +
+ +
+ +
@@ -14586,7 +14613,7 @@ loose text artifact
- + - + diff --git a/projects/SelfTest/UsageTests/Misc.tests.cpp b/projects/SelfTest/UsageTests/Misc.tests.cpp index 858607f8f4..36a80cebb5 100644 --- a/projects/SelfTest/UsageTests/Misc.tests.cpp +++ b/projects/SelfTest/UsageTests/Misc.tests.cpp @@ -135,14 +135,18 @@ TEST_CASE( "even more nested SECTION tests", "[sections]" ) { } } -TEST_CASE( "#1650 - failing nested section tests should not interfere with names of other sections", - "[.][failing][sections]"){ +TEST_CASE( "#1650 failing nested section tests should not interfere with names of other sections", + "[.][failing][sections]"){ + SECTION("Upper section"){ SECTION("Lower section with success"){ SUCCEED(); } SECTION("Lower section with failure"){ - FAIL(); + FAIL("Failure in lower section"); + } + SECTION("Second lower section with failure"){ + FAIL("Failure in second lower section"); } } } From f7af81561e791f8a618b8611522846ccd1180235 Mon Sep 17 00:00:00 2001 From: Daniel Bak Date: Thu, 6 Jun 2019 13:11:11 -0600 Subject: [PATCH 3/4] Added captures to errnoParser and nanParser as these were not capturing correctly on my machine. --- scripts/approvalTests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/approvalTests.py b/scripts/approvalTests.py index bb01e6d570..a6d077bd6a 100755 --- a/scripts/approvalTests.py +++ b/scripts/approvalTests.py @@ -48,6 +48,8 @@ \(\*__error\(\)\) | \(\*_errno\(\)\) + | + \(\*__errno\(\)\) ''', re.VERBOSE) sinceEpochParser = re.compile(r'\d+ .+ since epoch') infParser = re.compile(r''' @@ -66,6 +68,8 @@ | \(__builtin_nanf\ \(""\)\) # Linux (ubuntu) NAN macro | + \(__builtin_nanf \(""\)\) + | __builtin_nanf\("0x"\) # The weird content of the brackets is there because a different parser has already ran before this one ''', re.VERBOSE) From 271e033175ef48ba328c6e58b3c9b62c51d8925a Mon Sep 17 00:00:00 2001 From: Daniel Bak Date: Tue, 4 Jun 2019 19:20:00 -0600 Subject: [PATCH 4/4] In a failing test, the sectionend() corresponding to a sectionstart() could possibly pop different data than was pushed. This is because sectionEndedEarly() would push the failing section onto another stack (m_unfinishedSections), and then the next sectionEnded() would close the hanging sectionStarted(). Then the m_unfinishedSections would be popped, filling up the remaining hanging sections. This caused two issues: One: any reporter relying on SectionEnd() having accurate section data was broken. SectionEnd() would present different data than SectionStarted(). This caused an issue for the junit reporter, which relies on that data matching to manage the parralel m_sectionStack. However, this also applied to other reporters; having the xml reporter write the "name" attribute to the OverallResults written in sectionEnded revealed that it was not matching the section. Two: I noticed where tests containing failing sections would pass through again, but not hit any nodes. Since a failing section would not be popped, it would remain on the stack, and this causes the test runner to believe that the test requires another pass because there are unresolved open sections (when all sections were actually closed). --- include/internal/catch_section.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/include/internal/catch_section.cpp b/include/internal/catch_section.cpp index 0646d36fe0..1d9b2f9cbf 100644 --- a/include/internal/catch_section.cpp +++ b/include/internal/catch_section.cpp @@ -22,10 +22,7 @@ namespace Catch { Section::~Section() { if( m_sectionIncluded ) { SectionEndInfo endInfo{ m_info, m_assertions, m_timer.getElapsedSeconds() }; - if( uncaught_exceptions() ) - getResultCapture().sectionEndedEarly( endInfo ); - else - getResultCapture().sectionEnded( endInfo ); + getResultCapture().sectionEnded( endInfo ); } }