From 3d0b82a062127a17ee128093d17c6296cebe9a87 Mon Sep 17 00:00:00 2001 From: Daniel Bak Date: Wed, 5 Jun 2019 11:58:20 -0600 Subject: [PATCH 1/3] Added test displaying issue #1650 --- projects/SelfTest/UsageTests/Misc.tests.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/projects/SelfTest/UsageTests/Misc.tests.cpp b/projects/SelfTest/UsageTests/Misc.tests.cpp index 9a87169b93..858607f8f4 100644 --- a/projects/SelfTest/UsageTests/Misc.tests.cpp +++ b/projects/SelfTest/UsageTests/Misc.tests.cpp @@ -135,6 +135,18 @@ TEST_CASE( "even more nested SECTION tests", "[sections]" ) { } } +TEST_CASE( "#1650 - failing nested section tests should not interfere with names of other sections", + "[.][failing][sections]"){ + SECTION("Upper section"){ + SECTION("Lower section with success"){ + SUCCEED(); + } + SECTION("Lower section with failure"){ + FAIL(); + } + } +} + TEST_CASE( "looped SECTION tests", "[.][failing][sections]" ) { int a = 1; From 198548011196d04569edcdf39bd102de9542103b Mon Sep 17 00:00:00 2001 From: Daniel Bak Date: Wed, 5 Jun 2019 16:47:42 -0600 Subject: [PATCH 2/3] Added test to reveal bug #1650 Editied approval tests to expect correct behavior. --- .../Baselines/compact.sw.approved.txt | 5 +- .../Baselines/console.std.approved.txt | 30 ++- .../Baselines/console.sw.approved.txt | 41 +++- .../Baselines/console.swa4.approved.txt | 182 ++---------------- .../SelfTest/Baselines/junit.sw.approved.txt | 15 +- .../SelfTest/Baselines/xml.sw.approved.txt | 31 ++- projects/SelfTest/UsageTests/Misc.tests.cpp | 10 +- 7 files changed, 141 insertions(+), 173 deletions(-) diff --git a/projects/SelfTest/Baselines/compact.sw.approved.txt b/projects/SelfTest/Baselines/compact.sw.approved.txt index 89ebdfb1b5..7f3345b6c6 100644 --- a/projects/SelfTest/Baselines/compact.sw.approved.txt +++ b/projects/SelfTest/Baselines/compact.sw.approved.txt @@ -21,6 +21,9 @@ This would not be caught previously Nor would this Tricky.tests.cpp:: failed: explicitly with 1 message: '1514' Compilation.tests.cpp:: passed: std::is_same, TypeList>::value for: true +Misc.tests.cpp:: passed: +Misc.tests.cpp:: failed: explicitly with 1 message: 'Failure in lower section' +Misc.tests.cpp:: failed: explicitly with 1 message: 'Failure in second lower section' Exception.tests.cpp:: failed: unexpected exception with message: 'answer := 42' with 1 message: 'expected exception' Exception.tests.cpp:: failed: unexpected exception with message: 'answer := 42'; expression was: thisThrows() with 1 message: 'expected exception' Exception.tests.cpp:: passed: thisThrows() with 1 message: 'answer := 42' @@ -1629,5 +1632,5 @@ Misc.tests.cpp:: passed: v.size() == 5 for: 5 == 5 Misc.tests.cpp:: passed: v.capacity() >= 5 for: 5 >= 5 Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: -Failed 86 test cases, failed 148 assertions. +Failed 87 test cases, failed 150 assertions. diff --git a/projects/SelfTest/Baselines/console.std.approved.txt b/projects/SelfTest/Baselines/console.std.approved.txt index 5f8e036272..6685d18c0c 100644 --- a/projects/SelfTest/Baselines/console.std.approved.txt +++ b/projects/SelfTest/Baselines/console.std.approved.txt @@ -28,6 +28,32 @@ Tricky.tests.cpp:: FAILED: explicitly with message: 1514 +------------------------------------------------------------------------------- +#1650 failing nested section tests should not interfere with names of other +sections + Upper section + Lower section with failure +------------------------------------------------------------------------------- +Misc.tests.cpp: +............................................................................... + +Misc.tests.cpp:: FAILED: +explicitly with message: + Failure in lower section + +------------------------------------------------------------------------------- +#1650 failing nested section tests should not interfere with names of other +sections + Upper section + Second lower section with failure +------------------------------------------------------------------------------- +Misc.tests.cpp: +............................................................................... + +Misc.tests.cpp:: FAILED: +explicitly with message: + Failure in second lower section + ------------------------------------------------------------------------------- #748 - captures with unexpected exceptions outside assertions @@ -1380,6 +1406,6 @@ due to unexpected exception with message: Why would you throw a std::string? =============================================================================== -test cases: 289 | 215 passed | 70 failed | 4 failed as expected -assertions: 1539 | 1387 passed | 131 failed | 21 failed as expected +test cases: 290 | 215 passed | 71 failed | 4 failed as expected +assertions: 1542 | 1388 passed | 133 failed | 21 failed as expected diff --git a/projects/SelfTest/Baselines/console.sw.approved.txt b/projects/SelfTest/Baselines/console.sw.approved.txt index cd65bc1709..994926582e 100644 --- a/projects/SelfTest/Baselines/console.sw.approved.txt +++ b/projects/SelfTest/Baselines/console.sw.approved.txt @@ -172,6 +172,43 @@ Compilation.tests.cpp:: PASSED: with expansion: true +------------------------------------------------------------------------------- +#1650 failing nested section tests should not interfere with names of other +sections + Upper section + Lower section with success +------------------------------------------------------------------------------- +Misc.tests.cpp: +............................................................................... + +Misc.tests.cpp:: PASSED: + +------------------------------------------------------------------------------- +#1650 failing nested section tests should not interfere with names of other +sections + Upper section + Lower section with failure +------------------------------------------------------------------------------- +Misc.tests.cpp: +............................................................................... + +Misc.tests.cpp:: FAILED: +explicitly with message: + Failure in lower section + +------------------------------------------------------------------------------- +#1650 failing nested section tests should not interfere with names of other +sections + Upper section + Second lower section with failure +------------------------------------------------------------------------------- +Misc.tests.cpp: +............................................................................... + +Misc.tests.cpp:: FAILED: +explicitly with message: + Failure in second lower section + ------------------------------------------------------------------------------- #748 - captures with unexpected exceptions outside assertions @@ -12218,6 +12255,6 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 289 | 199 passed | 86 failed | 4 failed as expected -assertions: 1556 | 1387 passed | 148 failed | 21 failed as expected +test cases: 290 | 199 passed | 87 failed | 4 failed as expected +assertions: 1559 | 1388 passed | 150 failed | 21 failed as expected diff --git a/projects/SelfTest/Baselines/console.swa4.approved.txt b/projects/SelfTest/Baselines/console.swa4.approved.txt index bfa6675066..d6a30c8b37 100644 --- a/projects/SelfTest/Baselines/console.swa4.approved.txt +++ b/projects/SelfTest/Baselines/console.swa4.approved.txt @@ -173,185 +173,43 @@ with expansion: true ------------------------------------------------------------------------------- -#748 - captures with unexpected exceptions - outside assertions -------------------------------------------------------------------------------- -Exception.tests.cpp: -............................................................................... - -Exception.tests.cpp:: FAILED: -due to unexpected exception with messages: - answer := 42 - expected exception - -------------------------------------------------------------------------------- -#748 - captures with unexpected exceptions - inside REQUIRE_NOTHROW -------------------------------------------------------------------------------- -Exception.tests.cpp: -............................................................................... - -Exception.tests.cpp:: FAILED: - REQUIRE_NOTHROW( thisThrows() ) -due to unexpected exception with messages: - answer := 42 - expected exception - -------------------------------------------------------------------------------- -#748 - captures with unexpected exceptions - inside REQUIRE_THROWS -------------------------------------------------------------------------------- -Exception.tests.cpp: -............................................................................... - -Exception.tests.cpp:: PASSED: - REQUIRE_THROWS( thisThrows() ) -with message: - answer := 42 - -------------------------------------------------------------------------------- -#809 -------------------------------------------------------------------------------- -Compilation.tests.cpp: -............................................................................... - -Compilation.tests.cpp:: PASSED: - REQUIRE( 42 == f ) -with expansion: - 42 == {?} - -------------------------------------------------------------------------------- -#833 -------------------------------------------------------------------------------- -Compilation.tests.cpp: -............................................................................... - -Compilation.tests.cpp:: PASSED: - REQUIRE( a == t ) -with expansion: - 3 == 3 - -Compilation.tests.cpp:: PASSED: - CHECK( a == t ) -with expansion: - 3 == 3 - -Compilation.tests.cpp:: PASSED: - REQUIRE_THROWS( throws_int(true) ) - -Compilation.tests.cpp:: PASSED: - CHECK_THROWS_AS( throws_int(true), int ) - -Compilation.tests.cpp:: PASSED: - REQUIRE_NOTHROW( throws_int(false) ) - -Compilation.tests.cpp:: PASSED: - REQUIRE_THAT( "aaa", Catch::EndsWith("aaa") ) -with expansion: - "aaa" ends with: "aaa" - -Compilation.tests.cpp:: PASSED: - REQUIRE( templated_tests(3) ) -with expansion: - true - -------------------------------------------------------------------------------- -#835 -- errno should not be touched by Catch +#1650 failing nested section tests should not interfere with names of other +sections + Upper section + Lower section with success ------------------------------------------------------------------------------- Misc.tests.cpp: ............................................................................... -Misc.tests.cpp:: FAILED: - CHECK( f() == 0 ) -with expansion: - 1 == 0 - Misc.tests.cpp:: PASSED: - REQUIRE( errno == 1 ) -with expansion: - 1 == 1 ------------------------------------------------------------------------------- -#872 -------------------------------------------------------------------------------- -Compilation.tests.cpp: -............................................................................... - -Compilation.tests.cpp:: PASSED: - REQUIRE( x == 4 ) -with expansion: - {?} == 4 -with message: - dummy := 0 - -------------------------------------------------------------------------------- -#961 -- Dynamically created sections should all be reported - Looped section 0 +#1650 failing nested section tests should not interfere with names of other +sections + Upper section + Lower section with failure ------------------------------------------------------------------------------- Misc.tests.cpp: ............................................................................... -Misc.tests.cpp:: PASSED: -with message: - Everything is OK - -------------------------------------------------------------------------------- -#961 -- Dynamically created sections should all be reported - Looped section 1 -------------------------------------------------------------------------------- -Misc.tests.cpp: -............................................................................... - -Misc.tests.cpp:: PASSED: -with message: - Everything is OK - -------------------------------------------------------------------------------- -#961 -- Dynamically created sections should all be reported - Looped section 2 -------------------------------------------------------------------------------- -Misc.tests.cpp: -............................................................................... - -Misc.tests.cpp:: PASSED: -with message: - Everything is OK - -------------------------------------------------------------------------------- -#961 -- Dynamically created sections should all be reported - Looped section 3 -------------------------------------------------------------------------------- -Misc.tests.cpp: -............................................................................... - -Misc.tests.cpp:: PASSED: -with message: - Everything is OK +Misc.tests.cpp:: FAILED: +explicitly with message: + Failure in lower section ------------------------------------------------------------------------------- -#961 -- Dynamically created sections should all be reported - Looped section 4 +#1650 failing nested section tests should not interfere with names of other +sections + Upper section + Second lower section with failure ------------------------------------------------------------------------------- Misc.tests.cpp: ............................................................................... -Misc.tests.cpp:: PASSED: -with message: - Everything is OK - -------------------------------------------------------------------------------- -'Not' checks that should fail -------------------------------------------------------------------------------- -Condition.tests.cpp: -............................................................................... - -Condition.tests.cpp:: FAILED: - CHECK( false != false ) - -Condition.tests.cpp:: FAILED: - CHECK( true != true ) +Misc.tests.cpp:: FAILED: +explicitly with message: + Failure in second lower section =============================================================================== -test cases: 18 | 13 passed | 3 failed | 2 failed as expected -assertions: 40 | 33 passed | 4 failed | 3 failed as expected +test cases: 12 | 9 passed | 3 failed +assertions: 22 | 18 passed | 4 failed diff --git a/projects/SelfTest/Baselines/junit.sw.approved.txt b/projects/SelfTest/Baselines/junit.sw.approved.txt index 94f0138552..8461977119 100644 --- a/projects/SelfTest/Baselines/junit.sw.approved.txt +++ b/projects/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ - + @@ -28,6 +28,19 @@ Nor would this + + + +Failure in lower section +Misc.tests.cpp: + + + + +Failure in second lower section +Misc.tests.cpp: + + expected exception diff --git a/projects/SelfTest/Baselines/xml.sw.approved.txt b/projects/SelfTest/Baselines/xml.sw.approved.txt index dd2c3f4a96..a228d080ce 100644 --- a/projects/SelfTest/Baselines/xml.sw.approved.txt +++ b/projects/SelfTest/Baselines/xml.sw.approved.txt @@ -177,6 +177,33 @@ Nor would this + +
+
+ +
+ +
+
+
+ + Failure in lower section + + +
+ +
+
+
+ + Failure in second lower section + + +
+ +
+ +
@@ -14586,7 +14613,7 @@ loose text artifact
- + - + diff --git a/projects/SelfTest/UsageTests/Misc.tests.cpp b/projects/SelfTest/UsageTests/Misc.tests.cpp index 858607f8f4..36a80cebb5 100644 --- a/projects/SelfTest/UsageTests/Misc.tests.cpp +++ b/projects/SelfTest/UsageTests/Misc.tests.cpp @@ -135,14 +135,18 @@ TEST_CASE( "even more nested SECTION tests", "[sections]" ) { } } -TEST_CASE( "#1650 - failing nested section tests should not interfere with names of other sections", - "[.][failing][sections]"){ +TEST_CASE( "#1650 failing nested section tests should not interfere with names of other sections", + "[.][failing][sections]"){ + SECTION("Upper section"){ SECTION("Lower section with success"){ SUCCEED(); } SECTION("Lower section with failure"){ - FAIL(); + FAIL("Failure in lower section"); + } + SECTION("Second lower section with failure"){ + FAIL("Failure in second lower section"); } } } From f7af81561e791f8a618b8611522846ccd1180235 Mon Sep 17 00:00:00 2001 From: Daniel Bak Date: Thu, 6 Jun 2019 13:11:11 -0600 Subject: [PATCH 3/3] Added captures to errnoParser and nanParser as these were not capturing correctly on my machine. --- scripts/approvalTests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/approvalTests.py b/scripts/approvalTests.py index bb01e6d570..a6d077bd6a 100755 --- a/scripts/approvalTests.py +++ b/scripts/approvalTests.py @@ -48,6 +48,8 @@ \(\*__error\(\)\) | \(\*_errno\(\)\) + | + \(\*__errno\(\)\) ''', re.VERBOSE) sinceEpochParser = re.compile(r'\d+ .+ since epoch') infParser = re.compile(r''' @@ -66,6 +68,8 @@ | \(__builtin_nanf\ \(""\)\) # Linux (ubuntu) NAN macro | + \(__builtin_nanf \(""\)\) + | __builtin_nanf\("0x"\) # The weird content of the brackets is there because a different parser has already ran before this one ''', re.VERBOSE)