Merge branch 'section-OOO-test' into section-OOO-fix

catchorg · Jun 6, 2019 · 010393f · 010393f
2 parents dba9e01 + bb42ca6
commit 010393f
Show file tree

Hide file tree

Showing 8 changed files with 154 additions and 170 deletions.
diff --git a/projects/SelfTest/Baselines/compact.sw.approved.txt b/projects/SelfTest/Baselines/compact.sw.approved.txt
@@ -21,6 +21,9 @@ This would not be caught previously
 Nor would this
 Tricky.tests.cpp:<line number>: failed: explicitly with 1 message: '1514'
 Compilation.tests.cpp:<line number>: passed: std::is_same<TypeList<int>, TypeList<int>>::value for: true
+Misc.tests.cpp:<line number>: passed:
+Misc.tests.cpp:<line number>: failed: explicitly with 1 message: 'Failure in lower section'
+Misc.tests.cpp:<line number>: failed: explicitly with 1 message: 'Failure in second lower section'
 Exception.tests.cpp:<line number>: failed: unexpected exception with message: 'answer := 42' with 1 message: 'expected exception'
 Exception.tests.cpp:<line number>: failed: unexpected exception with message: 'answer := 42'; expression was: thisThrows() with 1 message: 'expected exception'
 Exception.tests.cpp:<line number>: passed: thisThrows() with 1 message: 'answer := 42'
@@ -1629,5 +1632,5 @@ Misc.tests.cpp:<line number>: passed: v.size() == 5 for: 5 == 5
 Misc.tests.cpp:<line number>: passed: v.capacity() >= 5 for: 5 >= 5
 Misc.tests.cpp:<line number>: passed:
 Misc.tests.cpp:<line number>: passed:
-Failed 86 test cases, failed 148 assertions.
+Failed 87 test cases, failed 150 assertions.
 
diff --git a/projects/SelfTest/Baselines/console.std.approved.txt b/projects/SelfTest/Baselines/console.std.approved.txt
@@ -28,6 +28,32 @@ Tricky.tests.cpp:<line number>: FAILED:
 explicitly with message:
   1514
 
+-------------------------------------------------------------------------------
+#1650 failing nested section tests should not interfere with names of other
+sections
+  Upper section
+  Lower section with failure
+-------------------------------------------------------------------------------
+Misc.tests.cpp:<line number>
+...............................................................................
+
+Misc.tests.cpp:<line number>: FAILED:
+explicitly with message:
+  Failure in lower section
+
+-------------------------------------------------------------------------------
+#1650 failing nested section tests should not interfere with names of other
+sections
+  Upper section
+  Second lower section with failure
+-------------------------------------------------------------------------------
+Misc.tests.cpp:<line number>
+...............................................................................
+
+Misc.tests.cpp:<line number>: FAILED:
+explicitly with message:
+  Failure in second lower section
+
 -------------------------------------------------------------------------------
 #748 - captures with unexpected exceptions
   outside assertions
@@ -1380,6 +1406,6 @@ due to unexpected exception with message:
   Why would you throw a std::string?
 
 ===============================================================================
-test cases:  289 |  215 passed |  70 failed |  4 failed as expected
-assertions: 1539 | 1387 passed | 131 failed | 21 failed as expected
+test cases:  290 |  215 passed |  71 failed |  4 failed as expected
+assertions: 1542 | 1388 passed | 133 failed | 21 failed as expected
 
diff --git a/projects/SelfTest/Baselines/console.sw.approved.txt b/projects/SelfTest/Baselines/console.sw.approved.txt
@@ -172,6 +172,43 @@ Compilation.tests.cpp:<line number>: PASSED:
 with expansion:
   true
 
+-------------------------------------------------------------------------------
+#1650 failing nested section tests should not interfere with names of other
+sections
+  Upper section
+  Lower section with success
+-------------------------------------------------------------------------------
+Misc.tests.cpp:<line number>
+...............................................................................
+
+Misc.tests.cpp:<line number>: PASSED:
+
+-------------------------------------------------------------------------------
+#1650 failing nested section tests should not interfere with names of other
+sections
+  Upper section
+  Lower section with failure
+-------------------------------------------------------------------------------
+Misc.tests.cpp:<line number>
+...............................................................................
+
+Misc.tests.cpp:<line number>: FAILED:
+explicitly with message:
+  Failure in lower section
+
+-------------------------------------------------------------------------------
+#1650 failing nested section tests should not interfere with names of other
+sections
+  Upper section
+  Second lower section with failure
+-------------------------------------------------------------------------------
+Misc.tests.cpp:<line number>
+...............................................................................
+
+Misc.tests.cpp:<line number>: FAILED:
+explicitly with message:
+  Failure in second lower section
+
 -------------------------------------------------------------------------------
 #748 - captures with unexpected exceptions
   outside assertions
@@ -12218,6 +12255,6 @@ Misc.tests.cpp:<line number>
 Misc.tests.cpp:<line number>: PASSED:
 
 ===============================================================================
-test cases:  289 |  199 passed |  86 failed |  4 failed as expected
-assertions: 1556 | 1387 passed | 148 failed | 21 failed as expected
+test cases:  290 |  199 passed |  87 failed |  4 failed as expected
+assertions: 1559 | 1388 passed | 150 failed | 21 failed as expected
 
diff --git a/projects/SelfTest/Baselines/console.swa4.approved.txt b/projects/SelfTest/Baselines/console.swa4.approved.txt
@@ -173,185 +173,43 @@ with expansion:
   true
 
 -------------------------------------------------------------------------------
-#748 - captures with unexpected exceptions
-  outside assertions
--------------------------------------------------------------------------------
-Exception.tests.cpp:<line number>
-...............................................................................
-
-Exception.tests.cpp:<line number>: FAILED:
-due to unexpected exception with messages:
-  answer := 42
-  expected exception
-
--------------------------------------------------------------------------------
-#748 - captures with unexpected exceptions
-  inside REQUIRE_NOTHROW
--------------------------------------------------------------------------------
-Exception.tests.cpp:<line number>
-...............................................................................
-
-Exception.tests.cpp:<line number>: FAILED:
-  REQUIRE_NOTHROW( thisThrows() )
-due to unexpected exception with messages:
-  answer := 42
-  expected exception
-
--------------------------------------------------------------------------------
-#748 - captures with unexpected exceptions
-  inside REQUIRE_THROWS
--------------------------------------------------------------------------------
-Exception.tests.cpp:<line number>
-...............................................................................
-
-Exception.tests.cpp:<line number>: PASSED:
-  REQUIRE_THROWS( thisThrows() )
-with message:
-  answer := 42
-
--------------------------------------------------------------------------------
-#809
--------------------------------------------------------------------------------
-Compilation.tests.cpp:<line number>
-...............................................................................
-
-Compilation.tests.cpp:<line number>: PASSED:
-  REQUIRE( 42 == f )
-with expansion:
-  42 == {?}
-
--------------------------------------------------------------------------------
-#833
--------------------------------------------------------------------------------
-Compilation.tests.cpp:<line number>
-...............................................................................
-
-Compilation.tests.cpp:<line number>: PASSED:
-  REQUIRE( a == t )
-with expansion:
-  3 == 3
-
-Compilation.tests.cpp:<line number>: PASSED:
-  CHECK( a == t )
-with expansion:
-  3 == 3
-
-Compilation.tests.cpp:<line number>: PASSED:
-  REQUIRE_THROWS( throws_int(true) )
-
-Compilation.tests.cpp:<line number>: PASSED:
-  CHECK_THROWS_AS( throws_int(true), int )
-
-Compilation.tests.cpp:<line number>: PASSED:
-  REQUIRE_NOTHROW( throws_int(false) )
-
-Compilation.tests.cpp:<line number>: PASSED:
-  REQUIRE_THAT( "aaa", Catch::EndsWith("aaa") )
-with expansion:
-  "aaa" ends with: "aaa"
-
-Compilation.tests.cpp:<line number>: PASSED:
-  REQUIRE( templated_tests<int>(3) )
-with expansion:
-  true
-
--------------------------------------------------------------------------------
-#835 -- errno should not be touched by Catch
+#1650 failing nested section tests should not interfere with names of other
+sections
+  Upper section
+  Lower section with success
 -------------------------------------------------------------------------------
 Misc.tests.cpp:<line number>
 ...............................................................................
 
-Misc.tests.cpp:<line number>: FAILED:
-  CHECK( f() == 0 )
-with expansion:
-  1 == 0
-
 Misc.tests.cpp:<line number>: PASSED:
-  REQUIRE( errno == 1 )
-with expansion:
-  1 == 1
 
 -------------------------------------------------------------------------------
-#872
--------------------------------------------------------------------------------
-Compilation.tests.cpp:<line number>
-...............................................................................
-
-Compilation.tests.cpp:<line number>: PASSED:
-  REQUIRE( x == 4 )
-with expansion:
-  {?} == 4
-with message:
-  dummy := 0
-
--------------------------------------------------------------------------------
-#961 -- Dynamically created sections should all be reported
-  Looped section 0
+#1650 failing nested section tests should not interfere with names of other
+sections
+  Upper section
+  Lower section with failure
 -------------------------------------------------------------------------------
 Misc.tests.cpp:<line number>
 ...............................................................................
 
-Misc.tests.cpp:<line number>: PASSED:
-with message:
-  Everything is OK
-
--------------------------------------------------------------------------------
-#961 -- Dynamically created sections should all be reported
-  Looped section 1
--------------------------------------------------------------------------------
-Misc.tests.cpp:<line number>
-...............................................................................
-
-Misc.tests.cpp:<line number>: PASSED:
-with message:
-  Everything is OK
-
--------------------------------------------------------------------------------
-#961 -- Dynamically created sections should all be reported
-  Looped section 2
--------------------------------------------------------------------------------
-Misc.tests.cpp:<line number>
-...............................................................................
-
-Misc.tests.cpp:<line number>: PASSED:
-with message:
-  Everything is OK
-
--------------------------------------------------------------------------------
-#961 -- Dynamically created sections should all be reported
-  Looped section 3
--------------------------------------------------------------------------------
-Misc.tests.cpp:<line number>
-...............................................................................
-
-Misc.tests.cpp:<line number>: PASSED:
-with message:
-  Everything is OK
+Misc.tests.cpp:<line number>: FAILED:
+explicitly with message:
+  Failure in lower section
 
 -------------------------------------------------------------------------------
-#961 -- Dynamically created sections should all be reported
-  Looped section 4
+#1650 failing nested section tests should not interfere with names of other
+sections
+  Upper section
+  Second lower section with failure
 -------------------------------------------------------------------------------
 Misc.tests.cpp:<line number>
 ...............................................................................
 
-Misc.tests.cpp:<line number>: PASSED:
-with message:
-  Everything is OK
-
--------------------------------------------------------------------------------
-'Not' checks that should fail
--------------------------------------------------------------------------------
-Condition.tests.cpp:<line number>
-...............................................................................
-
-Condition.tests.cpp:<line number>: FAILED:
-  CHECK( false != false )
-
-Condition.tests.cpp:<line number>: FAILED:
-  CHECK( true != true )
+Misc.tests.cpp:<line number>: FAILED:
+explicitly with message:
+  Failure in second lower section
 
 ===============================================================================
-test cases: 18 | 13 passed | 3 failed | 2 failed as expected
-assertions: 40 | 33 passed | 4 failed | 3 failed as expected
+test cases: 12 |  9 passed | 3 failed
+assertions: 22 | 18 passed | 4 failed
 
diff --git a/projects/SelfTest/Baselines/junit.sw.approved.txt b/projects/SelfTest/Baselines/junit.sw.approved.txt
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <testsuitesloose text artifact
 >
-  <testsuite name="<exe-name>" errors="17" failures="132" tests="1557" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+  <testsuite name="<exe-name>" errors="17" failures="134" tests="1560" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
     <properties>
       <property name="filters" value="~[!nonportable]~[!benchmark]~[approvals]"/>
       <property name="random-seed" value="1"/>
@@ -28,6 +28,19 @@ Nor would this
       </system-err>
     </testcase>
     <testcase classname="<exe-name>.global" name="#1548" time="{duration}"/>
+    <testcase classname="<exe-name>.global" name="#1650 failing nested section tests should not interfere with names of other sections/Upper section/Lower section with success" time="{duration}"/>
+    <testcase classname="<exe-name>.global" name="#1650 failing nested section tests should not interfere with names of other sections/Upper section/Lower section with failure" time="{duration}">
+      <failure type="FAIL">
+Failure in lower section
+Misc.tests.cpp:<line number>
+      </failure>
+    </testcase>
+    <testcase classname="<exe-name>.global" name="#1650 failing nested section tests should not interfere with names of other sections/Upper section/Second lower section with failure" time="{duration}">
+      <failure type="FAIL">
+Failure in second lower section
+Misc.tests.cpp:<line number>
+      </failure>
+    </testcase>
     <testcase classname="<exe-name>.global" name="#748 - captures with unexpected exceptions/outside assertions" time="{duration}">
       <error type="TEST_CASE">
 expected exception

diff --git a/projects/SelfTest/Baselines/xml.sw.approved.txt b/projects/SelfTest/Baselines/xml.sw.approved.txt
@@ -177,6 +177,33 @@ Nor would this
       </Expression>
       <OverallResult success="true"/>
     </TestCase>
+    <TestCase name="#1650 failing nested section tests should not interfere with names of other sections" tags="[.][failing][sections]" filename="projects/<exe-name>/UsageTests/Misc.tests.cpp" >
+      <Section name="Upper section" filename="projects/<exe-name>/UsageTests/Misc.tests.cpp" >
+        <Section name="Lower section with success" filename="projects/<exe-name>/UsageTests/Misc.tests.cpp" >
+          <OverallResults successes="1" failures="0" expectedFailures="0"/>
+        </Section>
+        <OverallResults successes="1" failures="0" expectedFailures="0"/>
+      </Section>
+      <Section name="Upper section" filename="projects/<exe-name>/UsageTests/Misc.tests.cpp" >
+        <Section name="Lower section with failure" filename="projects/<exe-name>/UsageTests/Misc.tests.cpp" >
+          <Failure filename="projects/<exe-name>/UsageTests/Misc.tests.cpp" >
+            Failure in lower section
+          </Failure>
+          <OverallResults successes="0" failures="1" expectedFailures="0"/>
+        </Section>
+        <OverallResults successes="0" failures="1" expectedFailures="0"/>
+      </Section>
+      <Section name="Upper section" filename="projects/<exe-name>/UsageTests/Misc.tests.cpp" >
+        <Section name="Second lower section with failure" filename="projects/<exe-name>/UsageTests/Misc.tests.cpp" >
+          <Failure filename="projects/<exe-name>/UsageTests/Misc.tests.cpp" >
+            Failure in second lower section
+          </Failure>
+          <OverallResults successes="0" failures="1" expectedFailures="0"/>
+        </Section>
+        <OverallResults successes="0" failures="1" expectedFailures="0"/>
+      </Section>
+      <OverallResult success="false"/>
+    </TestCase>
     <TestCase name="#748 - captures with unexpected exceptions" tags="[!shouldfail][!throws][.][failing]" filename="projects/<exe-name>/UsageTests/Exception.tests.cpp" >
       <Section name="outside assertions" filename="projects/<exe-name>/UsageTests/Exception.tests.cpp" >
         <Info>
@@ -14586,7 +14613,7 @@ loose text artifact
       </Section>
       <OverallResult success="true"/>
     </TestCase>
-    <OverallResults successes="1387" failures="149" expectedFailures="21"/>
+    <OverallResults successes="1388" failures="151" expectedFailures="21"/>
   </Group>
-  <OverallResults successes="1387" failures="148" expectedFailures="21"/>
+  <OverallResults successes="1388" failures="150" expectedFailures="21"/>
 </Catch>