catchorg#1175 - don't list hidden tests by default

CMakeLists.txt

@@ -357,16 +357,23 @@ if (BUILD_TESTING AND NOT_SUBPROJECT)
     add_test(NAME RunTests COMMAND $<TARGET_FILE:SelfTest>)
 
     add_test(NAME ListTests COMMAND $<TARGET_FILE:SelfTest> --list-tests --verbosity high)
-    set_tests_properties(ListTests PROPERTIES PASS_REGULAR_EXPRESSION "[0-9]+ test cases")
+    set_tests_properties(ListTests PROPERTIES 
+        PASS_REGULAR_EXPRESSION "[0-9]+ test cases"
+        FAIL_REGULAR_EXPRESSION "Hidden Test"
+    )
 
     add_test(NAME ListTags COMMAND $<TARGET_FILE:SelfTest> --list-tags)
-    set_tests_properties(ListTags PROPERTIES PASS_REGULAR_EXPRESSION "[0-9]+ tags")
+    set_tests_properties(ListTags PROPERTIES 
+        PASS_REGULAR_EXPRESSION "[0-9]+ tags" 
+        FAIL_REGULAR_EXPRESSION "[.]")
 
     add_test(NAME ListReporters COMMAND $<TARGET_FILE:SelfTest> --list-reporters)
     set_tests_properties(ListReporters PROPERTIES PASS_REGULAR_EXPRESSION "Available reporters:")
 
     add_test(NAME ListTestNamesOnly COMMAND $<TARGET_FILE:SelfTest> --list-test-names-only)
-    set_tests_properties(ListTestNamesOnly PROPERTIES PASS_REGULAR_EXPRESSION "Regex string matcher")
+    set_tests_properties(ListTestNamesOnly PROPERTIES 
+        PASS_REGULAR_EXPRESSION "Regex string matcher"
+        FAIL_REGULAR_EXPRESSION "Hidden Test")
 
     add_test(NAME NoAssertions COMMAND $<TARGET_FILE:SelfTest> -w NoAssertions)
     set_tests_properties(NoAssertions PROPERTIES PASS_REGULAR_EXPRESSION "No assertions in test case")

include/internal/catch_config.cpp

@@ -15,12 +15,16 @@ namespace Catch {
     :   m_data( data ),
         m_stream( openStream() )
     {
-        if( !data.testsOrTags.empty() ) {
-            TestSpecParser parser( ITagAliasRegistry::get() );
+        TestSpecParser parser(ITagAliasRegistry::get());
+        if (data.testsOrTags.empty()) {
+            parser.parse("~[.]"); // All not hidden tests
+        }
+        else {
+            m_hasTestFilters = true;
             for( auto const& testOrTags : data.testsOrTags )
                 parser.parse( testOrTags );
-            m_testSpec = parser.testSpec();
         }
+        m_testSpec = parser.testSpec();
     }
 
     std::string const& Config::getFilename() const {
@@ -39,6 +43,7 @@ namespace Catch {
     std::vector<std::string> const& Config::getSectionsToRun() const { return m_data.sectionsToRun; }
 
     TestSpec const& Config::testSpec() const { return m_testSpec; }
+    bool Config::hasTestFilters() const { return m_hasTestFilters; }
 
     bool Config::showHelp() const { return m_data.showHelp; }
 

include/internal/catch_config.hpp

@@ -82,6 +82,7 @@ namespace Catch {
         std::vector<std::string> const& getSectionsToRun() const override;
 
         virtual TestSpec const& testSpec() const override;
+        bool hasTestFilters() const override;
 
         bool showHelp() const;
 
@@ -109,6 +110,7 @@ namespace Catch {
 
         std::unique_ptr<IStream const> m_stream;
         TestSpec m_testSpec;
+        bool m_hasTestFilters = false;
     };
 
 } // end namespace Catch

include/internal/catch_interfaces_config.h

@@ -68,6 +68,7 @@ namespace Catch {
         virtual bool showInvisibles() const = 0;
         virtual ShowDurations::OrNot showDurations() const = 0;
         virtual TestSpec const& testSpec() const = 0;
+        virtual bool hasTestFilters() const = 0;
         virtual RunTests::InWhatOrder runOrder() const = 0;
         virtual unsigned int rngSeed() const = 0;
         virtual int benchmarkResolutionMultiple() const = 0;

include/internal/catch_list.cpp

@@ -28,11 +28,10 @@ namespace Catch {
 
     std::size_t listTests( Config const& config ) {
         TestSpec testSpec = config.testSpec();
-        if( config.testSpec().hasFilters() )
+        if( config.hasTestFilters() )
             Catch::cout() << "Matching test cases:\n";
         else {
             Catch::cout() << "All available test cases:\n";
-            testSpec = TestSpecParser( ITagAliasRegistry::get() ).parse( "*" ).testSpec();
         }
 
         auto matchedTestCases = filterTests( getAllTestCasesSorted( config ), testSpec, config );
@@ -54,7 +53,7 @@ namespace Catch {
                 Catch::cout() << Column( testCaseInfo.tagsAsString() ).indent( 6 ) << "\n";
         }
 
-        if( !config.testSpec().hasFilters() )
+        if( !config.hasTestFilters() )
             Catch::cout() << pluralise( matchedTestCases.size(), "test case" ) << '\n' << std::endl;
         else
             Catch::cout() << pluralise( matchedTestCases.size(), "matching test case" ) << '\n' << std::endl;
@@ -63,8 +62,6 @@ namespace Catch {
 
     std::size_t listTestsNamesOnly( Config const& config ) {
         TestSpec testSpec = config.testSpec();
-        if( !config.testSpec().hasFilters() )
-            testSpec = TestSpecParser( ITagAliasRegistry::get() ).parse( "*" ).testSpec();
         std::size_t matchedTests = 0;
         std::vector<TestCase> matchedTestCases = filterTests( getAllTestCasesSorted( config ), testSpec, config );
         for( auto const& testCaseInfo : matchedTestCases ) {
@@ -94,11 +91,10 @@ namespace Catch {
 
     std::size_t listTags( Config const& config ) {
         TestSpec testSpec = config.testSpec();
-        if( config.testSpec().hasFilters() )
+        if( config.hasTestFilters() )
             Catch::cout() << "Tags for matching test cases:\n";
         else {
             Catch::cout() << "All available tags:\n";
-            testSpec = TestSpecParser( ITagAliasRegistry::get() ).parse( "*" ).testSpec();
         }
 
         std::map<std::string, TagInfo> tagCounts;

include/internal/catch_session.cpp

@@ -70,8 +70,6 @@ namespace Catch {
             context.testGroupStarting(config->name(), 1, 1);
 
             TestSpec testSpec = config->testSpec();
-            if (!testSpec.hasFilters())
-                testSpec = TestSpecParser(ITagAliasRegistry::get()).parse("~[.]").testSpec(); // All not hidden tests
 
             auto const& allTestCases = getAllTestCasesSorted(*config);
             for (auto const& testCase : allTestCases) {

projects/SelfTest/Baselines/compact.sw.approved.txt

@@ -9,6 +9,7 @@ Compilation.tests.cpp:<line number>: passed: t1 < t2 for: {?} < {?}
 Compilation.tests.cpp:<line number>: passed: t1 > t2 for: {?} > {?}
 Compilation.tests.cpp:<line number>: passed: t1 <= t2 for: {?} <= {?}
 Compilation.tests.cpp:<line number>: passed: t1 >= t2 for: {?} >= {?}
+Misc.tests.cpp:<line number>: passed:
 Exception.tests.cpp:<line number>: failed: unexpected exception with message: 'answer := 42' with 1 message: 'expected exception'
 Exception.tests.cpp:<line number>: failed: unexpected exception with message: 'answer := 42'; expression was: thisThrows() with 1 message: 'expected exception'
 Exception.tests.cpp:<line number>: passed: thisThrows() with 1 message: 'answer := 42'
@@ -514,13 +515,17 @@ CmdLine.tests.cpp:<line number>: passed: config.shouldDebugBreak == false for: f
 CmdLine.tests.cpp:<line number>: passed: config.abortAfter == -1 for: -1 == -1
 CmdLine.tests.cpp:<line number>: passed: config.noThrow == false for: false == false
 CmdLine.tests.cpp:<line number>: passed: config.reporterNames.empty() for: true
+CmdLine.tests.cpp:<line number>: passed: !(cfg.hasTestFilters()) for: !false
 CmdLine.tests.cpp:<line number>: passed: result for: {?}
+CmdLine.tests.cpp:<line number>: passed: cfg.hasTestFilters() for: true
 CmdLine.tests.cpp:<line number>: passed: cfg.testSpec().matches(fakeTestCase("notIncluded")) == false for: false == false
 CmdLine.tests.cpp:<line number>: passed: cfg.testSpec().matches(fakeTestCase("test1")) for: true
 CmdLine.tests.cpp:<line number>: passed: result for: {?}
+CmdLine.tests.cpp:<line number>: passed: cfg.hasTestFilters() for: true
 CmdLine.tests.cpp:<line number>: passed: cfg.testSpec().matches(fakeTestCase("test1")) == false for: false == false
 CmdLine.tests.cpp:<line number>: passed: cfg.testSpec().matches(fakeTestCase("alwaysIncluded")) for: true
 CmdLine.tests.cpp:<line number>: passed: result for: {?}
+CmdLine.tests.cpp:<line number>: passed: cfg.hasTestFilters() for: true
 CmdLine.tests.cpp:<line number>: passed: cfg.testSpec().matches(fakeTestCase("test1")) == false for: false == false
 CmdLine.tests.cpp:<line number>: passed: cfg.testSpec().matches(fakeTestCase("alwaysIncluded")) for: true
 CmdLine.tests.cpp:<line number>: passed: cli.parse({"test", "-r", "console"}) for: {?}

projects/SelfTest/Baselines/console.std.approved.txt

@@ -1064,6 +1064,6 @@ with expansion:
   "first" == "second"
 
 ===============================================================================
-test cases: 198 | 147 passed |  47 failed |  4 failed as expected
-assertions: 999 | 873 passed | 105 failed | 21 failed as expected
+test cases:  199 | 148 passed |  47 failed |  4 failed as expected
+assertions: 1004 | 878 passed | 105 failed | 21 failed as expected
 

projects/SelfTest/Baselines/console.sw.approved.txt

@@ -93,6 +93,15 @@ PASSED:
 with expansion:
   {?} >= {?}
 
+-------------------------------------------------------------------------------
+#1175 - Hidden Test
+-------------------------------------------------------------------------------
+Misc.tests.cpp:<line number>
+...............................................................................
+
+Misc.tests.cpp:<line number>:
+PASSED:
+
 -------------------------------------------------------------------------------
 #748 - captures with unexpected exceptions
   outside assertions
@@ -3965,6 +3974,12 @@ PASSED:
 with expansion:
   true
 
+CmdLine.tests.cpp:<line number>:
+PASSED:
+  CHECK_FALSE( cfg.hasTestFilters() )
+with expansion:
+  !false
+
 -------------------------------------------------------------------------------
 Process can be configured on command line
   test lists
@@ -3979,6 +3994,12 @@ PASSED:
 with expansion:
   {?}
 
+CmdLine.tests.cpp:<line number>:
+PASSED:
+  REQUIRE( cfg.hasTestFilters() )
+with expansion:
+  true
+
 CmdLine.tests.cpp:<line number>:
 PASSED:
   REQUIRE( cfg.testSpec().matches(fakeTestCase("notIncluded")) == false )
@@ -4005,6 +4026,12 @@ PASSED:
 with expansion:
   {?}
 
+CmdLine.tests.cpp:<line number>:
+PASSED:
+  REQUIRE( cfg.hasTestFilters() )
+with expansion:
+  true
+
 CmdLine.tests.cpp:<line number>:
 PASSED:
   REQUIRE( cfg.testSpec().matches(fakeTestCase("test1")) == false )
@@ -4031,6 +4058,12 @@ PASSED:
 with expansion:
   {?}
 
+CmdLine.tests.cpp:<line number>:
+PASSED:
+  REQUIRE( cfg.hasTestFilters() )
+with expansion:
+  true
+
 CmdLine.tests.cpp:<line number>:
 PASSED:
   REQUIRE( cfg.testSpec().matches(fakeTestCase("test1")) == false )
@@ -8466,6 +8499,6 @@ Misc.tests.cpp:<line number>:
 PASSED:
 
 ===============================================================================
-test cases:  198 | 133 passed |  61 failed |  4 failed as expected
-assertions: 1010 | 869 passed | 120 failed | 21 failed as expected
+test cases:  199 | 134 passed |  61 failed |  4 failed as expected
+assertions: 1015 | 874 passed | 120 failed | 21 failed as expected
 

projects/SelfTest/Baselines/console.swa4.approved.txt

@@ -93,6 +93,15 @@ PASSED:
 with expansion:
   {?} >= {?}
 
+-------------------------------------------------------------------------------
+#1175 - Hidden Test
+-------------------------------------------------------------------------------
+Misc.tests.cpp:<line number>
+...............................................................................
+
+Misc.tests.cpp:<line number>:
+PASSED:
+
 -------------------------------------------------------------------------------
 #748 - captures with unexpected exceptions
   outside assertions
@@ -299,6 +308,6 @@ with expansion:
   !true
 
 ===============================================================================
-test cases: 11 |  8 passed | 1 failed | 2 failed as expected
-assertions: 34 | 27 passed | 4 failed | 3 failed as expected
+test cases: 12 |  9 passed | 1 failed | 2 failed as expected
+assertions: 35 | 28 passed | 4 failed | 3 failed as expected
 

projects/SelfTest/Baselines/junit.sw.approved.txt

@@ -1,11 +1,12 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <testsuitesloose text artifact
 >
-  <testsuite name="<exe-name>" errors="15" failures="106" tests="1011" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+  <testsuite name="<exe-name>" errors="15" failures="106" tests="1016" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
     <testcase classname="<exe-name>.global" name="# A test name that starts with a #" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="#1005: Comparing pointer to int and long (NULL can be either on various systems)" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="#1027" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="#1147" time="{duration}"/>
+    <testcase classname="<exe-name>.global" name="#1175 - Hidden Test" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="#748 - captures with unexpected exceptions/outside assertions" time="{duration}">
       <error type="TEST_CASE">
 expected exception