https://github.com/catchorg/Catch2/issues/1175 - don't list hidden tests by default

CMakeLists.txt

@@ -357,16 +357,23 @@ if (BUILD_TESTING AND NOT_SUBPROJECT)
     add_test(NAME RunTests COMMAND $<TARGET_FILE:SelfTest>)
 
     add_test(NAME ListTests COMMAND $<TARGET_FILE:SelfTest> --list-tests --verbosity high)
-    set_tests_properties(ListTests PROPERTIES PASS_REGULAR_EXPRESSION "[0-9]+ test cases")
+    set_tests_properties(ListTests PROPERTIES 
+        PASS_REGULAR_EXPRESSION "[0-9]+ test cases"
+        FAIL_REGULAR_EXPRESSION "Hidden Test"
+    )
 
     add_test(NAME ListTags COMMAND $<TARGET_FILE:SelfTest> --list-tags)
-    set_tests_properties(ListTags PROPERTIES PASS_REGULAR_EXPRESSION "[0-9]+ tags")
+    set_tests_properties(ListTags PROPERTIES 
+        PASS_REGULAR_EXPRESSION "[0-9]+ tags" 
+        FAIL_REGULAR_EXPRESSION "[.]")
 
     add_test(NAME ListReporters COMMAND $<TARGET_FILE:SelfTest> --list-reporters)
     set_tests_properties(ListReporters PROPERTIES PASS_REGULAR_EXPRESSION "Available reporters:")
 
     add_test(NAME ListTestNamesOnly COMMAND $<TARGET_FILE:SelfTest> --list-test-names-only)
-    set_tests_properties(ListTestNamesOnly PROPERTIES PASS_REGULAR_EXPRESSION "Regex string matcher")
+    set_tests_properties(ListTestNamesOnly PROPERTIES 
+        PASS_REGULAR_EXPRESSION "Regex string matcher"
+        FAIL_REGULAR_EXPRESSION "Hidden Test")
 
     add_test(NAME NoAssertions COMMAND $<TARGET_FILE:SelfTest> -w NoAssertions)
     set_tests_properties(NoAssertions PROPERTIES PASS_REGULAR_EXPRESSION "No assertions in test case")

include/internal/catch_config.cpp

@@ -15,12 +15,16 @@ namespace Catch {
     :   m_data( data ),
         m_stream( openStream() )
     {
-        if( !data.testsOrTags.empty() ) {
         TestSpecParser parser(ITagAliasRegistry::get());
+        if (data.testsOrTags.empty()) {
+            parser.parse("~[.]"); // All not hidden tests
+        }
+        else {
+            m_hasTestFilters = true;
             for( auto const& testOrTags : data.testsOrTags )
                 parser.parse( testOrTags );
-            m_testSpec = parser.testSpec();
         }
+        m_testSpec = parser.testSpec();
     }
 
     std::string const& Config::getFilename() const {
@@ -39,6 +43,7 @@ namespace Catch {
     std::vector<std::string> const& Config::getSectionsToRun() const { return m_data.sectionsToRun; }
 
     TestSpec const& Config::testSpec() const { return m_testSpec; }
+    bool Config::hasTestFilters() const { return m_hasTestFilters; }
 
     bool Config::showHelp() const { return m_data.showHelp; }
 

include/internal/catch_config.hpp

@@ -82,6 +82,7 @@ namespace Catch {
         std::vector<std::string> const& getSectionsToRun() const override;
 
         virtual TestSpec const& testSpec() const override;
+        bool hasTestFilters() const override;
 
         bool showHelp() const;
 
@@ -109,6 +110,7 @@ namespace Catch {
 
         std::unique_ptr<IStream const> m_stream;
         TestSpec m_testSpec;
+        bool m_hasTestFilters = false;
     };
 
 } // end namespace Catch

include/internal/catch_interfaces_config.h

@@ -68,6 +68,7 @@ namespace Catch {
         virtual bool showInvisibles() const = 0;
         virtual ShowDurations::OrNot showDurations() const = 0;
         virtual TestSpec const& testSpec() const = 0;
+        virtual bool hasTestFilters() const = 0;
         virtual RunTests::InWhatOrder runOrder() const = 0;
         virtual unsigned int rngSeed() const = 0;
         virtual int benchmarkResolutionMultiple() const = 0;

include/internal/catch_list.cpp

@@ -28,11 +28,10 @@ namespace Catch {
 
     std::size_t listTests( Config const& config ) {
         TestSpec testSpec = config.testSpec();
-        if( config.testSpec().hasFilters() )
+        if( config.hasTestFilters() )
             Catch::cout() << "Matching test cases:\n";
         else {
             Catch::cout() << "All available test cases:\n";
-            testSpec = TestSpecParser( ITagAliasRegistry::get() ).parse( "*" ).testSpec();
         }
 
         auto matchedTestCases = filterTests( getAllTestCasesSorted( config ), testSpec, config );
@@ -54,7 +53,7 @@ namespace Catch {
                 Catch::cout() << Column( testCaseInfo.tagsAsString() ).indent( 6 ) << "\n";
         }
 
-        if( !config.testSpec().hasFilters() )
+        if( !config.hasTestFilters() )
             Catch::cout() << pluralise( matchedTestCases.size(), "test case" ) << '\n' << std::endl;
         else
             Catch::cout() << pluralise( matchedTestCases.size(), "matching test case" ) << '\n' << std::endl;
@@ -63,8 +62,6 @@ namespace Catch {
 
     std::size_t listTestsNamesOnly( Config const& config ) {
         TestSpec testSpec = config.testSpec();
-        if( !config.testSpec().hasFilters() )
-            testSpec = TestSpecParser( ITagAliasRegistry::get() ).parse( "*" ).testSpec();
         std::size_t matchedTests = 0;
         std::vector<TestCase> matchedTestCases = filterTests( getAllTestCasesSorted( config ), testSpec, config );
         for( auto const& testCaseInfo : matchedTestCases ) {
@@ -94,11 +91,10 @@ namespace Catch {
 
     std::size_t listTags( Config const& config ) {
         TestSpec testSpec = config.testSpec();
-        if( config.testSpec().hasFilters() )
+        if( config.hasTestFilters() )
             Catch::cout() << "Tags for matching test cases:\n";
         else {
             Catch::cout() << "All available tags:\n";
-            testSpec = TestSpecParser( ITagAliasRegistry::get() ).parse( "*" ).testSpec();
         }
 
         std::map<std::string, TagInfo> tagCounts;

include/internal/catch_session.cpp

@@ -70,8 +70,6 @@ namespace Catch {
             context.testGroupStarting(config->name(), 1, 1);
 
             TestSpec testSpec = config->testSpec();
-            if (!testSpec.hasFilters())
-                testSpec = TestSpecParser(ITagAliasRegistry::get()).parse("~[.]").testSpec(); // All not hidden tests
 
             auto const& allTestCases = getAllTestCasesSorted(*config);
             for (auto const& testCase : allTestCases) {

projects/SelfTest/Baselines/compact.sw.approved.txt

@@ -9,6 +9,7 @@ Compilation.tests.cpp:<line number>: passed: t1 < t2 for: {?} < {?}
 Compilation.tests.cpp:<line number>: passed: t1 > t2 for: {?} > {?}
 Compilation.tests.cpp:<line number>: passed: t1 <= t2 for: {?} <= {?}
 Compilation.tests.cpp:<line number>: passed: t1 >= t2 for: {?} >= {?}
+Misc.tests.cpp:<line number>: passed:
 Exception.tests.cpp:<line number>: failed: unexpected exception with message: 'answer := 42' with 1 message: 'expected exception'
 Exception.tests.cpp:<line number>: failed: unexpected exception with message: 'answer := 42'; expression was: thisThrows() with 1 message: 'expected exception'
 Exception.tests.cpp:<line number>: passed: thisThrows() with 1 message: 'answer := 42'
@@ -514,13 +515,17 @@ CmdLine.tests.cpp:<line number>: passed: config.shouldDebugBreak == false for: f
 CmdLine.tests.cpp:<line number>: passed: config.abortAfter == -1 for: -1 == -1
 CmdLine.tests.cpp:<line number>: passed: config.noThrow == false for: false == false
 CmdLine.tests.cpp:<line number>: passed: config.reporterNames.empty() for: true
+CmdLine.tests.cpp:<line number>: passed: !(cfg.hasTestFilters()) for: !false
 CmdLine.tests.cpp:<line number>: passed: result for: {?}
+CmdLine.tests.cpp:<line number>: passed: cfg.hasTestFilters() for: true
 CmdLine.tests.cpp:<line number>: passed: cfg.testSpec().matches(fakeTestCase("notIncluded")) == false for: false == false
 CmdLine.tests.cpp:<line number>: passed: cfg.testSpec().matches(fakeTestCase("test1")) for: true
 CmdLine.tests.cpp:<line number>: passed: result for: {?}
+CmdLine.tests.cpp:<line number>: passed: cfg.hasTestFilters() for: true
 CmdLine.tests.cpp:<line number>: passed: cfg.testSpec().matches(fakeTestCase("test1")) == false for: false == false
 CmdLine.tests.cpp:<line number>: passed: cfg.testSpec().matches(fakeTestCase("alwaysIncluded")) for: true
 CmdLine.tests.cpp:<line number>: passed: result for: {?}
+CmdLine.tests.cpp:<line number>: passed: cfg.hasTestFilters() for: true
 CmdLine.tests.cpp:<line number>: passed: cfg.testSpec().matches(fakeTestCase("test1")) == false for: false == false
 CmdLine.tests.cpp:<line number>: passed: cfg.testSpec().matches(fakeTestCase("alwaysIncluded")) for: true
 CmdLine.tests.cpp:<line number>: passed: cli.parse({"test", "-r", "console"}) for: {?}

projects/SelfTest/Baselines/console.std.approved.txt

@@ -1064,6 +1064,6 @@ with expansion:
   "first" == "second"
 
 ===============================================================================
-test cases: 198 | 147 passed |  47 failed |  4 failed as expected
+test cases:  199 | 148 passed |  47 failed |  4 failed as expected
-assertions: 999 | 873 passed | 105 failed | 21 failed as expected
+assertions: 1004 | 878 passed | 105 failed | 21 failed as expected
 

projects/SelfTest/Baselines/console.sw.approved.txt

@@ -93,6 +93,15 @@ PASSED:
 with expansion:
   {?} >= {?}
 
+-------------------------------------------------------------------------------
+#1175 - Hidden Test
+-------------------------------------------------------------------------------
+Misc.tests.cpp:<line number>
+...............................................................................
+
+Misc.tests.cpp:<line number>:
+PASSED:
+
 -------------------------------------------------------------------------------
 #748 - captures with unexpected exceptions
   outside assertions
@@ -3965,6 +3974,12 @@ PASSED:
 with expansion:
   true
 
+CmdLine.tests.cpp:<line number>:
+PASSED:
+  CHECK_FALSE( cfg.hasTestFilters() )
+with expansion:
+  !false
+
 -------------------------------------------------------------------------------
 Process can be configured on command line
   test lists
@@ -3979,6 +3994,12 @@ PASSED:
 with expansion:
   {?}
 
+CmdLine.tests.cpp:<line number>:
+PASSED:
+  REQUIRE( cfg.hasTestFilters() )
+with expansion:
+  true
+
 CmdLine.tests.cpp:<line number>:
 PASSED:
   REQUIRE( cfg.testSpec().matches(fakeTestCase("notIncluded")) == false )
@@ -4005,6 +4026,12 @@ PASSED:
 with expansion:
   {?}
 
+CmdLine.tests.cpp:<line number>:
+PASSED:
+  REQUIRE( cfg.hasTestFilters() )
+with expansion:
+  true
+
 CmdLine.tests.cpp:<line number>:
 PASSED:
   REQUIRE( cfg.testSpec().matches(fakeTestCase("test1")) == false )
@@ -4031,6 +4058,12 @@ PASSED:
 with expansion:
   {?}
 
+CmdLine.tests.cpp:<line number>:
+PASSED:
+  REQUIRE( cfg.hasTestFilters() )
+with expansion:
+  true
+
 CmdLine.tests.cpp:<line number>:
 PASSED:
   REQUIRE( cfg.testSpec().matches(fakeTestCase("test1")) == false )
@@ -8466,6 +8499,6 @@ Misc.tests.cpp:<line number>:
 PASSED:
 
 ===============================================================================
-test cases:  198 | 133 passed |  61 failed |  4 failed as expected
+test cases:  199 | 134 passed |  61 failed |  4 failed as expected
-assertions: 1010 | 869 passed | 120 failed | 21 failed as expected
+assertions: 1015 | 874 passed | 120 failed | 21 failed as expected
 

projects/SelfTest/Baselines/console.swa4.approved.txt

@@ -93,6 +93,15 @@ PASSED:
 with expansion:
   {?} >= {?}
 
+-------------------------------------------------------------------------------
+#1175 - Hidden Test
+-------------------------------------------------------------------------------
+Misc.tests.cpp:<line number>
+...............................................................................
+
+Misc.tests.cpp:<line number>:
+PASSED:
+
 -------------------------------------------------------------------------------
 #748 - captures with unexpected exceptions
   outside assertions
@@ -299,6 +308,6 @@ with expansion:
   !true
 
 ===============================================================================
-test cases: 11 |  8 passed | 1 failed | 2 failed as expected
+test cases: 12 |  9 passed | 1 failed | 2 failed as expected
-assertions: 34 | 27 passed | 4 failed | 3 failed as expected
+assertions: 35 | 28 passed | 4 failed | 3 failed as expected
 

projects/SelfTest/Baselines/junit.sw.approved.txt

@@ -1,11 +1,12 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <testsuitesloose text artifact
 >
-  <testsuite name="<exe-name>" errors="15" failures="106" tests="1011" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+  <testsuite name="<exe-name>" errors="15" failures="106" tests="1016" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
     <testcase classname="<exe-name>.global" name="# A test name that starts with a #" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="#1005: Comparing pointer to int and long (NULL can be either on various systems)" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="#1027" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="#1147" time="{duration}"/>
+    <testcase classname="<exe-name>.global" name="#1175 - Hidden Test" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="#748 - captures with unexpected exceptions/outside assertions" time="{duration}">
       <error type="TEST_CASE">
 expected exception

projects/SelfTest/Baselines/xml.sw.approved.txt

@@ -93,6 +93,9 @@
       </Expression>
       <OverallResult success="true"/>
     </TestCase>
+    <TestCase name="#1175 - Hidden Test" tags="[.]" filename="projects/<exe-name>/UsageTests/Misc.tests.cpp" >
+      <OverallResult success="true"/>
+    </TestCase>
     <TestCase name="#748 - captures with unexpected exceptions" tags="[!shouldfail][!throws][.][failing]" filename="projects/<exe-name>/UsageTests/Exception.tests.cpp" >
       <Section name="outside assertions" filename="projects/<exe-name>/UsageTests/Exception.tests.cpp" >
         <Info>
@@ -4624,7 +4627,15 @@
             true
           </Expanded>
         </Expression>
-        <OverallResults successes="6" failures="0" expectedFailures="0"/>
+        <Expression success="true" type="CHECK_FALSE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
+          <Original>
+            !(cfg.hasTestFilters())
+          </Original>
+          <Expanded>
+            !false
+          </Expanded>
+        </Expression>
+        <OverallResults successes="7" failures="0" expectedFailures="0"/>
       </Section>
       <Section name="test lists" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
         <Section name="1 test" description="Specify one test case using" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
@@ -4636,6 +4647,14 @@
               {?}
             </Expanded>
           </Expression>
+          <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
+            <Original>
+              cfg.hasTestFilters()
+            </Original>
+            <Expanded>
+              true
+            </Expanded>
+          </Expression>
           <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
             <Original>
               cfg.testSpec().matches(fakeTestCase("notIncluded")) == false
@@ -4652,9 +4671,9 @@
               true
             </Expanded>
           </Expression>
-          <OverallResults successes="3" failures="0" expectedFailures="0"/>
+          <OverallResults successes="4" failures="0" expectedFailures="0"/>
         </Section>
-        <OverallResults successes="3" failures="0" expectedFailures="0"/>
+        <OverallResults successes="4" failures="0" expectedFailures="0"/>
       </Section>
       <Section name="test lists" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
         <Section name="Specify one test case exclusion using exclude:" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
@@ -4666,6 +4685,14 @@
               {?}
             </Expanded>
           </Expression>
+          <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
+            <Original>
+              cfg.hasTestFilters()
+            </Original>
+            <Expanded>
+              true
+            </Expanded>
+          </Expression>
           <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
             <Original>
               cfg.testSpec().matches(fakeTestCase("test1")) == false
@@ -4682,9 +4709,9 @@
               true
             </Expanded>
           </Expression>
-          <OverallResults successes="3" failures="0" expectedFailures="0"/>
+          <OverallResults successes="4" failures="0" expectedFailures="0"/>
         </Section>
-        <OverallResults successes="3" failures="0" expectedFailures="0"/>
+        <OverallResults successes="4" failures="0" expectedFailures="0"/>
       </Section>
       <Section name="test lists" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
         <Section name="Specify one test case exclusion using ~" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
@@ -4696,6 +4723,14 @@
               {?}
             </Expanded>
           </Expression>
+          <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
+            <Original>
+              cfg.hasTestFilters()
+            </Original>
+            <Expanded>
+              true
+            </Expanded>
+          </Expression>
           <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
             <Original>
               cfg.testSpec().matches(fakeTestCase("test1")) == false
@@ -4712,9 +4747,9 @@
               true
             </Expanded>
           </Expression>
-          <OverallResults successes="3" failures="0" expectedFailures="0"/>
+          <OverallResults successes="4" failures="0" expectedFailures="0"/>
         </Section>
-        <OverallResults successes="3" failures="0" expectedFailures="0"/>
+        <OverallResults successes="4" failures="0" expectedFailures="0"/>
       </Section>
       <Section name="reporter" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
         <Section name="-r/console" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
@@ -9323,7 +9358,7 @@ loose text artifact
       </Section>
       <OverallResult success="true"/>
     </TestCase>
-    <OverallResults successes="869" failures="121" expectedFailures="21"/>
+    <OverallResults successes="874" failures="121" expectedFailures="21"/>
   </Group>
-  <OverallResults successes="869" failures="120" expectedFailures="21"/>
+  <OverallResults successes="874" failures="120" expectedFailures="21"/>
 </Catch>

projects/SelfTest/IntrospectiveTests/CmdLine.tests.cpp

@@ -289,6 +289,9 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]"
         CHECK(config.abortAfter == -1);
         CHECK(config.noThrow == false);
         CHECK(config.reporterNames.empty());
+
+        Catch::Config cfg(config);
+        CHECK_FALSE(cfg.hasTestFilters());
     }
 
     SECTION("test lists") {
@@ -297,6 +300,7 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]"
             CHECK(result);
 
             Catch::Config cfg(config);
+            REQUIRE(cfg.hasTestFilters());
             REQUIRE(cfg.testSpec().matches(fakeTestCase("notIncluded")) == false);
             REQUIRE(cfg.testSpec().matches(fakeTestCase("test1")));
         }
@@ -305,6 +309,7 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]"
             CHECK(result);
 
             Catch::Config cfg(config);
+            REQUIRE(cfg.hasTestFilters());
             REQUIRE(cfg.testSpec().matches(fakeTestCase("test1")) == false);
             REQUIRE(cfg.testSpec().matches(fakeTestCase("alwaysIncluded")));
         }
@@ -314,6 +319,7 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]"
             CHECK(result);
 
             Catch::Config cfg(config);
+            REQUIRE(cfg.hasTestFilters());
             REQUIRE(cfg.testSpec().matches(fakeTestCase("test1")) == false);
             REQUIRE(cfg.testSpec().matches(fakeTestCase("alwaysIncluded")));
         }

projects/SelfTest/UsageTests/Misc.tests.cpp

@@ -348,4 +348,9 @@ TEST_CASE( "#961 -- Dynamically created sections should all be reported", "[.]"
     }
 }
 
+TEST_CASE( "#1175 - Hidden Test", "[.]" ) {
+  // Just for checking that hidden test is not listed by default
+  SUCCEED();
+}
+
 }} // namespace MiscTests