Separated getting test cases to run from running them

include/catch_runner.hpp

@@ -83,22 +83,22 @@ namespace Catch {
         {
             Runner runner( configWrapper, reporter );
 
+            Totals totals;
             // Run test specs specified on the command line - or default to all
             if( config.testSpecs.empty() ) {
-                runner.runAll();
+                totals = runner.runAllNonHidden();
             }
             else {
-                // !TBD We should get all the testcases upfront, report any missing,
-                // then just run them
                 std::vector<std::string>::const_iterator it = config.testSpecs.begin();
                 std::vector<std::string>::const_iterator itEnd = config.testSpecs.end();
                 for(; it != itEnd; ++it ) {
-                    if( runner.runMatching( *it ) == 0 ) {
+                    Totals groupTotals = runner.runMatching( *it );
+                    if( groupTotals.testCases.total() == 0 )
                         std::cerr << "\n[No test cases matched with: " << *it << "]" << std::endl;
-                    }
+                    totals += groupTotals;
                 }
             }
-            result = static_cast<int>( runner.getTotals().assertions.failed );
+            result = static_cast<int>( totals.assertions.failed );
         }
         Catch::cleanUp();
         return result;

include/internal/catch_interfaces_runner.h

@@ -17,9 +17,18 @@ namespace Catch {
     
     struct IRunner {
         virtual ~IRunner();
-        virtual void runAll( bool runHiddenTests = false ) = 0;        
+
-        virtual std::size_t runMatching( const std::string& rawTestSpec ) = 0;        
+        /// Runs all tests, even if hidden
-        virtual Totals getTotals() const = 0;
+        virtual Totals runAll() = 0;
+
+        /// Runs all tests unless 'hidden' by ./ prefix
+        virtual Totals runAllNonHidden() = 0;
+
+        /// Runs all test that match the spec string
+        virtual Totals runMatching( const std::string& rawTestSpec ) = 0;
+
+        /// Runs all the tests passed in
+        virtual Totals runTests( const std::string& groupName, const std::vector<TestCaseInfo>& testCases ) = 0;
     };
 }
 

include/internal/catch_interfaces_testcase.h

@@ -22,7 +22,10 @@ namespace Catch {
     struct ITestCaseRegistry {
         virtual ~ITestCaseRegistry();
         virtual const std::vector<TestCaseInfo>& getAllTests() const = 0;
+        virtual const std::vector<TestCaseInfo>& getAllNonHiddenTests() const = 0;        
+
         virtual std::vector<TestCaseInfo> getMatchingTestCases( const std::string& rawTestSpec ) const = 0;
+        virtual void getMatchingTestCases( const std::string& rawTestSpec, std::vector<TestCaseInfo>& matchingTestsOut ) const = 0;
     };
 }
 

include/internal/catch_runner_impl.hpp

@@ -79,45 +79,37 @@ namespace Catch {
             m_context.setConfig( m_prevConfig );
         }
         
-        virtual void runAll( bool runHiddenTests = false ) {
+        virtual Totals runAll() {
-            m_reporter->StartGroup( "" );
+            return runTests( "", getRegistryHub().getTestCaseRegistry().getAllTests() );
-            const std::vector<TestCaseInfo>& allTests = getRegistryHub().getTestCaseRegistry().getAllTests();
-            for( std::size_t i=0; i < allTests.size(); ++i ) {
-                if( runHiddenTests || !allTests[i].isHidden() ) {
-                    if( aborting() ) {
-                        m_reporter->Aborted();
-                        break;
-                    }
-                    runTest( allTests[i] );
-                }
-            }
-            m_reporter->EndGroup( "", getTotals() );
         }
-        
-        virtual std::size_t runMatching( const std::string& rawTestSpec ) {
 
-            Totals prevTotals = getTotals();
+        virtual Totals runAllNonHidden() {
-            m_reporter->StartGroup( rawTestSpec );
+            return runTests( "", getRegistryHub().getTestCaseRegistry().getAllNonHiddenTests() );
-
-            TestSpec testSpec( rawTestSpec );
-            
-            const std::vector<TestCaseInfo>& allTests = getRegistryHub().getTestCaseRegistry().getAllTests();
-            std::size_t testsRun = 0;
-            for( std::size_t i=0; i < allTests.size(); ++i ) {
-                if( testSpec.matches( allTests[i].getName() ) ) {
-                    if( aborting() ) {
-                        m_reporter->Aborted();
-                        break;
-                    }
-                    runTest( allTests[i] );
-                    testsRun++;
-                }
-            }
-            m_reporter->EndGroup( rawTestSpec, getTotals() - prevTotals );
-            return testsRun;
         }
-        
+
-        void runTest( const TestCaseInfo& testInfo ) {
+        virtual Totals runMatching( const std::string& rawTestSpec ) {
+
+            const std::vector<TestCaseInfo>& matchingTests = getRegistryHub().getTestCaseRegistry().getMatchingTestCases( rawTestSpec );
+            return runTests( rawTestSpec, matchingTests );
+        }
+
+        virtual Totals runTests( const std::string& groupName, const std::vector<TestCaseInfo>& testCases ) {
+
+            Totals totals;
+            m_reporter->StartGroup( groupName );
+
+            for( std::size_t i=0; i < testCases.size(); ++i ) {
+                if( aborting() ) {
+                    m_reporter->Aborted();
+                    break;
+                }
+                totals += runTest( testCases[i] );
+            }
+            m_reporter->EndGroup( groupName, totals );
+            return totals;
+        }
+
+        Totals runTest( const TestCaseInfo& testInfo ) {
             Totals prevTotals = m_totals;
 
             std::string redirectedCout;
@@ -129,10 +121,8 @@ namespace Catch {
 
             do {
                 do {
-//                    m_reporter->StartGroup( "test case run" );
                     m_currentResult.setLineInfo( m_runningTest->getTestCaseInfo().getLineInfo() );
                     runCurrentTest( redirectedCout, redirectedCerr );
-//                    m_reporter->EndGroup( "test case run", m_totals.delta( prevTotals ) );
                 }
                 while( m_runningTest->hasUntestedSections() && !aborting() );
             }
@@ -144,12 +134,9 @@ namespace Catch {
             Totals deltaTotals = m_totals.delta( prevTotals );
             m_totals.testCases += deltaTotals.testCases;            
             m_reporter->EndTestCase( testInfo, deltaTotals, redirectedCout, redirectedCerr );
+            return deltaTotals;
         }
 
-        virtual Totals getTotals() const {
-            return m_totals;
-        }
-        
         const Config& config() const {
             return m_config;
         }

include/internal/catch_test_case_registry_impl.hpp

@@ -7,6 +7,7 @@
  */
 #include "catch_test_registry.hpp"
 #include "catch_test_case_info.h"
+#include "catch_test_spec.h"
 #include "catch_context.h"
 
 #include <vector>
@@ -31,6 +32,8 @@ namespace Catch {
             if( m_functions.find( testInfo ) == m_functions.end() ) {
                 m_functions.insert( testInfo );
                 m_functionsInOrder.push_back( testInfo );
+                if( !testInfo.isHidden() )
+                    m_nonHiddenFunctions.push_back( testInfo );
             }
             else {
                 const TestCaseInfo& prev = *m_functions.find( testInfo );
@@ -45,24 +48,35 @@ namespace Catch {
             return m_functionsInOrder;
         }
 
+        virtual const std::vector<TestCaseInfo>& getAllNonHiddenTests() const {
+            return m_nonHiddenFunctions;
+        }
+
         virtual std::vector<TestCaseInfo> getMatchingTestCases( const std::string& rawTestSpec ) const {
             TestSpec testSpec( rawTestSpec );
             
-            std::vector<TestCaseInfo> testList;
+            std::vector<TestCaseInfo> matchingTests;
+            getMatchingTestCases( rawTestSpec, matchingTests );
+            return matchingTests;
+        }
+
+        virtual void getMatchingTestCases( const std::string& rawTestSpec, std::vector<TestCaseInfo>& matchingTestsOut ) const {
+            TestSpec testSpec( rawTestSpec );
+
             std::vector<TestCaseInfo>::const_iterator it = m_functionsInOrder.begin();
             std::vector<TestCaseInfo>::const_iterator itEnd = m_functionsInOrder.end();
             for(; it != itEnd; ++it ) {
                 if( testSpec.matches( it->getName() ) ) {
-                    testList.push_back( *it );
+                    matchingTestsOut.push_back( *it );
                 }
             }
-            return testList;
         }
-        
+
     private:
         
         std::set<TestCaseInfo> m_functions;
         std::vector<TestCaseInfo> m_functionsInOrder;
+        std::vector<TestCaseInfo> m_nonHiddenFunctions;        
         size_t m_unnamedCount;
     };
 

include/internal/catch_totals.hpp

@@ -50,7 +50,13 @@ namespace Catch {
                 ++diff.testCases.passed;
             return diff;
         }
-        
+
+        Totals& operator += ( const Totals& other ) {
+            assertions += other.assertions;
+            testCases += other.testCases;
+            return *this;
+        }
+
         Counts assertions;
         Counts testCases;
     };

projects/SelfTest/TestMain.cpp

@@ -34,16 +34,16 @@ TEST_CASE( "selftest/main", "Runs all Catch self tests and checks their results"
         
         SECTION(    "selftest/test counts/succeeding tests", 
                     "Number of 'succeeding' tests is fixed" ) {
-            runner.runMatching( "./succeeding/*" );
+            Totals totals = runner.runMatching( "./succeeding/*" );
-            CHECK( runner.getTotals().assertions.passed == 285 );
+            CHECK( totals.assertions.passed == 285 );
-            CHECK( runner.getTotals().assertions.failed == 0 );
+            CHECK( totals.assertions.failed == 0 );
         }
 
         SECTION(    "selftest/test counts/failing tests", 
                     "Number of 'failing' tests is fixed" ) {
-            runner.runMatching( "./failing/*" );        
+            Totals totals = runner.runMatching( "./failing/*" );        
-            CHECK( runner.getTotals().assertions.passed == 0 );
+            CHECK( totals.assertions.passed == 0 );
-            CHECK( runner.getTotals().assertions.failed == 72 );
+            CHECK( totals.assertions.failed == 72 );
         }
     }
 }
@@ -51,9 +51,9 @@ TEST_CASE( "selftest/main", "Runs all Catch self tests and checks their results"
 TEST_CASE( "meta/Misc/Sections", "looped tests" ) {
     Catch::EmbeddedRunner runner;
     
-    runner.runMatching( "./mixed/Misc/Sections/nested2" );
+    Catch::Totals totals = runner.runMatching( "./mixed/Misc/Sections/nested2" );
-    CHECK( runner.getTotals().assertions.passed == 2 );
+    CHECK( totals.assertions.passed == 2 );
-    CHECK( runner.getTotals().assertions.failed == 1 );
+    CHECK( totals.assertions.failed == 1 );
 }
 
 #pragma clang diagnostic ignored "-Wweak-vtables"

projects/SelfTest/catch_self_test.cpp

@@ -13,21 +13,20 @@
 
 namespace Catch{
     
-    std::size_t EmbeddedRunner::runMatching( const std::string& rawTestSpec, const std::string& ) {
+    Totals EmbeddedRunner::runMatching( const std::string& rawTestSpec, const std::string& ) {
         std::ostringstream oss;
         Config config;
         config.setStreamBuf( oss.rdbuf() );
         
-        std::size_t result;
+        Totals totals;
-        
+
         // Scoped because Runner doesn't report EndTesting until its destructor
         {
             Runner runner( config, m_reporter.get() );
-            result = runner.runMatching( rawTestSpec );
+            totals = runner.runMatching( rawTestSpec );
-            m_totals = runner.getTotals();
         }
         m_output = oss.str();
-        return result;
+        return totals;
     }
     
     void MockReporter::Result( const ResultInfo& resultInfo ) {

projects/SelfTest/catch_self_test.hpp

@@ -114,17 +114,13 @@ namespace Catch {
     public:
         EmbeddedRunner() : m_reporter( new MockReporter() ) {}
         
-        std::size_t runMatching(    const std::string& rawTestSpec,
+        Totals runMatching( const std::string& rawTestSpec,
-                                    const std::string& reporter = "basic" );
+                            const std::string& reporter = "basic" );
         
         std::string getOutput() {
             return m_output;
         }
 
-        const Totals& getTotals() const {
-            return m_totals;
-        }
-
         void addRecorder( const std::string& recorder ) {
             m_reporter->addRecorder( recorder );
         }
@@ -134,7 +130,6 @@ namespace Catch {
         }
 
     private:
-        Totals m_totals;
         std::string m_output;
         Ptr<MockReporter> m_reporter;
     };
@@ -157,8 +152,7 @@ namespace Catch {
         
         void operator()( const TestCaseInfo& testCase ) {
             EmbeddedRunner runner;
-            runner.runMatching( testCase.getName() );
+            Totals totals = runner.runMatching( testCase.getName() );
-            Totals totals = runner.getTotals();
             switch( m_expectedResult ) {
                 case Expected::ToSucceed:
                     if( totals.assertions.failed > 0 ) {