Improvements to self test to give finer grained feedback

Test/TestMain.cpp

@@ -11,22 +11,51 @@
  */
 
 #include "../catch_with_main.hpp"
-
 #include "../internal/catch_self_test.hpp"
 
 TEST_CASE( "selftest/main", "Runs all Catch self tests and checks their results" )
 {
-    Catch::EmbeddedRunner runner;
+    using namespace Catch;
     
+    ///////////////////////////////////////////////////////////////////////////
+    SECTION(    "selftest/expected result",
+                "Tests do what they claim" )
+    {
+        SECTION(    "selftest/expected result/failing tests", 
+                    "Tests in the 'failing' branch fail" )
+        {
+            MetaTestRunner::runMatching( "./failing/*",  MetaTestRunner::Expected::ToFail );
+        }
+        
+        SECTION(    "selftest/expected result/succeeding tests", 
+                    "Tests in the 'succeeding' branch succeed" )
+        {
+            MetaTestRunner::runMatching( "./succeeding/*",  MetaTestRunner::Expected::ToSucceed );
+        }
+    }
+    
+    ///////////////////////////////////////////////////////////////////////////
+    SECTION(    "selftest/test counts", 
+                "Number of test cases that run is fixed" )
+    {
+        EmbeddedRunner runner;
+        
+        SECTION(    "selftest/test counts/succeeding tests", 
+                    "Number of 'succeeding' tests is fixed" )
+        {
             runner.runMatching( "./succeeding/*" );
-    INFO( runner.getOutput() );
             CHECK( runner.getSuccessCount() == 213 );
             CHECK( runner.getFailureCount() == 0 );
+        }
 
+        SECTION(    "selftest/test counts/failing tests", 
+                    "Number of 'failing' tests is fixed" )
+        {
             runner.runMatching( "./failing/*" );        
-    INFO( runner.getOutput() );
             CHECK( runner.getSuccessCount() == 0 );
             CHECK( runner.getFailureCount() == 54 );
+        }
+    }
 }
 
 TEST_CASE( "meta/Misc/Sections", "looped tests" )

internal/catch_common.h

@@ -63,6 +63,18 @@ namespace Catch
         }
     }
     
+    template<typename ContainerT, typename Function>
+    inline void forEach( ContainerT& container, Function function )
+    {
+        std::for_each( container.begin(), container.end(), function );
+    }
+    
+    template<typename ContainerT, typename Function>
+    inline void forEach( const ContainerT& container, Function function )
+    {
+        std::for_each( container.begin(), container.end(), function );
+    }
+    
     ATTRIBUTE_NORETURN
     inline void throwLogicError( const std::string& message, const std::string& file, long line )
     {

internal/catch_interfaces_runner.h

@@ -16,6 +16,8 @@
 
 namespace Catch
 {
+    class TestCaseInfo;
+    
     struct IRunner
     {
         virtual ~IRunner

internal/catch_interfaces_testcase.h

@@ -52,6 +52,10 @@ namespace Catch
         
         virtual const std::vector<TestCaseInfo>& getAllTests
             () const = 0;
+
+        virtual std::vector<TestCaseInfo> getMatchingTestCases
+            ( const std::string& rawTestSpec 
+            ) = 0;
     };
 }
 

internal/catch_runner_impl.hpp

@@ -56,44 +56,6 @@ namespace Catch
         std::string& m_targetString;
     };
     
-    ///////////////////////////////////////////////////////////////////////////
-    ///////////////////////////////////////////////////////////////////////////    
-    
-    class TestSpec
-    {
-    public:
-        ///////////////////////////////////////////////////////////////////////        
-        TestSpec
-        (
-            const std::string& rawSpec
-        )
-        :   m_rawSpec( rawSpec ),
-            m_isWildcarded( false )
-        {
-            if( m_rawSpec[m_rawSpec.size()-1] == '*' )
-            {
-                m_rawSpec = m_rawSpec.substr( 0, m_rawSpec.size()-1 );
-                m_isWildcarded = true;
-            }
-        }
-        
-        ///////////////////////////////////////////////////////////////////////        
-        bool matches
-        (
-            const std::string& testName
-        )
-        const
-        {
-            if( !m_isWildcarded )
-                return m_rawSpec == testName;
-            else
-                return testName.size() >= m_rawSpec.size() && testName.substr( 0, m_rawSpec.size() ) == m_rawSpec;            
-        }
-        
-    private:
-        std::string m_rawSpec;
-        bool m_isWildcarded;
-    };
     
     ///////////////////////////////////////////////////////////////////////////
     ///////////////////////////////////////////////////////////////////////////    

internal/catch_self_test.hpp

@@ -79,6 +79,76 @@ namespace Catch
         std::size_t m_failures;
         std::string m_output;
     };
+
+    class MetaTestRunner
+    {
+    public:
+        struct Expected
+        {
+            enum Result
+            {
+                ToSucceed,
+                ToFail
+            };
+        };
+        
+        ///////////////////////////////////////////////////////////////////////////
+        static void runMatching
+        (
+            const std::string& testSpec, 
+            Expected::Result expectedResult
+        )
+        {
+            forEach(    Hub::getTestCaseRegistry().getMatchingTestCases( testSpec ), 
+                        MetaTestRunner( expectedResult ) );
+        }
+        
+        ///////////////////////////////////////////////////////////////////////////
+        MetaTestRunner
+        (
+            Expected::Result expectedResult
+        )
+        : m_expectedResult( expectedResult )
+        {        
+        }
+        
+        ///////////////////////////////////////////////////////////////////////////
+        void operator()
+        (
+            const TestCaseInfo& testCase
+        )
+        {
+            EmbeddedRunner runner;
+            runner.runMatching( testCase.getName() );
+            switch( m_expectedResult )
+            {
+                case Expected::ToSucceed:
+                    if( runner.getFailureCount() > 0 )
+                    {
+                        INFO( runner.getOutput() );
+                        FAIL( "Expected test case '" 
+                             << testCase.getName() 
+                             << "' to succeed but there was/ were " 
+                             << runner.getFailureCount() << " failure(s)" );
+                    }
+                    break;
+                case Expected::ToFail:
+                    if( runner.getSuccessCount() > 0 )
+                    {
+                        INFO( runner.getOutput() );
+                        FAIL( "Expected test case '" 
+                             << testCase.getName() 
+                             << "' to fail but there was/ were " 
+                             << runner.getSuccessCount() << " success(es)" );
+                    }
+                    break;
+            }        
+        };
+        
+    private:
+        Expected::Result m_expectedResult;
+    };
+    
 }
 
 #endif // TWOBLUECUBES_CATCH_SELF_TEST_HPP_INCLUDED

internal/catch_test_case_info.hpp

@@ -157,6 +157,44 @@ namespace Catch
         std::string m_description;
     };
     
+    ///////////////////////////////////////////////////////////////////////////
+    ///////////////////////////////////////////////////////////////////////////    
+    
+    class TestSpec
+    {
+    public:
+        ///////////////////////////////////////////////////////////////////////        
+        TestSpec
+        (
+            const std::string& rawSpec
+        )
+        :   m_rawSpec( rawSpec ),
+            m_isWildcarded( false )
+        {
+            if( m_rawSpec[m_rawSpec.size()-1] == '*' )
+            {
+                m_rawSpec = m_rawSpec.substr( 0, m_rawSpec.size()-1 );
+                m_isWildcarded = true;
+            }
+        }
+        
+        ///////////////////////////////////////////////////////////////////////        
+        bool matches
+        (
+            const std::string& testName
+        )
+        const
+        {
+            if( !m_isWildcarded )
+                return m_rawSpec == testName;
+            else
+                return testName.size() >= m_rawSpec.size() && testName.substr( 0, m_rawSpec.size() ) == m_rawSpec;            
+        }
+        
+    private:
+        std::string m_rawSpec;
+        bool m_isWildcarded;
+    };
 }
 
 #endif // TWOBLUECUBES_CATCH_TESTCASEINFO_HPP_INCLUDED

internal/catch_test_case_registry_impl.hpp

@@ -18,6 +18,7 @@
 #include <set>
 #include <sstream>
 
+#include <iostream> // !TBD DBG
 namespace Catch
 {
     class TestRegistry : public ITestCaseRegistry
@@ -57,6 +58,28 @@ namespace Catch
             return m_functionsInOrder;
         }
 
+        ///////////////////////////////////////////////////////////////////////////
+        virtual std::vector<TestCaseInfo> getMatchingTestCases
+        ( 
+            const std::string& rawTestSpec 
+        )
+        {
+            TestSpec testSpec( rawTestSpec );
+            
+            std::vector<TestCaseInfo> testList;
+            std::vector<TestCaseInfo>::const_iterator it = m_functionsInOrder.begin();
+            std::vector<TestCaseInfo>::const_iterator itEnd = m_functionsInOrder.end();
+            for(; it != itEnd; ++it )
+            {
+                if( testSpec.matches( it->getName() ) )
+                {
+                    testList.push_back( *it );
+                    std::cout << it->getName() << std::endl;
+                }
+            }
+            return testList;
+        }
+        
     private:
         
         std::set<TestCaseInfo> m_functions;