From c1dbc9c22bda91b586919333fd0b946450ad3067 Mon Sep 17 00:00:00 2001 From: Phil Nash Date: Wed, 15 Aug 2012 19:12:51 +0100 Subject: [PATCH] Separated getting test cases to run from running them --- include/catch_runner.hpp | 12 ++-- include/internal/catch_interfaces_runner.h | 15 +++- include/internal/catch_interfaces_testcase.h | 3 + include/internal/catch_runner_impl.hpp | 71 ++++++++----------- .../catch_test_case_registry_impl.hpp | 22 ++++-- include/internal/catch_totals.hpp | 8 ++- projects/SelfTest/TestMain.cpp | 18 ++--- projects/SelfTest/catch_self_test.cpp | 11 ++- projects/SelfTest/catch_self_test.hpp | 12 +--- 9 files changed, 92 insertions(+), 80 deletions(-) diff --git a/include/catch_runner.hpp b/include/catch_runner.hpp index ca63aa0d..eed0fca7 100644 --- a/include/catch_runner.hpp +++ b/include/catch_runner.hpp @@ -83,22 +83,22 @@ namespace Catch { { Runner runner( configWrapper, reporter ); + Totals totals; // Run test specs specified on the command line - or default to all if( config.testSpecs.empty() ) { - runner.runAll(); + totals = runner.runAllNonHidden(); } else { - // !TBD We should get all the testcases upfront, report any missing, - // then just run them std::vector::const_iterator it = config.testSpecs.begin(); std::vector::const_iterator itEnd = config.testSpecs.end(); for(; it != itEnd; ++it ) { - if( runner.runMatching( *it ) == 0 ) { + Totals groupTotals = runner.runMatching( *it ); + if( groupTotals.testCases.total() == 0 ) std::cerr << "\n[No test cases matched with: " << *it << "]" << std::endl; - } + totals += groupTotals; } } - result = static_cast( runner.getTotals().assertions.failed ); + result = static_cast( totals.assertions.failed ); } Catch::cleanUp(); return result; diff --git a/include/internal/catch_interfaces_runner.h b/include/internal/catch_interfaces_runner.h index b932e65f..f3825626 100644 --- a/include/internal/catch_interfaces_runner.h +++ b/include/internal/catch_interfaces_runner.h @@ -17,9 +17,18 @@ namespace Catch { struct IRunner { virtual ~IRunner(); - virtual void runAll( bool runHiddenTests = false ) = 0; - virtual std::size_t runMatching( const std::string& rawTestSpec ) = 0; - virtual Totals getTotals() const = 0; + + /// Runs all tests, even if hidden + virtual Totals runAll() = 0; + + /// Runs all tests unless 'hidden' by ./ prefix + virtual Totals runAllNonHidden() = 0; + + /// Runs all test that match the spec string + virtual Totals runMatching( const std::string& rawTestSpec ) = 0; + + /// Runs all the tests passed in + virtual Totals runTests( const std::string& groupName, const std::vector& testCases ) = 0; }; } diff --git a/include/internal/catch_interfaces_testcase.h b/include/internal/catch_interfaces_testcase.h index 0781e646..992b71d3 100644 --- a/include/internal/catch_interfaces_testcase.h +++ b/include/internal/catch_interfaces_testcase.h @@ -22,7 +22,10 @@ namespace Catch { struct ITestCaseRegistry { virtual ~ITestCaseRegistry(); virtual const std::vector& getAllTests() const = 0; + virtual const std::vector& getAllNonHiddenTests() const = 0; + virtual std::vector getMatchingTestCases( const std::string& rawTestSpec ) const = 0; + virtual void getMatchingTestCases( const std::string& rawTestSpec, std::vector& matchingTestsOut ) const = 0; }; } diff --git a/include/internal/catch_runner_impl.hpp b/include/internal/catch_runner_impl.hpp index da60a90c..7a22b3b9 100644 --- a/include/internal/catch_runner_impl.hpp +++ b/include/internal/catch_runner_impl.hpp @@ -79,45 +79,37 @@ namespace Catch { m_context.setConfig( m_prevConfig ); } - virtual void runAll( bool runHiddenTests = false ) { - m_reporter->StartGroup( "" ); - const std::vector& allTests = getRegistryHub().getTestCaseRegistry().getAllTests(); - for( std::size_t i=0; i < allTests.size(); ++i ) { - if( runHiddenTests || !allTests[i].isHidden() ) { - if( aborting() ) { - m_reporter->Aborted(); - break; - } - runTest( allTests[i] ); - } - } - m_reporter->EndGroup( "", getTotals() ); + virtual Totals runAll() { + return runTests( "", getRegistryHub().getTestCaseRegistry().getAllTests() ); } - - virtual std::size_t runMatching( const std::string& rawTestSpec ) { - Totals prevTotals = getTotals(); - m_reporter->StartGroup( rawTestSpec ); - - TestSpec testSpec( rawTestSpec ); - - const std::vector& allTests = getRegistryHub().getTestCaseRegistry().getAllTests(); - std::size_t testsRun = 0; - for( std::size_t i=0; i < allTests.size(); ++i ) { - if( testSpec.matches( allTests[i].getName() ) ) { - if( aborting() ) { - m_reporter->Aborted(); - break; - } - runTest( allTests[i] ); - testsRun++; - } - } - m_reporter->EndGroup( rawTestSpec, getTotals() - prevTotals ); - return testsRun; + virtual Totals runAllNonHidden() { + return runTests( "", getRegistryHub().getTestCaseRegistry().getAllNonHiddenTests() ); } - - void runTest( const TestCaseInfo& testInfo ) { + + virtual Totals runMatching( const std::string& rawTestSpec ) { + + const std::vector& matchingTests = getRegistryHub().getTestCaseRegistry().getMatchingTestCases( rawTestSpec ); + return runTests( rawTestSpec, matchingTests ); + } + + virtual Totals runTests( const std::string& groupName, const std::vector& testCases ) { + + Totals totals; + m_reporter->StartGroup( groupName ); + + for( std::size_t i=0; i < testCases.size(); ++i ) { + if( aborting() ) { + m_reporter->Aborted(); + break; + } + totals += runTest( testCases[i] ); + } + m_reporter->EndGroup( groupName, totals ); + return totals; + } + + Totals runTest( const TestCaseInfo& testInfo ) { Totals prevTotals = m_totals; std::string redirectedCout; @@ -129,10 +121,8 @@ namespace Catch { do { do { -// m_reporter->StartGroup( "test case run" ); m_currentResult.setLineInfo( m_runningTest->getTestCaseInfo().getLineInfo() ); runCurrentTest( redirectedCout, redirectedCerr ); -// m_reporter->EndGroup( "test case run", m_totals.delta( prevTotals ) ); } while( m_runningTest->hasUntestedSections() && !aborting() ); } @@ -144,12 +134,9 @@ namespace Catch { Totals deltaTotals = m_totals.delta( prevTotals ); m_totals.testCases += deltaTotals.testCases; m_reporter->EndTestCase( testInfo, deltaTotals, redirectedCout, redirectedCerr ); + return deltaTotals; } - virtual Totals getTotals() const { - return m_totals; - } - const Config& config() const { return m_config; } diff --git a/include/internal/catch_test_case_registry_impl.hpp b/include/internal/catch_test_case_registry_impl.hpp index 10202f64..a17e6033 100644 --- a/include/internal/catch_test_case_registry_impl.hpp +++ b/include/internal/catch_test_case_registry_impl.hpp @@ -7,6 +7,7 @@ */ #include "catch_test_registry.hpp" #include "catch_test_case_info.h" +#include "catch_test_spec.h" #include "catch_context.h" #include @@ -31,6 +32,8 @@ namespace Catch { if( m_functions.find( testInfo ) == m_functions.end() ) { m_functions.insert( testInfo ); m_functionsInOrder.push_back( testInfo ); + if( !testInfo.isHidden() ) + m_nonHiddenFunctions.push_back( testInfo ); } else { const TestCaseInfo& prev = *m_functions.find( testInfo ); @@ -45,24 +48,35 @@ namespace Catch { return m_functionsInOrder; } + virtual const std::vector& getAllNonHiddenTests() const { + return m_nonHiddenFunctions; + } + virtual std::vector getMatchingTestCases( const std::string& rawTestSpec ) const { TestSpec testSpec( rawTestSpec ); - std::vector testList; + std::vector matchingTests; + getMatchingTestCases( rawTestSpec, matchingTests ); + return matchingTests; + } + + virtual void getMatchingTestCases( const std::string& rawTestSpec, std::vector& matchingTestsOut ) const { + TestSpec testSpec( rawTestSpec ); + std::vector::const_iterator it = m_functionsInOrder.begin(); std::vector::const_iterator itEnd = m_functionsInOrder.end(); for(; it != itEnd; ++it ) { if( testSpec.matches( it->getName() ) ) { - testList.push_back( *it ); + matchingTestsOut.push_back( *it ); } } - return testList; } - + private: std::set m_functions; std::vector m_functionsInOrder; + std::vector m_nonHiddenFunctions; size_t m_unnamedCount; }; diff --git a/include/internal/catch_totals.hpp b/include/internal/catch_totals.hpp index 557288dd..8940d0fc 100644 --- a/include/internal/catch_totals.hpp +++ b/include/internal/catch_totals.hpp @@ -50,7 +50,13 @@ namespace Catch { ++diff.testCases.passed; return diff; } - + + Totals& operator += ( const Totals& other ) { + assertions += other.assertions; + testCases += other.testCases; + return *this; + } + Counts assertions; Counts testCases; }; diff --git a/projects/SelfTest/TestMain.cpp b/projects/SelfTest/TestMain.cpp index 324fc28c..4dd59c8f 100644 --- a/projects/SelfTest/TestMain.cpp +++ b/projects/SelfTest/TestMain.cpp @@ -34,16 +34,16 @@ TEST_CASE( "selftest/main", "Runs all Catch self tests and checks their results" SECTION( "selftest/test counts/succeeding tests", "Number of 'succeeding' tests is fixed" ) { - runner.runMatching( "./succeeding/*" ); - CHECK( runner.getTotals().assertions.passed == 285 ); - CHECK( runner.getTotals().assertions.failed == 0 ); + Totals totals = runner.runMatching( "./succeeding/*" ); + CHECK( totals.assertions.passed == 285 ); + CHECK( totals.assertions.failed == 0 ); } SECTION( "selftest/test counts/failing tests", "Number of 'failing' tests is fixed" ) { - runner.runMatching( "./failing/*" ); - CHECK( runner.getTotals().assertions.passed == 0 ); - CHECK( runner.getTotals().assertions.failed == 72 ); + Totals totals = runner.runMatching( "./failing/*" ); + CHECK( totals.assertions.passed == 0 ); + CHECK( totals.assertions.failed == 72 ); } } } @@ -51,9 +51,9 @@ TEST_CASE( "selftest/main", "Runs all Catch self tests and checks their results" TEST_CASE( "meta/Misc/Sections", "looped tests" ) { Catch::EmbeddedRunner runner; - runner.runMatching( "./mixed/Misc/Sections/nested2" ); - CHECK( runner.getTotals().assertions.passed == 2 ); - CHECK( runner.getTotals().assertions.failed == 1 ); + Catch::Totals totals = runner.runMatching( "./mixed/Misc/Sections/nested2" ); + CHECK( totals.assertions.passed == 2 ); + CHECK( totals.assertions.failed == 1 ); } #pragma clang diagnostic ignored "-Wweak-vtables" diff --git a/projects/SelfTest/catch_self_test.cpp b/projects/SelfTest/catch_self_test.cpp index d0065a78..a5d9cb34 100644 --- a/projects/SelfTest/catch_self_test.cpp +++ b/projects/SelfTest/catch_self_test.cpp @@ -13,21 +13,20 @@ namespace Catch{ - std::size_t EmbeddedRunner::runMatching( const std::string& rawTestSpec, const std::string& ) { + Totals EmbeddedRunner::runMatching( const std::string& rawTestSpec, const std::string& ) { std::ostringstream oss; Config config; config.setStreamBuf( oss.rdbuf() ); - std::size_t result; - + Totals totals; + // Scoped because Runner doesn't report EndTesting until its destructor { Runner runner( config, m_reporter.get() ); - result = runner.runMatching( rawTestSpec ); - m_totals = runner.getTotals(); + totals = runner.runMatching( rawTestSpec ); } m_output = oss.str(); - return result; + return totals; } void MockReporter::Result( const ResultInfo& resultInfo ) { diff --git a/projects/SelfTest/catch_self_test.hpp b/projects/SelfTest/catch_self_test.hpp index 1b79a44f..c223e931 100644 --- a/projects/SelfTest/catch_self_test.hpp +++ b/projects/SelfTest/catch_self_test.hpp @@ -114,17 +114,13 @@ namespace Catch { public: EmbeddedRunner() : m_reporter( new MockReporter() ) {} - std::size_t runMatching( const std::string& rawTestSpec, - const std::string& reporter = "basic" ); + Totals runMatching( const std::string& rawTestSpec, + const std::string& reporter = "basic" ); std::string getOutput() { return m_output; } - const Totals& getTotals() const { - return m_totals; - } - void addRecorder( const std::string& recorder ) { m_reporter->addRecorder( recorder ); } @@ -134,7 +130,6 @@ namespace Catch { } private: - Totals m_totals; std::string m_output; Ptr m_reporter; }; @@ -157,8 +152,7 @@ namespace Catch { void operator()( const TestCaseInfo& testCase ) { EmbeddedRunner runner; - runner.runMatching( testCase.getName() ); - Totals totals = runner.getTotals(); + Totals totals = runner.runMatching( testCase.getName() ); switch( m_expectedResult ) { case Expected::ToSucceed: if( totals.assertions.failed > 0 ) {