Separated getting test cases to run from running them

This commit is contained in:
Phil Nash 2012-08-15 19:12:51 +01:00
parent d0a5461add
commit c1dbc9c22b
9 changed files with 92 additions and 80 deletions

View File

@ -83,22 +83,22 @@ namespace Catch {
{ {
Runner runner( configWrapper, reporter ); Runner runner( configWrapper, reporter );
Totals totals;
// Run test specs specified on the command line - or default to all // Run test specs specified on the command line - or default to all
if( config.testSpecs.empty() ) { if( config.testSpecs.empty() ) {
runner.runAll(); totals = runner.runAllNonHidden();
} }
else { else {
// !TBD We should get all the testcases upfront, report any missing,
// then just run them
std::vector<std::string>::const_iterator it = config.testSpecs.begin(); std::vector<std::string>::const_iterator it = config.testSpecs.begin();
std::vector<std::string>::const_iterator itEnd = config.testSpecs.end(); std::vector<std::string>::const_iterator itEnd = config.testSpecs.end();
for(; it != itEnd; ++it ) { for(; it != itEnd; ++it ) {
if( runner.runMatching( *it ) == 0 ) { Totals groupTotals = runner.runMatching( *it );
if( groupTotals.testCases.total() == 0 )
std::cerr << "\n[No test cases matched with: " << *it << "]" << std::endl; std::cerr << "\n[No test cases matched with: " << *it << "]" << std::endl;
} totals += groupTotals;
} }
} }
result = static_cast<int>( runner.getTotals().assertions.failed ); result = static_cast<int>( totals.assertions.failed );
} }
Catch::cleanUp(); Catch::cleanUp();
return result; return result;

View File

@ -17,9 +17,18 @@ namespace Catch {
struct IRunner { struct IRunner {
virtual ~IRunner(); virtual ~IRunner();
virtual void runAll( bool runHiddenTests = false ) = 0;
virtual std::size_t runMatching( const std::string& rawTestSpec ) = 0; /// Runs all tests, even if hidden
virtual Totals getTotals() const = 0; virtual Totals runAll() = 0;
/// Runs all tests unless 'hidden' by ./ prefix
virtual Totals runAllNonHidden() = 0;
/// Runs all test that match the spec string
virtual Totals runMatching( const std::string& rawTestSpec ) = 0;
/// Runs all the tests passed in
virtual Totals runTests( const std::string& groupName, const std::vector<TestCaseInfo>& testCases ) = 0;
}; };
} }

View File

@ -22,7 +22,10 @@ namespace Catch {
struct ITestCaseRegistry { struct ITestCaseRegistry {
virtual ~ITestCaseRegistry(); virtual ~ITestCaseRegistry();
virtual const std::vector<TestCaseInfo>& getAllTests() const = 0; virtual const std::vector<TestCaseInfo>& getAllTests() const = 0;
virtual const std::vector<TestCaseInfo>& getAllNonHiddenTests() const = 0;
virtual std::vector<TestCaseInfo> getMatchingTestCases( const std::string& rawTestSpec ) const = 0; virtual std::vector<TestCaseInfo> getMatchingTestCases( const std::string& rawTestSpec ) const = 0;
virtual void getMatchingTestCases( const std::string& rawTestSpec, std::vector<TestCaseInfo>& matchingTestsOut ) const = 0;
}; };
} }

View File

@ -79,45 +79,37 @@ namespace Catch {
m_context.setConfig( m_prevConfig ); m_context.setConfig( m_prevConfig );
} }
virtual void runAll( bool runHiddenTests = false ) { virtual Totals runAll() {
m_reporter->StartGroup( "" ); return runTests( "", getRegistryHub().getTestCaseRegistry().getAllTests() );
const std::vector<TestCaseInfo>& allTests = getRegistryHub().getTestCaseRegistry().getAllTests();
for( std::size_t i=0; i < allTests.size(); ++i ) {
if( runHiddenTests || !allTests[i].isHidden() ) {
if( aborting() ) {
m_reporter->Aborted();
break;
}
runTest( allTests[i] );
}
}
m_reporter->EndGroup( "", getTotals() );
} }
virtual std::size_t runMatching( const std::string& rawTestSpec ) {
Totals prevTotals = getTotals(); virtual Totals runAllNonHidden() {
m_reporter->StartGroup( rawTestSpec ); return runTests( "", getRegistryHub().getTestCaseRegistry().getAllNonHiddenTests() );
TestSpec testSpec( rawTestSpec );
const std::vector<TestCaseInfo>& allTests = getRegistryHub().getTestCaseRegistry().getAllTests();
std::size_t testsRun = 0;
for( std::size_t i=0; i < allTests.size(); ++i ) {
if( testSpec.matches( allTests[i].getName() ) ) {
if( aborting() ) {
m_reporter->Aborted();
break;
}
runTest( allTests[i] );
testsRun++;
}
}
m_reporter->EndGroup( rawTestSpec, getTotals() - prevTotals );
return testsRun;
} }
void runTest( const TestCaseInfo& testInfo ) { virtual Totals runMatching( const std::string& rawTestSpec ) {
const std::vector<TestCaseInfo>& matchingTests = getRegistryHub().getTestCaseRegistry().getMatchingTestCases( rawTestSpec );
return runTests( rawTestSpec, matchingTests );
}
virtual Totals runTests( const std::string& groupName, const std::vector<TestCaseInfo>& testCases ) {
Totals totals;
m_reporter->StartGroup( groupName );
for( std::size_t i=0; i < testCases.size(); ++i ) {
if( aborting() ) {
m_reporter->Aborted();
break;
}
totals += runTest( testCases[i] );
}
m_reporter->EndGroup( groupName, totals );
return totals;
}
Totals runTest( const TestCaseInfo& testInfo ) {
Totals prevTotals = m_totals; Totals prevTotals = m_totals;
std::string redirectedCout; std::string redirectedCout;
@ -129,10 +121,8 @@ namespace Catch {
do { do {
do { do {
// m_reporter->StartGroup( "test case run" );
m_currentResult.setLineInfo( m_runningTest->getTestCaseInfo().getLineInfo() ); m_currentResult.setLineInfo( m_runningTest->getTestCaseInfo().getLineInfo() );
runCurrentTest( redirectedCout, redirectedCerr ); runCurrentTest( redirectedCout, redirectedCerr );
// m_reporter->EndGroup( "test case run", m_totals.delta( prevTotals ) );
} }
while( m_runningTest->hasUntestedSections() && !aborting() ); while( m_runningTest->hasUntestedSections() && !aborting() );
} }
@ -144,12 +134,9 @@ namespace Catch {
Totals deltaTotals = m_totals.delta( prevTotals ); Totals deltaTotals = m_totals.delta( prevTotals );
m_totals.testCases += deltaTotals.testCases; m_totals.testCases += deltaTotals.testCases;
m_reporter->EndTestCase( testInfo, deltaTotals, redirectedCout, redirectedCerr ); m_reporter->EndTestCase( testInfo, deltaTotals, redirectedCout, redirectedCerr );
return deltaTotals;
} }
virtual Totals getTotals() const {
return m_totals;
}
const Config& config() const { const Config& config() const {
return m_config; return m_config;
} }

View File

@ -7,6 +7,7 @@
*/ */
#include "catch_test_registry.hpp" #include "catch_test_registry.hpp"
#include "catch_test_case_info.h" #include "catch_test_case_info.h"
#include "catch_test_spec.h"
#include "catch_context.h" #include "catch_context.h"
#include <vector> #include <vector>
@ -31,6 +32,8 @@ namespace Catch {
if( m_functions.find( testInfo ) == m_functions.end() ) { if( m_functions.find( testInfo ) == m_functions.end() ) {
m_functions.insert( testInfo ); m_functions.insert( testInfo );
m_functionsInOrder.push_back( testInfo ); m_functionsInOrder.push_back( testInfo );
if( !testInfo.isHidden() )
m_nonHiddenFunctions.push_back( testInfo );
} }
else { else {
const TestCaseInfo& prev = *m_functions.find( testInfo ); const TestCaseInfo& prev = *m_functions.find( testInfo );
@ -45,24 +48,35 @@ namespace Catch {
return m_functionsInOrder; return m_functionsInOrder;
} }
virtual const std::vector<TestCaseInfo>& getAllNonHiddenTests() const {
return m_nonHiddenFunctions;
}
virtual std::vector<TestCaseInfo> getMatchingTestCases( const std::string& rawTestSpec ) const { virtual std::vector<TestCaseInfo> getMatchingTestCases( const std::string& rawTestSpec ) const {
TestSpec testSpec( rawTestSpec ); TestSpec testSpec( rawTestSpec );
std::vector<TestCaseInfo> testList; std::vector<TestCaseInfo> matchingTests;
getMatchingTestCases( rawTestSpec, matchingTests );
return matchingTests;
}
virtual void getMatchingTestCases( const std::string& rawTestSpec, std::vector<TestCaseInfo>& matchingTestsOut ) const {
TestSpec testSpec( rawTestSpec );
std::vector<TestCaseInfo>::const_iterator it = m_functionsInOrder.begin(); std::vector<TestCaseInfo>::const_iterator it = m_functionsInOrder.begin();
std::vector<TestCaseInfo>::const_iterator itEnd = m_functionsInOrder.end(); std::vector<TestCaseInfo>::const_iterator itEnd = m_functionsInOrder.end();
for(; it != itEnd; ++it ) { for(; it != itEnd; ++it ) {
if( testSpec.matches( it->getName() ) ) { if( testSpec.matches( it->getName() ) ) {
testList.push_back( *it ); matchingTestsOut.push_back( *it );
} }
} }
return testList;
} }
private: private:
std::set<TestCaseInfo> m_functions; std::set<TestCaseInfo> m_functions;
std::vector<TestCaseInfo> m_functionsInOrder; std::vector<TestCaseInfo> m_functionsInOrder;
std::vector<TestCaseInfo> m_nonHiddenFunctions;
size_t m_unnamedCount; size_t m_unnamedCount;
}; };

View File

@ -50,7 +50,13 @@ namespace Catch {
++diff.testCases.passed; ++diff.testCases.passed;
return diff; return diff;
} }
Totals& operator += ( const Totals& other ) {
assertions += other.assertions;
testCases += other.testCases;
return *this;
}
Counts assertions; Counts assertions;
Counts testCases; Counts testCases;
}; };

View File

@ -34,16 +34,16 @@ TEST_CASE( "selftest/main", "Runs all Catch self tests and checks their results"
SECTION( "selftest/test counts/succeeding tests", SECTION( "selftest/test counts/succeeding tests",
"Number of 'succeeding' tests is fixed" ) { "Number of 'succeeding' tests is fixed" ) {
runner.runMatching( "./succeeding/*" ); Totals totals = runner.runMatching( "./succeeding/*" );
CHECK( runner.getTotals().assertions.passed == 285 ); CHECK( totals.assertions.passed == 285 );
CHECK( runner.getTotals().assertions.failed == 0 ); CHECK( totals.assertions.failed == 0 );
} }
SECTION( "selftest/test counts/failing tests", SECTION( "selftest/test counts/failing tests",
"Number of 'failing' tests is fixed" ) { "Number of 'failing' tests is fixed" ) {
runner.runMatching( "./failing/*" ); Totals totals = runner.runMatching( "./failing/*" );
CHECK( runner.getTotals().assertions.passed == 0 ); CHECK( totals.assertions.passed == 0 );
CHECK( runner.getTotals().assertions.failed == 72 ); CHECK( totals.assertions.failed == 72 );
} }
} }
} }
@ -51,9 +51,9 @@ TEST_CASE( "selftest/main", "Runs all Catch self tests and checks their results"
TEST_CASE( "meta/Misc/Sections", "looped tests" ) { TEST_CASE( "meta/Misc/Sections", "looped tests" ) {
Catch::EmbeddedRunner runner; Catch::EmbeddedRunner runner;
runner.runMatching( "./mixed/Misc/Sections/nested2" ); Catch::Totals totals = runner.runMatching( "./mixed/Misc/Sections/nested2" );
CHECK( runner.getTotals().assertions.passed == 2 ); CHECK( totals.assertions.passed == 2 );
CHECK( runner.getTotals().assertions.failed == 1 ); CHECK( totals.assertions.failed == 1 );
} }
#pragma clang diagnostic ignored "-Wweak-vtables" #pragma clang diagnostic ignored "-Wweak-vtables"

View File

@ -13,21 +13,20 @@
namespace Catch{ namespace Catch{
std::size_t EmbeddedRunner::runMatching( const std::string& rawTestSpec, const std::string& ) { Totals EmbeddedRunner::runMatching( const std::string& rawTestSpec, const std::string& ) {
std::ostringstream oss; std::ostringstream oss;
Config config; Config config;
config.setStreamBuf( oss.rdbuf() ); config.setStreamBuf( oss.rdbuf() );
std::size_t result; Totals totals;
// Scoped because Runner doesn't report EndTesting until its destructor // Scoped because Runner doesn't report EndTesting until its destructor
{ {
Runner runner( config, m_reporter.get() ); Runner runner( config, m_reporter.get() );
result = runner.runMatching( rawTestSpec ); totals = runner.runMatching( rawTestSpec );
m_totals = runner.getTotals();
} }
m_output = oss.str(); m_output = oss.str();
return result; return totals;
} }
void MockReporter::Result( const ResultInfo& resultInfo ) { void MockReporter::Result( const ResultInfo& resultInfo ) {

View File

@ -114,17 +114,13 @@ namespace Catch {
public: public:
EmbeddedRunner() : m_reporter( new MockReporter() ) {} EmbeddedRunner() : m_reporter( new MockReporter() ) {}
std::size_t runMatching( const std::string& rawTestSpec, Totals runMatching( const std::string& rawTestSpec,
const std::string& reporter = "basic" ); const std::string& reporter = "basic" );
std::string getOutput() { std::string getOutput() {
return m_output; return m_output;
} }
const Totals& getTotals() const {
return m_totals;
}
void addRecorder( const std::string& recorder ) { void addRecorder( const std::string& recorder ) {
m_reporter->addRecorder( recorder ); m_reporter->addRecorder( recorder );
} }
@ -134,7 +130,6 @@ namespace Catch {
} }
private: private:
Totals m_totals;
std::string m_output; std::string m_output;
Ptr<MockReporter> m_reporter; Ptr<MockReporter> m_reporter;
}; };
@ -157,8 +152,7 @@ namespace Catch {
void operator()( const TestCaseInfo& testCase ) { void operator()( const TestCaseInfo& testCase ) {
EmbeddedRunner runner; EmbeddedRunner runner;
runner.runMatching( testCase.getName() ); Totals totals = runner.runMatching( testCase.getName() );
Totals totals = runner.getTotals();
switch( m_expectedResult ) { switch( m_expectedResult ) {
case Expected::ToSucceed: case Expected::ToSucceed:
if( totals.assertions.failed > 0 ) { if( totals.assertions.failed > 0 ) {