mirror of
https://github.com/catchorg/Catch2.git
synced 2024-12-23 11:43:29 +01:00
Separated getting test cases to run from running them
This commit is contained in:
parent
d0a5461add
commit
c1dbc9c22b
@ -83,22 +83,22 @@ namespace Catch {
|
|||||||
{
|
{
|
||||||
Runner runner( configWrapper, reporter );
|
Runner runner( configWrapper, reporter );
|
||||||
|
|
||||||
|
Totals totals;
|
||||||
// Run test specs specified on the command line - or default to all
|
// Run test specs specified on the command line - or default to all
|
||||||
if( config.testSpecs.empty() ) {
|
if( config.testSpecs.empty() ) {
|
||||||
runner.runAll();
|
totals = runner.runAllNonHidden();
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
// !TBD We should get all the testcases upfront, report any missing,
|
|
||||||
// then just run them
|
|
||||||
std::vector<std::string>::const_iterator it = config.testSpecs.begin();
|
std::vector<std::string>::const_iterator it = config.testSpecs.begin();
|
||||||
std::vector<std::string>::const_iterator itEnd = config.testSpecs.end();
|
std::vector<std::string>::const_iterator itEnd = config.testSpecs.end();
|
||||||
for(; it != itEnd; ++it ) {
|
for(; it != itEnd; ++it ) {
|
||||||
if( runner.runMatching( *it ) == 0 ) {
|
Totals groupTotals = runner.runMatching( *it );
|
||||||
|
if( groupTotals.testCases.total() == 0 )
|
||||||
std::cerr << "\n[No test cases matched with: " << *it << "]" << std::endl;
|
std::cerr << "\n[No test cases matched with: " << *it << "]" << std::endl;
|
||||||
}
|
totals += groupTotals;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
result = static_cast<int>( runner.getTotals().assertions.failed );
|
result = static_cast<int>( totals.assertions.failed );
|
||||||
}
|
}
|
||||||
Catch::cleanUp();
|
Catch::cleanUp();
|
||||||
return result;
|
return result;
|
||||||
|
@ -17,9 +17,18 @@ namespace Catch {
|
|||||||
|
|
||||||
struct IRunner {
|
struct IRunner {
|
||||||
virtual ~IRunner();
|
virtual ~IRunner();
|
||||||
virtual void runAll( bool runHiddenTests = false ) = 0;
|
|
||||||
virtual std::size_t runMatching( const std::string& rawTestSpec ) = 0;
|
/// Runs all tests, even if hidden
|
||||||
virtual Totals getTotals() const = 0;
|
virtual Totals runAll() = 0;
|
||||||
|
|
||||||
|
/// Runs all tests unless 'hidden' by ./ prefix
|
||||||
|
virtual Totals runAllNonHidden() = 0;
|
||||||
|
|
||||||
|
/// Runs all test that match the spec string
|
||||||
|
virtual Totals runMatching( const std::string& rawTestSpec ) = 0;
|
||||||
|
|
||||||
|
/// Runs all the tests passed in
|
||||||
|
virtual Totals runTests( const std::string& groupName, const std::vector<TestCaseInfo>& testCases ) = 0;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -22,7 +22,10 @@ namespace Catch {
|
|||||||
struct ITestCaseRegistry {
|
struct ITestCaseRegistry {
|
||||||
virtual ~ITestCaseRegistry();
|
virtual ~ITestCaseRegistry();
|
||||||
virtual const std::vector<TestCaseInfo>& getAllTests() const = 0;
|
virtual const std::vector<TestCaseInfo>& getAllTests() const = 0;
|
||||||
|
virtual const std::vector<TestCaseInfo>& getAllNonHiddenTests() const = 0;
|
||||||
|
|
||||||
virtual std::vector<TestCaseInfo> getMatchingTestCases( const std::string& rawTestSpec ) const = 0;
|
virtual std::vector<TestCaseInfo> getMatchingTestCases( const std::string& rawTestSpec ) const = 0;
|
||||||
|
virtual void getMatchingTestCases( const std::string& rawTestSpec, std::vector<TestCaseInfo>& matchingTestsOut ) const = 0;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -79,45 +79,37 @@ namespace Catch {
|
|||||||
m_context.setConfig( m_prevConfig );
|
m_context.setConfig( m_prevConfig );
|
||||||
}
|
}
|
||||||
|
|
||||||
virtual void runAll( bool runHiddenTests = false ) {
|
virtual Totals runAll() {
|
||||||
m_reporter->StartGroup( "" );
|
return runTests( "", getRegistryHub().getTestCaseRegistry().getAllTests() );
|
||||||
const std::vector<TestCaseInfo>& allTests = getRegistryHub().getTestCaseRegistry().getAllTests();
|
|
||||||
for( std::size_t i=0; i < allTests.size(); ++i ) {
|
|
||||||
if( runHiddenTests || !allTests[i].isHidden() ) {
|
|
||||||
if( aborting() ) {
|
|
||||||
m_reporter->Aborted();
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
runTest( allTests[i] );
|
|
||||||
}
|
|
||||||
}
|
|
||||||
m_reporter->EndGroup( "", getTotals() );
|
|
||||||
}
|
}
|
||||||
|
|
||||||
virtual std::size_t runMatching( const std::string& rawTestSpec ) {
|
|
||||||
|
|
||||||
Totals prevTotals = getTotals();
|
virtual Totals runAllNonHidden() {
|
||||||
m_reporter->StartGroup( rawTestSpec );
|
return runTests( "", getRegistryHub().getTestCaseRegistry().getAllNonHiddenTests() );
|
||||||
|
|
||||||
TestSpec testSpec( rawTestSpec );
|
|
||||||
|
|
||||||
const std::vector<TestCaseInfo>& allTests = getRegistryHub().getTestCaseRegistry().getAllTests();
|
|
||||||
std::size_t testsRun = 0;
|
|
||||||
for( std::size_t i=0; i < allTests.size(); ++i ) {
|
|
||||||
if( testSpec.matches( allTests[i].getName() ) ) {
|
|
||||||
if( aborting() ) {
|
|
||||||
m_reporter->Aborted();
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
runTest( allTests[i] );
|
|
||||||
testsRun++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
m_reporter->EndGroup( rawTestSpec, getTotals() - prevTotals );
|
|
||||||
return testsRun;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void runTest( const TestCaseInfo& testInfo ) {
|
virtual Totals runMatching( const std::string& rawTestSpec ) {
|
||||||
|
|
||||||
|
const std::vector<TestCaseInfo>& matchingTests = getRegistryHub().getTestCaseRegistry().getMatchingTestCases( rawTestSpec );
|
||||||
|
return runTests( rawTestSpec, matchingTests );
|
||||||
|
}
|
||||||
|
|
||||||
|
virtual Totals runTests( const std::string& groupName, const std::vector<TestCaseInfo>& testCases ) {
|
||||||
|
|
||||||
|
Totals totals;
|
||||||
|
m_reporter->StartGroup( groupName );
|
||||||
|
|
||||||
|
for( std::size_t i=0; i < testCases.size(); ++i ) {
|
||||||
|
if( aborting() ) {
|
||||||
|
m_reporter->Aborted();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
totals += runTest( testCases[i] );
|
||||||
|
}
|
||||||
|
m_reporter->EndGroup( groupName, totals );
|
||||||
|
return totals;
|
||||||
|
}
|
||||||
|
|
||||||
|
Totals runTest( const TestCaseInfo& testInfo ) {
|
||||||
Totals prevTotals = m_totals;
|
Totals prevTotals = m_totals;
|
||||||
|
|
||||||
std::string redirectedCout;
|
std::string redirectedCout;
|
||||||
@ -129,10 +121,8 @@ namespace Catch {
|
|||||||
|
|
||||||
do {
|
do {
|
||||||
do {
|
do {
|
||||||
// m_reporter->StartGroup( "test case run" );
|
|
||||||
m_currentResult.setLineInfo( m_runningTest->getTestCaseInfo().getLineInfo() );
|
m_currentResult.setLineInfo( m_runningTest->getTestCaseInfo().getLineInfo() );
|
||||||
runCurrentTest( redirectedCout, redirectedCerr );
|
runCurrentTest( redirectedCout, redirectedCerr );
|
||||||
// m_reporter->EndGroup( "test case run", m_totals.delta( prevTotals ) );
|
|
||||||
}
|
}
|
||||||
while( m_runningTest->hasUntestedSections() && !aborting() );
|
while( m_runningTest->hasUntestedSections() && !aborting() );
|
||||||
}
|
}
|
||||||
@ -144,12 +134,9 @@ namespace Catch {
|
|||||||
Totals deltaTotals = m_totals.delta( prevTotals );
|
Totals deltaTotals = m_totals.delta( prevTotals );
|
||||||
m_totals.testCases += deltaTotals.testCases;
|
m_totals.testCases += deltaTotals.testCases;
|
||||||
m_reporter->EndTestCase( testInfo, deltaTotals, redirectedCout, redirectedCerr );
|
m_reporter->EndTestCase( testInfo, deltaTotals, redirectedCout, redirectedCerr );
|
||||||
|
return deltaTotals;
|
||||||
}
|
}
|
||||||
|
|
||||||
virtual Totals getTotals() const {
|
|
||||||
return m_totals;
|
|
||||||
}
|
|
||||||
|
|
||||||
const Config& config() const {
|
const Config& config() const {
|
||||||
return m_config;
|
return m_config;
|
||||||
}
|
}
|
||||||
|
@ -7,6 +7,7 @@
|
|||||||
*/
|
*/
|
||||||
#include "catch_test_registry.hpp"
|
#include "catch_test_registry.hpp"
|
||||||
#include "catch_test_case_info.h"
|
#include "catch_test_case_info.h"
|
||||||
|
#include "catch_test_spec.h"
|
||||||
#include "catch_context.h"
|
#include "catch_context.h"
|
||||||
|
|
||||||
#include <vector>
|
#include <vector>
|
||||||
@ -31,6 +32,8 @@ namespace Catch {
|
|||||||
if( m_functions.find( testInfo ) == m_functions.end() ) {
|
if( m_functions.find( testInfo ) == m_functions.end() ) {
|
||||||
m_functions.insert( testInfo );
|
m_functions.insert( testInfo );
|
||||||
m_functionsInOrder.push_back( testInfo );
|
m_functionsInOrder.push_back( testInfo );
|
||||||
|
if( !testInfo.isHidden() )
|
||||||
|
m_nonHiddenFunctions.push_back( testInfo );
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
const TestCaseInfo& prev = *m_functions.find( testInfo );
|
const TestCaseInfo& prev = *m_functions.find( testInfo );
|
||||||
@ -45,24 +48,35 @@ namespace Catch {
|
|||||||
return m_functionsInOrder;
|
return m_functionsInOrder;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
virtual const std::vector<TestCaseInfo>& getAllNonHiddenTests() const {
|
||||||
|
return m_nonHiddenFunctions;
|
||||||
|
}
|
||||||
|
|
||||||
virtual std::vector<TestCaseInfo> getMatchingTestCases( const std::string& rawTestSpec ) const {
|
virtual std::vector<TestCaseInfo> getMatchingTestCases( const std::string& rawTestSpec ) const {
|
||||||
TestSpec testSpec( rawTestSpec );
|
TestSpec testSpec( rawTestSpec );
|
||||||
|
|
||||||
std::vector<TestCaseInfo> testList;
|
std::vector<TestCaseInfo> matchingTests;
|
||||||
|
getMatchingTestCases( rawTestSpec, matchingTests );
|
||||||
|
return matchingTests;
|
||||||
|
}
|
||||||
|
|
||||||
|
virtual void getMatchingTestCases( const std::string& rawTestSpec, std::vector<TestCaseInfo>& matchingTestsOut ) const {
|
||||||
|
TestSpec testSpec( rawTestSpec );
|
||||||
|
|
||||||
std::vector<TestCaseInfo>::const_iterator it = m_functionsInOrder.begin();
|
std::vector<TestCaseInfo>::const_iterator it = m_functionsInOrder.begin();
|
||||||
std::vector<TestCaseInfo>::const_iterator itEnd = m_functionsInOrder.end();
|
std::vector<TestCaseInfo>::const_iterator itEnd = m_functionsInOrder.end();
|
||||||
for(; it != itEnd; ++it ) {
|
for(; it != itEnd; ++it ) {
|
||||||
if( testSpec.matches( it->getName() ) ) {
|
if( testSpec.matches( it->getName() ) ) {
|
||||||
testList.push_back( *it );
|
matchingTestsOut.push_back( *it );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return testList;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
||||||
std::set<TestCaseInfo> m_functions;
|
std::set<TestCaseInfo> m_functions;
|
||||||
std::vector<TestCaseInfo> m_functionsInOrder;
|
std::vector<TestCaseInfo> m_functionsInOrder;
|
||||||
|
std::vector<TestCaseInfo> m_nonHiddenFunctions;
|
||||||
size_t m_unnamedCount;
|
size_t m_unnamedCount;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -50,7 +50,13 @@ namespace Catch {
|
|||||||
++diff.testCases.passed;
|
++diff.testCases.passed;
|
||||||
return diff;
|
return diff;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Totals& operator += ( const Totals& other ) {
|
||||||
|
assertions += other.assertions;
|
||||||
|
testCases += other.testCases;
|
||||||
|
return *this;
|
||||||
|
}
|
||||||
|
|
||||||
Counts assertions;
|
Counts assertions;
|
||||||
Counts testCases;
|
Counts testCases;
|
||||||
};
|
};
|
||||||
|
@ -34,16 +34,16 @@ TEST_CASE( "selftest/main", "Runs all Catch self tests and checks their results"
|
|||||||
|
|
||||||
SECTION( "selftest/test counts/succeeding tests",
|
SECTION( "selftest/test counts/succeeding tests",
|
||||||
"Number of 'succeeding' tests is fixed" ) {
|
"Number of 'succeeding' tests is fixed" ) {
|
||||||
runner.runMatching( "./succeeding/*" );
|
Totals totals = runner.runMatching( "./succeeding/*" );
|
||||||
CHECK( runner.getTotals().assertions.passed == 285 );
|
CHECK( totals.assertions.passed == 285 );
|
||||||
CHECK( runner.getTotals().assertions.failed == 0 );
|
CHECK( totals.assertions.failed == 0 );
|
||||||
}
|
}
|
||||||
|
|
||||||
SECTION( "selftest/test counts/failing tests",
|
SECTION( "selftest/test counts/failing tests",
|
||||||
"Number of 'failing' tests is fixed" ) {
|
"Number of 'failing' tests is fixed" ) {
|
||||||
runner.runMatching( "./failing/*" );
|
Totals totals = runner.runMatching( "./failing/*" );
|
||||||
CHECK( runner.getTotals().assertions.passed == 0 );
|
CHECK( totals.assertions.passed == 0 );
|
||||||
CHECK( runner.getTotals().assertions.failed == 72 );
|
CHECK( totals.assertions.failed == 72 );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -51,9 +51,9 @@ TEST_CASE( "selftest/main", "Runs all Catch self tests and checks their results"
|
|||||||
TEST_CASE( "meta/Misc/Sections", "looped tests" ) {
|
TEST_CASE( "meta/Misc/Sections", "looped tests" ) {
|
||||||
Catch::EmbeddedRunner runner;
|
Catch::EmbeddedRunner runner;
|
||||||
|
|
||||||
runner.runMatching( "./mixed/Misc/Sections/nested2" );
|
Catch::Totals totals = runner.runMatching( "./mixed/Misc/Sections/nested2" );
|
||||||
CHECK( runner.getTotals().assertions.passed == 2 );
|
CHECK( totals.assertions.passed == 2 );
|
||||||
CHECK( runner.getTotals().assertions.failed == 1 );
|
CHECK( totals.assertions.failed == 1 );
|
||||||
}
|
}
|
||||||
|
|
||||||
#pragma clang diagnostic ignored "-Wweak-vtables"
|
#pragma clang diagnostic ignored "-Wweak-vtables"
|
||||||
|
@ -13,21 +13,20 @@
|
|||||||
|
|
||||||
namespace Catch{
|
namespace Catch{
|
||||||
|
|
||||||
std::size_t EmbeddedRunner::runMatching( const std::string& rawTestSpec, const std::string& ) {
|
Totals EmbeddedRunner::runMatching( const std::string& rawTestSpec, const std::string& ) {
|
||||||
std::ostringstream oss;
|
std::ostringstream oss;
|
||||||
Config config;
|
Config config;
|
||||||
config.setStreamBuf( oss.rdbuf() );
|
config.setStreamBuf( oss.rdbuf() );
|
||||||
|
|
||||||
std::size_t result;
|
Totals totals;
|
||||||
|
|
||||||
// Scoped because Runner doesn't report EndTesting until its destructor
|
// Scoped because Runner doesn't report EndTesting until its destructor
|
||||||
{
|
{
|
||||||
Runner runner( config, m_reporter.get() );
|
Runner runner( config, m_reporter.get() );
|
||||||
result = runner.runMatching( rawTestSpec );
|
totals = runner.runMatching( rawTestSpec );
|
||||||
m_totals = runner.getTotals();
|
|
||||||
}
|
}
|
||||||
m_output = oss.str();
|
m_output = oss.str();
|
||||||
return result;
|
return totals;
|
||||||
}
|
}
|
||||||
|
|
||||||
void MockReporter::Result( const ResultInfo& resultInfo ) {
|
void MockReporter::Result( const ResultInfo& resultInfo ) {
|
||||||
|
@ -114,17 +114,13 @@ namespace Catch {
|
|||||||
public:
|
public:
|
||||||
EmbeddedRunner() : m_reporter( new MockReporter() ) {}
|
EmbeddedRunner() : m_reporter( new MockReporter() ) {}
|
||||||
|
|
||||||
std::size_t runMatching( const std::string& rawTestSpec,
|
Totals runMatching( const std::string& rawTestSpec,
|
||||||
const std::string& reporter = "basic" );
|
const std::string& reporter = "basic" );
|
||||||
|
|
||||||
std::string getOutput() {
|
std::string getOutput() {
|
||||||
return m_output;
|
return m_output;
|
||||||
}
|
}
|
||||||
|
|
||||||
const Totals& getTotals() const {
|
|
||||||
return m_totals;
|
|
||||||
}
|
|
||||||
|
|
||||||
void addRecorder( const std::string& recorder ) {
|
void addRecorder( const std::string& recorder ) {
|
||||||
m_reporter->addRecorder( recorder );
|
m_reporter->addRecorder( recorder );
|
||||||
}
|
}
|
||||||
@ -134,7 +130,6 @@ namespace Catch {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
Totals m_totals;
|
|
||||||
std::string m_output;
|
std::string m_output;
|
||||||
Ptr<MockReporter> m_reporter;
|
Ptr<MockReporter> m_reporter;
|
||||||
};
|
};
|
||||||
@ -157,8 +152,7 @@ namespace Catch {
|
|||||||
|
|
||||||
void operator()( const TestCaseInfo& testCase ) {
|
void operator()( const TestCaseInfo& testCase ) {
|
||||||
EmbeddedRunner runner;
|
EmbeddedRunner runner;
|
||||||
runner.runMatching( testCase.getName() );
|
Totals totals = runner.runMatching( testCase.getName() );
|
||||||
Totals totals = runner.getTotals();
|
|
||||||
switch( m_expectedResult ) {
|
switch( m_expectedResult ) {
|
||||||
case Expected::ToSucceed:
|
case Expected::ToSucceed:
|
||||||
if( totals.assertions.failed > 0 ) {
|
if( totals.assertions.failed > 0 ) {
|
||||||
|
Loading…
Reference in New Issue
Block a user