3rd-party
/
catch2
mirror of https://github.com/catchorg/Catch2.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add abiiity to stop after n tests. Add ordered list of tests to check sw4

This commit is contained in:
Malcolm Noyes 2013-12-09 23:12:05 +00:00
parent 15cc1fc185
commit e8c32e819c
8 changed files with 274 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/catch.hpp
View File

@ -208,10 +208,12 @@
#define CATCH_MAP_CATEGORY_TO_TAG( Category, Tag ) INTERNAL_CATCH_MAP_CATEGORY_TO_TAG( Category, Tag ) #define CATCH_MAP_CATEGORY_TO_TAG( Category, Tag ) INTERNAL_CATCH_MAP_CATEGORY_TO_TAG( Category, Tag )
#define CATCH_CONFIG_SHOW_SUCCESS( v ) CATCH_INTERNAL_CONFIG_SHOW_SUCCESS( v ) #define CATCH_CONFIG_SHOW_SUCCESS( v ) CATCH_INTERNAL_CONFIG_SHOW_SUCCESS( v )
#define CATCH_CONFIG_WARN_MISSING_ASSERTIONS( v ) CATCH_INTERNAL_CONFIG_WARN_MISSING_ASSERTIONS( v ) #define CATCH_CONFIG_WARN_MISSING_ASSERTIONS( v ) CATCH_INTERNAL_CONFIG_WARN_MISSING_ASSERTIONS( v )
#define CATCH_CONFIG_ABORT_AFTER( v ) CATCH_INTERNAL_CONFIG_ABORT_AFTER( v )
#else #else
#define CATCH_MAP_CATEGORY_TO_TAG( Category, Tag ) #define CATCH_MAP_CATEGORY_TO_TAG( Category, Tag )
#define CATCH_CONFIG_SHOW_SUCCESS( v ) #define CATCH_CONFIG_SHOW_SUCCESS( v )
#define CATCH_CONFIG_WARN_MISSING_ASSERTIONS( v ) #define CATCH_CONFIG_WARN_MISSING_ASSERTIONS( v )
#define CATCH_CONFIG_ABORT_AFTER( v )
#endif #endif
using Catch::Detail::Approx; using Catch::Detail::Approx;

100
include/internal/catch_config.hpp
View File

@ -45,8 +45,10 @@ namespace CatchOverrides {
template <typename T> template <typename T>
class Config class Config
{ {
typedef std::map<int, bool> LineData; typedef std::map<int, int> LineData;
typedef std::map<std::string, LineData> FileLineData; typedef std::map<std::string, LineData> FileLineData;
typedef std::map<int, std::string> StringLineData;
typedef std::map<std::string, StringLineData> FileStringLineData;
public: public:
bool includeSuccessfulResults(const std::string& file, int c) const bool includeSuccessfulResults(const std::string& file, int c) const
{ {
@ -58,7 +60,7 @@ namespace CatchOverrides {
{ {
if( c <= lineIt->first ) if( c <= lineIt->first )
break; break;
result = lineIt->second; result = lineIt->second ? true : false;
} }
} }
return result; return result;
@ -69,12 +71,12 @@ namespace CatchOverrides {
if( it == showSuccessfulTestsData.end() ) if( it == showSuccessfulTestsData.end() )
{ {
LineData tmp; LineData tmp;
tmp.insert(std::make_pair(c,v)); tmp.insert(std::make_pair(c,(v ? 1 : 0)));
showSuccessfulTestsData.insert(std::make_pair(file, tmp)); showSuccessfulTestsData.insert(std::make_pair(file, tmp));
} }
else else
{ {
it->second.insert(std::make_pair(c,v)); it->second.insert(std::make_pair(c,(v ? 1 : 0)));
} }
} }
bool warnAboutMissingAssertions(const std::string& file, int c) const bool warnAboutMissingAssertions(const std::string& file, int c) const
@ -87,7 +89,7 @@ namespace CatchOverrides {
{ {
if( c <= lineIt->first ) if( c <= lineIt->first )
break; break;
result = lineIt->second; result = lineIt->second ? true : false;
} }
} }
return result; return result;
@ -98,10 +100,72 @@ namespace CatchOverrides {
if( it == missingAssertionData.end() ) if( it == missingAssertionData.end() )
{ {
LineData tmp; LineData tmp;
tmp.insert(std::make_pair(c,v)); tmp.insert(std::make_pair(c,(v ? 1 : 0)));
missingAssertionData.insert(std::make_pair(file, tmp)); missingAssertionData.insert(std::make_pair(file, tmp));
} }
else else
{
it->second.insert(std::make_pair(c,(v ? 1 : 0)));
}
}
int abortAfter(const std::string& file, int c) const
{
int result(-1);
FileLineData::const_iterator it = abortAfterData.find(file);
if( it != abortAfterData.end() )
{
for( LineData::const_iterator lineIt = it->second.begin(); lineIt != it->second.end(); ++lineIt )
{
if( c <= lineIt->first )
break;
result = lineIt->second;
}
}
return result;
}
void insertAbortAfter(const std::string& file, int c, int v)
{
FileLineData::iterator it = abortAfterData.find(file);
if( it == abortAfterData.end() )
{
LineData tmp;
tmp.insert(std::make_pair(c,v));
abortAfterData.insert(std::make_pair(file, tmp));
}
else
{
it->second.insert(std::make_pair(c,v));
}
}
std::vector<std::string> listOfTests(const std::string& file, int c) const
{
std::vector<std::string> result;
FileStringLineData::const_iterator it = testData.find(file);
if( it != testData.end() )
{
for( StringLineData::const_iterator lineIt = it->second.begin(); lineIt != it->second.end(); ++lineIt )
{
if( lineIt->second.empty() && c > lineIt->first)
result.clear();
else {
if( c <= lineIt->first )
break;
result.push_back(lineIt->second);
}
}
}
return result;
}
void insertTest(const std::string& file, int c, const std::string& v)
{
FileStringLineData::iterator it = testData.find(file);
if( it == testData.end() )
{
StringLineData tmp;
tmp.insert(std::make_pair(c,v));
testData.insert(std::make_pair(file, tmp));
}
else
{ {
it->second.insert(std::make_pair(c,v)); it->second.insert(std::make_pair(c,v));
} }
@ -117,6 +181,8 @@ namespace CatchOverrides {
private: private:
FileLineData showSuccessfulTestsData; FileLineData showSuccessfulTestsData;
FileLineData missingAssertionData; FileLineData missingAssertionData;
FileLineData abortAfterData;
FileStringLineData testData;
static Config<T>* s_instance; static Config<T>* s_instance;
}; };
@ -130,6 +196,8 @@ namespace CatchOverrides {
{ {
Config<T>::instance().insertSuccessfulResults(file, c, false); Config<T>::instance().insertSuccessfulResults(file, c, false);
Config<T>::instance().insertMissingAssertions(file, c, false); Config<T>::instance().insertMissingAssertions(file, c, false);
Config<T>::instance().insertAbortAfter(file, c, -1);
Config<T>::instance().insertTest(file, c, "");
} }
}; };
@ -152,6 +220,26 @@ namespace CatchOverrides {
Config<T>::instance().insertMissingAssertions(file, c, v ? true : false); Config<T>::instance().insertMissingAssertions(file, c, v ? true : false);
} }
}; };
template <typename T>
struct ConfigAbortAfter
{
template <typename U>
ConfigAbortAfter( const std::string& file, int c, U v )
{
Config<T>::instance().insertAbortAfter(file, c, v);
}
};
template <typename T>
struct ConfigAddTest
{
template <typename U>
ConfigAddTest( const std::string& file, int c, U v )
{
Config<T>::instance().insertTest(file, c, v);
}
};
} }
namespace Catch { namespace Catch {

49
include/internal/catch_vs_test_registry.hpp
View File

@ -192,6 +192,9 @@ private:
#define INTERNAL_CATCH_MAP_CATEGORY_TO_TAG( Category, Tag ) \ #define INTERNAL_CATCH_MAP_CATEGORY_TO_TAG( Category, Tag ) \
INTERNAL_CATCH_MAP_CATEGORY_TO_TAG2( #Category, Tag, __COUNTER__ ) INTERNAL_CATCH_MAP_CATEGORY_TO_TAG2( #Category, Tag, __COUNTER__ )
#define INTERNAL_CATCH_MAP_CATEGORY_TO_LIST( Category ) \
INTERNAL_CATCH_MAP_CATEGORY_TO_LIST2( #Category, __COUNTER__ )
#define FAIL_STRING( str ) _T( str ) #define FAIL_STRING( str ) _T( str )
#else // detect CLR #else // detect CLR
@ -238,6 +241,9 @@ private:
#define INTERNAL_CATCH_MAP_CATEGORY_TO_TAG( Category, Tag ) \ #define INTERNAL_CATCH_MAP_CATEGORY_TO_TAG( Category, Tag ) \
INTERNAL_CATCH_MAP_CATEGORY_TO_TAG2( Category, Tag, __COUNTER__ ) INTERNAL_CATCH_MAP_CATEGORY_TO_TAG2( Category, Tag, __COUNTER__ )
#define INTERNAL_CATCH_MAP_CATEGORY_TO_LIST( Category, List ) \
INTERNAL_CATCH_MAP_CATEGORY_TO_LIST2( Category, List, __COUNTER__ )
#define FAIL_STRING( str ) WIDEN( str ) #define FAIL_STRING( str ) WIDEN( str )
#endif // detect CLR #endif // detect CLR
@ -250,12 +256,24 @@ private:
#define CATCH_INTERNAL_CONFIG_WARN_MISSING_ASSERTIONS2( v, Count ) \ #define CATCH_INTERNAL_CONFIG_WARN_MISSING_ASSERTIONS2( v, Count ) \
namespace { CatchOverrides::ConfigWarnMissingAssertions<Catch::IConfig const*> INTERNAL_CATCH_UNIQUE_NAME_LINE( C_A_T_C_H_____O_V_E_R_R_I_D_E____, INTERNAL_CATCH_CONCAT_LINE_COUNTER( Count ) )(__FILE__, Count, v); } namespace { CatchOverrides::ConfigWarnMissingAssertions<Catch::IConfig const*> INTERNAL_CATCH_UNIQUE_NAME_LINE( C_A_T_C_H_____O_V_E_R_R_I_D_E____, INTERNAL_CATCH_CONCAT_LINE_COUNTER( Count ) )(__FILE__, Count, v); }
#define CATCH_INTERNAL_CONFIG_ABORT_AFTER2( v, Count ) \
namespace { CatchOverrides::ConfigAbortAfter<Catch::IConfig const*> INTERNAL_CATCH_UNIQUE_NAME_LINE( C_A_T_C_H_____O_V_E_R_R_I_D_E____, INTERNAL_CATCH_CONCAT_LINE_COUNTER( Count ) )(__FILE__, Count, v); }
#define CATCH_INTERNAL_CONFIG_ADD_TEST2( v, Count ) \
namespace { CatchOverrides::ConfigAddTest<Catch::IConfig const*> INTERNAL_CATCH_UNIQUE_NAME_LINE( C_A_T_C_H_____O_V_E_R_R_I_D_E____, INTERNAL_CATCH_CONCAT_LINE_COUNTER( Count ) )(__FILE__, Count, v); }
#define CATCH_INTERNAL_CONFIG_SHOW_SUCCESS( v ) \ #define CATCH_INTERNAL_CONFIG_SHOW_SUCCESS( v ) \
CATCH_INTERNAL_CONFIG_SHOW_SUCCESS2( v, __COUNTER__) CATCH_INTERNAL_CONFIG_SHOW_SUCCESS2( v, __COUNTER__)
#define CATCH_INTERNAL_CONFIG_WARN_MISSING_ASSERTIONS( v ) \ #define CATCH_INTERNAL_CONFIG_WARN_MISSING_ASSERTIONS( v ) \
CATCH_INTERNAL_CONFIG_WARN_MISSING_ASSERTIONS2( v, __COUNTER__) CATCH_INTERNAL_CONFIG_WARN_MISSING_ASSERTIONS2( v, __COUNTER__)
#define CATCH_INTERNAL_CONFIG_ABORT_AFTER( v ) \
CATCH_INTERNAL_CONFIG_ABORT_AFTER2( v, __COUNTER__)
#define CATCH_INTERNAL_CONFIG_ADD_TEST( v ) \
CATCH_INTERNAL_CONFIG_ADD_TEST2( v, __COUNTER__)
#define CATCH_INTERNAL_RUN_SINGLE_TEST( Count ) \ #define CATCH_INTERNAL_RUN_SINGLE_TEST( Count ) \
{ CatchOverrides::ConfigGuard cg; \ { CatchOverrides::ConfigGuard cg; \
Catch::ConfigData cd(cg.value().get()); \ Catch::ConfigData cd(cg.value().get()); \
@ -263,6 +281,7 @@ private:
cd.abortAfter = 1; \ cd.abortAfter = 1; \
cd.showSuccessfulTests = CatchOverrides::Config<Catch::IConfig const*>::instance().includeSuccessfulResults(__FILE__, Count ); \ cd.showSuccessfulTests = CatchOverrides::Config<Catch::IConfig const*>::instance().includeSuccessfulResults(__FILE__, Count ); \
cd.warnings = (CatchOverrides::Config<Catch::IConfig const*>::instance().warnAboutMissingAssertions(__FILE__, Count ) ? Catch::WarnAbout::NoAssertions : Catch::WarnAbout::Nothing); \ cd.warnings = (CatchOverrides::Config<Catch::IConfig const*>::instance().warnAboutMissingAssertions(__FILE__, Count ) ? Catch::WarnAbout::NoAssertions : Catch::WarnAbout::Nothing); \
cd.abortAfter = CatchOverrides::Config<Catch::IConfig const*>::instance().abortAfter(__FILE__, Count ); \
Catch::Ptr<Catch::Config> config(new Catch::Config(cd)); \ Catch::Ptr<Catch::Config> config(new Catch::Config(cd)); \
Catch::MSTestReporter* rep = new Catch::MSTestReporter(config.get()); \ Catch::MSTestReporter* rep = new Catch::MSTestReporter(config.get()); \
Catch::RunContext tr(config.get(), rep); \ Catch::RunContext tr(config.get(), rep); \
@ -351,6 +370,7 @@ private:
Catch::ConfigData cd; \ Catch::ConfigData cd; \
cd.showSuccessfulTests = CatchOverrides::Config<Catch::IConfig const*>::instance().includeSuccessfulResults(__FILE__, Count ); \ cd.showSuccessfulTests = CatchOverrides::Config<Catch::IConfig const*>::instance().includeSuccessfulResults(__FILE__, Count ); \
cd.warnings = (CatchOverrides::Config<Catch::IConfig const*>::instance().warnAboutMissingAssertions(__FILE__, Count ) ? Catch::WarnAbout::NoAssertions : Catch::WarnAbout::Nothing); \ cd.warnings = (CatchOverrides::Config<Catch::IConfig const*>::instance().warnAboutMissingAssertions(__FILE__, Count ) ? Catch::WarnAbout::NoAssertions : Catch::WarnAbout::Nothing); \
cd.abortAfter = CatchOverrides::Config<Catch::IConfig const*>::instance().abortAfter(__FILE__, Count ); \
cd.reporterName = "vs_reporter"; \ cd.reporterName = "vs_reporter"; \
cd.name = "Batch run using tag : " Tag; \ cd.name = "Batch run using tag : " Tag; \
cd.testsOrTags.push_back( Tag ); \ cd.testsOrTags.push_back( Tag ); \
@ -365,6 +385,35 @@ private:
}; \ }; \
} }
#define INTERNAL_CATCH_MAP_CATEGORY_TO_LIST2( Category, Count ) \
CHECK_FOR_TEST_CASE_CLASH \
namespace CATCH_INTERNAL_NAMESPACE( INTERNAL_CATCH_CONCAT_LINE_COUNTER( Count ) ) { \
CatchOverrides::ConfigReset<Catch::IConfig const*> INTERNAL_CATCH_UNIQUE_NAME_LINE( C_A_T_C_H____T_E_S_T____C_O_N_F_I_G___, INTERNAL_CATCH_CONCAT_LINE_COUNTER( Count ) )(__FILE__, Count); \
INTERNAL_CATCH_CLASS_DEFINITION( INTERNAL_CATCH_UNIQUE_NAME_LINE( C_A_T_C_H____T_E_S_T____C_L_A_S_S___, INTERNAL_CATCH_CONCAT_LINE_COUNTER( Count ) ) ) \
{ \
INTERNAL_CATCH_CLASS_CONTEXT \
BEGIN_INTERNAL_CATCH_BATCH_METHOD( Category, INTERNAL_CATCH_CONCAT_LINE_COUNTER( Count ) ) \
{ \
Catch::ConfigData cd; \
cd.showSuccessfulTests = CatchOverrides::Config<Catch::IConfig const*>::instance().includeSuccessfulResults(__FILE__, Count ); \
cd.warnings = (CatchOverrides::Config<Catch::IConfig const*>::instance().warnAboutMissingAssertions(__FILE__, Count ) ? Catch::WarnAbout::NoAssertions : Catch::WarnAbout::Nothing); \
cd.abortAfter = CatchOverrides::Config<Catch::IConfig const*>::instance().abortAfter(__FILE__, Count ); \
cd.reporterName = "vs_reporter"; \
cd.name = "Batch run using category : " Category; \
std::vector<std::string> stringNames = CatchOverrides::Config<Catch::IConfig const*>::instance().listOfTests(__FILE__, Count ); \
Catch::Ptr<Catch::Config> config(new Catch::Config(cd)); \
Catch::MSTestReporter* rep = new Catch::MSTestReporter(config.get()); \
Catch::RunContext tr(config.get(), rep); \
for( std::vector<std::string>::iterator it = stringNames.begin(); it != stringNames.end(); ++it ) { \
std::vector<Catch::TestCase> testCase = Catch::getRegistryHub().getTestCaseRegistry().getMatchingTestCases(*it); \
if( testCase.empty() ) Assert::Fail(FAIL_STRING("No tests match")); \
if( testCase.size() > 1 ) Assert::Fail(FAIL_STRING("More than one test with the same name")); \
tr.runTest(*testCase.begin()); \
} \
} \
}; \
}
//#undef CATCH_CONFIG_VARIADIC_MACROS //#undef CATCH_CONFIG_VARIADIC_MACROS
#ifdef CATCH_CONFIG_VARIADIC_MACROS #ifdef CATCH_CONFIG_VARIADIC_MACROS

4
projects/SelfTest/MessageInstantiationTests1.cpp
View File

@ -23,7 +23,7 @@ namespace MI1
// greedy instantiation (or whatever process it uses) eliminate all other // greedy instantiation (or whatever process it uses) eliminate all other
// references to the globalCount // references to the globalCount
TEST_CASE("message counting1","") TEST_CASE("message counting1","[vs]")
{ {
if( Counter::g_haveCountedMessages > 0 ) { if( Counter::g_haveCountedMessages > 0 ) {
REQUIRE( Catch::MessageInfoCounter<unsigned int>::globalCount > 0 ); REQUIRE( Catch::MessageInfoCounter<unsigned int>::globalCount > 0 );
@ -42,7 +42,7 @@ int g_haveCountedMessagesLong = 0;
namespace MI1 namespace MI1
{ {
TEST_CASE("long message counting1","") TEST_CASE("long message counting1","[vs]")
{ {
if( LongCounter::g_haveCountedMessagesLong > 0 ) { if( LongCounter::g_haveCountedMessagesLong > 0 ) {
REQUIRE( Catch::MessageInfoCounter<long>::globalCount > 0 ); REQUIRE( Catch::MessageInfoCounter<long>::globalCount > 0 );

4
projects/SelfTest/MessageInstantiationTests2.cpp
View File

@ -22,7 +22,7 @@ namespace MI2
// greedy instantiation (or whatever process it uses) eliminate all other // greedy instantiation (or whatever process it uses) eliminate all other
// references to the globalCount // references to the globalCount
TEST_CASE("message counting2","") TEST_CASE("message counting2","[vs]")
{ {
if( Counter::g_haveCountedMessages > 0 ) { if( Counter::g_haveCountedMessages > 0 ) {
REQUIRE( Catch::MessageInfoCounter<unsigned int>::globalCount > 0 ); REQUIRE( Catch::MessageInfoCounter<unsigned int>::globalCount > 0 );
@ -41,7 +41,7 @@ extern int g_haveCountedMessagesLong;
namespace MI2 namespace MI2
{ {
TEST_CASE("long message counting2","") TEST_CASE("long message counting2","[vs]")
{ {
if( LongCounter::g_haveCountedMessagesLong > 0 ) { if( LongCounter::g_haveCountedMessagesLong > 0 ) {
REQUIRE( Catch::MessageInfoCounter<long>::globalCount > 0 ); REQUIRE( Catch::MessageInfoCounter<long>::globalCount > 0 );

99
projects/SelfTest/RunAllTests.cpp
View File

@ -12,6 +12,7 @@
namespace AllTestsRunner { namespace AllTestsRunner {
#ifdef OLD_RUNNER
class NullStreamingReporter : public Catch::SharedImpl<Catch::IStreamingReporter> { class NullStreamingReporter : public Catch::SharedImpl<Catch::IStreamingReporter> {
public: public:
@ -54,6 +55,23 @@ namespace AllTestsRunner {
Catch::Ptr<Catch::IStreamingReporter> m_reporter; Catch::Ptr<Catch::IStreamingReporter> m_reporter;
}; };
NullStreamingReporter::~NullStreamingReporter() {}
Catch::Totals EmbeddedRunner::runMatching( const std::string& rawTestSpec, std::size_t groupIndex, std::size_t groupsCount, const std::string& ) {
std::ostringstream oss;
Catch::Ptr<Catch::Config> config = new Catch::Config();
config->setStreamBuf( oss.rdbuf() );
Catch::Totals totals;
// Scoped because RunContext doesn't report EndTesting until its destructor
{
Catch::RunContext runner( config.get(), m_reporter.get() );
totals = runner.runMatching( rawTestSpec, groupIndex, groupsCount );
}
return totals;
}
class MetaTestRunner { class MetaTestRunner {
public: public:
@ -116,23 +134,6 @@ namespace AllTestsRunner {
std::size_t m_groupsCount; std::size_t m_groupsCount;
}; };
NullStreamingReporter::~NullStreamingReporter() {}
Catch::Totals EmbeddedRunner::runMatching( const std::string& rawTestSpec, std::size_t groupIndex, std::size_t groupsCount, const std::string& ) {
std::ostringstream oss;
Catch::Ptr<Catch::Config> config = new Catch::Config();
config->setStreamBuf( oss.rdbuf() );
Catch::Totals totals;
// Scoped because RunContext doesn't report EndTesting until its destructor
{
Catch::RunContext runner( config.get(), m_reporter.get() );
totals = runner.runMatching( rawTestSpec, groupIndex, groupsCount );
}
return totals;
}
TEST_CASE( "Run all failing and succeeding tests", "[vsall]" ) { TEST_CASE( "Run all failing and succeeding tests", "[vsall]" ) {
/////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////
@ -181,8 +182,66 @@ namespace AllTestsRunner {
} }
} }
// mstest /TestContainer:Debug\ManagedTestCatch.dll /category:"all" TEST_CASE( "Run all failing and succeeding tests", "[sw4][vs]" ) {
#if defined(INTERNAL_CATCH_VS_MANAGED) || defined(INTERNAL_CATCH_VS_NATIVE) CatchOverrides::ConfigGuard cg;
CATCH_MAP_CATEGORY_TO_TAG(all, "[vsall]"); Catch::ConfigData cd(cg.value().get());
cd.name = "Test sw4";
cd.abortAfter = 1;
cd.showSuccessfulTests = true;
cd.warnings = Catch::WarnAbout::NoAssertions;
cd.abortAfter = -1;
Catch::Ptr<Catch::Config> config(new Catch::Config(cd));
Catch::MSTestReporter* rep = new Catch::MSTestReporter(config.get());
Catch::RunContext tr(config.get(), rep);
std::string names[] = {"one","two","three"};
std::vector<std::string> stringNames(names, names + (sizeof(names)/sizeof(std::string)));
std::vector<Catch::TestCase> testCase = Catch::getRegistryHub().getTestCaseRegistry().getMatchingTestCases( "Some simple comparisons between doubles" );
//std::vector<Catch::TestCase> testCase = Catch::getRegistryHub().getTestCaseRegistry().getMatchingTestCases(name_desc.name);
if( testCase.empty() ) Assert::Fail(FAIL_STRING("No tests match"));
if( testCase.size() > 1 ) Assert::Fail(FAIL_STRING("More than one test with the same name"));
Catch::Totals totals = tr.runTest(*testCase.begin());
if( totals.assertions.failed > 0 ) {
INTERNAL_CATCH_TEST_THROW_FAILURE
}
/*for(std::vector<Catch::TestCase>::iterator it = tests.begin(); it != tests.end(); ++it )
{
Catch::Totals totals;
std::size_t groupIndex(0);
std::size_t groupsCount(0);
{
EmbeddedRunner runner;
std::string name = it->getTestCaseInfo().name;
totals = runner.runMatching( name, groupIndex, groupsCount );
}
}*/
}
#endif #endif
// mstest /TestContainer:Debug\ManagedTestCatch.dll /category:"all"
#if defined(INTERNAL_CATCH_VS_MANAGED) || defined(INTERNAL_CATCH_VS_NATIVE)
CATCH_MAP_CATEGORY_TO_TAG(all, "~[vs]");
CATCH_CONFIG_SHOW_SUCCESS(true)
CATCH_CONFIG_WARN_MISSING_ASSERTIONS(true)
CATCH_MAP_CATEGORY_TO_TAG(allSucceeding, "~[vs]");
CATCH_CONFIG_SHOW_SUCCESS(true)
CATCH_CONFIG_WARN_MISSING_ASSERTIONS(true)
CATCH_CONFIG_ABORT_AFTER(4)
CATCH_INTERNAL_CONFIG_ADD_TEST("Some simple comparisons between doubles")
CATCH_INTERNAL_CONFIG_ADD_TEST("Approximate comparisons with different epsilons")
CATCH_INTERNAL_CONFIG_ADD_TEST("Approximate comparisons with floats")
CATCH_INTERNAL_CONFIG_ADD_TEST("Approximate comparisons with ints")
CATCH_INTERNAL_CONFIG_ADD_TEST("Approximate comparisons with mixed numeric types")
CATCH_INTERNAL_CONFIG_ADD_TEST("Use a custom approx")
CATCH_INTERNAL_CONFIG_ADD_TEST("Approximate PI")
CATCH_INTERNAL_CONFIG_ADD_TEST("A METHOD_AS_TEST_CASE based test run that succeeds")
CATCH_INTERNAL_CONFIG_ADD_TEST("A METHOD_AS_TEST_CASE based test run that fails")
CATCH_INTERNAL_CONFIG_ADD_TEST("A TEST_CASE_METHOD based test run that succeeds")
CATCH_INTERNAL_CONFIG_ADD_TEST("A TEST_CASE_METHOD based test run that fails")
CATCH_INTERNAL_CONFIG_ADD_TEST("Equality checks that should succeed")
CATCH_INTERNAL_CONFIG_ADD_TEST("Equality checks that should fail]")
INTERNAL_CATCH_MAP_CATEGORY_TO_LIST(allSucceedingAborting);
#endif
} }

26
projects/SelfTest/VisualStudioTests.cpp
View File

@ -24,26 +24,26 @@ namespace VisualStudioTests
}; };
int UniqueTestsFixture::uniqueID = 0; int UniqueTestsFixture::uniqueID = 0;
TEST_CASE("M00", "[m_off]") TEST_CASE("M00", "[m_off][vs]")
{ {
bool show = Catch::getCurrentContext().getConfig()->includeSuccessfulResults(); bool show = Catch::getCurrentContext().getConfig()->includeSuccessfulResults();
REQUIRE(!show); REQUIRE(!show);
} }
CATCH_CONFIG_SHOW_SUCCESS(true) CATCH_CONFIG_SHOW_SUCCESS(true)
TEST_CASE("M01", "[m_on]") TEST_CASE("M01", "[m_on][vs]")
{ {
bool show = Catch::getCurrentContext().getConfig()->includeSuccessfulResults(); bool show = Catch::getCurrentContext().getConfig()->includeSuccessfulResults();
REQUIRE(show); REQUIRE(show);
} }
TEST_CASE("M02", "[m_off]") TEST_CASE("M02", "[m_off][vs]")
{ {
bool show = Catch::getCurrentContext().getConfig()->includeSuccessfulResults(); bool show = Catch::getCurrentContext().getConfig()->includeSuccessfulResults();
REQUIRE(!show); REQUIRE(!show);
} }
TEST_CASE_METHOD(UniqueTestsFixture, "M10", "[m_off]") TEST_CASE_METHOD(UniqueTestsFixture, "M10", "[m_off][vs]")
{ {
bool show = Catch::getCurrentContext().getConfig()->includeSuccessfulResults(); bool show = Catch::getCurrentContext().getConfig()->includeSuccessfulResults();
REQUIRE(!show); REQUIRE(!show);
@ -52,7 +52,7 @@ namespace VisualStudioTests
CATCH_CONFIG_WARN_MISSING_ASSERTIONS(true) CATCH_CONFIG_WARN_MISSING_ASSERTIONS(true)
CATCH_CONFIG_SHOW_SUCCESS(true) CATCH_CONFIG_SHOW_SUCCESS(true)
TEST_CASE_METHOD(UniqueTestsFixture, "M11", "[m_on]") TEST_CASE_METHOD(UniqueTestsFixture, "M11", "[m_on][vs]")
{ {
bool show = Catch::getCurrentContext().getConfig()->includeSuccessfulResults(); bool show = Catch::getCurrentContext().getConfig()->includeSuccessfulResults();
REQUIRE(show); REQUIRE(show);
@ -61,7 +61,7 @@ namespace VisualStudioTests
CATCH_CONFIG_WARN_MISSING_ASSERTIONS(true) CATCH_CONFIG_WARN_MISSING_ASSERTIONS(true)
CATCH_CONFIG_SHOW_SUCCESS(true) CATCH_CONFIG_SHOW_SUCCESS(true)
TEST_CASE_METHOD(UniqueTestsFixture, "M99", "[m_on]") TEST_CASE_METHOD(UniqueTestsFixture, "M99", "[m_on][vs]")
{ {
bool show = Catch::getCurrentContext().getConfig()->includeSuccessfulResults(); bool show = Catch::getCurrentContext().getConfig()->includeSuccessfulResults();
REQUIRE(show); REQUIRE(show);
@ -69,7 +69,7 @@ namespace VisualStudioTests
getID(); getID();
} }
TEST_CASE_METHOD(UniqueTestsFixture, "M12", "[m_off]") TEST_CASE_METHOD(UniqueTestsFixture, "M12", "[m_off][vs]")
{ {
bool show = Catch::getCurrentContext().getConfig()->includeSuccessfulResults(); bool show = Catch::getCurrentContext().getConfig()->includeSuccessfulResults();
REQUIRE(!show); REQUIRE(!show);
@ -95,16 +95,16 @@ namespace VisualStudioTests
REQUIRE(!show); REQUIRE(!show);
} }
}; };
METHOD_AS_TEST_CASE(ConfigTest::run1,"M20", "[m_off]"); METHOD_AS_TEST_CASE(ConfigTest::run1,"M20", "[m_off][vs]");
CATCH_CONFIG_SHOW_SUCCESS(true) CATCH_CONFIG_SHOW_SUCCESS(true)
METHOD_AS_TEST_CASE(ConfigTest::run2,"M21", "[m_on]"); METHOD_AS_TEST_CASE(ConfigTest::run2,"M21", "[m_on][vs]");
METHOD_AS_TEST_CASE(ConfigTest::run3,"M22", "[m_off]"); METHOD_AS_TEST_CASE(ConfigTest::run3,"M22", "[m_off][vs]");
CATCH_MAP_CATEGORY_TO_TAG(vstestsCheckOutputOff, "[m_off]"); CATCH_MAP_CATEGORY_TO_TAG(vstestsCheckOutputOff, "[m_off][vs]");
CATCH_CONFIG_SHOW_SUCCESS(true) CATCH_CONFIG_SHOW_SUCCESS(true)
CATCH_MAP_CATEGORY_TO_TAG(vstestsCheckOutputOn, "[m_on]"); CATCH_MAP_CATEGORY_TO_TAG(vstestsCheckOutputOn, "[m_on][vs]");
CATCH_MAP_CATEGORY_TO_TAG(vstestsCheckOutputOff2, "[m_off]"); CATCH_MAP_CATEGORY_TO_TAG(vstestsCheckOutputOff2, "[m_off][vs]");
} }
#endif #endif

46
scripts/approvalVSTests.py → scripts/validateVSTests.py
View File

@ -23,7 +23,8 @@ if len(sys.argv) == 2:
else: else:
if sys.platform == 'win32': if sys.platform == 'win32':
cmdPath = os.path.join( catchPath, 'projects\\VS2010\\TestCatch\\Release\\TestCatch.exe' ) cmdPath = os.path.join( catchPath, 'projects\\VS2010\\TestCatch\\Release\\TestCatch.exe' )
dllPath = os.path.join( catchPath, 'projects\\VS2010\\ManagedTestCatch\\Release\\ManagedTestCatch.dll' ) #dllPath = os.path.join( catchPath, 'projects\\VS2010\\ManagedTestCatch\\Release\\ManagedTestCatch.dll' )
dllPath = os.path.join( catchPath, 'projects\\VS2010\\ManagedTestCatch\\Debug\\ManagedTestCatch.dll' )
else: else:
cmdPath = os.path.join( catchPath, 'projects/XCode4/CatchSelfTest/DerivedData/CatchSelfTest/Build/Products/Debug/CatchSelfTest' ) cmdPath = os.path.join( catchPath, 'projects/XCode4/CatchSelfTest/DerivedData/CatchSelfTest/Build/Products/Debug/CatchSelfTest' )
@ -62,6 +63,7 @@ def approve( baseName, args ):
else: else:
raise Exception("Results file does not exist: '" + rawResultsPath + "'") raise Exception("Results file does not exist: '" + rawResultsPath + "'")
def callDiff():
#os.remove( rawResultsPath ) #os.remove( rawResultsPath )
print print
print baseName + ":" print baseName + ":"
@ -593,7 +595,7 @@ def approveXml( baseName, args ):
rawWriteFile.write(line + "\n") rawWriteFile.write(line + "\n")
rawWriteFile.close() rawWriteFile.close()
def parseTrxFile(trxFile): def parseTrxFile(baseName, trxFile):
print "TRX file:" ,trxFile print "TRX file:" ,trxFile
if os.path.exists( trxFile ): if os.path.exists( trxFile ):
xml = "" xml = ""
@ -657,19 +659,26 @@ def parseTrxFile(trxFile):
if tag != None and tag == "StdOut": if tag != None and tag == "StdOut":
desc = sub.text desc = sub.text
lines = desc.splitlines() lines = desc.splitlines()
if (len(lines) > 2 and found = False
lines[0].startswith("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") and index = 0
lines[1].startswith("Using Catch v") ): for tmp in lines:
lines = lines[2:-1] if (len(lines) >= (index + 2) and
lines[index].startswith("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") and
lines[index + 1].startswith("Using Catch v") ):
found = True
break
index += 1
lines = lines[index + 2:-1]
#print "*******",desc #print "*******",desc
#print lines #print lines
if found:
for line in lines: for line in lines:
testcase = resultParser.parseResultLine(line) testcase = resultParser.parseResultLine(line)
if isinstance(testcase, TestCaseData): if isinstance(testcase, TestCaseData):
testRun.testcases.append(testcase) testRun.testcases.append(testcase)
lines = testRun.generateSortedUnapprovedLines(0) lines = testRun.generateSortedUnapprovedLines(0)
rawSortedPath = os.path.join( rootPath, 'mstest.trx.sorted.unapproved.txt' ) rawSortedPath = os.path.join( rootPath, '{0}.sorted.unapproved.txt'.format( baseName ) )
rawWriteFile = open( rawSortedPath, 'wb' ) rawWriteFile = open( rawSortedPath, 'wb' )
for line in lines: for line in lines:
#print "L:",line #print "L:",line
@ -677,7 +686,7 @@ def parseTrxFile(trxFile):
rawWriteFile.close() rawWriteFile.close()
def approveMsTest( baseName ): def approveMsTest( baseName, filter ):
rawResultsPath = os.path.join( rootPath, '_{0}.tmp'.format( baseName ) ) rawResultsPath = os.path.join( rootPath, '_{0}.tmp'.format( baseName ) )
if not(os.path.exists( dllPath )): if not(os.path.exists( dllPath )):
raise Exception("Managed DLL does not exist: '" + dllPath + "'") raise Exception("Managed DLL does not exist: '" + dllPath + "'")
@ -685,18 +694,19 @@ def approveMsTest( baseName ):
args = [] args = []
args.append("MSTest.exe") args.append("MSTest.exe")
args.append("/testcontainer:" + dllPath) args.append("/testcontainer:" + dllPath)
#f = open( rawResultsPath, 'w' ) args.append("/category:\"" + filter + "\"")
#subprocess.call( args, stdout=f, stderr=f ) f = open( rawResultsPath, 'w' )
#f.close() subprocess.call( args, stdout=f, stderr=f )
f.close()
#if os.path.exists( rawResultsPath ): if os.path.exists( rawResultsPath ):
# f = open( rawResultsPath, 'r' ) f = open( rawResultsPath, 'r' )
# for line in f: for line in f:
line = "Results file: c:\Projects\Catch\TestResults\NoyesMa_SACHDEW7 2013-12-09 11_43_57.trx" #line = "Results file: c:\Projects\Catch\TestResults\NoyesMa_SACHDEW7 2013-12-09 11_43_57.trx"
if line.startswith("Results file:"): if line.startswith("Results file:"):
trxFile = line[13:].strip() trxFile = line[13:].strip()
parseTrxFile(trxFile) parseTrxFile(baseName, trxFile)
# Standard console reporter # Standard console reporter
#approve( "console.std", ["~_"] ) #approve( "console.std", ["~_"] )
@ -709,7 +719,9 @@ def approveMsTest( baseName ):
# xml reporter, include passes, warn about No Assertions # xml reporter, include passes, warn about No Assertions
#approveXml( "xml.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "xml"] ) #approveXml( "xml.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "xml"] )
#mstest runner, xml output #mstest runner, xml output
approveMsTest( "mstest.sw") #approveMsTest( "mstest.std", "all")
#approveMsTest( "mstest.sw", "allSucceeding")
approveMsTest( "mstest.swa4", "allSucceedingAborting")
if overallResult <> 0: if overallResult <> 0:
print "run approve.py to approve new baselines" print "run approve.py to approve new baselines"