From 8774268140a901e95b3f156b114f40767fe976ac Mon Sep 17 00:00:00 2001 From: Malcolm Noyes Date: Tue, 10 Dec 2013 13:25:31 +0000 Subject: [PATCH] Fix cout/cerr outut to reporter --- include/internal/catch_vs_test_registry.hpp | 26 +- include/reporters/catch_vs_reporter.hpp | 331 ++++++++++++-------- projects/SelfTest/RunAllTests.cpp | 211 +------------ scripts/validateVSTests.py | 43 ++- 4 files changed, 256 insertions(+), 355 deletions(-) diff --git a/include/internal/catch_vs_test_registry.hpp b/include/internal/catch_vs_test_registry.hpp index f4d44b34..58d748e1 100644 --- a/include/internal/catch_vs_test_registry.hpp +++ b/include/internal/catch_vs_test_registry.hpp @@ -283,12 +283,14 @@ private: cd.warnings = (CatchOverrides::Config::instance().warnAboutMissingAssertions(__FILE__, Count ) ? Catch::WarnAbout::NoAssertions : Catch::WarnAbout::Nothing); \ cd.abortAfter = CatchOverrides::Config::instance().abortAfter(__FILE__, Count ); \ Catch::Ptr config(new Catch::Config(cd)); \ - Catch::MSTestReporter* rep = new Catch::MSTestReporter(config.get()); \ - Catch::RunContext tr(config.get(), rep); \ + Catch::ReporterRegistrar reporterReg("vs_reporter"); \ + Catch::RunContext context(config.get(), Catch::getRegistryHub().getReporterRegistry().create( "vs_reporter", config.get())); \ std::vector testCase = Catch::getRegistryHub().getTestCaseRegistry().getMatchingTestCases(name_desc.name); \ if( testCase.empty() ) Assert::Fail(FAIL_STRING("No tests match")); \ if( testCase.size() > 1 ) Assert::Fail(FAIL_STRING("More than one test with the same name")); \ - Catch::Totals totals = tr.runTest(*testCase.begin()); \ + context.testGroupStarting( "", 0, 1 ); \ + Catch::Totals totals = context.runTest(*testCase.begin()); \ + context.testGroupEnded( "", totals, 0, 1 ); \ if( totals.assertions.failed > 0 ) { \ INTERNAL_CATCH_TEST_THROW_FAILURE \ } \ @@ -371,11 +373,11 @@ private: cd.showSuccessfulTests = CatchOverrides::Config::instance().includeSuccessfulResults(__FILE__, Count ); \ cd.warnings = (CatchOverrides::Config::instance().warnAboutMissingAssertions(__FILE__, Count ) ? Catch::WarnAbout::NoAssertions : Catch::WarnAbout::Nothing); \ cd.abortAfter = CatchOverrides::Config::instance().abortAfter(__FILE__, Count ); \ - cd.reporterName = "vs_reporter"; \ + cd.reporterName = "vs_reporterlf"; \ cd.name = "Batch run using tag : " Tag; \ cd.testsOrTags.push_back( Tag ); \ Catch::Ptr config(new Catch::Config(cd)); \ - Catch::ReporterRegistrar reporterReg("vs_reporter"); \ + Catch::ReporterRegistrar reporterReg("vs_reporterlf"); \ Catch::Runner runner(config); \ Catch::Totals totals = runner.runTests(); \ if( totals.assertions.failed > 0 ) { \ @@ -398,17 +400,23 @@ private: cd.showSuccessfulTests = CatchOverrides::Config::instance().includeSuccessfulResults(__FILE__, Count ); \ cd.warnings = (CatchOverrides::Config::instance().warnAboutMissingAssertions(__FILE__, Count ) ? Catch::WarnAbout::NoAssertions : Catch::WarnAbout::Nothing); \ cd.abortAfter = CatchOverrides::Config::instance().abortAfter(__FILE__, Count ); \ - cd.reporterName = "vs_reporter"; \ + cd.reporterName = "vs_reporterlf"; \ cd.name = "Batch run using category : " Category; \ std::vector stringNames = CatchOverrides::Config::instance().listOfTests(__FILE__, Count ); \ Catch::Ptr config(new Catch::Config(cd)); \ - Catch::MSTestReporter* rep = new Catch::MSTestReporter(config.get()); \ - Catch::RunContext tr(config.get(), rep); \ + Catch::ReporterRegistrar reporterReg("vs_reporterlf"); \ + Catch::RunContext context(config.get(), Catch::getRegistryHub().getReporterRegistry().create( "vs_reporterlf", config.get())); \ + Catch::Totals totals; \ + context.testGroupStarting( "", 0, 1 ); \ for( std::vector::iterator it = stringNames.begin(); it != stringNames.end(); ++it ) { \ std::vector testCase = Catch::getRegistryHub().getTestCaseRegistry().getMatchingTestCases(*it); \ if( testCase.empty() ) Assert::Fail(FAIL_STRING("No tests match")); \ if( testCase.size() > 1 ) Assert::Fail(FAIL_STRING("More than one test with the same name")); \ - tr.runTest(*testCase.begin()); \ + totals += context.runTest(*testCase.begin()); \ + } \ + context.testGroupEnded( "", totals, 0, 1 ); \ + if( totals.assertions.failed > 0 ) { \ + INTERNAL_CATCH_TEST_REPORT_BATCH_FAILURE(totals.assertions.failed) \ } \ } \ }; \ diff --git a/include/reporters/catch_vs_reporter.hpp b/include/reporters/catch_vs_reporter.hpp index 9b7d2368..8a158133 100644 --- a/include/reporters/catch_vs_reporter.hpp +++ b/include/reporters/catch_vs_reporter.hpp @@ -11,6 +11,7 @@ #include "../internal/catch_interfaces_reporter.h" #include "../internal/catch_text.h" #include "../internal/catch_version.h" +#include "../internal/catch_console_colour.hpp" namespace Catch { @@ -44,42 +45,33 @@ namespace Catch { #endif // _WINDLL #endif // detect CLR - - struct MSTestReporter : SharedImpl { - MSTestReporter( ReporterConfig const& _config ) - : m_config( _config.fullConfig() ), - m_headerPrinted( false ), - m_atLeastOneTestCasePrinted( false ), - m_failed(0) - {} - - MSTestReporter( Ptr const& _fullConfig ) - : m_config( _fullConfig ), - m_headerPrinted( false ), - m_atLeastOneTestCasePrinted( false ), - m_failed(0) - {} - - virtual ~MSTestReporter() { - if( m_atLeastOneTestCasePrinted ) { - write_output_message(stream.str()); - /*if( m_failed ) - { - Assert::IsTrue(false, L"At least one test failed - examine output for failures."); - }*/ + inline void replaceSingleLinefeed(const std::string& s, std::string& result) + { + bool needr(false); + for(std::string::const_iterator it = s.begin(); it != s.end(); ++it ) { + if( *it == '\r' ) { + needr = false; } + else if( *it == '\n' && needr ) { + needr = false; + result += '\r'; + result += *it; + } + else { + needr = true; + } + result += *it; } - - static std::string getDescription() { - return "Reports test results to MSTest"; - } - virtual ReporterPreferences getPreferences() const { - ReporterPreferences prefs; - prefs.shouldRedirectStdOut = true; - return prefs; - } + } + + struct VSStreamingReporterBase : SharedImpl { + + VSStreamingReporterBase( ReporterConfig const& _config ) + : m_config( _config.fullConfig() ) + {} + + virtual ~VSStreamingReporterBase() {} - //base virtual void noMatchingTestCases( std::string const& ) {} virtual void testRunStarting( TestRunInfo const& _testRunInfo ) { @@ -93,69 +85,76 @@ namespace Catch { currentTestCaseInfo = _testInfo; } virtual void sectionStarting( SectionInfo const& _sectionInfo ) { - m_headerPrinted = false; m_sectionStack.push_back( _sectionInfo ); } - virtual void sectionEnded( SectionStats const& _sectionStats ) { - if( _sectionStats.missingAssertions ) { - lazyPrint(); - if( m_sectionStack.size() > 1 ) - stream << "\r\n" << "No assertions in section"; - else - stream << "\r\n" << "No assertions in test case"; - stream << " '" << _sectionStats.sectionInfo.name << "'" << "\r\n" << "\r\n"; - } - if( m_headerPrinted ) { - if( m_config->showDurations() == ShowDurations::Always ) - stream << "Completed in " << _sectionStats.durationInSeconds << "s" << "\r\n"; - m_headerPrinted = false; - } - else { - if( m_config->showDurations() == ShowDurations::Always ) - stream << _sectionStats.sectionInfo.name << " completed in " << _sectionStats.durationInSeconds << "s" << "\r\n"; - } + virtual void sectionEnded( SectionStats const& /* _sectionStats */ ) { m_sectionStack.pop_back(); } - virtual void testCaseEnded( TestCaseStats const& _testCaseStats ) { - if( !_testCaseStats.stdOut.empty() ) { - write_output_message(getDoubleDashes()); - write_output_message("Output to std::cout :"); - write_output_message(getDashes()); - write_output_message(_testCaseStats.stdOut); - write_output_message(getDoubleDashes()); - } - if( !_testCaseStats.stdErr.empty() ) { - write_output_message(getDoubleDashes()); - write_output_message("Output to std::cerr :"); - write_output_message(getDashes()); - write_output_message(_testCaseStats.stdErr); - write_output_message(getDoubleDashes()); - } - m_headerPrinted = false; + virtual void testCaseEnded( TestCaseStats const& /* _testCaseStats */ ) { currentTestCaseInfo.reset(); assert( m_sectionStack.empty() ); } - virtual void testGroupEnded( TestGroupStats const& _testGroupStats ) { - if( currentGroupInfo.used ) { - printSummaryDivider(); - stream << "Summary for group '" << _testGroupStats.groupInfo.name << "':" << "\r\n"; - printTotals( _testGroupStats.totals ); - stream << "\r\n" << "\r\n"; - } + virtual void testGroupEnded( TestGroupStats const& /* _testGroupStats */ ) { currentGroupInfo.reset(); } - virtual void testRunEnded( TestRunStats const& _testRunStats ) { - if( m_atLeastOneTestCasePrinted ) - printTotalsDivider(); - printTotals( _testRunStats.totals ); - stream << "\r\n" << "\r\n"; - m_failed = _testRunStats.totals.testCases.failed; + virtual void testRunEnded( TestRunStats const& /* _testRunStats */ ) { currentTestCaseInfo.reset(); currentGroupInfo.reset(); currentTestRunInfo.reset(); } - // base end + + Ptr m_config; + + LazyStat currentTestRunInfo; + LazyStat currentGroupInfo; + LazyStat currentTestCaseInfo; + + std::vector m_sectionStack; + }; + + struct MSTestReporter : VSStreamingReporterBase { + typedef VSStreamingReporterBase StreamingReporterBase; + MSTestReporter( ReporterConfig const& _config ) + : VSStreamingReporterBase( _config ), + m_prevCout( std::cout.rdbuf() ), + m_prevCerr( std::cerr.rdbuf() ), + m_addLineFeeds(true), + m_headerPrinted( false ), + m_atLeastOneTestCasePrinted( false ) + { + std::cout.rdbuf( stream.rdbuf() ); + std::cerr.rdbuf( stream.rdbuf() ); + } + + virtual ~MSTestReporter() { + std::string output = stream.str(); + if( !output.empty() ) { + if( m_addLineFeeds ) { + std::string revised; + replaceSingleLinefeed(output, revised); + write_output_message(revised); + } + else { + write_output_message(output); + } + } + std::cout.rdbuf( m_prevCout ); + std::cerr.rdbuf( m_prevCerr ); + } + + static std::string getDescription() { + return "Reports test results as plain lines of text"; + } + virtual ReporterPreferences getPreferences() const { + ReporterPreferences prefs; + prefs.shouldRedirectStdOut = false; + return prefs; + } + + virtual void noMatchingTestCases( std::string const& spec ) { + stream << "No test cases matched '" << spec << "'" << std::endl; + } virtual void assertionStarting( AssertionInfo const& ) { } @@ -163,32 +162,87 @@ namespace Catch { virtual bool assertionEnded( AssertionStats const& _assertionStats ) { AssertionResult const& result = _assertionStats.assertionResult; + bool printInfoMessages = true; + // Drop out if result was successful and we're not printing those - if( !m_config->includeSuccessfulResults() && result.isOk() ) - return false; + if( !m_config->includeSuccessfulResults() && result.isOk() ) { + if( result.getResultType() != ResultWas::Warning ) + return false; + printInfoMessages = false; + } lazyPrint(); - AssertionPrinter printer( stream, _assertionStats ); + AssertionPrinter printer( stream, _assertionStats, printInfoMessages ); printer.print(); - stream << "\r\n"; + stream << std::endl; return true; } + virtual void sectionStarting( SectionInfo const& _sectionInfo ) { + m_headerPrinted = false; + StreamingReporterBase::sectionStarting( _sectionInfo ); + } + virtual void sectionEnded( SectionStats const& _sectionStats ) { + if( _sectionStats.missingAssertions ) { + lazyPrint(); + Colour colour( Colour::ResultError ); + if( m_sectionStack.size() > 1 ) + stream << "\nNo assertions in section"; + else + stream << "\nNo assertions in test case"; + stream << " '" << _sectionStats.sectionInfo.name << "'\n" << std::endl; + } + if( m_headerPrinted ) { + if( m_config->showDurations() == ShowDurations::Always ) + stream << "Completed in " << _sectionStats.durationInSeconds << "s" << std::endl; + m_headerPrinted = false; + } + else { + if( m_config->showDurations() == ShowDurations::Always ) + stream << _sectionStats.sectionInfo.name << " completed in " << _sectionStats.durationInSeconds << "s" << std::endl; + } + StreamingReporterBase::sectionEnded( _sectionStats ); + } + + virtual void testCaseEnded( TestCaseStats const& _testCaseStats ) { + StreamingReporterBase::testCaseEnded( _testCaseStats ); + m_headerPrinted = false; + } + virtual void testGroupEnded( TestGroupStats const& _testGroupStats ) { + if( currentGroupInfo.used ) { + printSummaryDivider(); + stream << "Summary for group '" << _testGroupStats.groupInfo.name << "':\n"; + printTotals( _testGroupStats.totals ); + stream << "\n" << std::endl; + } + StreamingReporterBase::testGroupEnded( _testGroupStats ); + } + virtual void testRunEnded( TestRunStats const& _testRunStats ) { + if( m_atLeastOneTestCasePrinted ) + printTotalsDivider(); + printTotals( _testRunStats.totals ); + stream << "\n" << std::endl; + StreamingReporterBase::testRunEnded( _testRunStats ); + } + private: class AssertionPrinter { void operator= ( AssertionPrinter const& ); public: - AssertionPrinter( std::ostream& _stream, AssertionStats const& _stats ) + AssertionPrinter( std::ostream& _stream, AssertionStats const& _stats, bool _printInfoMessages ) : stream( _stream ), stats( _stats ), result( _stats.assertionResult ), + colour( Colour::None ), message( result.getMessage() ), - messages( _stats.infoMessages ) + messages( _stats.infoMessages ), + printInfoMessages( _printInfoMessages ) { switch( result.getResultType() ) { case ResultWas::Ok: + colour = Colour::Success; passOrFail = "PASSED"; //if( result.hasMessage() ) if( _stats.infoMessages.size() == 1 ) @@ -198,9 +252,11 @@ namespace Catch { break; case ResultWas::ExpressionFailed: if( result.isOk() ) { + colour = Colour::Success; passOrFail = "FAILED - but was ok"; } else { + colour = Colour::Error; passOrFail = "FAILED"; } if( _stats.infoMessages.size() == 1 ) @@ -209,10 +265,12 @@ namespace Catch { messageLabel = "with messages"; break; case ResultWas::ThrewException: + colour = Colour::Error; passOrFail = "FAILED"; messageLabel = "due to unexpected exception with message"; break; case ResultWas::DidntThrowException: + colour = Colour::Error; passOrFail = "FAILED"; messageLabel = "because no exception was thrown where one was expected"; break; @@ -224,6 +282,7 @@ namespace Catch { break; case ResultWas::ExplicitFailure: passOrFail = "FAILED"; + colour = Colour::Error; if( _stats.infoMessages.size() == 1 ) messageLabel = "explicitly with message"; if( _stats.infoMessages.size() > 1 ) @@ -234,6 +293,7 @@ namespace Catch { case ResultWas::FailureBit: case ResultWas::Exception: passOrFail = "** internal error **"; + colour = Colour::Error; break; } } @@ -242,13 +302,13 @@ namespace Catch { printSourceInfo(); if( stats.totals.assertions.total() > 0 ) { if( result.isOk() ) - stream << "\r\n"; + stream << "\n"; printResultType(); printOriginalExpression(); printReconstructedExpression(); } else { - stream << "\r\n"; + stream << "\n"; } printMessage(); } @@ -256,42 +316,50 @@ namespace Catch { private: void printResultType() const { if( !passOrFail.empty() ) { - stream << passOrFail << ":" << "\r\n"; + Colour colourGuard( colour ); + stream << passOrFail << ":\n"; } } void printOriginalExpression() const { if( result.hasExpression() ) { + Colour colourGuard( Colour::OriginalExpression ); stream << " "; stream << result.getExpressionInMacro(); - stream << "\r\n"; + stream << "\n"; } } void printReconstructedExpression() const { if( result.hasExpandedExpression() ) { - stream << "with expansion:" << "\r\n"; - stream << Text( result.getExpandedExpression(), TextAttributes().setIndent(2) ) << "\r\n"; + stream << "with expansion:\n"; + Colour colourGuard( Colour::ReconstructedExpression ); + stream << Text( result.getExpandedExpression(), TextAttributes().setIndent(2) ) << "\n"; } } void printMessage() const { if( !messageLabel.empty() ) - stream << messageLabel << ":" << "\r\n"; + stream << messageLabel << ":" << "\n"; for( std::vector::const_iterator it = messages.begin(), itEnd = messages.end(); it != itEnd; ++it ) { - stream << Text( it->message, TextAttributes().setIndent(2) ) << "\r\n"; + // If this assertion is a warning ignore any INFO messages + if( printInfoMessages || it->type != ResultWas::Info ) + stream << Text( it->message, TextAttributes().setIndent(2) ) << "\n"; } } void printSourceInfo() const { + Colour colourGuard( Colour::FileName ); stream << result.getSourceInfo() << ": "; } std::ostream& stream; AssertionStats const& stats; AssertionResult const& result; + Colour::Code colour; std::string passOrFail; std::string messageLabel; std::string message; std::vector messages; + bool printInfoMessages; }; void lazyPrint() { @@ -308,29 +376,27 @@ namespace Catch { m_atLeastOneTestCasePrinted = true; } void lazyPrintRunInfo() { - stream << getTildes() << "\r\n"; + stream << "\n" << getTildes() << "\n"; + Colour colour( Colour::SecondaryText ); stream << "Using Catch v" << libraryVersion::value.majorVersion << "." << libraryVersion::value.minorVersion << " b" << libraryVersion::value.buildNumber; if( libraryVersion::value.branchName != "master" ) stream << " (" << libraryVersion::value.branchName << ")"; #if (_MANAGED == 1) || (_M_CEE == 1) // detect CLR - stream << " for a managed MSTest project." << "\r\n"; + stream << " for a managed MSTest project." << "\n"; #else #ifdef _WINDLL - stream << " for a native MSTest project." << "\r\n"; + stream << " for a native MSTest project." << "\n"; #endif #endif currentTestRunInfo.used = true; } void lazyPrintGroupInfo() { - if( currentGroupInfo.some() ) - { - if( !currentGroupInfo->name.empty() && currentGroupInfo->groupsCounts > 1 ) { - printClosedHeader( "Group: " + currentGroupInfo->name ); - currentGroupInfo.used = true; - } + if( !currentGroupInfo->name.empty() && currentGroupInfo->groupsCounts > 1 ) { + printClosedHeader( "Group: " + currentGroupInfo->name ); + currentGroupInfo.used = true; } } void printTestCaseAndSectionHeader() { @@ -338,6 +404,7 @@ namespace Catch { printOpenHeader( currentTestCaseInfo->name ); if( m_sectionStack.size() > 1 ) { + Colour colourGuard( Colour::Headers ); std::vector::const_iterator it = m_sectionStack.begin()+1, // Skip first section (test case) @@ -349,19 +416,21 @@ namespace Catch { SourceLineInfo lineInfo = m_sectionStack.front().lineInfo; if( !lineInfo.empty() ){ - stream << getDashes() << "\r\n"; - stream << lineInfo << "\r\n"; + stream << getDashes() << "\n"; + Colour colourGuard( Colour::FileName ); + stream << lineInfo << "\n"; } - stream << getDots() << "\r\n" << "\r\n"; + stream << getDots() << "\n" << std::endl; } void printClosedHeader( std::string const& _name ) { printOpenHeader( _name ); - stream << getDots() << "\r\n"; + stream << getDots() << "\n"; } void printOpenHeader( std::string const& _name ) { - stream << getDashes() << "\r\n"; + stream << getDashes() << "\n"; { + Colour colourGuard( Colour::Headers ); printHeaderString( _name ); } } @@ -376,14 +445,20 @@ namespace Catch { i = 0; stream << Text( _string, TextAttributes() .setIndent( indent+i) - .setInitialIndent( indent ) ) << "\r\n"; + .setInitialIndent( indent ) ) << "\n"; } void printTotals( const Totals& totals ) { - if( totals.assertions.total() == 0 ) { + if( totals.testCases.total() == 0 ) { stream << "No tests ran"; } + else if( totals.assertions.total() == 0 ) { + Colour colour( Colour::Yellow ); + printCounts( "test case", totals.testCases ); + stream << " (no assertions)"; + } else if( totals.assertions.failed ) { + Colour colour( Colour::ResultError ); printCounts( "test case", totals.testCases ); if( totals.testCases.failed > 0 ) { stream << " ("; @@ -392,6 +467,7 @@ namespace Catch { } } else { + Colour colour( Colour::ResultSuccess ); stream << "All tests passed (" << pluralise( totals.assertions.passed, "assertion" ) << " in " << pluralise( totals.testCases.passed, "test case" ) << ")"; @@ -425,42 +501,47 @@ namespace Catch { } void printTotalsDivider() { - stream << getDoubleDashes() << "\r\n"; + stream << getDoubleDashes() << "\n"; } void printSummaryDivider() { - stream << getDashes() << "\r\n"; + stream << getDashes() << "\n"; } static std::string getDashes() { - std::string dashes( CATCH_CONFIG_CONSOLE_WIDTH-1, '-' ); + const std::string dashes( CATCH_CONFIG_CONSOLE_WIDTH-1, '-' ); return dashes; } static std::string getDots() { - std::string dots( CATCH_CONFIG_CONSOLE_WIDTH-1, '.' ); + const std::string dots( CATCH_CONFIG_CONSOLE_WIDTH-1, '.' ); return dots; } static std::string getDoubleDashes() { - std::string doubleDashes( CATCH_CONFIG_CONSOLE_WIDTH-1, '=' ); + const std::string doubleDashes( CATCH_CONFIG_CONSOLE_WIDTH-1, '=' ); return doubleDashes; } static std::string getTildes() { - std::string dots( CATCH_CONFIG_CONSOLE_WIDTH-1, '~' ); + const std::string dots( CATCH_CONFIG_CONSOLE_WIDTH-1, '~' ); return dots; } private: - Ptr m_config; std::ostringstream stream; + std::streambuf* m_prevCout; + std::streambuf* m_prevCerr; + protected: + bool m_addLineFeeds; - LazyStat currentTestRunInfo; - LazyStat currentGroupInfo; - LazyStat currentTestCaseInfo; - - std::vector m_sectionStack; + private: bool m_headerPrinted; bool m_atLeastOneTestCasePrinted; - size_t m_failed; }; + struct MSTestReporterLineFeed : MSTestReporter { + MSTestReporterLineFeed( ReporterConfig const& _config ) + : MSTestReporter( _config ) + { + m_addLineFeeds = false; + } + }; } // end namespace Catch #endif // TWOBLUECUBES_CATCH_REPORTER_MSTEST_HPP_INCLUDED diff --git a/projects/SelfTest/RunAllTests.cpp b/projects/SelfTest/RunAllTests.cpp index f7c0f399..3da197f4 100644 --- a/projects/SelfTest/RunAllTests.cpp +++ b/projects/SelfTest/RunAllTests.cpp @@ -12,211 +12,6 @@ namespace AllTestsRunner { -#ifdef OLD_RUNNER - class NullStreamingReporter : public Catch::SharedImpl { - public: - - virtual ~NullStreamingReporter(); - - static std::string getDescription() { - return "null reporter"; - } - - private: // IStreamingReporter - - virtual Catch::ReporterPreferences getPreferences() const { - return Catch::ReporterPreferences(); - } - - virtual void noMatchingTestCases( std::string const& ) {} - virtual void testRunStarting( Catch::TestRunInfo const& ) {} - virtual void testGroupStarting( Catch::GroupInfo const& ) {} - virtual void testCaseStarting( Catch::TestCaseInfo const& ) {} - virtual void sectionStarting( Catch::SectionInfo const& ) {} - virtual void assertionStarting( Catch::AssertionInfo const& ) {} - virtual bool assertionEnded( Catch::AssertionStats const& ) { return false; } - virtual void sectionEnded( Catch::SectionStats const& ) {} - virtual void testCaseEnded( Catch::TestCaseStats const& ) {} - virtual void testGroupEnded( Catch::TestGroupStats const& ) {} - virtual void testRunEnded( Catch::TestRunStats const& ) {} - }; - - class EmbeddedRunner { - - public: - EmbeddedRunner() : m_reporter( new NullStreamingReporter() ) {} - - Catch::Totals runMatching( const std::string& rawTestSpec, - std::size_t groupIndex, - std::size_t groupsCount, - const std::string& reporter = "console" ); - - private: - Catch::Ptr m_reporter; - }; - - NullStreamingReporter::~NullStreamingReporter() {} - - Catch::Totals EmbeddedRunner::runMatching( const std::string& rawTestSpec, std::size_t groupIndex, std::size_t groupsCount, const std::string& ) { - std::ostringstream oss; - Catch::Ptr config = new Catch::Config(); - config->setStreamBuf( oss.rdbuf() ); - - Catch::Totals totals; - - // Scoped because RunContext doesn't report EndTesting until its destructor - { - Catch::RunContext runner( config.get(), m_reporter.get() ); - totals = runner.runMatching( rawTestSpec, groupIndex, groupsCount ); - } - return totals; - } - - class MetaTestRunner { - - public: - struct Expected { enum Result { - ToSucceed, - ToFail - }; }; - - MetaTestRunner( Expected::Result expectedResult, std::size_t groupIndex, std::size_t groupsCount ) - : m_expectedResult( expectedResult ), - m_groupIndex( groupIndex ), - m_groupsCount( groupsCount ) - {} - - static void runMatching( const std::string& testSpec, - Expected::Result expectedResult, - std::size_t groupIndex, - std::size_t groupsCount ) { - forEach( Catch::getRegistryHub().getTestCaseRegistry().getMatchingTestCases( testSpec ), - MetaTestRunner( expectedResult, groupIndex, groupsCount ) ); - } - - void operator()( const Catch::TestCase& testCase ) { - std::string name; - Catch::Totals totals; - { - EmbeddedRunner runner; - name = testCase.getTestCaseInfo().name; - totals = runner.runMatching( name, m_groupIndex, m_groupsCount ); - } - switch( m_expectedResult ) { - case Expected::ToSucceed: - if( totals.assertions.failed > 0 ) { - FAIL( "Expected test case '" - << name - << "' to succeed but there was/ were " - << totals.assertions.failed << " failure(s)" ); - } - else { - SUCCEED( "Tests passed, as expected" ); - } - break; - case Expected::ToFail: - if( totals.assertions.failed == 0 ) { - FAIL( "Expected test case '" - << name - << "' to fail but there was/ were " - << totals.assertions.passed << " success(es)" ); - } - else { - SUCCEED( "Tests failed, as expected" ); - } - break; - } - } - - private: - Expected::Result m_expectedResult; - std::size_t m_groupIndex; - std::size_t m_groupsCount; - }; - - TEST_CASE( "Run all failing and succeeding tests", "[vsall]" ) { - - /////////////////////////////////////////////////////////////////////////// - SECTION( "selftest/expected result", - "Tests do what they claim" ) { - -#ifdef _UNICODE - std::cout << "using Unicode..." << std::endl; -#else - std::cout << "using Mbcs..." << std::endl; -#endif - - SECTION( "selftest/expected result/failing tests", - "Tests in the 'failing' branch fail" ) { - std::cout << "Tests in the 'failing' branch fail" << std::endl; - MetaTestRunner::runMatching( "./failing/*", MetaTestRunner::Expected::ToFail, 0, 2 ); - } - - SECTION( "selftest/expected result/succeeding tests", - "Tests in the 'succeeding' branch succeed" ) { - std::cout << "Tests in the 'succeeding' branch succeed" << std::endl; - MetaTestRunner::runMatching( "./succeeding/*", MetaTestRunner::Expected::ToSucceed, 1, 2 ); - } - } - - /////////////////////////////////////////////////////////////////////////// - SECTION( "selftest/test counts", - "Number of test cases that run is fixed" ) { - EmbeddedRunner runner; - - SECTION( "selftest/test counts/succeeding tests", - "Number of 'succeeding' tests is fixed" ) { - std::cout << "Number of 'succeeding' tests is fixed" << std::endl; - Catch::Totals totals = runner.runMatching( "./succeeding/*", 0, 2 ); - CHECK( totals.assertions.passed == 298 ); - CHECK( totals.assertions.failed == 0 ); - } - - SECTION( "selftest/test counts/failing tests", - "Number of 'failing' tests is fixed" ) { - std::cout << "Number of 'failing' tests is fixed" << std::endl; - Catch::Totals totals = runner.runMatching( "./failing/*", 1, 2 ); - CHECK( totals.assertions.passed == 2 ); - CHECK( totals.assertions.failed == 77 ); - } - } - } - - TEST_CASE( "Run all failing and succeeding tests", "[sw4][vs]" ) { - CatchOverrides::ConfigGuard cg; - Catch::ConfigData cd(cg.value().get()); - cd.name = "Test sw4"; - cd.abortAfter = 1; - cd.showSuccessfulTests = true; - cd.warnings = Catch::WarnAbout::NoAssertions; - cd.abortAfter = -1; - Catch::Ptr config(new Catch::Config(cd)); - Catch::MSTestReporter* rep = new Catch::MSTestReporter(config.get()); - Catch::RunContext tr(config.get(), rep); - std::string names[] = {"one","two","three"}; - std::vector stringNames(names, names + (sizeof(names)/sizeof(std::string))); - std::vector testCase = Catch::getRegistryHub().getTestCaseRegistry().getMatchingTestCases( "Some simple comparisons between doubles" ); - //std::vector testCase = Catch::getRegistryHub().getTestCaseRegistry().getMatchingTestCases(name_desc.name); - if( testCase.empty() ) Assert::Fail(FAIL_STRING("No tests match")); - if( testCase.size() > 1 ) Assert::Fail(FAIL_STRING("More than one test with the same name")); - Catch::Totals totals = tr.runTest(*testCase.begin()); - if( totals.assertions.failed > 0 ) { - INTERNAL_CATCH_TEST_THROW_FAILURE - } - /*for(std::vector::iterator it = tests.begin(); it != tests.end(); ++it ) - { - Catch::Totals totals; - std::size_t groupIndex(0); - std::size_t groupsCount(0); - { - EmbeddedRunner runner; - std::string name = it->getTestCaseInfo().name; - totals = runner.runMatching( name, groupIndex, groupsCount ); - } - }*/ - } -#endif - // mstest /TestContainer:Debug\ManagedTestCatch.dll /category:"all" #if defined(INTERNAL_CATCH_VS_MANAGED) || defined(INTERNAL_CATCH_VS_NATIVE) CATCH_MAP_CATEGORY_TO_TAG(all, "~[vs]"); @@ -242,6 +37,10 @@ namespace AllTestsRunner { CATCH_INTERNAL_CONFIG_ADD_TEST("Equality checks that should succeed") CATCH_INTERNAL_CONFIG_ADD_TEST("Equality checks that should fail]") INTERNAL_CATCH_MAP_CATEGORY_TO_LIST(allSucceedingAborting); - #endif + + CATCH_INTERNAL_CONFIG_ADD_TEST("Output from all sections is reported") + CATCH_INTERNAL_CONFIG_ADD_TEST("Standard output from all sections is reported") + INTERNAL_CATCH_MAP_CATEGORY_TO_LIST(OutputFromAllSectionsIsReported); +#endif } diff --git a/scripts/validateVSTests.py b/scripts/validateVSTests.py index 37e864e1..53ca4920 100644 --- a/scripts/validateVSTests.py +++ b/scripts/validateVSTests.py @@ -8,13 +8,12 @@ import re import xml.etree.cElementTree as etree from scriptCommon import catchPath -#from rawfile import writeRawFile -#from rawfile import parseRawFileIntoTree from catch_test_run import TestRunApprovedHandler from catch_test_run import TestRunData from catch_test_run import TestRunResultHandler from catch_test_case import TestCaseResultParser from catch_test_case import TestCaseData +from catch_conditions import RandomOutput rootPath = os.path.join(os.path.join(os.path.join( catchPath, 'projects'), 'SelfTest'), 'Baselines' ) @@ -638,6 +637,8 @@ def parseTrxFile(baseName, trxFile): #print ids ids = dict(ids) #print ids["87ec526a-e414-1a3f-ba0f-e210b204bb42"] + + lineNumber = 0 resultParser = TestCaseResultParser() for tc in ts: m = qname.match(tc.tag) @@ -668,15 +669,27 @@ def parseTrxFile(baseName, trxFile): found = True break index += 1 - lines = lines[index + 2:-1] + lines = lines[index + 2:] #print "*******",desc #print lines if found: + endOfRun = False for line in lines: - testcase = resultParser.parseResultLine(line) - if isinstance(testcase, TestCaseData): - testRun.testcases.append(testcase) - lines = testRun.generateSortedUnapprovedLines(0) + if endOfRun: + testRun.results = line.strip() + else: + try: + testcase = resultParser.parseResultLine(line) + except RandomOutput as e: + #print "E:", self.lineNumber, ", ",e.output + testRun.output = e.output + testRun.outputLine = lineNumber - len(e.output) + if isinstance(testcase, TestCaseData): + testRun.testcases.append(testcase) + if line.startswith("==============================================================================="): + endOfRun = True + lineNumber += 1 + lines = testRun.generateSortedUnapprovedLines(testRun.outputLine) rawSortedPath = os.path.join( rootPath, '{0}.sorted.unapproved.txt'.format( baseName ) ) rawWriteFile = open( rawSortedPath, 'wb' ) @@ -709,18 +722,18 @@ def approveMsTest( baseName, filter ): parseTrxFile(baseName, trxFile) # Standard console reporter -#approve( "console.std", ["~_"] ) +approve( "console.std", ["~_"] ) # console reporter, include passes, warn about No Assertions -#approve( "console.sw", ["~_", "-s", "-w", "NoAssertions"] ) +approve( "console.sw", ["~_", "-s", "-w", "NoAssertions"] ) # console reporter, include passes, warn about No Assertions, limit failures to first 4 -#approve( "console.swa4", ["~_", "-s", "-w", "NoAssertions", "-x", "4"] ) +approve( "console.swa4", ["~_", "-s", "-w", "NoAssertions", "-x", "4"] ) # junit reporter, include passes, warn about No Assertions -#approveJunit( "junit.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "junit"] ) +approveJunit( "junit.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "junit"] ) # xml reporter, include passes, warn about No Assertions -#approveXml( "xml.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "xml"] ) -#mstest runner, xml output -#approveMsTest( "mstest.std", "all") -#approveMsTest( "mstest.sw", "allSucceeding") +approveXml( "xml.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "xml"] ) +# mstest runner, xml output +approveMsTest( "mstest.std", "all") +approveMsTest( "mstest.sw", "allSucceeding") approveMsTest( "mstest.swa4", "allSucceedingAborting") if overallResult <> 0: