mirror of
https://github.com/catchorg/Catch2.git
synced 2024-11-04 05:09:53 +01:00
e92b9c07c3
Unlike the relatively non-invasive old way of capturing stdout/stderr, this new way is also able to capture output from C's stdlib functions such as `printf`. This is done by redirecting stdout and stderr file descriptors to a file, and then reading this file back. This approach has two sizeable drawbacks: 1) Performance, obviously. Previously an installed capture made the program run faster (as long as it was then discarded), because a call to `std::cout` did not result in text output to the console. This new capture method in fact forces disk IO. While it is likely that any modern OS will keep this file in memory-cache and might never actually issue the IO to the backing storage, it is still a possibility and calls to the file system are not free. 2) Nonportability. While POSIX is usually assumed portable, and this implementation relies only on a very common parts of it, it is no longer standard C++ (or just plain C) and thus might not be available on some obscure platforms. Different C libs might also implement the relevant functions in a less-than-useful ways (e.g. MS's `tmpfile` generates a temp file inside system folder, so it will not work without elevated privileges and thus is useless). These two drawbacks mean that, at least for now, the new capture is opt-in. To opt-in, `CATCH_CONFIG_EXPERIMENTAL_REDIRECT` needs to be defined in the implementation file. Closes #1243
431 lines
15 KiB
C++
431 lines
15 KiB
C++
#include "catch_run_context.h"
|
|
#include "catch_context.h"
|
|
#include "catch_enforce.h"
|
|
#include "catch_random_number_generator.h"
|
|
#include "catch_stream.h"
|
|
#include "catch_output_redirect.h"
|
|
|
|
#include <cassert>
|
|
#include <algorithm>
|
|
#include <sstream>
|
|
|
|
namespace Catch {
|
|
|
|
RunContext::RunContext(IConfigPtr const& _config, IStreamingReporterPtr&& reporter)
|
|
: m_runInfo(_config->name()),
|
|
m_context(getCurrentMutableContext()),
|
|
m_config(_config),
|
|
m_reporter(std::move(reporter)),
|
|
m_lastAssertionInfo{ StringRef(), SourceLineInfo("",0), StringRef(), ResultDisposition::Normal },
|
|
m_includeSuccessfulResults( m_config->includeSuccessfulResults() )
|
|
{
|
|
m_context.setRunner(this);
|
|
m_context.setConfig(m_config);
|
|
m_context.setResultCapture(this);
|
|
m_reporter->testRunStarting(m_runInfo);
|
|
}
|
|
|
|
RunContext::~RunContext() {
|
|
m_reporter->testRunEnded(TestRunStats(m_runInfo, m_totals, aborting()));
|
|
}
|
|
|
|
void RunContext::testGroupStarting(std::string const& testSpec, std::size_t groupIndex, std::size_t groupsCount) {
|
|
m_reporter->testGroupStarting(GroupInfo(testSpec, groupIndex, groupsCount));
|
|
}
|
|
|
|
void RunContext::testGroupEnded(std::string const& testSpec, Totals const& totals, std::size_t groupIndex, std::size_t groupsCount) {
|
|
m_reporter->testGroupEnded(TestGroupStats(GroupInfo(testSpec, groupIndex, groupsCount), totals, aborting()));
|
|
}
|
|
|
|
Totals RunContext::runTest(TestCase const& testCase) {
|
|
Totals prevTotals = m_totals;
|
|
|
|
std::string redirectedCout;
|
|
std::string redirectedCerr;
|
|
|
|
auto const& testInfo = testCase.getTestCaseInfo();
|
|
|
|
m_reporter->testCaseStarting(testInfo);
|
|
|
|
m_activeTestCase = &testCase;
|
|
|
|
|
|
ITracker& rootTracker = m_trackerContext.startRun();
|
|
assert(rootTracker.isSectionTracker());
|
|
static_cast<SectionTracker&>(rootTracker).addInitialFilters(m_config->getSectionsToRun());
|
|
do {
|
|
m_trackerContext.startCycle();
|
|
m_testCaseTracker = &SectionTracker::acquire(m_trackerContext, TestCaseTracking::NameAndLocation(testInfo.name, testInfo.lineInfo));
|
|
runCurrentTest(redirectedCout, redirectedCerr);
|
|
} while (!m_testCaseTracker->isSuccessfullyCompleted() && !aborting());
|
|
|
|
Totals deltaTotals = m_totals.delta(prevTotals);
|
|
if (testInfo.expectedToFail() && deltaTotals.testCases.passed > 0) {
|
|
deltaTotals.assertions.failed++;
|
|
deltaTotals.testCases.passed--;
|
|
deltaTotals.testCases.failed++;
|
|
}
|
|
m_totals.testCases += deltaTotals.testCases;
|
|
m_reporter->testCaseEnded(TestCaseStats(testInfo,
|
|
deltaTotals,
|
|
redirectedCout,
|
|
redirectedCerr,
|
|
aborting()));
|
|
|
|
m_activeTestCase = nullptr;
|
|
m_testCaseTracker = nullptr;
|
|
|
|
return deltaTotals;
|
|
}
|
|
|
|
IConfigPtr RunContext::config() const {
|
|
return m_config;
|
|
}
|
|
|
|
IStreamingReporter& RunContext::reporter() const {
|
|
return *m_reporter;
|
|
}
|
|
|
|
void RunContext::assertionEnded(AssertionResult const & result) {
|
|
if (result.getResultType() == ResultWas::Ok) {
|
|
m_totals.assertions.passed++;
|
|
m_lastAssertionPassed = true;
|
|
} else if (!result.isOk()) {
|
|
m_lastAssertionPassed = false;
|
|
if( m_activeTestCase->getTestCaseInfo().okToFail() )
|
|
m_totals.assertions.failedButOk++;
|
|
else
|
|
m_totals.assertions.failed++;
|
|
}
|
|
else {
|
|
m_lastAssertionPassed = true;
|
|
}
|
|
|
|
// We have no use for the return value (whether messages should be cleared), because messages were made scoped
|
|
// and should be let to clear themselves out.
|
|
static_cast<void>(m_reporter->assertionEnded(AssertionStats(result, m_messages, m_totals)));
|
|
|
|
// Reset working state
|
|
resetAssertionInfo();
|
|
m_lastResult = result;
|
|
}
|
|
void RunContext::resetAssertionInfo() {
|
|
m_lastAssertionInfo.macroName = StringRef();
|
|
m_lastAssertionInfo.capturedExpression = "{Unknown expression after the reported line}"_sr;
|
|
}
|
|
|
|
bool RunContext::sectionStarted(SectionInfo const & sectionInfo, Counts & assertions) {
|
|
ITracker& sectionTracker = SectionTracker::acquire(m_trackerContext, TestCaseTracking::NameAndLocation(sectionInfo.name, sectionInfo.lineInfo));
|
|
if (!sectionTracker.isOpen())
|
|
return false;
|
|
m_activeSections.push_back(§ionTracker);
|
|
|
|
m_lastAssertionInfo.lineInfo = sectionInfo.lineInfo;
|
|
|
|
m_reporter->sectionStarting(sectionInfo);
|
|
|
|
assertions = m_totals.assertions;
|
|
|
|
return true;
|
|
}
|
|
|
|
bool RunContext::testForMissingAssertions(Counts& assertions) {
|
|
if (assertions.total() != 0)
|
|
return false;
|
|
if (!m_config->warnAboutMissingAssertions())
|
|
return false;
|
|
if (m_trackerContext.currentTracker().hasChildren())
|
|
return false;
|
|
m_totals.assertions.failed++;
|
|
assertions.failed++;
|
|
return true;
|
|
}
|
|
|
|
void RunContext::sectionEnded(SectionEndInfo const & endInfo) {
|
|
Counts assertions = m_totals.assertions - endInfo.prevAssertions;
|
|
bool missingAssertions = testForMissingAssertions(assertions);
|
|
|
|
if (!m_activeSections.empty()) {
|
|
m_activeSections.back()->close();
|
|
m_activeSections.pop_back();
|
|
}
|
|
|
|
m_reporter->sectionEnded(SectionStats(endInfo.sectionInfo, assertions, endInfo.durationInSeconds, missingAssertions));
|
|
m_messages.clear();
|
|
}
|
|
|
|
void RunContext::sectionEndedEarly(SectionEndInfo const & endInfo) {
|
|
if (m_unfinishedSections.empty())
|
|
m_activeSections.back()->fail();
|
|
else
|
|
m_activeSections.back()->close();
|
|
m_activeSections.pop_back();
|
|
|
|
m_unfinishedSections.push_back(endInfo);
|
|
}
|
|
void RunContext::benchmarkStarting( BenchmarkInfo const& info ) {
|
|
m_reporter->benchmarkStarting( info );
|
|
}
|
|
void RunContext::benchmarkEnded( BenchmarkStats const& stats ) {
|
|
m_reporter->benchmarkEnded( stats );
|
|
}
|
|
|
|
void RunContext::pushScopedMessage(MessageInfo const & message) {
|
|
m_messages.push_back(message);
|
|
}
|
|
|
|
void RunContext::popScopedMessage(MessageInfo const & message) {
|
|
m_messages.erase(std::remove(m_messages.begin(), m_messages.end(), message), m_messages.end());
|
|
}
|
|
|
|
std::string RunContext::getCurrentTestName() const {
|
|
return m_activeTestCase
|
|
? m_activeTestCase->getTestCaseInfo().name
|
|
: std::string();
|
|
}
|
|
|
|
const AssertionResult * RunContext::getLastResult() const {
|
|
return &(*m_lastResult);
|
|
}
|
|
|
|
void RunContext::exceptionEarlyReported() {
|
|
m_shouldReportUnexpected = false;
|
|
}
|
|
|
|
void RunContext::handleFatalErrorCondition( StringRef message ) {
|
|
// First notify reporter that bad things happened
|
|
m_reporter->fatalErrorEncountered(message);
|
|
|
|
// Don't rebuild the result -- the stringification itself can cause more fatal errors
|
|
// Instead, fake a result data.
|
|
AssertionResultData tempResult( ResultWas::FatalErrorCondition, { false } );
|
|
tempResult.message = message;
|
|
AssertionResult result(m_lastAssertionInfo, tempResult);
|
|
|
|
assertionEnded(result);
|
|
|
|
handleUnfinishedSections();
|
|
|
|
// Recreate section for test case (as we will lose the one that was in scope)
|
|
auto const& testCaseInfo = m_activeTestCase->getTestCaseInfo();
|
|
SectionInfo testCaseSection(testCaseInfo.lineInfo, testCaseInfo.name, testCaseInfo.description);
|
|
|
|
Counts assertions;
|
|
assertions.failed = 1;
|
|
SectionStats testCaseSectionStats(testCaseSection, assertions, 0, false);
|
|
m_reporter->sectionEnded(testCaseSectionStats);
|
|
|
|
auto const& testInfo = m_activeTestCase->getTestCaseInfo();
|
|
|
|
Totals deltaTotals;
|
|
deltaTotals.testCases.failed = 1;
|
|
deltaTotals.assertions.failed = 1;
|
|
m_reporter->testCaseEnded(TestCaseStats(testInfo,
|
|
deltaTotals,
|
|
std::string(),
|
|
std::string(),
|
|
false));
|
|
m_totals.testCases.failed++;
|
|
testGroupEnded(std::string(), m_totals, 1, 1);
|
|
m_reporter->testRunEnded(TestRunStats(m_runInfo, m_totals, false));
|
|
}
|
|
|
|
bool RunContext::lastAssertionPassed() {
|
|
return m_lastAssertionPassed;
|
|
}
|
|
|
|
void RunContext::assertionPassed() {
|
|
m_lastAssertionPassed = true;
|
|
++m_totals.assertions.passed;
|
|
resetAssertionInfo();
|
|
}
|
|
|
|
bool RunContext::aborting() const {
|
|
return m_totals.assertions.failed == static_cast<std::size_t>(m_config->abortAfter());
|
|
}
|
|
|
|
void RunContext::runCurrentTest(std::string & redirectedCout, std::string & redirectedCerr) {
|
|
auto const& testCaseInfo = m_activeTestCase->getTestCaseInfo();
|
|
SectionInfo testCaseSection(testCaseInfo.lineInfo, testCaseInfo.name, testCaseInfo.description);
|
|
m_reporter->sectionStarting(testCaseSection);
|
|
Counts prevAssertions = m_totals.assertions;
|
|
double duration = 0;
|
|
m_shouldReportUnexpected = true;
|
|
m_lastAssertionInfo = { "TEST_CASE"_sr, testCaseInfo.lineInfo, StringRef(), ResultDisposition::Normal };
|
|
|
|
seedRng(*m_config);
|
|
|
|
Timer timer;
|
|
try {
|
|
if (m_reporter->getPreferences().shouldRedirectStdOut) {
|
|
#if !defined(CATCH_CONFIG_EXPERIMENTAL_REDIRECT)
|
|
RedirectedStdOut redirectedStdOut;
|
|
RedirectedStdErr redirectedStdErr;
|
|
|
|
timer.start();
|
|
invokeActiveTestCase();
|
|
redirectedCout += redirectedStdOut.str();
|
|
redirectedCerr += redirectedStdErr.str();
|
|
#else
|
|
OutputRedirect r(redirectedCout, redirectedCerr);
|
|
timer.start();
|
|
invokeActiveTestCase();
|
|
#endif
|
|
} else {
|
|
timer.start();
|
|
invokeActiveTestCase();
|
|
}
|
|
duration = timer.getElapsedSeconds();
|
|
} catch (TestFailureException&) {
|
|
// This just means the test was aborted due to failure
|
|
} catch (...) {
|
|
// Under CATCH_CONFIG_FAST_COMPILE, unexpected exceptions under REQUIRE assertions
|
|
// are reported without translation at the point of origin.
|
|
if( m_shouldReportUnexpected ) {
|
|
AssertionReaction dummyReaction;
|
|
handleUnexpectedInflightException( m_lastAssertionInfo, translateActiveException(), dummyReaction );
|
|
}
|
|
}
|
|
Counts assertions = m_totals.assertions - prevAssertions;
|
|
bool missingAssertions = testForMissingAssertions(assertions);
|
|
|
|
m_testCaseTracker->close();
|
|
handleUnfinishedSections();
|
|
m_messages.clear();
|
|
|
|
SectionStats testCaseSectionStats(testCaseSection, assertions, duration, missingAssertions);
|
|
m_reporter->sectionEnded(testCaseSectionStats);
|
|
}
|
|
|
|
void RunContext::invokeActiveTestCase() {
|
|
FatalConditionHandler fatalConditionHandler; // Handle signals
|
|
m_activeTestCase->invoke();
|
|
fatalConditionHandler.reset();
|
|
}
|
|
|
|
void RunContext::handleUnfinishedSections() {
|
|
// If sections ended prematurely due to an exception we stored their
|
|
// infos here so we can tear them down outside the unwind process.
|
|
for (auto it = m_unfinishedSections.rbegin(),
|
|
itEnd = m_unfinishedSections.rend();
|
|
it != itEnd;
|
|
++it)
|
|
sectionEnded(*it);
|
|
m_unfinishedSections.clear();
|
|
}
|
|
|
|
void RunContext::handleExpr(
|
|
AssertionInfo const& info,
|
|
ITransientExpression const& expr,
|
|
AssertionReaction& reaction
|
|
) {
|
|
m_reporter->assertionStarting( info );
|
|
|
|
bool negated = isFalseTest( info.resultDisposition );
|
|
bool result = expr.getResult() != negated;
|
|
|
|
if( result ) {
|
|
if (!m_includeSuccessfulResults) {
|
|
assertionPassed();
|
|
}
|
|
else {
|
|
reportExpr(info, ResultWas::Ok, &expr, negated);
|
|
}
|
|
}
|
|
else {
|
|
reportExpr(info, ResultWas::ExpressionFailed, &expr, negated );
|
|
populateReaction( reaction );
|
|
}
|
|
}
|
|
void RunContext::reportExpr(
|
|
AssertionInfo const &info,
|
|
ResultWas::OfType resultType,
|
|
ITransientExpression const *expr,
|
|
bool negated ) {
|
|
|
|
m_lastAssertionInfo = info;
|
|
AssertionResultData data( resultType, LazyExpression( negated ) );
|
|
|
|
AssertionResult assertionResult{ info, data };
|
|
assertionResult.m_resultData.lazyExpression.m_transientExpression = expr;
|
|
|
|
assertionEnded( assertionResult );
|
|
}
|
|
|
|
void RunContext::handleMessage(
|
|
AssertionInfo const& info,
|
|
ResultWas::OfType resultType,
|
|
StringRef const& message,
|
|
AssertionReaction& reaction
|
|
) {
|
|
m_reporter->assertionStarting( info );
|
|
|
|
m_lastAssertionInfo = info;
|
|
|
|
AssertionResultData data( resultType, LazyExpression( false ) );
|
|
data.message = message;
|
|
AssertionResult assertionResult{ m_lastAssertionInfo, data };
|
|
assertionEnded( assertionResult );
|
|
if( !assertionResult.isOk() )
|
|
populateReaction( reaction );
|
|
}
|
|
void RunContext::handleUnexpectedExceptionNotThrown(
|
|
AssertionInfo const& info,
|
|
AssertionReaction& reaction
|
|
) {
|
|
handleNonExpr(info, Catch::ResultWas::DidntThrowException, reaction);
|
|
}
|
|
|
|
void RunContext::handleUnexpectedInflightException(
|
|
AssertionInfo const& info,
|
|
std::string const& message,
|
|
AssertionReaction& reaction
|
|
) {
|
|
m_lastAssertionInfo = info;
|
|
|
|
AssertionResultData data( ResultWas::ThrewException, LazyExpression( false ) );
|
|
data.message = message;
|
|
AssertionResult assertionResult{ info, data };
|
|
assertionEnded( assertionResult );
|
|
populateReaction( reaction );
|
|
}
|
|
|
|
void RunContext::populateReaction( AssertionReaction& reaction ) {
|
|
reaction.shouldDebugBreak = m_config->shouldDebugBreak();
|
|
reaction.shouldThrow = aborting() || (m_lastAssertionInfo.resultDisposition & ResultDisposition::Normal);
|
|
}
|
|
|
|
void RunContext::handleIncomplete(
|
|
AssertionInfo const& info
|
|
) {
|
|
m_lastAssertionInfo = info;
|
|
|
|
AssertionResultData data( ResultWas::ThrewException, LazyExpression( false ) );
|
|
data.message = "Exception translation was disabled by CATCH_CONFIG_FAST_COMPILE";
|
|
AssertionResult assertionResult{ info, data };
|
|
assertionEnded( assertionResult );
|
|
}
|
|
void RunContext::handleNonExpr(
|
|
AssertionInfo const &info,
|
|
ResultWas::OfType resultType,
|
|
AssertionReaction &reaction
|
|
) {
|
|
m_lastAssertionInfo = info;
|
|
|
|
AssertionResultData data( resultType, LazyExpression( false ) );
|
|
AssertionResult assertionResult{ info, data };
|
|
assertionEnded( assertionResult );
|
|
|
|
if( !assertionResult.isOk() )
|
|
populateReaction( reaction );
|
|
}
|
|
|
|
|
|
IResultCapture& getResultCapture() {
|
|
if (auto* capture = getCurrentContext().getResultCapture())
|
|
return *capture;
|
|
else
|
|
CATCH_INTERNAL_ERROR("No result capture instance");
|
|
}
|
|
}
|