3rd-party
/
catch2
mirror of https://github.com/catchorg/Catch2.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add new reporter event called for each test case enter/exit 

This means that e.g. for `TEST_CASE` with two sibling `SECTION`s
the event will fire twice, because the `TEST_CASE` will be entered
twice.

Closes #2107 (the event mentioned there already exists, but this
is its counterpart that we also want to provide to users)
This commit is contained in:
Martin Hořeňovský 2021-09-12 00:00:47 +02:00
parent 4dcf8382c7
commit 10fb93cce8
No known key found for this signature in database
GPG Key ID: DE48307B8B0D381A
11 changed files with 207 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/catch2/interfaces/catch_interfaces_reporter.hpp
View File

@ -177,7 +177,10 @@ namespace Catch {
virtual void testRunStarting( TestRunInfo const& testRunInfo ) = 0;
//! Called _once_ for each TEST_CASE, no matter how many times it is entered
virtual void testCaseStarting( TestCaseInfo const& testInfo ) = 0;
//! Called _every time_ a TEST_CASE is entered, including repeats (due to sections)
virtual void testCasePartialStarting( TestCaseInfo const& testInfo, uint64_t partNumber ) = 0;
virtual void sectionStarting( SectionInfo const& sectionInfo ) = 0;
virtual void benchmarkPreparing( StringRef ) {}
@ -191,6 +194,9 @@ namespace Catch {
virtual bool assertionEnded( AssertionStats const& assertionStats ) = 0;
virtual void sectionEnded( SectionStats const& sectionStats ) = 0;
//! Called _every time_ a TEST_CASE is entered, including repeats (due to sections)
virtual void testCasePartialEnded(TestCaseStats const& testCaseStats, uint64_t partNumber ) = 0;
//! Called _once_ for each TEST_CASE, no matter how many times it is entered
virtual void testCaseEnded( TestCaseStats const& testCaseStats ) = 0;
virtual void testRunEnded( TestRunStats const& testRunStats ) = 0;

19
src/catch2/internal/catch_run_context.cpp
View File

@ -176,7 +176,7 @@ namespace Catch {
}
Totals RunContext::runTest(TestCaseHandle const& testCase) {
Totals prevTotals = m_totals;
const Totals prevTotals = m_totals;
std::string redirectedCout;
std::string redirectedCerr;
@ -191,10 +191,25 @@ namespace Catch {
ITracker& rootTracker = m_trackerContext.startRun();
assert(rootTracker.isSectionTracker());
static_cast<SectionTracker&>(rootTracker).addInitialFilters(m_config->getSectionsToRun());
uint64_t testRuns = 0;
do {
m_trackerContext.startCycle();
m_testCaseTracker = &SectionTracker::acquire(m_trackerContext, TestCaseTracking::NameAndLocation(testInfo.name, testInfo.lineInfo));
runCurrentTest(redirectedCout, redirectedCerr);
m_reporter->testCasePartialStarting(testInfo, testRuns);
const auto beforeRunTotals = m_totals;
std::string oneRunCout, oneRunCerr;
runCurrentTest(oneRunCout, oneRunCerr);
redirectedCout += oneRunCout;
redirectedCerr += oneRunCerr;
const auto singleRunTotals = m_totals.delta(beforeRunTotals);
auto statsForOneRun = TestCaseStats(testInfo, singleRunTotals, redirectedCout, oneRunCerr, aborting());
m_reporter->testCasePartialEnded(statsForOneRun, testRuns);
++testRuns;
} while (!m_testCaseTracker->isSuccessfullyCompleted() && !aborting());
Totals deltaTotals = m_totals.delta(prevTotals);

2
src/catch2/reporters/catch_reporter_combined_tu.cpp
View File

@ -232,8 +232,10 @@ namespace Catch {
void EventListenerBase::noMatchingTestCases( std::string const& ) {}
void EventListenerBase::testRunStarting( TestRunInfo const& ) {}
void EventListenerBase::testCaseStarting( TestCaseInfo const& ) {}
void EventListenerBase::testCasePartialStarting(TestCaseInfo const&, uint64_t) {}
void EventListenerBase::sectionStarting( SectionInfo const& ) {}
void EventListenerBase::sectionEnded( SectionStats const& ) {}
void EventListenerBase::testCasePartialEnded(TestCaseStats const&, uint64_t) {}
void EventListenerBase::testCaseEnded( TestCaseStats const& ) {}
void EventListenerBase::testRunEnded( TestRunStats const& ) {}
void EventListenerBase::skipTest( TestCaseInfo const& ) {}

3
src/catch2/reporters/catch_reporter_cumulative_base.hpp
View File

@ -52,13 +52,14 @@ namespace Catch {
void testRunStarting( TestRunInfo const& ) override {}
void testCaseStarting( TestCaseInfo const& ) override {}
void testCasePartialStarting( TestCaseInfo const&, uint64_t ) override {}
void sectionStarting( SectionInfo const& sectionInfo ) override;
void assertionStarting( AssertionInfo const& ) override {}
bool assertionEnded( AssertionStats const& assertionStats ) override;
void sectionEnded( SectionStats const& sectionStats ) override;
void testCasePartialEnded( TestCaseStats const&, uint64_t ) override {}
void testCaseEnded( TestCaseStats const& testCaseStats ) override;
void testRunEnded( TestRunStats const& testRunStats ) override;
//! Customization point: called after last test finishes (testRunEnded has been handled)

5
src/catch2/reporters/catch_reporter_event_listener.hpp
View File

@ -35,11 +35,16 @@ namespace Catch {
void noMatchingTestCases( std::string const& spec ) override;
void testRunStarting( TestRunInfo const& testRunInfo ) override;
void testCaseStarting( TestCaseInfo const& testInfo ) override;
void testCasePartialStarting( TestCaseInfo const& testInfo,
uint64_t partNumber ) override;
void sectionStarting( SectionInfo const& sectionInfo ) override;
void sectionEnded( SectionStats const& sectionStats ) override;
void testCasePartialEnded( TestCaseStats const& testCaseStats,
uint64_t partNumber ) override;
void testCaseEnded( TestCaseStats const& testCaseStats ) override;
void testRunEnded( TestRunStats const& testRunStats ) override;
void skipTest( TestCaseInfo const& testInfo ) override;
};
} // end namespace Catch

17
src/catch2/reporters/catch_reporter_listening.cpp
View File

@ -76,6 +76,15 @@ namespace Catch {
m_reporter->testCaseStarting( testInfo );
}
void
ListeningReporter::testCasePartialStarting( TestCaseInfo const& testInfo,
uint64_t partNumber ) {
for ( auto& listener : m_listeners ) {
listener->testCasePartialStarting( testInfo, partNumber );
}
m_reporter->testCasePartialStarting( testInfo, partNumber );
}
void ListeningReporter::sectionStarting( SectionInfo const& sectionInfo ) {
for ( auto& listener : m_listeners ) {
listener->sectionStarting( sectionInfo );
@ -105,6 +114,14 @@ namespace Catch {
m_reporter->sectionEnded( sectionStats );
}
void ListeningReporter::testCasePartialEnded( TestCaseStats const& testInfo,
uint64_t partNumber ) {
for ( auto& listener : m_listeners ) {
listener->testCasePartialEnded( testInfo, partNumber );
}
m_reporter->testCasePartialEnded( testInfo, partNumber );
}
void ListeningReporter::testCaseEnded( TestCaseStats const& testCaseStats ) {
for ( auto& listener : m_listeners ) {
listener->testCaseEnded( testCaseStats );

2
src/catch2/reporters/catch_reporter_listening.hpp
View File

@ -41,12 +41,14 @@ namespace Catch {
void testRunStarting( TestRunInfo const& testRunInfo ) override;
void testCaseStarting( TestCaseInfo const& testInfo ) override;
void testCasePartialStarting(TestCaseInfo const& testInfo, uint64_t partNumber) override;
void sectionStarting( SectionInfo const& sectionInfo ) override;
void assertionStarting( AssertionInfo const& assertionInfo ) override;
// The return value indicates if the messages buffer should be cleared:
bool assertionEnded( AssertionStats const& assertionStats ) override;
void sectionEnded( SectionStats const& sectionStats ) override;
void testCasePartialEnded(TestCaseStats const& testInfo, uint64_t partNumber) override;
void testCaseEnded( TestCaseStats const& testCaseStats ) override;
void testRunEnded( TestRunStats const& testRunStats ) override;

2
src/catch2/reporters/catch_reporter_streaming_base.hpp
View File

@ -51,6 +51,7 @@ namespace Catch {
void testCaseStarting(TestCaseInfo const& _testInfo) override {
currentTestCaseInfo = &_testInfo;
}
void testCasePartialStarting( TestCaseInfo const&, uint64_t ) override {}
void sectionStarting(SectionInfo const& _sectionInfo) override {
m_sectionStack.push_back(_sectionInfo);
}
@ -61,6 +62,7 @@ namespace Catch {
void sectionEnded(SectionStats const& /* _sectionStats */) override {
m_sectionStack.pop_back();
}
void testCasePartialEnded( TestCaseStats const&, uint64_t ) override {}
void testCaseEnded(TestCaseStats const& /* _testCaseStats */) override {
currentTestCaseInfo = nullptr;
}

8
tests/ExtraTests/CMakeLists.txt
View File

@ -172,6 +172,14 @@ if (MSVC)
list(APPEND CATCH_WARNING_TARGETS ${EXTRA_TEST_BINARIES} WindowsHeader)
endif()
add_executable(PartialTestCaseEvents ${TESTS_DIR}/X21-PartialTestCaseEvents.cpp)
target_link_libraries(PartialTestCaseEvents PRIVATE Catch2WithMain)
add_test(
NAME PartialTestCaseEvents
COMMAND ${PYTHON_EXECUTABLE} ${CATCH_DIR}/tests/TestScripts/testPartialTestCaseEvent.py $<TARGET_FILE:PartialTestCaseEvents>
)
#add_executable(DebugBreakMacros ${TESTS_DIR}/X12-CustomDebugBreakMacro.cpp)
#target_link_libraries(DebugBreakMacros Catch2)
#add_test(NAME DebugBreakMacros COMMAND DebugBreakMacros --break)

74
tests/ExtraTests/X21-PartialTestCaseEvents.cpp Normal file
View File

@ -0,0 +1,74 @@
// Copyright Catch2 Authors
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
// https://www.boost.org/LICENSE_1_0.txt)
// SPDX-License-Identifier: BSL-1.0
/**\file
* Registers custom reporter that reports testCase* events
*
* The resulting executable can then be used by an external Python script
* to verify that testCase{Starting,Ended} and testCasePartial{Starting,Ended}
* events are properly nested.
*/
#include <catch2/catch_test_macros.hpp>
#include <catch2/reporters/catch_reporter_streaming_base.hpp>
#include <catch2/catch_test_case_info.hpp>
#include <catch2/catch_reporter_registrars.hpp>
#include <catch2/generators/catch_generators.hpp>
#include <iostream>
using Catch::TestCaseInfo;
using Catch::TestCaseStats;
class PartialReporter : public Catch::StreamingReporterBase {
public:
using StreamingReporterBase::StreamingReporterBase;
~PartialReporter() override; // = default
static std::string getDescription() {
return "Special reporter for testing TestCasePartialStarting/Ended events";
}
//! Called _once_ for each TEST_CASE, no matter how many times it is entered
void testCaseStarting(TestCaseInfo const& testInfo) override {
std::cout << "TestCaseStarting: " << testInfo.name << '\n';
}
//! Called _every time_ a TEST_CASE is entered, including repeats (due to sections)
void testCasePartialStarting(TestCaseInfo const& testInfo, uint64_t partNumber) override {
std::cout << "TestCaseStartingPartial: " << testInfo.name << '#' << partNumber << '\n';
}
//! Called _every time_ a TEST_CASE is entered, including repeats (due to sections)
void testCasePartialEnded(TestCaseStats const& testCaseStats, uint64_t partNumber) override {
std::cout << "TestCasePartialEnded: " << testCaseStats.testInfo->name << '#' << partNumber << '\n';
}
//! Called _once_ for each TEST_CASE, no matter how many times it is entered
void testCaseEnded(TestCaseStats const& testCaseStats) override {
std::cout << "TestCaseEnded: " << testCaseStats.testInfo->name << '\n';
}
};
PartialReporter::~PartialReporter() = default;
CATCH_REGISTER_REPORTER("partial", PartialReporter)
TEST_CASE("section") {
SECTION("A") {}
SECTION("B") {}
SECTION("C") {}
SECTION("D") {}
}
TEST_CASE("generator") {
auto _ = GENERATE(1, 2, 3, 4);
(void)_;
}

72
tests/TestScripts/testPartialTestCaseEvent.py Executable file
View File

@ -0,0 +1,72 @@
#!/usr/bin/env python3
"""
This test script verifies that the testCasePartial{Starting,Ended} reporter
events fire properly. This is done by calling a test binary compiled with
reporter that reports specifically testCase* events, and verifying the
outputs match what we expect.
"""
import subprocess
import sys
expected_section_output = '''\
TestCaseStarting: section
TestCaseStartingPartial: section#0
TestCasePartialEnded: section#0
TestCaseStartingPartial: section#1
TestCasePartialEnded: section#1
TestCaseStartingPartial: section#2
TestCasePartialEnded: section#2
TestCaseStartingPartial: section#3
TestCasePartialEnded: section#3
TestCaseEnded: section
'''
expected_generator_output = '''\
TestCaseStarting: generator
TestCaseStartingPartial: generator#0
TestCasePartialEnded: generator#0
TestCaseStartingPartial: generator#1
TestCasePartialEnded: generator#1
TestCaseStartingPartial: generator#2
TestCasePartialEnded: generator#2
TestCaseStartingPartial: generator#3
TestCasePartialEnded: generator#3
TestCaseEnded: generator
'''
from typing import List
def get_test_output(test_exe: str, sections: bool) -> List[str]:
cmd = [test_exe, '--reporter', 'partial']
if sections:
cmd.append('section')
else:
cmd.append('generator')
ret = subprocess.run(cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE,
timeout = 10,
check = True,
universal_newlines = True)
return ret.stdout
def main():
test_exe, = sys.argv[1:]
actual_section_output = get_test_output(test_exe, sections = True)
assert actual_section_output == expected_section_output, (
'Sections\nActual:\n{}\nExpected:\n{}\n'.format(actual_section_output, expected_section_output))
actual_generator_output = get_test_output(test_exe, sections = False)
assert actual_generator_output == expected_generator_output, (
'Generators\nActual:\n{}\nExpected:\n{}\n'.format(actual_generator_output, expected_generator_output))
if __name__ == '__main__':
sys.exit(main())