diff --git a/src/catch2/interfaces/catch_interfaces_reporter.hpp b/src/catch2/interfaces/catch_interfaces_reporter.hpp index 09480f04..0056c8ad 100644 --- a/src/catch2/interfaces/catch_interfaces_reporter.hpp +++ b/src/catch2/interfaces/catch_interfaces_reporter.hpp @@ -177,7 +177,10 @@ namespace Catch { virtual void testRunStarting( TestRunInfo const& testRunInfo ) = 0; + //! Called _once_ for each TEST_CASE, no matter how many times it is entered virtual void testCaseStarting( TestCaseInfo const& testInfo ) = 0; + //! Called _every time_ a TEST_CASE is entered, including repeats (due to sections) + virtual void testCasePartialStarting( TestCaseInfo const& testInfo, uint64_t partNumber ) = 0; virtual void sectionStarting( SectionInfo const& sectionInfo ) = 0; virtual void benchmarkPreparing( StringRef ) {} @@ -191,6 +194,9 @@ namespace Catch { virtual bool assertionEnded( AssertionStats const& assertionStats ) = 0; virtual void sectionEnded( SectionStats const& sectionStats ) = 0; + //! Called _every time_ a TEST_CASE is entered, including repeats (due to sections) + virtual void testCasePartialEnded(TestCaseStats const& testCaseStats, uint64_t partNumber ) = 0; + //! Called _once_ for each TEST_CASE, no matter how many times it is entered virtual void testCaseEnded( TestCaseStats const& testCaseStats ) = 0; virtual void testRunEnded( TestRunStats const& testRunStats ) = 0; diff --git a/src/catch2/internal/catch_run_context.cpp b/src/catch2/internal/catch_run_context.cpp index 222ee932..1e4cc6b1 100644 --- a/src/catch2/internal/catch_run_context.cpp +++ b/src/catch2/internal/catch_run_context.cpp @@ -176,7 +176,7 @@ namespace Catch { } Totals RunContext::runTest(TestCaseHandle const& testCase) { - Totals prevTotals = m_totals; + const Totals prevTotals = m_totals; std::string redirectedCout; std::string redirectedCerr; @@ -191,10 +191,25 @@ namespace Catch { ITracker& rootTracker = m_trackerContext.startRun(); assert(rootTracker.isSectionTracker()); static_cast(rootTracker).addInitialFilters(m_config->getSectionsToRun()); + + uint64_t testRuns = 0; do { m_trackerContext.startCycle(); m_testCaseTracker = &SectionTracker::acquire(m_trackerContext, TestCaseTracking::NameAndLocation(testInfo.name, testInfo.lineInfo)); - runCurrentTest(redirectedCout, redirectedCerr); + + m_reporter->testCasePartialStarting(testInfo, testRuns); + + const auto beforeRunTotals = m_totals; + std::string oneRunCout, oneRunCerr; + runCurrentTest(oneRunCout, oneRunCerr); + redirectedCout += oneRunCout; + redirectedCerr += oneRunCerr; + + const auto singleRunTotals = m_totals.delta(beforeRunTotals); + auto statsForOneRun = TestCaseStats(testInfo, singleRunTotals, redirectedCout, oneRunCerr, aborting()); + + m_reporter->testCasePartialEnded(statsForOneRun, testRuns); + ++testRuns; } while (!m_testCaseTracker->isSuccessfullyCompleted() && !aborting()); Totals deltaTotals = m_totals.delta(prevTotals); diff --git a/src/catch2/reporters/catch_reporter_combined_tu.cpp b/src/catch2/reporters/catch_reporter_combined_tu.cpp index 084f46c5..dc5dc427 100644 --- a/src/catch2/reporters/catch_reporter_combined_tu.cpp +++ b/src/catch2/reporters/catch_reporter_combined_tu.cpp @@ -232,8 +232,10 @@ namespace Catch { void EventListenerBase::noMatchingTestCases( std::string const& ) {} void EventListenerBase::testRunStarting( TestRunInfo const& ) {} void EventListenerBase::testCaseStarting( TestCaseInfo const& ) {} + void EventListenerBase::testCasePartialStarting(TestCaseInfo const&, uint64_t) {} void EventListenerBase::sectionStarting( SectionInfo const& ) {} void EventListenerBase::sectionEnded( SectionStats const& ) {} + void EventListenerBase::testCasePartialEnded(TestCaseStats const&, uint64_t) {} void EventListenerBase::testCaseEnded( TestCaseStats const& ) {} void EventListenerBase::testRunEnded( TestRunStats const& ) {} void EventListenerBase::skipTest( TestCaseInfo const& ) {} diff --git a/src/catch2/reporters/catch_reporter_cumulative_base.hpp b/src/catch2/reporters/catch_reporter_cumulative_base.hpp index 8340aadb..0c7f7def 100644 --- a/src/catch2/reporters/catch_reporter_cumulative_base.hpp +++ b/src/catch2/reporters/catch_reporter_cumulative_base.hpp @@ -52,13 +52,14 @@ namespace Catch { void testRunStarting( TestRunInfo const& ) override {} void testCaseStarting( TestCaseInfo const& ) override {} - + void testCasePartialStarting( TestCaseInfo const&, uint64_t ) override {} void sectionStarting( SectionInfo const& sectionInfo ) override; void assertionStarting( AssertionInfo const& ) override {} bool assertionEnded( AssertionStats const& assertionStats ) override; void sectionEnded( SectionStats const& sectionStats ) override; + void testCasePartialEnded( TestCaseStats const&, uint64_t ) override {} void testCaseEnded( TestCaseStats const& testCaseStats ) override; void testRunEnded( TestRunStats const& testRunStats ) override; //! Customization point: called after last test finishes (testRunEnded has been handled) diff --git a/src/catch2/reporters/catch_reporter_event_listener.hpp b/src/catch2/reporters/catch_reporter_event_listener.hpp index 034c6f3c..855acc43 100644 --- a/src/catch2/reporters/catch_reporter_event_listener.hpp +++ b/src/catch2/reporters/catch_reporter_event_listener.hpp @@ -35,11 +35,16 @@ namespace Catch { void noMatchingTestCases( std::string const& spec ) override; void testRunStarting( TestRunInfo const& testRunInfo ) override; void testCaseStarting( TestCaseInfo const& testInfo ) override; + void testCasePartialStarting( TestCaseInfo const& testInfo, + uint64_t partNumber ) override; void sectionStarting( SectionInfo const& sectionInfo ) override; void sectionEnded( SectionStats const& sectionStats ) override; + void testCasePartialEnded( TestCaseStats const& testCaseStats, + uint64_t partNumber ) override; void testCaseEnded( TestCaseStats const& testCaseStats ) override; void testRunEnded( TestRunStats const& testRunStats ) override; void skipTest( TestCaseInfo const& testInfo ) override; + }; } // end namespace Catch diff --git a/src/catch2/reporters/catch_reporter_listening.cpp b/src/catch2/reporters/catch_reporter_listening.cpp index 74cfd0aa..62f6af55 100644 --- a/src/catch2/reporters/catch_reporter_listening.cpp +++ b/src/catch2/reporters/catch_reporter_listening.cpp @@ -76,6 +76,15 @@ namespace Catch { m_reporter->testCaseStarting( testInfo ); } + void + ListeningReporter::testCasePartialStarting( TestCaseInfo const& testInfo, + uint64_t partNumber ) { + for ( auto& listener : m_listeners ) { + listener->testCasePartialStarting( testInfo, partNumber ); + } + m_reporter->testCasePartialStarting( testInfo, partNumber ); + } + void ListeningReporter::sectionStarting( SectionInfo const& sectionInfo ) { for ( auto& listener : m_listeners ) { listener->sectionStarting( sectionInfo ); @@ -105,6 +114,14 @@ namespace Catch { m_reporter->sectionEnded( sectionStats ); } + void ListeningReporter::testCasePartialEnded( TestCaseStats const& testInfo, + uint64_t partNumber ) { + for ( auto& listener : m_listeners ) { + listener->testCasePartialEnded( testInfo, partNumber ); + } + m_reporter->testCasePartialEnded( testInfo, partNumber ); + } + void ListeningReporter::testCaseEnded( TestCaseStats const& testCaseStats ) { for ( auto& listener : m_listeners ) { listener->testCaseEnded( testCaseStats ); diff --git a/src/catch2/reporters/catch_reporter_listening.hpp b/src/catch2/reporters/catch_reporter_listening.hpp index d00320b0..0e317653 100644 --- a/src/catch2/reporters/catch_reporter_listening.hpp +++ b/src/catch2/reporters/catch_reporter_listening.hpp @@ -41,12 +41,14 @@ namespace Catch { void testRunStarting( TestRunInfo const& testRunInfo ) override; void testCaseStarting( TestCaseInfo const& testInfo ) override; + void testCasePartialStarting(TestCaseInfo const& testInfo, uint64_t partNumber) override; void sectionStarting( SectionInfo const& sectionInfo ) override; void assertionStarting( AssertionInfo const& assertionInfo ) override; // The return value indicates if the messages buffer should be cleared: bool assertionEnded( AssertionStats const& assertionStats ) override; void sectionEnded( SectionStats const& sectionStats ) override; + void testCasePartialEnded(TestCaseStats const& testInfo, uint64_t partNumber) override; void testCaseEnded( TestCaseStats const& testCaseStats ) override; void testRunEnded( TestRunStats const& testRunStats ) override; diff --git a/src/catch2/reporters/catch_reporter_streaming_base.hpp b/src/catch2/reporters/catch_reporter_streaming_base.hpp index c51595e6..3051294e 100644 --- a/src/catch2/reporters/catch_reporter_streaming_base.hpp +++ b/src/catch2/reporters/catch_reporter_streaming_base.hpp @@ -51,6 +51,7 @@ namespace Catch { void testCaseStarting(TestCaseInfo const& _testInfo) override { currentTestCaseInfo = &_testInfo; } + void testCasePartialStarting( TestCaseInfo const&, uint64_t ) override {} void sectionStarting(SectionInfo const& _sectionInfo) override { m_sectionStack.push_back(_sectionInfo); } @@ -61,6 +62,7 @@ namespace Catch { void sectionEnded(SectionStats const& /* _sectionStats */) override { m_sectionStack.pop_back(); } + void testCasePartialEnded( TestCaseStats const&, uint64_t ) override {} void testCaseEnded(TestCaseStats const& /* _testCaseStats */) override { currentTestCaseInfo = nullptr; } diff --git a/tests/ExtraTests/CMakeLists.txt b/tests/ExtraTests/CMakeLists.txt index a861345d..bd7a575c 100644 --- a/tests/ExtraTests/CMakeLists.txt +++ b/tests/ExtraTests/CMakeLists.txt @@ -172,6 +172,14 @@ if (MSVC) list(APPEND CATCH_WARNING_TARGETS ${EXTRA_TEST_BINARIES} WindowsHeader) endif() + +add_executable(PartialTestCaseEvents ${TESTS_DIR}/X21-PartialTestCaseEvents.cpp) +target_link_libraries(PartialTestCaseEvents PRIVATE Catch2WithMain) +add_test( + NAME PartialTestCaseEvents + COMMAND ${PYTHON_EXECUTABLE} ${CATCH_DIR}/tests/TestScripts/testPartialTestCaseEvent.py $ +) + #add_executable(DebugBreakMacros ${TESTS_DIR}/X12-CustomDebugBreakMacro.cpp) #target_link_libraries(DebugBreakMacros Catch2) #add_test(NAME DebugBreakMacros COMMAND DebugBreakMacros --break) diff --git a/tests/ExtraTests/X21-PartialTestCaseEvents.cpp b/tests/ExtraTests/X21-PartialTestCaseEvents.cpp new file mode 100644 index 00000000..997b9b9f --- /dev/null +++ b/tests/ExtraTests/X21-PartialTestCaseEvents.cpp @@ -0,0 +1,74 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 + +/**\file + * Registers custom reporter that reports testCase* events + * + * The resulting executable can then be used by an external Python script + * to verify that testCase{Starting,Ended} and testCasePartial{Starting,Ended} + * events are properly nested. + */ + + +#include +#include +#include +#include +#include + + +#include + +using Catch::TestCaseInfo; +using Catch::TestCaseStats; + +class PartialReporter : public Catch::StreamingReporterBase { +public: + using StreamingReporterBase::StreamingReporterBase; + + ~PartialReporter() override; // = default + + static std::string getDescription() { + return "Special reporter for testing TestCasePartialStarting/Ended events"; + } + + //! Called _once_ for each TEST_CASE, no matter how many times it is entered + void testCaseStarting(TestCaseInfo const& testInfo) override { + std::cout << "TestCaseStarting: " << testInfo.name << '\n'; + } + //! Called _every time_ a TEST_CASE is entered, including repeats (due to sections) + void testCasePartialStarting(TestCaseInfo const& testInfo, uint64_t partNumber) override { + std::cout << "TestCaseStartingPartial: " << testInfo.name << '#' << partNumber << '\n'; + } + + + //! Called _every time_ a TEST_CASE is entered, including repeats (due to sections) + void testCasePartialEnded(TestCaseStats const& testCaseStats, uint64_t partNumber) override { + std::cout << "TestCasePartialEnded: " << testCaseStats.testInfo->name << '#' << partNumber << '\n'; + } + //! Called _once_ for each TEST_CASE, no matter how many times it is entered + void testCaseEnded(TestCaseStats const& testCaseStats) override { + std::cout << "TestCaseEnded: " << testCaseStats.testInfo->name << '\n'; + } +}; +PartialReporter::~PartialReporter() = default; + + +CATCH_REGISTER_REPORTER("partial", PartialReporter) + +TEST_CASE("section") { + SECTION("A") {} + SECTION("B") {} + SECTION("C") {} + SECTION("D") {} +} + +TEST_CASE("generator") { + auto _ = GENERATE(1, 2, 3, 4); + (void)_; +} diff --git a/tests/TestScripts/testPartialTestCaseEvent.py b/tests/TestScripts/testPartialTestCaseEvent.py new file mode 100755 index 00000000..d119e6da --- /dev/null +++ b/tests/TestScripts/testPartialTestCaseEvent.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 + +""" +This test script verifies that the testCasePartial{Starting,Ended} reporter +events fire properly. This is done by calling a test binary compiled with +reporter that reports specifically testCase* events, and verifying the +outputs match what we expect. +""" + +import subprocess +import sys + +expected_section_output = '''\ +TestCaseStarting: section +TestCaseStartingPartial: section#0 +TestCasePartialEnded: section#0 +TestCaseStartingPartial: section#1 +TestCasePartialEnded: section#1 +TestCaseStartingPartial: section#2 +TestCasePartialEnded: section#2 +TestCaseStartingPartial: section#3 +TestCasePartialEnded: section#3 +TestCaseEnded: section +''' + +expected_generator_output = '''\ +TestCaseStarting: generator +TestCaseStartingPartial: generator#0 +TestCasePartialEnded: generator#0 +TestCaseStartingPartial: generator#1 +TestCasePartialEnded: generator#1 +TestCaseStartingPartial: generator#2 +TestCasePartialEnded: generator#2 +TestCaseStartingPartial: generator#3 +TestCasePartialEnded: generator#3 +TestCaseEnded: generator +''' + + +from typing import List + +def get_test_output(test_exe: str, sections: bool) -> List[str]: + cmd = [test_exe, '--reporter', 'partial'] + if sections: + cmd.append('section') + else: + cmd.append('generator') + + ret = subprocess.run(cmd, + stdout = subprocess.PIPE, + stderr = subprocess.PIPE, + timeout = 10, + check = True, + universal_newlines = True) + + return ret.stdout + +def main(): + test_exe, = sys.argv[1:] + actual_section_output = get_test_output(test_exe, sections = True) + + assert actual_section_output == expected_section_output, ( + 'Sections\nActual:\n{}\nExpected:\n{}\n'.format(actual_section_output, expected_section_output)) + + actual_generator_output = get_test_output(test_exe, sections = False) + assert actual_generator_output == expected_generator_output, ( + 'Generators\nActual:\n{}\nExpected:\n{}\n'.format(actual_generator_output, expected_generator_output)) + + + +if __name__ == '__main__': + sys.exit(main())