diff --git a/docs/release-notes.md b/docs/release-notes.md index 8fd5b9a1..88893ea3 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -179,6 +179,7 @@ new design. * Universal ADL-found operators should no longer break decomposition (#2121) * Reporter selection is properly case-insensitive * Previously it forced lower cased name, which would fail for reporters with upper case characters in name +* The cumulative reporter base stores benchmark results alongside assertion results ### Other changes diff --git a/src/catch2/reporters/catch_reporter_cumulative_base.cpp b/src/catch2/reporters/catch_reporter_cumulative_base.cpp index 64afac18..b8ebcad0 100644 --- a/src/catch2/reporters/catch_reporter_cumulative_base.cpp +++ b/src/catch2/reporters/catch_reporter_cumulative_base.cpp @@ -32,9 +32,41 @@ namespace Catch { } // namespace + namespace Detail { + AssertionOrBenchmarkResult::AssertionOrBenchmarkResult( + AssertionStats const& assertion ): + m_assertion( assertion ) {} + + AssertionOrBenchmarkResult::AssertionOrBenchmarkResult( + BenchmarkStats<> const& benchmark ): + m_benchmark( benchmark ) {} + + bool AssertionOrBenchmarkResult::isAssertion() const { + return m_assertion.some(); + } + bool AssertionOrBenchmarkResult::isBenchmark() const { + return m_benchmark.some(); + } + + AssertionStats const& AssertionOrBenchmarkResult::asAssertion() const { + assert(m_assertion.some()); + + return *m_assertion; + } + BenchmarkStats<> const& AssertionOrBenchmarkResult::asBenchmark() const { + assert(m_benchmark.some()); + + return *m_benchmark; + } + + } CumulativeReporterBase::~CumulativeReporterBase() = default; + void CumulativeReporterBase::benchmarkEnded(BenchmarkStats<> const& benchmarkStats) { + m_sectionStack.back()->assertionsAndBenchmarks.emplace_back(benchmarkStats); + } + void CumulativeReporterBase::sectionStarting( SectionInfo const& sectionInfo ) { SectionStats incompleteStats( sectionInfo, Counts(), 0, false ); @@ -75,7 +107,7 @@ namespace Catch { static_cast( assertionStats.assertionResult.getExpandedExpression() ); SectionNode& sectionNode = *m_sectionStack.back(); - sectionNode.assertions.push_back( assertionStats ); + sectionNode.assertionsAndBenchmarks.emplace_back( assertionStats ); } void CumulativeReporterBase::sectionEnded( SectionStats const& sectionStats ) { @@ -120,4 +152,13 @@ namespace Catch { defaultListTags( stream, tags, m_config->hasTestFilters() ); } + bool CumulativeReporterBase::SectionNode::hasAnyAssertions() const { + return std::any_of( + assertionsAndBenchmarks.begin(), + assertionsAndBenchmarks.end(), + []( Detail::AssertionOrBenchmarkResult const& res ) { + return res.isAssertion(); + } ); + } + } // end namespace Catch diff --git a/src/catch2/reporters/catch_reporter_cumulative_base.hpp b/src/catch2/reporters/catch_reporter_cumulative_base.hpp index a6acc54f..2db14b09 100644 --- a/src/catch2/reporters/catch_reporter_cumulative_base.hpp +++ b/src/catch2/reporters/catch_reporter_cumulative_base.hpp @@ -10,6 +10,7 @@ #include #include +#include #include #include @@ -17,6 +18,38 @@ namespace Catch { + namespace Detail { + + //! Represents either an assertion or a benchmark result to be handled by cumulative reporter later + class AssertionOrBenchmarkResult { + // This should really be a variant, but this is much faster + // to write and the data layout here is already terrible + // enough that we do not have to care about the object size. + Optional m_assertion; + Optional> m_benchmark; + public: + AssertionOrBenchmarkResult(AssertionStats const& assertion); + AssertionOrBenchmarkResult(BenchmarkStats<> const& benchmark); + + bool isAssertion() const; + bool isBenchmark() const; + + AssertionStats const& asAssertion() const; + BenchmarkStats<> const& asBenchmark() const; + }; + } + + /** + * Utility base for reporters that need to handle all results at once + * + * It stores tree of all test cases, sections and assertions, and after the + * test run is finished, calls into `testRunEndedCumulative` to pass the + * control to the deriving class. + * + * If you are deriving from this class and override any testing related + * member functions, you should first call into the base's implementation to + * avoid breaking the tree construction. + */ struct CumulativeReporterBase : IStreamingReporter { template struct Node { @@ -33,9 +66,11 @@ namespace Catch { return stats.sectionInfo.lineInfo == other.stats.sectionInfo.lineInfo; } + bool hasAnyAssertions() const; + SectionStats stats; std::vector> childSections; - std::vector assertions; + std::vector assertionsAndBenchmarks; std::string stdOut; std::string stdErr; }; @@ -51,14 +86,13 @@ namespace Catch { void benchmarkPreparing( StringRef ) override {} void benchmarkStarting( BenchmarkInfo const& ) override {} - void benchmarkEnded( BenchmarkStats<> const& ) override {} + void benchmarkEnded( BenchmarkStats<> const& benchmarkStats ) override; void benchmarkFailed( StringRef ) override {} void noMatchingTestCases( StringRef ) override {} void reportInvalidArguments( StringRef ) override {} void fatalErrorEncountered( StringRef /*error*/ ) override {} - void testRunStarting( TestRunInfo const& ) override {} void testCaseStarting( TestCaseInfo const& ) override {} diff --git a/src/catch2/reporters/catch_reporter_junit.cpp b/src/catch2/reporters/catch_reporter_junit.cpp index 00b00b47..57b86909 100644 --- a/src/catch2/reporters/catch_reporter_junit.cpp +++ b/src/catch2/reporters/catch_reporter_junit.cpp @@ -179,9 +179,9 @@ namespace Catch { if( !rootName.empty() ) name = rootName + '/' + name; - if( !sectionNode.assertions.empty() || - !sectionNode.stdOut.empty() || - !sectionNode.stdErr.empty() ) { + if( sectionNode.hasAnyAssertions() + || !sectionNode.stdOut.empty() + || !sectionNode.stdErr.empty() ) { XmlWriter::ScopedElement e = xml.scopedElement( "testcase" ); if( className.empty() ) { xml.writeAttribute( "classname"_sr, name ); @@ -219,8 +219,11 @@ namespace Catch { } void JunitReporter::writeAssertions( SectionNode const& sectionNode ) { - for( auto const& assertion : sectionNode.assertions ) - writeAssertion( assertion ); + for (auto const& assertionOrBenchmark : sectionNode.assertionsAndBenchmarks) { + if (assertionOrBenchmark.isAssertion()) { + writeAssertion(assertionOrBenchmark.asAssertion()); + } + } } void JunitReporter::writeAssertion( AssertionStats const& stats ) { diff --git a/src/catch2/reporters/catch_reporter_sonarqube.cpp b/src/catch2/reporters/catch_reporter_sonarqube.cpp index e09b5109..d45bbfe1 100644 --- a/src/catch2/reporters/catch_reporter_sonarqube.cpp +++ b/src/catch2/reporters/catch_reporter_sonarqube.cpp @@ -54,7 +54,9 @@ namespace Catch { if (!rootName.empty()) name = rootName + '/' + name; - if (!sectionNode.assertions.empty() || !sectionNode.stdOut.empty() || !sectionNode.stdErr.empty()) { + if ( sectionNode.hasAnyAssertions() + || !sectionNode.stdOut.empty() + || !sectionNode.stdErr.empty() ) { XmlWriter::ScopedElement e = xml.scopedElement("testCase"); xml.writeAttribute("name"_sr, name); xml.writeAttribute("duration"_sr, static_cast(sectionNode.stats.durationInSeconds * 1000)); @@ -67,8 +69,11 @@ namespace Catch { } void SonarQubeReporter::writeAssertions(SectionNode const& sectionNode, bool okToFail) { - for (auto const& assertion : sectionNode.assertions) - writeAssertion(assertion, okToFail); + for (auto const& assertionOrBenchmark : sectionNode.assertionsAndBenchmarks) { + if (assertionOrBenchmark.isAssertion()) { + writeAssertion(assertionOrBenchmark.asAssertion(), okToFail); + } + } } void SonarQubeReporter::writeAssertion(AssertionStats const& stats, bool okToFail) { diff --git a/tests/ExtraTests/CMakeLists.txt b/tests/ExtraTests/CMakeLists.txt index 0dd0f0cf..7efa7721 100644 --- a/tests/ExtraTests/CMakeLists.txt +++ b/tests/ExtraTests/CMakeLists.txt @@ -196,6 +196,18 @@ add_test( COMMAND ${PYTHON_EXECUTABLE} ${CATCH_DIR}/tests/TestScripts/testPartialTestCaseEvent.py $ ) +add_executable(BenchmarksInCumulativeReporter ${TESTS_DIR}/X22-BenchmarksInCumulativeReporter.cpp) +target_link_libraries(BenchmarksInCumulativeReporter PRIVATE Catch2::Catch2WithMain) +add_test( + NAME BenchmarksInCumulativeReporter + COMMAND BenchmarksInCumulativeReporter --reporter testReporter +) +set_tests_properties( + BenchmarksInCumulativeReporter + PROPERTIES + PASS_REGULAR_EXPRESSION "1\n2\n3\n4\n5\n" +) + add_executable(CasingInReporterNames ${TESTS_DIR}/X23-CasingInReporterNames.cpp) target_link_libraries(CasingInReporterNames PRIVATE Catch2::Catch2WithMain) diff --git a/tests/ExtraTests/X22-BenchmarksInCumulativeReporter.cpp b/tests/ExtraTests/X22-BenchmarksInCumulativeReporter.cpp new file mode 100644 index 00000000..1ab25c77 --- /dev/null +++ b/tests/ExtraTests/X22-BenchmarksInCumulativeReporter.cpp @@ -0,0 +1,78 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 + +/**\file + * Test that the cumulative reporter base stores both assertions and + * benchmarks, and stores them in the right order. + * + * This is done through a custom reporter that writes out the assertions + * and benchmarks and checking that the output is in right order. + */ + +#include +#include +#include +#include + +#include + +class CumulativeBenchmarkReporter final : public Catch::CumulativeReporterBase { + +public: + CumulativeBenchmarkReporter(Catch::ReporterConfig const& _config) : + CumulativeReporterBase(_config) { + m_preferences.shouldReportAllAssertions = true; + } + + static std::string getDescription() { + return "Custom reporter for testing cumulative reporter base"; + } + + virtual void testRunEndedCumulative() override; +}; + +CATCH_REGISTER_REPORTER("testReporter", CumulativeBenchmarkReporter) + +#include +#include + +TEST_CASE("Some assertions and benchmarks") { + using namespace std::chrono_literals; + + REQUIRE(1); + BENCHMARK("2") { + std::this_thread::sleep_for(1ms); + }; + REQUIRE(3); + BENCHMARK("4") { + std::this_thread::sleep_for(1ms); + }; + REQUIRE(5); +} + +void CumulativeBenchmarkReporter::testRunEndedCumulative() { + auto const& testCases = m_testRun->children; + assert(testCases.size() == 1); + + auto const& testCase = *testCases.front(); + auto const& sections = testCase.children; + assert(sections.size() == 1); + + auto const& section = *sections.front(); + assert(section.childSections.empty()); + for (auto const& aob : section.assertionsAndBenchmarks) { + if (aob.isAssertion()) { + auto const& assertion = aob.asAssertion(); + std::cout << assertion.assertionResult.getExpandedExpression() << '\n'; + } + if (aob.isBenchmark()) { + auto const& bench = aob.asBenchmark(); + std::cout << bench.info.name << '\n'; + } + } +}