Cumulative reporter base records benchmark results

docs/release-notes.md

@@ -179,6 +179,7 @@ new design.
 * Universal ADL-found operators should no longer break decomposition (#2121)
 * Reporter selection is properly case-insensitive
   * Previously it forced lower cased name, which would fail for reporters with upper case characters in name
+* The cumulative reporter base stores benchmark results alongside assertion results
 
 
 ### Other changes

src/catch2/reporters/catch_reporter_cumulative_base.cpp

@@ -32,9 +32,41 @@ namespace Catch {
 
     } // namespace
 
+    namespace Detail {
+        AssertionOrBenchmarkResult::AssertionOrBenchmarkResult(
+            AssertionStats const& assertion ):
+            m_assertion( assertion ) {}
+
+        AssertionOrBenchmarkResult::AssertionOrBenchmarkResult(
+            BenchmarkStats<> const& benchmark ):
+            m_benchmark( benchmark ) {}
+
+        bool AssertionOrBenchmarkResult::isAssertion() const {
+            return m_assertion.some();
+        }
+        bool AssertionOrBenchmarkResult::isBenchmark() const {
+            return m_benchmark.some();
+        }
+
+        AssertionStats const& AssertionOrBenchmarkResult::asAssertion() const {
+            assert(m_assertion.some());
+
+            return *m_assertion;
+        }
+        BenchmarkStats<> const& AssertionOrBenchmarkResult::asBenchmark() const {
+            assert(m_benchmark.some());
+
+            return *m_benchmark;
+        }
+
+    }
 
     CumulativeReporterBase::~CumulativeReporterBase() = default;
 
+    void CumulativeReporterBase::benchmarkEnded(BenchmarkStats<> const& benchmarkStats) {
+        m_sectionStack.back()->assertionsAndBenchmarks.emplace_back(benchmarkStats);
+    }
+
     void
     CumulativeReporterBase::sectionStarting( SectionInfo const& sectionInfo ) {
         SectionStats incompleteStats( sectionInfo, Counts(), 0, false );
@@ -75,7 +107,7 @@ namespace Catch {
         static_cast<void>(
             assertionStats.assertionResult.getExpandedExpression() );
         SectionNode& sectionNode = *m_sectionStack.back();
-        sectionNode.assertions.push_back( assertionStats );
+        sectionNode.assertionsAndBenchmarks.emplace_back( assertionStats );
     }
 
     void CumulativeReporterBase::sectionEnded( SectionStats const& sectionStats ) {
@@ -120,4 +152,13 @@ namespace Catch {
         defaultListTags( stream, tags, m_config->hasTestFilters() );
     }
 
+    bool CumulativeReporterBase::SectionNode::hasAnyAssertions() const {
+        return std::any_of(
+            assertionsAndBenchmarks.begin(),
+            assertionsAndBenchmarks.end(),
+            []( Detail::AssertionOrBenchmarkResult const& res ) {
+                return res.isAssertion();
+            } );
+    }
+
 } // end namespace Catch

src/catch2/reporters/catch_reporter_cumulative_base.hpp

@@ -10,6 +10,7 @@
 
 #include <catch2/interfaces/catch_interfaces_reporter.hpp>
 #include <catch2/internal/catch_unique_ptr.hpp>
+#include <catch2/internal/catch_optional.hpp>
 
 #include <iosfwd>
 #include <string>
@@ -17,6 +18,38 @@
 
 namespace Catch {
 
+    namespace Detail {
+
+        //! Represents either an assertion or a benchmark result to be handled by cumulative reporter later
+        class AssertionOrBenchmarkResult {
+            // This should really be a variant, but this is much faster
+            // to write and the data layout here is already terrible
+            // enough that we do not have to care about the object size.
+            Optional<AssertionStats> m_assertion;
+            Optional<BenchmarkStats<>> m_benchmark;
+        public:
+            AssertionOrBenchmarkResult(AssertionStats const& assertion);
+            AssertionOrBenchmarkResult(BenchmarkStats<> const& benchmark);
+
+            bool isAssertion() const;
+            bool isBenchmark() const;
+
+            AssertionStats const& asAssertion() const;
+            BenchmarkStats<> const& asBenchmark() const;
+        };
+    }
+
+    /**
+     * Utility base for reporters that need to handle all results at once
+     *
+     * It stores tree of all test cases, sections and assertions, and after the
+     * test run is finished, calls into `testRunEndedCumulative` to pass the
+     * control to the deriving class.
+     *
+     * If you are deriving from this class and override any testing related
+     * member functions, you should first call into the base's implementation to
+     * avoid breaking the tree construction.
+     */
     struct CumulativeReporterBase : IStreamingReporter {
         template<typename T, typename ChildNodeT>
         struct Node {
@@ -33,9 +66,11 @@ namespace Catch {
                 return stats.sectionInfo.lineInfo == other.stats.sectionInfo.lineInfo;
             }
 
+            bool hasAnyAssertions() const;
+
             SectionStats stats;
             std::vector<Detail::unique_ptr<SectionNode>> childSections;
-            std::vector<AssertionStats> assertions;
+            std::vector<Detail::AssertionOrBenchmarkResult> assertionsAndBenchmarks;
             std::string stdOut;
             std::string stdErr;
         };
@@ -51,14 +86,13 @@ namespace Catch {
 
         void benchmarkPreparing( StringRef ) override {}
         void benchmarkStarting( BenchmarkInfo const& ) override {}
-        void benchmarkEnded( BenchmarkStats<> const& ) override {}
+        void benchmarkEnded( BenchmarkStats<> const& benchmarkStats ) override;
         void benchmarkFailed( StringRef ) override {}
 
         void noMatchingTestCases( StringRef ) override {}
         void reportInvalidArguments( StringRef ) override {}
         void fatalErrorEncountered( StringRef /*error*/ ) override {}
 
-
         void testRunStarting( TestRunInfo const& ) override {}
 
         void testCaseStarting( TestCaseInfo const& ) override {}

src/catch2/reporters/catch_reporter_junit.cpp

@@ -179,9 +179,9 @@ namespace Catch {
         if( !rootName.empty() )
             name = rootName + '/' + name;
 
-        if( !sectionNode.assertions.empty() ||
+        if( sectionNode.hasAnyAssertions()
-            !sectionNode.stdOut.empty() ||
+           || !sectionNode.stdOut.empty()
-            !sectionNode.stdErr.empty() ) {
+           || !sectionNode.stdErr.empty() ) {
             XmlWriter::ScopedElement e = xml.scopedElement( "testcase" );
             if( className.empty() ) {
                 xml.writeAttribute( "classname"_sr, name );
@@ -219,8 +219,11 @@ namespace Catch {
     }
 
     void JunitReporter::writeAssertions( SectionNode const& sectionNode ) {
-        for( auto const& assertion : sectionNode.assertions )
+        for (auto const& assertionOrBenchmark : sectionNode.assertionsAndBenchmarks) {
-            writeAssertion( assertion );
+            if (assertionOrBenchmark.isAssertion()) {
+                writeAssertion(assertionOrBenchmark.asAssertion());
+            }
+        }
     }
 
     void JunitReporter::writeAssertion( AssertionStats const& stats ) {

src/catch2/reporters/catch_reporter_sonarqube.cpp

@@ -54,7 +54,9 @@ namespace Catch {
         if (!rootName.empty())
             name = rootName + '/' + name;
 
-        if (!sectionNode.assertions.empty() || !sectionNode.stdOut.empty() || !sectionNode.stdErr.empty()) {
+        if ( sectionNode.hasAnyAssertions()
+            || !sectionNode.stdOut.empty()
+            ||  !sectionNode.stdErr.empty() ) {
             XmlWriter::ScopedElement e = xml.scopedElement("testCase");
             xml.writeAttribute("name"_sr, name);
             xml.writeAttribute("duration"_sr, static_cast<long>(sectionNode.stats.durationInSeconds * 1000));
@@ -67,8 +69,11 @@ namespace Catch {
     }
 
     void SonarQubeReporter::writeAssertions(SectionNode const& sectionNode, bool okToFail) {
-        for (auto const& assertion : sectionNode.assertions)
+        for (auto const& assertionOrBenchmark : sectionNode.assertionsAndBenchmarks) {
-            writeAssertion(assertion, okToFail);
+            if (assertionOrBenchmark.isAssertion()) {
+                writeAssertion(assertionOrBenchmark.asAssertion(), okToFail);
+            }
+        }
     }
 
     void SonarQubeReporter::writeAssertion(AssertionStats const& stats, bool okToFail) {

tests/ExtraTests/CMakeLists.txt

@@ -196,6 +196,18 @@ add_test(
   COMMAND ${PYTHON_EXECUTABLE} ${CATCH_DIR}/tests/TestScripts/testPartialTestCaseEvent.py $<TARGET_FILE:PartialTestCaseEvents>
 )
 
+add_executable(BenchmarksInCumulativeReporter ${TESTS_DIR}/X22-BenchmarksInCumulativeReporter.cpp)
+target_link_libraries(BenchmarksInCumulativeReporter PRIVATE Catch2::Catch2WithMain)
+add_test(
+  NAME BenchmarksInCumulativeReporter
+  COMMAND BenchmarksInCumulativeReporter --reporter testReporter
+)
+set_tests_properties(
+    BenchmarksInCumulativeReporter
+  PROPERTIES
+    PASS_REGULAR_EXPRESSION "1\n2\n3\n4\n5\n"
+)
+
 
 add_executable(CasingInReporterNames ${TESTS_DIR}/X23-CasingInReporterNames.cpp)
 target_link_libraries(CasingInReporterNames PRIVATE Catch2::Catch2WithMain)

tests/ExtraTests/X22-BenchmarksInCumulativeReporter.cpp

@@ -0,0 +1,78 @@
+
+//              Copyright Catch2 Authors
+// Distributed under the Boost Software License, Version 1.0.
+//   (See accompanying file LICENSE_1_0.txt or copy at
+//        https://www.boost.org/LICENSE_1_0.txt)
+
+// SPDX-License-Identifier: BSL-1.0
+
+/**\file
+ * Test that the cumulative reporter base stores both assertions and
+ * benchmarks, and stores them in the right order.
+ *
+ * This is done through a custom reporter that writes out the assertions
+ * and benchmarks and checking that the output is in right order.
+ */
+
+#include <catch2/catch_test_macros.hpp>
+#include <catch2/benchmark/catch_benchmark.hpp>
+#include <catch2/reporters/catch_reporter_cumulative_base.hpp>
+#include <catch2/catch_reporter_registrars.hpp>
+
+#include <iostream>
+
+class CumulativeBenchmarkReporter final : public Catch::CumulativeReporterBase {
+
+public:
+    CumulativeBenchmarkReporter(Catch::ReporterConfig const& _config) :
+        CumulativeReporterBase(_config) {
+        m_preferences.shouldReportAllAssertions = true;
+    }
+
+    static std::string getDescription() {
+        return "Custom reporter for testing cumulative reporter base";
+    }
+
+    virtual void testRunEndedCumulative() override;
+};
+
+CATCH_REGISTER_REPORTER("testReporter", CumulativeBenchmarkReporter)
+
+#include <chrono>
+#include <thread>
+
+TEST_CASE("Some assertions and benchmarks") {
+    using namespace std::chrono_literals;
+
+    REQUIRE(1);
+    BENCHMARK("2") {
+        std::this_thread::sleep_for(1ms);
+    };
+    REQUIRE(3);
+    BENCHMARK("4") {
+        std::this_thread::sleep_for(1ms);
+    };
+    REQUIRE(5);
+}
+
+void CumulativeBenchmarkReporter::testRunEndedCumulative() {
+    auto const& testCases = m_testRun->children;
+    assert(testCases.size() == 1);
+
+    auto const& testCase = *testCases.front();
+    auto const& sections = testCase.children;
+    assert(sections.size() == 1);
+
+    auto const& section = *sections.front();
+    assert(section.childSections.empty());
+    for (auto const& aob : section.assertionsAndBenchmarks) {
+        if (aob.isAssertion()) {
+            auto const& assertion = aob.asAssertion();
+            std::cout << assertion.assertionResult.getExpandedExpression() << '\n';
+        }
+        if (aob.isBenchmark()) {
+            auto const& bench = aob.asBenchmark();
+            std::cout << bench.info.name << '\n';
+        }
+    }
+}