Cumulative reporter base records benchmark results

src/catch2/reporters/catch_reporter_cumulative_base.cpp

@@ -32,9 +32,41 @@ namespace Catch {
 
     } // namespace
 
+    namespace Detail {
+        AssertionOrBenchmarkResult::AssertionOrBenchmarkResult(
+            AssertionStats const& assertion ):
+            m_assertion( assertion ) {}
+
+        AssertionOrBenchmarkResult::AssertionOrBenchmarkResult(
+            BenchmarkStats<> const& benchmark ):
+            m_benchmark( benchmark ) {}
+
+        bool AssertionOrBenchmarkResult::isAssertion() const {
+            return m_assertion.some();
+        }
+        bool AssertionOrBenchmarkResult::isBenchmark() const {
+            return m_benchmark.some();
+        }
+
+        AssertionStats const& AssertionOrBenchmarkResult::asAssertion() const {
+            assert(m_assertion.some());
+
+            return *m_assertion;
+        }
+        BenchmarkStats<> const& AssertionOrBenchmarkResult::asBenchmark() const {
+            assert(m_benchmark.some());
+
+            return *m_benchmark;
+        }
+
+    }
 
     CumulativeReporterBase::~CumulativeReporterBase() = default;
 
+    void CumulativeReporterBase::benchmarkEnded(BenchmarkStats<> const& benchmarkStats) {
+        m_sectionStack.back()->assertionsAndBenchmarks.emplace_back(benchmarkStats);
+    }
+
     void
     CumulativeReporterBase::sectionStarting( SectionInfo const& sectionInfo ) {
         SectionStats incompleteStats( sectionInfo, Counts(), 0, false );
@@ -75,7 +107,7 @@ namespace Catch {
         static_cast<void>(
             assertionStats.assertionResult.getExpandedExpression() );
         SectionNode& sectionNode = *m_sectionStack.back();
-        sectionNode.assertions.push_back( assertionStats );
+        sectionNode.assertionsAndBenchmarks.emplace_back( assertionStats );
     }
 
     void CumulativeReporterBase::sectionEnded( SectionStats const& sectionStats ) {
@@ -120,4 +152,13 @@ namespace Catch {
         defaultListTags( stream, tags, m_config->hasTestFilters() );
     }
 
+    bool CumulativeReporterBase::SectionNode::hasAnyAssertions() const {
+        return std::any_of(
+            assertionsAndBenchmarks.begin(),
+            assertionsAndBenchmarks.end(),
+            []( Detail::AssertionOrBenchmarkResult const& res ) {
+                return res.isAssertion();
+            } );
+    }
+
 } // end namespace Catch

src/catch2/reporters/catch_reporter_cumulative_base.hpp

@@ -10,6 +10,7 @@
 
 #include <catch2/interfaces/catch_interfaces_reporter.hpp>
 #include <catch2/internal/catch_unique_ptr.hpp>
+#include <catch2/internal/catch_optional.hpp>
 
 #include <iosfwd>
 #include <string>
@@ -17,6 +18,38 @@
 
 namespace Catch {
 
+    namespace Detail {
+
+        //! Represents either an assertion or a benchmark result to be handled by cumulative reporter later
+        class AssertionOrBenchmarkResult {
+            // This should really be a variant, but this is much faster
+            // to write and the data layout here is already terrible
+            // enough that we do not have to care about the object size.
+            Optional<AssertionStats> m_assertion;
+            Optional<BenchmarkStats<>> m_benchmark;
+        public:
+            AssertionOrBenchmarkResult(AssertionStats const& assertion);
+            AssertionOrBenchmarkResult(BenchmarkStats<> const& benchmark);
+
+            bool isAssertion() const;
+            bool isBenchmark() const;
+
+            AssertionStats const& asAssertion() const;
+            BenchmarkStats<> const& asBenchmark() const;
+        };
+    }
+
+    /**
+     * Utility base for reporters that need to handle all results at once
+     *
+     * It stores tree of all test cases, sections and assertions, and after the
+     * test run is finished, calls into `testRunEndedCumulative` to pass the
+     * control to the deriving class.
+     *
+     * If you are deriving from this class and override any testing related
+     * member functions, you should first call into the base's implementation to
+     * avoid breaking the tree construction.
+     */
     struct CumulativeReporterBase : IStreamingReporter {
         template<typename T, typename ChildNodeT>
         struct Node {
@@ -33,9 +66,11 @@ namespace Catch {
                 return stats.sectionInfo.lineInfo == other.stats.sectionInfo.lineInfo;
             }
 
+            bool hasAnyAssertions() const;
+
             SectionStats stats;
             std::vector<Detail::unique_ptr<SectionNode>> childSections;
-            std::vector<AssertionStats> assertions;
+            std::vector<Detail::AssertionOrBenchmarkResult> assertionsAndBenchmarks;
             std::string stdOut;
             std::string stdErr;
         };
@@ -51,14 +86,13 @@ namespace Catch {
 
         void benchmarkPreparing( StringRef ) override {}
         void benchmarkStarting( BenchmarkInfo const& ) override {}
-        void benchmarkEnded( BenchmarkStats<> const& ) override {}
+        void benchmarkEnded( BenchmarkStats<> const& benchmarkStats ) override;
         void benchmarkFailed( StringRef ) override {}
 
         void noMatchingTestCases( StringRef ) override {}
         void reportInvalidArguments( StringRef ) override {}
         void fatalErrorEncountered( StringRef /*error*/ ) override {}
 
-
         void testRunStarting( TestRunInfo const& ) override {}
 
         void testCaseStarting( TestCaseInfo const& ) override {}

src/catch2/reporters/catch_reporter_junit.cpp

@@ -179,9 +179,9 @@ namespace Catch {
         if( !rootName.empty() )
             name = rootName + '/' + name;
 
-        if( !sectionNode.assertions.empty() ||
-            !sectionNode.stdOut.empty() ||
-            !sectionNode.stdErr.empty() ) {
+        if( sectionNode.hasAnyAssertions()
+           || !sectionNode.stdOut.empty()
+           || !sectionNode.stdErr.empty() ) {
             XmlWriter::ScopedElement e = xml.scopedElement( "testcase" );
             if( className.empty() ) {
                 xml.writeAttribute( "classname"_sr, name );
@@ -219,8 +219,11 @@ namespace Catch {
     }
 
     void JunitReporter::writeAssertions( SectionNode const& sectionNode ) {
-        for( auto const& assertion : sectionNode.assertions )
-            writeAssertion( assertion );
+        for (auto const& assertionOrBenchmark : sectionNode.assertionsAndBenchmarks) {
+            if (assertionOrBenchmark.isAssertion()) {
+                writeAssertion(assertionOrBenchmark.asAssertion());
+            }
+        }
     }
 
     void JunitReporter::writeAssertion( AssertionStats const& stats ) {

src/catch2/reporters/catch_reporter_sonarqube.cpp

@@ -54,7 +54,9 @@ namespace Catch {
         if (!rootName.empty())
             name = rootName + '/' + name;
 
-        if (!sectionNode.assertions.empty() || !sectionNode.stdOut.empty() || !sectionNode.stdErr.empty()) {
+        if ( sectionNode.hasAnyAssertions()
+            || !sectionNode.stdOut.empty()
+            ||  !sectionNode.stdErr.empty() ) {
             XmlWriter::ScopedElement e = xml.scopedElement("testCase");
             xml.writeAttribute("name"_sr, name);
             xml.writeAttribute("duration"_sr, static_cast<long>(sectionNode.stats.durationInSeconds * 1000));
@@ -67,8 +69,11 @@ namespace Catch {
     }
 
     void SonarQubeReporter::writeAssertions(SectionNode const& sectionNode, bool okToFail) {
-        for (auto const& assertion : sectionNode.assertions)
-            writeAssertion(assertion, okToFail);
+        for (auto const& assertionOrBenchmark : sectionNode.assertionsAndBenchmarks) {
+            if (assertionOrBenchmark.isAssertion()) {
+                writeAssertion(assertionOrBenchmark.asAssertion(), okToFail);
+            }
+        }
     }
 
     void SonarQubeReporter::writeAssertion(AssertionStats const& stats, bool okToFail) {