mirror of
				https://github.com/catchorg/Catch2.git
				synced 2025-11-03 21:49:32 +01:00 
			
		
		
		
	Fix inconsistencies in reporting benchmarking failures
With these changes, all these benchmarks
```cpp
BENCHMARK("Empty benchmark") {};
BENCHMARK("Throwing benchmark") {
    throw "just a plain literal, bleh";
};
BENCHMARK("Asserting benchmark") {
    REQUIRE(1 == 2);
};
BENCHMARK("FAIL'd benchmark") {
    FAIL("This benchmark only fails, nothing else");
};
```
report the respective failure and mark the outer `TEST_CASE` as
failed. Previously, the first two would not fail the `TEST_CASE`,
and the latter two would break xml reporter's formatting, because
`benchmarkFailed`, `benchmarkEnded` etc would not be be called
properly in failure cases.
			
			
This commit is contained in:
		@@ -81,10 +81,13 @@ namespace Catch {
 | 
			
		||||
                    auto analysis = Detail::analyse(*cfg, env, samples.begin(), samples.end());
 | 
			
		||||
                    BenchmarkStats<FloatDuration<Clock>> stats{ info, analysis.samples, analysis.mean, analysis.standard_deviation, analysis.outliers, analysis.outlier_variance };
 | 
			
		||||
                    getResultCapture().benchmarkEnded(stats);
 | 
			
		||||
 | 
			
		||||
                } CATCH_CATCH_ANON (TestFailureException) {
 | 
			
		||||
                    getResultCapture().benchmarkFailed("Benchmark failed due to failed assertion"_sr);
 | 
			
		||||
                } CATCH_CATCH_ALL{
 | 
			
		||||
                    if (translateActiveException() != Detail::benchmarkErrorMsg) // benchmark errors have been reported, otherwise rethrow.
 | 
			
		||||
                        std::rethrow_exception(std::current_exception());
 | 
			
		||||
                    getResultCapture().benchmarkFailed(translateActiveException());
 | 
			
		||||
                    // We let the exception go further up so that the
 | 
			
		||||
                    // test case is marked as failed.
 | 
			
		||||
                    std::rethrow_exception(std::current_exception());
 | 
			
		||||
                }
 | 
			
		||||
            }
 | 
			
		||||
 | 
			
		||||
 
 | 
			
		||||
@@ -10,6 +10,7 @@
 | 
			
		||||
#ifndef CATCH_COMPLETE_INVOKE_HPP_INCLUDED
 | 
			
		||||
#define CATCH_COMPLETE_INVOKE_HPP_INCLUDED
 | 
			
		||||
 | 
			
		||||
#include <catch2/internal/catch_test_failure_exception.hpp>
 | 
			
		||||
#include <catch2/internal/catch_enforce.hpp>
 | 
			
		||||
#include <catch2/internal/catch_meta.hpp>
 | 
			
		||||
#include <catch2/interfaces/catch_interfaces_capture.hpp>
 | 
			
		||||
@@ -51,17 +52,11 @@ namespace Catch {
 | 
			
		||||
                return CompleteInvoker<FunctionReturnType<Fun, Args...>>::invoke(std::forward<Fun>(fun), std::forward<Args>(args)...);
 | 
			
		||||
            }
 | 
			
		||||
 | 
			
		||||
            extern const std::string benchmarkErrorMsg;
 | 
			
		||||
        } // namespace Detail
 | 
			
		||||
 | 
			
		||||
        template <typename Fun>
 | 
			
		||||
        Detail::CompleteType_t<FunctionReturnType<Fun>> user_code(Fun&& fun) {
 | 
			
		||||
            CATCH_TRY{
 | 
			
		||||
                return Detail::complete_invoke(std::forward<Fun>(fun));
 | 
			
		||||
            } CATCH_CATCH_ALL{
 | 
			
		||||
                getResultCapture().benchmarkFailed(translateActiveException());
 | 
			
		||||
                CATCH_RUNTIME_ERROR(Detail::benchmarkErrorMsg);
 | 
			
		||||
            }
 | 
			
		||||
            return Detail::complete_invoke(std::forward<Fun>(fun));
 | 
			
		||||
        }
 | 
			
		||||
    } // namespace Benchmark
 | 
			
		||||
} // namespace Catch
 | 
			
		||||
 
 | 
			
		||||
@@ -47,26 +47,6 @@ namespace Catch {
 | 
			
		||||
} // namespace Catch
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
////////////////////////////////////////////////
 | 
			
		||||
// vvv formerly catch_complete_invoke.cpp vvv //
 | 
			
		||||
////////////////////////////////////////////////
 | 
			
		||||
 | 
			
		||||
#include <catch2/benchmark/detail/catch_complete_invoke.hpp>
 | 
			
		||||
 | 
			
		||||
namespace Catch {
 | 
			
		||||
    namespace Benchmark {
 | 
			
		||||
        namespace Detail {
 | 
			
		||||
            CATCH_INTERNAL_START_WARNINGS_SUPPRESSION
 | 
			
		||||
            CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS
 | 
			
		||||
            const std::string benchmarkErrorMsg = "a benchmark failed to run successfully";
 | 
			
		||||
            CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION
 | 
			
		||||
        } // namespace Detail
 | 
			
		||||
    } // namespace Benchmark
 | 
			
		||||
} // namespace Catch
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
/////////////////////////////////////////////////
 | 
			
		||||
// vvv formerly catch_run_for_at_least.cpp vvv //
 | 
			
		||||
/////////////////////////////////////////////////
 | 
			
		||||
 
 | 
			
		||||
		Reference in New Issue
	
	Block a user