From 47a2c969380259e899f471edde84f06a6d687dc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ho=C5=99e=C5=88ovsk=C3=BD?= Date: Thu, 31 Aug 2023 21:52:08 +0200 Subject: [PATCH] Reduce the number of templates in Benchmarking The basic idea was to reduce the number of things dependent on the `Clock` type. To that end, I replaced `Duration` with `IDuration` typedef for `std::nanoseconds`, and `FloatDuration` with `FDuration` typedef for `Duration`. We can generally assume that any clock's duration can be expressed in nanoseconds, as long as we insert `duration_cast`s into the right places. Note that we cannot remove all dependence on `Clock` as a template arguments, because functions that actually measure the elapsed time have to use the Clock. We also changed some template function arguments to pass plain function pointers, so that the actual implementation can be placed into a cpp file. --- src/CMakeLists.txt | 1 + src/catch2/benchmark/catch_benchmark.hpp | 12 +-- src/catch2/benchmark/catch_chronometer.hpp | 5 +- src/catch2/benchmark/catch_clock.hpp | 16 +--- src/catch2/benchmark/catch_environment.hpp | 13 +-- src/catch2/benchmark/catch_estimate.hpp | 13 +-- src/catch2/benchmark/catch_execution_plan.hpp | 24 +++--- .../benchmark/catch_sample_analysis.hpp | 25 +----- src/catch2/benchmark/detail/catch_analyse.cpp | 85 +++++++++++++++++++ src/catch2/benchmark/detail/catch_analyse.hpp | 68 +-------------- .../detail/catch_benchmark_stats.hpp | 28 ++---- .../detail/catch_benchmark_stats_fwd.hpp | 4 +- .../benchmark/detail/catch_estimate_clock.hpp | 22 ++--- src/catch2/benchmark/detail/catch_measure.hpp | 2 +- .../detail/catch_run_for_at_least.hpp | 8 +- src/catch2/benchmark/detail/catch_stats.cpp | 81 ++++++++++++++++++ src/catch2/benchmark/detail/catch_stats.hpp | 79 +++-------------- src/catch2/benchmark/detail/catch_timing.hpp | 8 +- src/catch2/meson.build | 1 + .../InternalBenchmark.tests.cpp | 20 ++--- 20 files changed, 254 insertions(+), 261 deletions(-) create mode 100644 src/catch2/benchmark/detail/catch_analyse.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0fdf931e..4e3b3566 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -33,6 +33,7 @@ set(BENCHMARK_HEADERS ) set(BENCHMARK_SOURCES ${SOURCES_DIR}/benchmark/catch_chronometer.cpp + ${SOURCES_DIR}/benchmark/detail/catch_analyse.cpp ${SOURCES_DIR}/benchmark/detail/catch_benchmark_function.cpp ${SOURCES_DIR}/benchmark/detail/catch_run_for_at_least.cpp ${SOURCES_DIR}/benchmark/detail/catch_stats.cpp diff --git a/src/catch2/benchmark/catch_benchmark.hpp b/src/catch2/benchmark/catch_benchmark.hpp index ea8ffa75..3db40bb0 100644 --- a/src/catch2/benchmark/catch_benchmark.hpp +++ b/src/catch2/benchmark/catch_benchmark.hpp @@ -45,16 +45,18 @@ namespace Catch { : fun(CATCH_MOVE(func)), name(CATCH_MOVE(benchmarkName)) {} template - ExecutionPlan> prepare(const IConfig &cfg, Environment> env) const { + ExecutionPlan prepare(const IConfig &cfg, Environment env) const { auto min_time = env.clock_resolution.mean * Detail::minimum_ticks; auto run_time = std::max(min_time, std::chrono::duration_cast(cfg.benchmarkWarmupTime())); - auto&& test = Detail::run_for_at_least(std::chrono::duration_cast>(run_time), 1, fun); + auto&& test = Detail::run_for_at_least(std::chrono::duration_cast(run_time), 1, fun); int new_iters = static_cast(std::ceil(min_time * test.iterations / test.elapsed)); - return { new_iters, test.elapsed / test.iterations * new_iters * cfg.benchmarkSamples(), fun, std::chrono::duration_cast>(cfg.benchmarkWarmupTime()), Detail::warmup_iterations }; + return { new_iters, test.elapsed / test.iterations * new_iters * cfg.benchmarkSamples(), fun, std::chrono::duration_cast(cfg.benchmarkWarmupTime()), Detail::warmup_iterations }; } template void run() { + static_assert( Clock::is_steady, + "Benchmarking clock should be steady" ); auto const* cfg = getCurrentContext().getConfig(); auto env = Detail::measure_environment(); @@ -81,8 +83,8 @@ namespace Catch { return plan.template run(*cfg, env); }); - auto analysis = Detail::analyse(*cfg, env, samples.begin(), samples.end()); - BenchmarkStats> stats{ CATCH_MOVE(info), CATCH_MOVE(analysis.samples), analysis.mean, analysis.standard_deviation, analysis.outliers, analysis.outlier_variance }; + auto analysis = Detail::analyse(*cfg, samples.data(), samples.data() + samples.size()); + BenchmarkStats<> stats{ CATCH_MOVE(info), CATCH_MOVE(analysis.samples), analysis.mean, analysis.standard_deviation, analysis.outliers, analysis.outlier_variance }; getResultCapture().benchmarkEnded(stats); } CATCH_CATCH_ANON (TestFailureException const&) { getResultCapture().benchmarkFailed("Benchmark failed due to failed assertion"_sr); diff --git a/src/catch2/benchmark/catch_chronometer.hpp b/src/catch2/benchmark/catch_chronometer.hpp index c3f81306..95498e6b 100644 --- a/src/catch2/benchmark/catch_chronometer.hpp +++ b/src/catch2/benchmark/catch_chronometer.hpp @@ -32,7 +32,10 @@ namespace Catch { void start() override { started = Clock::now(); } void finish() override { finished = Clock::now(); } - ClockDuration elapsed() const { return finished - started; } + IDuration elapsed() const { + return std::chrono::duration_cast( + finished - started ); + } TimePoint started; TimePoint finished; diff --git a/src/catch2/benchmark/catch_clock.hpp b/src/catch2/benchmark/catch_clock.hpp index cee46097..4068c4d2 100644 --- a/src/catch2/benchmark/catch_clock.hpp +++ b/src/catch2/benchmark/catch_clock.hpp @@ -11,28 +11,16 @@ #define CATCH_CLOCK_HPP_INCLUDED #include -#include namespace Catch { namespace Benchmark { - template - using ClockDuration = typename Clock::duration; - template - using FloatDuration = std::chrono::duration; + using IDuration = std::chrono::nanoseconds; + using FDuration = std::chrono::duration; template using TimePoint = typename Clock::time_point; using default_clock = std::chrono::steady_clock; - - template - struct now { - TimePoint operator()() const { - return Clock::now(); - } - }; - - using fp_seconds = std::chrono::duration>; } // namespace Benchmark } // namespace Catch diff --git a/src/catch2/benchmark/catch_environment.hpp b/src/catch2/benchmark/catch_environment.hpp index 5d3816a5..da3f2fa9 100644 --- a/src/catch2/benchmark/catch_environment.hpp +++ b/src/catch2/benchmark/catch_environment.hpp @@ -15,20 +15,13 @@ namespace Catch { namespace Benchmark { - template struct EnvironmentEstimate { - Duration mean; + FDuration mean; OutlierClassification outliers; - - template - operator EnvironmentEstimate() const { - return { mean, outliers }; - } }; - template struct Environment { - EnvironmentEstimate> clock_resolution; - EnvironmentEstimate> clock_cost; + EnvironmentEstimate clock_resolution; + EnvironmentEstimate clock_cost; }; } // namespace Benchmark } // namespace Catch diff --git a/src/catch2/benchmark/catch_estimate.hpp b/src/catch2/benchmark/catch_estimate.hpp index be594a18..64383a2e 100644 --- a/src/catch2/benchmark/catch_estimate.hpp +++ b/src/catch2/benchmark/catch_estimate.hpp @@ -12,17 +12,12 @@ namespace Catch { namespace Benchmark { - template + template struct Estimate { - Duration point; - Duration lower_bound; - Duration upper_bound; + Type point; + Type lower_bound; + Type upper_bound; double confidence_interval; - - template - operator Estimate() const { - return { point, lower_bound, upper_bound, confidence_interval }; - } }; } // namespace Benchmark } // namespace Catch diff --git a/src/catch2/benchmark/catch_execution_plan.hpp b/src/catch2/benchmark/catch_execution_plan.hpp index 4f60a646..17ca589f 100644 --- a/src/catch2/benchmark/catch_execution_plan.hpp +++ b/src/catch2/benchmark/catch_execution_plan.hpp @@ -21,33 +21,31 @@ namespace Catch { namespace Benchmark { - template struct ExecutionPlan { int iterations_per_sample; - Duration estimated_duration; + FDuration estimated_duration; Detail::BenchmarkFunction benchmark; - Duration warmup_time; + FDuration warmup_time; int warmup_iterations; - template - operator ExecutionPlan() const { - return { iterations_per_sample, estimated_duration, benchmark, warmup_time, warmup_iterations }; - } - template - std::vector> run(const IConfig &cfg, Environment> env) const { + std::vector run(const IConfig &cfg, Environment env) const { // warmup a bit - Detail::run_for_at_least(std::chrono::duration_cast>(warmup_time), warmup_iterations, Detail::repeat(now{})); + Detail::run_for_at_least( + std::chrono::duration_cast( warmup_time ), + warmup_iterations, + Detail::repeat( []() { return Clock::now(); } ) + ); - std::vector> times; + std::vector times; const auto num_samples = cfg.benchmarkSamples(); times.reserve( num_samples ); for ( size_t i = 0; i < num_samples; ++i ) { Detail::ChronometerModel model; this->benchmark( Chronometer( model, iterations_per_sample ) ); auto sample_time = model.elapsed() - env.clock_cost.mean; - if ( sample_time < FloatDuration::zero() ) { - sample_time = FloatDuration::zero(); + if ( sample_time < FDuration::zero() ) { + sample_time = FDuration::zero(); } times.push_back(sample_time / iterations_per_sample); } diff --git a/src/catch2/benchmark/catch_sample_analysis.hpp b/src/catch2/benchmark/catch_sample_analysis.hpp index 97b8fe50..aeb87d05 100644 --- a/src/catch2/benchmark/catch_sample_analysis.hpp +++ b/src/catch2/benchmark/catch_sample_analysis.hpp @@ -12,35 +12,18 @@ #include #include -#include +#include #include namespace Catch { namespace Benchmark { - template struct SampleAnalysis { - std::vector samples; - Estimate mean; - Estimate standard_deviation; + std::vector samples; + Estimate mean; + Estimate standard_deviation; OutlierClassification outliers; double outlier_variance; - - template - operator SampleAnalysis() const { - std::vector samples2; - samples2.reserve(samples.size()); - for (auto const& d : samples) { - samples2.push_back(Duration2(d)); - } - return { - CATCH_MOVE(samples2), - mean, - standard_deviation, - outliers, - outlier_variance, - }; - } }; } // namespace Benchmark } // namespace Catch diff --git a/src/catch2/benchmark/detail/catch_analyse.cpp b/src/catch2/benchmark/detail/catch_analyse.cpp new file mode 100644 index 00000000..7d27daf1 --- /dev/null +++ b/src/catch2/benchmark/detail/catch_analyse.cpp @@ -0,0 +1,85 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +// Adapted from donated nonius code. + +#include +#include +#include +#include +#include +#include + +#include + +namespace Catch { + namespace Benchmark { + namespace Detail { + SampleAnalysis analyse(const IConfig &cfg, FDuration* first, FDuration* last) { + if (!cfg.benchmarkNoAnalysis()) { + std::vector samples; + samples.reserve(static_cast(last - first)); + for (auto current = first; current != last; ++current) { + samples.push_back( current->count() ); + } + + auto analysis = Catch::Benchmark::Detail::analyse_samples( + cfg.benchmarkConfidenceInterval(), + cfg.benchmarkResamples(), + samples.data(), + samples.data() + samples.size() ); + auto outliers = Catch::Benchmark::Detail::classify_outliers( + samples.data(), samples.data() + samples.size() ); + + auto wrap_estimate = [](Estimate e) { + return Estimate { + FDuration(e.point), + FDuration(e.lower_bound), + FDuration(e.upper_bound), + e.confidence_interval, + }; + }; + std::vector samples2; + samples2.reserve(samples.size()); + for (auto s : samples) { + samples2.push_back( FDuration( s ) ); + } + + return { + CATCH_MOVE(samples2), + wrap_estimate(analysis.mean), + wrap_estimate(analysis.standard_deviation), + outliers, + analysis.outlier_variance, + }; + } else { + std::vector samples; + samples.reserve(static_cast(last - first)); + + FDuration mean = FDuration(0); + int i = 0; + for (auto it = first; it < last; ++it, ++i) { + samples.push_back(FDuration(*it)); + mean += FDuration(*it); + } + mean /= i; + + return SampleAnalysis{ + CATCH_MOVE(samples), + Estimate{ mean, mean, mean, 0.0 }, + Estimate{ FDuration( 0 ), + FDuration( 0 ), + FDuration( 0 ), + 0.0 }, + OutlierClassification{}, + 0.0 + }; + } + } + } // namespace Detail + } // namespace Benchmark +} // namespace Catch diff --git a/src/catch2/benchmark/detail/catch_analyse.hpp b/src/catch2/benchmark/detail/catch_analyse.hpp index 60db7d87..5e3f7b0f 100644 --- a/src/catch2/benchmark/detail/catch_analyse.hpp +++ b/src/catch2/benchmark/detail/catch_analyse.hpp @@ -10,76 +10,16 @@ #ifndef CATCH_ANALYSE_HPP_INCLUDED #define CATCH_ANALYSE_HPP_INCLUDED -#include +#include #include -#include -#include -#include -#include namespace Catch { + class IConfig; + namespace Benchmark { namespace Detail { - template - SampleAnalysis analyse(const IConfig &cfg, Environment, Iterator first, Iterator last) { - if (!cfg.benchmarkNoAnalysis()) { - std::vector samples; - samples.reserve(static_cast(last - first)); - for (auto current = first; current != last; ++current) { - samples.push_back( current->count() ); - } - - auto analysis = Catch::Benchmark::Detail::analyse_samples( - cfg.benchmarkConfidenceInterval(), - cfg.benchmarkResamples(), - samples.data(), - samples.data() + samples.size() ); - auto outliers = Catch::Benchmark::Detail::classify_outliers( - samples.data(), samples.data() + samples.size() ); - - auto wrap_estimate = [](Estimate e) { - return Estimate { - Duration(e.point), - Duration(e.lower_bound), - Duration(e.upper_bound), - e.confidence_interval, - }; - }; - std::vector samples2; - samples2.reserve(samples.size()); - for (auto s : samples) { - samples2.push_back( Duration( s ) ); - } - - return { - CATCH_MOVE(samples2), - wrap_estimate(analysis.mean), - wrap_estimate(analysis.standard_deviation), - outliers, - analysis.outlier_variance, - }; - } else { - std::vector samples; - samples.reserve(static_cast(last - first)); - - Duration mean = Duration(0); - int i = 0; - for (auto it = first; it < last; ++it, ++i) { - samples.push_back(Duration(*it)); - mean += Duration(*it); - } - mean /= i; - - return { - CATCH_MOVE(samples), - Estimate{mean, mean, mean, 0.0}, - Estimate{Duration(0), Duration(0), Duration(0), 0.0}, - OutlierClassification{}, - 0.0 - }; - } - } + SampleAnalysis analyse(const IConfig &cfg, FDuration* first, FDuration* last); } // namespace Detail } // namespace Benchmark } // namespace Catch diff --git a/src/catch2/benchmark/detail/catch_benchmark_stats.hpp b/src/catch2/benchmark/detail/catch_benchmark_stats.hpp index a8b3494e..3633bc9f 100644 --- a/src/catch2/benchmark/detail/catch_benchmark_stats.hpp +++ b/src/catch2/benchmark/detail/catch_benchmark_stats.hpp @@ -8,7 +8,6 @@ #ifndef CATCH_BENCHMARK_STATS_HPP_INCLUDED #define CATCH_BENCHMARK_STATS_HPP_INCLUDED -#include #include #include // The fwd decl & default specialization needs to be seen by VS2017 before @@ -30,32 +29,17 @@ namespace Catch { double clockCost; }; - template + // We need to keep template parameter for backwards compatibility, + // but we also do not want to use the template paraneter. + template struct BenchmarkStats { BenchmarkInfo info; - std::vector samples; - Benchmark::Estimate mean; - Benchmark::Estimate standardDeviation; + std::vector samples; + Benchmark::Estimate mean; + Benchmark::Estimate standardDeviation; Benchmark::OutlierClassification outliers; double outlierVariance; - - template - operator BenchmarkStats() const { - std::vector samples2; - samples2.reserve(samples.size()); - for (auto const& sample : samples) { - samples2.push_back(Duration2(sample)); - } - return { - info, - CATCH_MOVE(samples2), - mean, - standardDeviation, - outliers, - outlierVariance, - }; - } }; diff --git a/src/catch2/benchmark/detail/catch_benchmark_stats_fwd.hpp b/src/catch2/benchmark/detail/catch_benchmark_stats_fwd.hpp index 60772561..2ccc25d5 100644 --- a/src/catch2/benchmark/detail/catch_benchmark_stats_fwd.hpp +++ b/src/catch2/benchmark/detail/catch_benchmark_stats_fwd.hpp @@ -8,14 +8,14 @@ #ifndef CATCH_BENCHMARK_STATS_FWD_HPP_INCLUDED #define CATCH_BENCHMARK_STATS_FWD_HPP_INCLUDED -#include +#include namespace Catch { // We cannot forward declare the type with default template argument // multiple times, so it is split out into a separate header so that // we can prevent multiple declarations in dependees - template > + template struct BenchmarkStats; } // end namespace Catch diff --git a/src/catch2/benchmark/detail/catch_estimate_clock.hpp b/src/catch2/benchmark/detail/catch_estimate_clock.hpp index a0dc2d3d..918484a9 100644 --- a/src/catch2/benchmark/detail/catch_estimate_clock.hpp +++ b/src/catch2/benchmark/detail/catch_estimate_clock.hpp @@ -55,23 +55,23 @@ namespace Catch { template int warmup() { - return run_for_at_least(std::chrono::duration_cast>(warmup_time), warmup_seed, &resolution) + return run_for_at_least(std::chrono::duration_cast(warmup_time), warmup_seed, &resolution) .iterations; } template - EnvironmentEstimate> estimate_clock_resolution(int iterations) { - auto r = run_for_at_least(std::chrono::duration_cast>(clock_resolution_estimation_time), iterations, &resolution) + EnvironmentEstimate estimate_clock_resolution(int iterations) { + auto r = run_for_at_least(std::chrono::duration_cast(clock_resolution_estimation_time), iterations, &resolution) .result; return { - FloatDuration(mean(r.data(), r.data() + r.size())), + FDuration(mean(r.data(), r.data() + r.size())), classify_outliers(r.data(), r.data() + r.size()), }; } template - EnvironmentEstimate> estimate_clock_cost(FloatDuration resolution) { + EnvironmentEstimate estimate_clock_cost(FDuration resolution) { auto time_limit = (std::min)( resolution * clock_cost_estimation_tick_limit, - FloatDuration(clock_cost_estimation_time_limit)); + FDuration(clock_cost_estimation_time_limit)); auto time_clock = [](int k) { return Detail::measure([k] { for (int i = 0; i < k; ++i) { @@ -82,7 +82,7 @@ namespace Catch { }; time_clock(1); int iters = clock_cost_estimation_iterations; - auto&& r = run_for_at_least(std::chrono::duration_cast>(clock_cost_estimation_time), iters, time_clock); + auto&& r = run_for_at_least(std::chrono::duration_cast(clock_cost_estimation_time), iters, time_clock); std::vector times; int nsamples = static_cast(std::ceil(time_limit / r.elapsed)); times.reserve(static_cast(nsamples)); @@ -92,18 +92,18 @@ namespace Catch { .count() ) ); } return { - FloatDuration(mean(times.data(), times.data() + times.size())), + FDuration(mean(times.data(), times.data() + times.size())), classify_outliers(times.data(), times.data() + times.size()), }; } template - Environment> measure_environment() { + Environment measure_environment() { #if defined(__clang__) # pragma clang diagnostic push # pragma clang diagnostic ignored "-Wexit-time-destructors" #endif - static Catch::Detail::unique_ptr>> env; + static Catch::Detail::unique_ptr env; #if defined(__clang__) # pragma clang diagnostic pop #endif @@ -115,7 +115,7 @@ namespace Catch { auto resolution = Detail::estimate_clock_resolution(iters); auto cost = Detail::estimate_clock_cost(resolution.mean); - env = Catch::Detail::make_unique>>( Environment>{resolution, cost} ); + env = Catch::Detail::make_unique( Environment{resolution, cost} ); return *env; } } // namespace Detail diff --git a/src/catch2/benchmark/detail/catch_measure.hpp b/src/catch2/benchmark/detail/catch_measure.hpp index 1a30efab..37494a68 100644 --- a/src/catch2/benchmark/detail/catch_measure.hpp +++ b/src/catch2/benchmark/detail/catch_measure.hpp @@ -18,7 +18,7 @@ namespace Catch { namespace Benchmark { namespace Detail { template - TimingOf measure(Fun&& fun, Args&&... args) { + TimingOf measure(Fun&& fun, Args&&... args) { auto start = Clock::now(); auto&& r = Detail::complete_invoke(fun, CATCH_FORWARD(args)...); auto end = Clock::now(); diff --git a/src/catch2/benchmark/detail/catch_run_for_at_least.hpp b/src/catch2/benchmark/detail/catch_run_for_at_least.hpp index 976a4b24..4dfa8bbb 100644 --- a/src/catch2/benchmark/detail/catch_run_for_at_least.hpp +++ b/src/catch2/benchmark/detail/catch_run_for_at_least.hpp @@ -24,11 +24,11 @@ namespace Catch { namespace Benchmark { namespace Detail { template - TimingOf measure_one(Fun&& fun, int iters, std::false_type) { + TimingOf measure_one(Fun&& fun, int iters, std::false_type) { return Detail::measure(fun, iters); } template - TimingOf measure_one(Fun&& fun, int iters, std::true_type) { + TimingOf measure_one(Fun&& fun, int iters, std::true_type) { Detail::ChronometerModel meter; auto&& result = Detail::complete_invoke(fun, Chronometer(meter, iters)); @@ -43,8 +43,8 @@ namespace Catch { void throw_optimized_away_error(); template - TimingOf> - run_for_at_least(ClockDuration how_long, + TimingOf> + run_for_at_least(IDuration how_long, const int initial_iterations, Fun&& fun) { auto iters = initial_iterations; diff --git a/src/catch2/benchmark/detail/catch_stats.cpp b/src/catch2/benchmark/detail/catch_stats.cpp index caad0167..fe64b1c5 100644 --- a/src/catch2/benchmark/detail/catch_stats.cpp +++ b/src/catch2/benchmark/detail/catch_stats.cpp @@ -11,7 +11,9 @@ #include +#include #include +#include #include #include #include @@ -257,6 +259,27 @@ namespace Catch { return sum / static_cast(count); } + sample jackknife( double ( *estimator )( double const*, + double const* ), + double* first, + double* last ) { + auto n = static_cast( last - first ); + auto second = first; + ++second; + sample results; + results.reserve( n ); + + for ( auto it = first; it != last; ++it ) { + std::iter_swap( it, first ); + results.push_back( estimator( second, last ) ); + } + + return results; + } + + double normal_cdf( double x ) { + return std::erfc( -x / std::sqrt( 2.0 ) ) / 2.0; + } double erfc_inv(double x) { return erf_inv(1.0 - x); @@ -278,6 +301,64 @@ namespace Catch { return result; } + Estimate + bootstrap( double confidence_level, + double* first, + double* last, + sample const& resample, + double ( *estimator )( double const*, double const* ) ) { + auto n_samples = last - first; + + double point = estimator( first, last ); + // Degenerate case with a single sample + if ( n_samples == 1 ) + return { point, point, point, confidence_level }; + + sample jack = jackknife( estimator, first, last ); + double jack_mean = + mean( jack.data(), jack.data() + jack.size() ); + double sum_squares = 0, sum_cubes = 0; + for ( double x : jack ) { + auto difference = jack_mean - x; + auto square = difference * difference; + auto cube = square * difference; + sum_squares += square; + sum_cubes += cube; + } + + double accel = sum_cubes / ( 6 * std::pow( sum_squares, 1.5 ) ); + long n = static_cast( resample.size() ); + double prob_n = + std::count_if( resample.begin(), + resample.end(), + [point]( double x ) { return x < point; } ) / + static_cast( n ); + // degenerate case with uniform samples + if ( directCompare( prob_n, 0. ) ) { + return { point, point, point, confidence_level }; + } + + double bias = normal_quantile( prob_n ); + double z1 = normal_quantile( ( 1. - confidence_level ) / 2. ); + + auto cumn = [n]( double x ) -> long { + return std::lround( normal_cdf( x ) * + static_cast( n ) ); + }; + auto a = [bias, accel]( double b ) { + return bias + b / ( 1. - accel * b ); + }; + double b1 = bias + z1; + double b2 = bias - z1; + double a1 = a( b1 ); + double a2 = a( b2 ); + auto lo = static_cast( (std::max)( cumn( a1 ), 0l ) ); + auto hi = + static_cast( (std::min)( cumn( a2 ), n - 1 ) ); + + return { point, resample[lo], resample[hi], confidence_level }; + } + bootstrap_analysis analyse_samples(double confidence_level, unsigned int n_resamples, double* first, diff --git a/src/catch2/benchmark/detail/catch_stats.hpp b/src/catch2/benchmark/detail/catch_stats.hpp index 54621670..9450406b 100644 --- a/src/catch2/benchmark/detail/catch_stats.hpp +++ b/src/catch2/benchmark/detail/catch_stats.hpp @@ -13,9 +13,7 @@ #include #include -#include #include -#include namespace Catch { namespace Benchmark { @@ -36,78 +34,23 @@ namespace Catch { double mean( double const* first, double const* last ); - template - sample jackknife(Estimator&& estimator, - double* first, - double* last) { - auto n = static_cast(last - first); - auto second = first; - ++second; - sample results; - results.reserve(n); + sample jackknife( double ( *estimator )( double const*, + double const* ), + double* first, + double* last ); - for (auto it = first; it != last; ++it) { - std::iter_swap(it, first); - results.push_back(estimator(second, last)); - } - - return results; - } - - inline double normal_cdf(double x) { - return std::erfc(-x / std::sqrt(2.0)) / 2.0; - } + double normal_cdf( double x ); double erfc_inv(double x); double normal_quantile(double p); - template - Estimate bootstrap( double confidence_level, - double* first, - double* last, - sample const& resample, - Estimator&& estimator ) { - auto n_samples = last - first; - - double point = estimator(first, last); - // Degenerate case with a single sample - if (n_samples == 1) return { point, point, point, confidence_level }; - - sample jack = jackknife(estimator, first, last); - double jack_mean = mean(jack.data(), jack.data() + jack.size()); - double sum_squares = 0, sum_cubes = 0; - for (double x : jack) { - auto difference = jack_mean - x; - auto square = difference * difference; - auto cube = square * difference; - sum_squares += square; sum_cubes += cube; - } - - double accel = sum_cubes / (6 * std::pow(sum_squares, 1.5)); - long n = static_cast(resample.size()); - double prob_n = std::count_if(resample.begin(), resample.end(), [point](double x) { return x < point; }) / static_cast(n); - // degenerate case with uniform samples - if ( directCompare( prob_n, 0. ) ) { - return { point, point, point, confidence_level }; - } - - double bias = normal_quantile(prob_n); - double z1 = normal_quantile((1. - confidence_level) / 2.); - - auto cumn = [n]( double x ) -> long { - return std::lround( normal_cdf( x ) * static_cast(n) ); - }; - auto a = [bias, accel](double b) { return bias + b / (1. - accel * b); }; - double b1 = bias + z1; - double b2 = bias - z1; - double a1 = a(b1); - double a2 = a(b2); - auto lo = static_cast((std::max)(cumn(a1), 0l)); - auto hi = static_cast((std::min)(cumn(a2), n - 1)); - - return { point, resample[lo], resample[hi], confidence_level }; - } + Estimate + bootstrap( double confidence_level, + double* first, + double* last, + sample const& resample, + double ( *estimator )( double const*, double const* ) ); struct bootstrap_analysis { Estimate mean; diff --git a/src/catch2/benchmark/detail/catch_timing.hpp b/src/catch2/benchmark/detail/catch_timing.hpp index f5c25571..da567190 100644 --- a/src/catch2/benchmark/detail/catch_timing.hpp +++ b/src/catch2/benchmark/detail/catch_timing.hpp @@ -17,14 +17,14 @@ namespace Catch { namespace Benchmark { - template + template struct Timing { - Duration elapsed; + IDuration elapsed; Result result; int iterations; }; - template - using TimingOf = Timing, Detail::CompleteType_t>>; + template + using TimingOf = Timing>>; } // namespace Benchmark } // namespace Catch diff --git a/src/catch2/meson.build b/src/catch2/meson.build index 2e9469d8..f1c32219 100644 --- a/src/catch2/meson.build +++ b/src/catch2/meson.build @@ -45,6 +45,7 @@ benchmark_headers = [ benchmark_sources = files( 'benchmark/catch_chronometer.cpp', + 'benchmark/detail/catch_analyse.cpp', 'benchmark/detail/catch_benchmark_function.cpp', 'benchmark/detail/catch_run_for_at_least.cpp', 'benchmark/detail/catch_stats.cpp', diff --git a/tests/SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp b/tests/SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp index 34ed8814..bc8d715b 100644 --- a/tests/SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp +++ b/tests/SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp @@ -292,15 +292,13 @@ TEST_CASE("analyse", "[approvals][benchmark]") { data.benchmarkSamples = 99; Catch::Config config{data}; - using Duration = Catch::Benchmark::FloatDuration; - - Catch::Benchmark::Environment env; - std::vector samples(99); + using FDuration = Catch::Benchmark::FDuration; + std::vector samples(99); for (size_t i = 0; i < samples.size(); ++i) { - samples[i] = Duration(23 + (i % 3 - 1)); + samples[i] = FDuration(23 + (i % 3 - 1)); } - auto analysis = Catch::Benchmark::Detail::analyse(config, env, samples.begin(), samples.end()); + auto analysis = Catch::Benchmark::Detail::analyse(config, samples.data(), samples.data() + samples.size()); CHECK( analysis.mean.point.count() == 23 ); CHECK( analysis.mean.lower_bound.count() < 23 ); CHECK(analysis.mean.lower_bound.count() > 22); @@ -333,15 +331,13 @@ TEST_CASE("analyse no analysis", "[benchmark]") { data.benchmarkSamples = 99; Catch::Config config{ data }; - using Duration = Catch::Benchmark::FloatDuration; - - Catch::Benchmark::Environment env; - std::vector samples(99); + using FDuration = Catch::Benchmark::FDuration; + std::vector samples(99); for (size_t i = 0; i < samples.size(); ++i) { - samples[i] = Duration(23 + (i % 3 - 1)); + samples[i] = FDuration(23 + (i % 3 - 1)); } - auto analysis = Catch::Benchmark::Detail::analyse(config, env, samples.begin(), samples.end()); + auto analysis = Catch::Benchmark::Detail::analyse(config, samples.data(), samples.data() + samples.size()); CHECK(analysis.mean.point.count() == 23); CHECK(analysis.mean.lower_bound.count() == 23); CHECK(analysis.mean.upper_bound.count() == 23);