diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0fdf931e..4e3b3566 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -33,6 +33,7 @@ set(BENCHMARK_HEADERS ) set(BENCHMARK_SOURCES ${SOURCES_DIR}/benchmark/catch_chronometer.cpp + ${SOURCES_DIR}/benchmark/detail/catch_analyse.cpp ${SOURCES_DIR}/benchmark/detail/catch_benchmark_function.cpp ${SOURCES_DIR}/benchmark/detail/catch_run_for_at_least.cpp ${SOURCES_DIR}/benchmark/detail/catch_stats.cpp diff --git a/src/catch2/benchmark/catch_benchmark.hpp b/src/catch2/benchmark/catch_benchmark.hpp index ea8ffa75..3db40bb0 100644 --- a/src/catch2/benchmark/catch_benchmark.hpp +++ b/src/catch2/benchmark/catch_benchmark.hpp @@ -45,16 +45,18 @@ namespace Catch { : fun(CATCH_MOVE(func)), name(CATCH_MOVE(benchmarkName)) {} template - ExecutionPlan> prepare(const IConfig &cfg, Environment> env) const { + ExecutionPlan prepare(const IConfig &cfg, Environment env) const { auto min_time = env.clock_resolution.mean * Detail::minimum_ticks; auto run_time = std::max(min_time, std::chrono::duration_cast(cfg.benchmarkWarmupTime())); - auto&& test = Detail::run_for_at_least(std::chrono::duration_cast>(run_time), 1, fun); + auto&& test = Detail::run_for_at_least(std::chrono::duration_cast(run_time), 1, fun); int new_iters = static_cast(std::ceil(min_time * test.iterations / test.elapsed)); - return { new_iters, test.elapsed / test.iterations * new_iters * cfg.benchmarkSamples(), fun, std::chrono::duration_cast>(cfg.benchmarkWarmupTime()), Detail::warmup_iterations }; + return { new_iters, test.elapsed / test.iterations * new_iters * cfg.benchmarkSamples(), fun, std::chrono::duration_cast(cfg.benchmarkWarmupTime()), Detail::warmup_iterations }; } template void run() { + static_assert( Clock::is_steady, + "Benchmarking clock should be steady" ); auto const* cfg = getCurrentContext().getConfig(); auto env = Detail::measure_environment(); @@ -81,8 +83,8 @@ namespace Catch { return plan.template run(*cfg, env); }); - auto analysis = Detail::analyse(*cfg, env, samples.begin(), samples.end()); - BenchmarkStats> stats{ CATCH_MOVE(info), CATCH_MOVE(analysis.samples), analysis.mean, analysis.standard_deviation, analysis.outliers, analysis.outlier_variance }; + auto analysis = Detail::analyse(*cfg, samples.data(), samples.data() + samples.size()); + BenchmarkStats<> stats{ CATCH_MOVE(info), CATCH_MOVE(analysis.samples), analysis.mean, analysis.standard_deviation, analysis.outliers, analysis.outlier_variance }; getResultCapture().benchmarkEnded(stats); } CATCH_CATCH_ANON (TestFailureException const&) { getResultCapture().benchmarkFailed("Benchmark failed due to failed assertion"_sr); diff --git a/src/catch2/benchmark/catch_chronometer.hpp b/src/catch2/benchmark/catch_chronometer.hpp index c3f81306..95498e6b 100644 --- a/src/catch2/benchmark/catch_chronometer.hpp +++ b/src/catch2/benchmark/catch_chronometer.hpp @@ -32,7 +32,10 @@ namespace Catch { void start() override { started = Clock::now(); } void finish() override { finished = Clock::now(); } - ClockDuration elapsed() const { return finished - started; } + IDuration elapsed() const { + return std::chrono::duration_cast( + finished - started ); + } TimePoint started; TimePoint finished; diff --git a/src/catch2/benchmark/catch_clock.hpp b/src/catch2/benchmark/catch_clock.hpp index cee46097..4068c4d2 100644 --- a/src/catch2/benchmark/catch_clock.hpp +++ b/src/catch2/benchmark/catch_clock.hpp @@ -11,28 +11,16 @@ #define CATCH_CLOCK_HPP_INCLUDED #include -#include namespace Catch { namespace Benchmark { - template - using ClockDuration = typename Clock::duration; - template - using FloatDuration = std::chrono::duration; + using IDuration = std::chrono::nanoseconds; + using FDuration = std::chrono::duration; template using TimePoint = typename Clock::time_point; using default_clock = std::chrono::steady_clock; - - template - struct now { - TimePoint operator()() const { - return Clock::now(); - } - }; - - using fp_seconds = std::chrono::duration>; } // namespace Benchmark } // namespace Catch diff --git a/src/catch2/benchmark/catch_environment.hpp b/src/catch2/benchmark/catch_environment.hpp index 5d3816a5..da3f2fa9 100644 --- a/src/catch2/benchmark/catch_environment.hpp +++ b/src/catch2/benchmark/catch_environment.hpp @@ -15,20 +15,13 @@ namespace Catch { namespace Benchmark { - template struct EnvironmentEstimate { - Duration mean; + FDuration mean; OutlierClassification outliers; - - template - operator EnvironmentEstimate() const { - return { mean, outliers }; - } }; - template struct Environment { - EnvironmentEstimate> clock_resolution; - EnvironmentEstimate> clock_cost; + EnvironmentEstimate clock_resolution; + EnvironmentEstimate clock_cost; }; } // namespace Benchmark } // namespace Catch diff --git a/src/catch2/benchmark/catch_estimate.hpp b/src/catch2/benchmark/catch_estimate.hpp index be594a18..64383a2e 100644 --- a/src/catch2/benchmark/catch_estimate.hpp +++ b/src/catch2/benchmark/catch_estimate.hpp @@ -12,17 +12,12 @@ namespace Catch { namespace Benchmark { - template + template struct Estimate { - Duration point; - Duration lower_bound; - Duration upper_bound; + Type point; + Type lower_bound; + Type upper_bound; double confidence_interval; - - template - operator Estimate() const { - return { point, lower_bound, upper_bound, confidence_interval }; - } }; } // namespace Benchmark } // namespace Catch diff --git a/src/catch2/benchmark/catch_execution_plan.hpp b/src/catch2/benchmark/catch_execution_plan.hpp index 4f60a646..17ca589f 100644 --- a/src/catch2/benchmark/catch_execution_plan.hpp +++ b/src/catch2/benchmark/catch_execution_plan.hpp @@ -21,33 +21,31 @@ namespace Catch { namespace Benchmark { - template struct ExecutionPlan { int iterations_per_sample; - Duration estimated_duration; + FDuration estimated_duration; Detail::BenchmarkFunction benchmark; - Duration warmup_time; + FDuration warmup_time; int warmup_iterations; - template - operator ExecutionPlan() const { - return { iterations_per_sample, estimated_duration, benchmark, warmup_time, warmup_iterations }; - } - template - std::vector> run(const IConfig &cfg, Environment> env) const { + std::vector run(const IConfig &cfg, Environment env) const { // warmup a bit - Detail::run_for_at_least(std::chrono::duration_cast>(warmup_time), warmup_iterations, Detail::repeat(now{})); + Detail::run_for_at_least( + std::chrono::duration_cast( warmup_time ), + warmup_iterations, + Detail::repeat( []() { return Clock::now(); } ) + ); - std::vector> times; + std::vector times; const auto num_samples = cfg.benchmarkSamples(); times.reserve( num_samples ); for ( size_t i = 0; i < num_samples; ++i ) { Detail::ChronometerModel model; this->benchmark( Chronometer( model, iterations_per_sample ) ); auto sample_time = model.elapsed() - env.clock_cost.mean; - if ( sample_time < FloatDuration::zero() ) { - sample_time = FloatDuration::zero(); + if ( sample_time < FDuration::zero() ) { + sample_time = FDuration::zero(); } times.push_back(sample_time / iterations_per_sample); } diff --git a/src/catch2/benchmark/catch_sample_analysis.hpp b/src/catch2/benchmark/catch_sample_analysis.hpp index 97b8fe50..aeb87d05 100644 --- a/src/catch2/benchmark/catch_sample_analysis.hpp +++ b/src/catch2/benchmark/catch_sample_analysis.hpp @@ -12,35 +12,18 @@ #include #include -#include +#include #include namespace Catch { namespace Benchmark { - template struct SampleAnalysis { - std::vector samples; - Estimate mean; - Estimate standard_deviation; + std::vector samples; + Estimate mean; + Estimate standard_deviation; OutlierClassification outliers; double outlier_variance; - - template - operator SampleAnalysis() const { - std::vector samples2; - samples2.reserve(samples.size()); - for (auto const& d : samples) { - samples2.push_back(Duration2(d)); - } - return { - CATCH_MOVE(samples2), - mean, - standard_deviation, - outliers, - outlier_variance, - }; - } }; } // namespace Benchmark } // namespace Catch diff --git a/src/catch2/benchmark/detail/catch_analyse.cpp b/src/catch2/benchmark/detail/catch_analyse.cpp new file mode 100644 index 00000000..7d27daf1 --- /dev/null +++ b/src/catch2/benchmark/detail/catch_analyse.cpp @@ -0,0 +1,85 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +// Adapted from donated nonius code. + +#include +#include +#include +#include +#include +#include + +#include + +namespace Catch { + namespace Benchmark { + namespace Detail { + SampleAnalysis analyse(const IConfig &cfg, FDuration* first, FDuration* last) { + if (!cfg.benchmarkNoAnalysis()) { + std::vector samples; + samples.reserve(static_cast(last - first)); + for (auto current = first; current != last; ++current) { + samples.push_back( current->count() ); + } + + auto analysis = Catch::Benchmark::Detail::analyse_samples( + cfg.benchmarkConfidenceInterval(), + cfg.benchmarkResamples(), + samples.data(), + samples.data() + samples.size() ); + auto outliers = Catch::Benchmark::Detail::classify_outliers( + samples.data(), samples.data() + samples.size() ); + + auto wrap_estimate = [](Estimate e) { + return Estimate { + FDuration(e.point), + FDuration(e.lower_bound), + FDuration(e.upper_bound), + e.confidence_interval, + }; + }; + std::vector samples2; + samples2.reserve(samples.size()); + for (auto s : samples) { + samples2.push_back( FDuration( s ) ); + } + + return { + CATCH_MOVE(samples2), + wrap_estimate(analysis.mean), + wrap_estimate(analysis.standard_deviation), + outliers, + analysis.outlier_variance, + }; + } else { + std::vector samples; + samples.reserve(static_cast(last - first)); + + FDuration mean = FDuration(0); + int i = 0; + for (auto it = first; it < last; ++it, ++i) { + samples.push_back(FDuration(*it)); + mean += FDuration(*it); + } + mean /= i; + + return SampleAnalysis{ + CATCH_MOVE(samples), + Estimate{ mean, mean, mean, 0.0 }, + Estimate{ FDuration( 0 ), + FDuration( 0 ), + FDuration( 0 ), + 0.0 }, + OutlierClassification{}, + 0.0 + }; + } + } + } // namespace Detail + } // namespace Benchmark +} // namespace Catch diff --git a/src/catch2/benchmark/detail/catch_analyse.hpp b/src/catch2/benchmark/detail/catch_analyse.hpp index 60db7d87..5e3f7b0f 100644 --- a/src/catch2/benchmark/detail/catch_analyse.hpp +++ b/src/catch2/benchmark/detail/catch_analyse.hpp @@ -10,76 +10,16 @@ #ifndef CATCH_ANALYSE_HPP_INCLUDED #define CATCH_ANALYSE_HPP_INCLUDED -#include +#include #include -#include -#include -#include -#include namespace Catch { + class IConfig; + namespace Benchmark { namespace Detail { - template - SampleAnalysis analyse(const IConfig &cfg, Environment, Iterator first, Iterator last) { - if (!cfg.benchmarkNoAnalysis()) { - std::vector samples; - samples.reserve(static_cast(last - first)); - for (auto current = first; current != last; ++current) { - samples.push_back( current->count() ); - } - - auto analysis = Catch::Benchmark::Detail::analyse_samples( - cfg.benchmarkConfidenceInterval(), - cfg.benchmarkResamples(), - samples.data(), - samples.data() + samples.size() ); - auto outliers = Catch::Benchmark::Detail::classify_outliers( - samples.data(), samples.data() + samples.size() ); - - auto wrap_estimate = [](Estimate e) { - return Estimate { - Duration(e.point), - Duration(e.lower_bound), - Duration(e.upper_bound), - e.confidence_interval, - }; - }; - std::vector samples2; - samples2.reserve(samples.size()); - for (auto s : samples) { - samples2.push_back( Duration( s ) ); - } - - return { - CATCH_MOVE(samples2), - wrap_estimate(analysis.mean), - wrap_estimate(analysis.standard_deviation), - outliers, - analysis.outlier_variance, - }; - } else { - std::vector samples; - samples.reserve(static_cast(last - first)); - - Duration mean = Duration(0); - int i = 0; - for (auto it = first; it < last; ++it, ++i) { - samples.push_back(Duration(*it)); - mean += Duration(*it); - } - mean /= i; - - return { - CATCH_MOVE(samples), - Estimate{mean, mean, mean, 0.0}, - Estimate{Duration(0), Duration(0), Duration(0), 0.0}, - OutlierClassification{}, - 0.0 - }; - } - } + SampleAnalysis analyse(const IConfig &cfg, FDuration* first, FDuration* last); } // namespace Detail } // namespace Benchmark } // namespace Catch diff --git a/src/catch2/benchmark/detail/catch_benchmark_stats.hpp b/src/catch2/benchmark/detail/catch_benchmark_stats.hpp index a8b3494e..3633bc9f 100644 --- a/src/catch2/benchmark/detail/catch_benchmark_stats.hpp +++ b/src/catch2/benchmark/detail/catch_benchmark_stats.hpp @@ -8,7 +8,6 @@ #ifndef CATCH_BENCHMARK_STATS_HPP_INCLUDED #define CATCH_BENCHMARK_STATS_HPP_INCLUDED -#include #include #include // The fwd decl & default specialization needs to be seen by VS2017 before @@ -30,32 +29,17 @@ namespace Catch { double clockCost; }; - template + // We need to keep template parameter for backwards compatibility, + // but we also do not want to use the template paraneter. + template struct BenchmarkStats { BenchmarkInfo info; - std::vector samples; - Benchmark::Estimate mean; - Benchmark::Estimate standardDeviation; + std::vector samples; + Benchmark::Estimate mean; + Benchmark::Estimate standardDeviation; Benchmark::OutlierClassification outliers; double outlierVariance; - - template - operator BenchmarkStats() const { - std::vector samples2; - samples2.reserve(samples.size()); - for (auto const& sample : samples) { - samples2.push_back(Duration2(sample)); - } - return { - info, - CATCH_MOVE(samples2), - mean, - standardDeviation, - outliers, - outlierVariance, - }; - } }; diff --git a/src/catch2/benchmark/detail/catch_benchmark_stats_fwd.hpp b/src/catch2/benchmark/detail/catch_benchmark_stats_fwd.hpp index 60772561..2ccc25d5 100644 --- a/src/catch2/benchmark/detail/catch_benchmark_stats_fwd.hpp +++ b/src/catch2/benchmark/detail/catch_benchmark_stats_fwd.hpp @@ -8,14 +8,14 @@ #ifndef CATCH_BENCHMARK_STATS_FWD_HPP_INCLUDED #define CATCH_BENCHMARK_STATS_FWD_HPP_INCLUDED -#include +#include namespace Catch { // We cannot forward declare the type with default template argument // multiple times, so it is split out into a separate header so that // we can prevent multiple declarations in dependees - template > + template struct BenchmarkStats; } // end namespace Catch diff --git a/src/catch2/benchmark/detail/catch_estimate_clock.hpp b/src/catch2/benchmark/detail/catch_estimate_clock.hpp index a0dc2d3d..918484a9 100644 --- a/src/catch2/benchmark/detail/catch_estimate_clock.hpp +++ b/src/catch2/benchmark/detail/catch_estimate_clock.hpp @@ -55,23 +55,23 @@ namespace Catch { template int warmup() { - return run_for_at_least(std::chrono::duration_cast>(warmup_time), warmup_seed, &resolution) + return run_for_at_least(std::chrono::duration_cast(warmup_time), warmup_seed, &resolution) .iterations; } template - EnvironmentEstimate> estimate_clock_resolution(int iterations) { - auto r = run_for_at_least(std::chrono::duration_cast>(clock_resolution_estimation_time), iterations, &resolution) + EnvironmentEstimate estimate_clock_resolution(int iterations) { + auto r = run_for_at_least(std::chrono::duration_cast(clock_resolution_estimation_time), iterations, &resolution) .result; return { - FloatDuration(mean(r.data(), r.data() + r.size())), + FDuration(mean(r.data(), r.data() + r.size())), classify_outliers(r.data(), r.data() + r.size()), }; } template - EnvironmentEstimate> estimate_clock_cost(FloatDuration resolution) { + EnvironmentEstimate estimate_clock_cost(FDuration resolution) { auto time_limit = (std::min)( resolution * clock_cost_estimation_tick_limit, - FloatDuration(clock_cost_estimation_time_limit)); + FDuration(clock_cost_estimation_time_limit)); auto time_clock = [](int k) { return Detail::measure([k] { for (int i = 0; i < k; ++i) { @@ -82,7 +82,7 @@ namespace Catch { }; time_clock(1); int iters = clock_cost_estimation_iterations; - auto&& r = run_for_at_least(std::chrono::duration_cast>(clock_cost_estimation_time), iters, time_clock); + auto&& r = run_for_at_least(std::chrono::duration_cast(clock_cost_estimation_time), iters, time_clock); std::vector times; int nsamples = static_cast(std::ceil(time_limit / r.elapsed)); times.reserve(static_cast(nsamples)); @@ -92,18 +92,18 @@ namespace Catch { .count() ) ); } return { - FloatDuration(mean(times.data(), times.data() + times.size())), + FDuration(mean(times.data(), times.data() + times.size())), classify_outliers(times.data(), times.data() + times.size()), }; } template - Environment> measure_environment() { + Environment measure_environment() { #if defined(__clang__) # pragma clang diagnostic push # pragma clang diagnostic ignored "-Wexit-time-destructors" #endif - static Catch::Detail::unique_ptr>> env; + static Catch::Detail::unique_ptr env; #if defined(__clang__) # pragma clang diagnostic pop #endif @@ -115,7 +115,7 @@ namespace Catch { auto resolution = Detail::estimate_clock_resolution(iters); auto cost = Detail::estimate_clock_cost(resolution.mean); - env = Catch::Detail::make_unique>>( Environment>{resolution, cost} ); + env = Catch::Detail::make_unique( Environment{resolution, cost} ); return *env; } } // namespace Detail diff --git a/src/catch2/benchmark/detail/catch_measure.hpp b/src/catch2/benchmark/detail/catch_measure.hpp index 1a30efab..37494a68 100644 --- a/src/catch2/benchmark/detail/catch_measure.hpp +++ b/src/catch2/benchmark/detail/catch_measure.hpp @@ -18,7 +18,7 @@ namespace Catch { namespace Benchmark { namespace Detail { template - TimingOf measure(Fun&& fun, Args&&... args) { + TimingOf measure(Fun&& fun, Args&&... args) { auto start = Clock::now(); auto&& r = Detail::complete_invoke(fun, CATCH_FORWARD(args)...); auto end = Clock::now(); diff --git a/src/catch2/benchmark/detail/catch_run_for_at_least.hpp b/src/catch2/benchmark/detail/catch_run_for_at_least.hpp index 976a4b24..4dfa8bbb 100644 --- a/src/catch2/benchmark/detail/catch_run_for_at_least.hpp +++ b/src/catch2/benchmark/detail/catch_run_for_at_least.hpp @@ -24,11 +24,11 @@ namespace Catch { namespace Benchmark { namespace Detail { template - TimingOf measure_one(Fun&& fun, int iters, std::false_type) { + TimingOf measure_one(Fun&& fun, int iters, std::false_type) { return Detail::measure(fun, iters); } template - TimingOf measure_one(Fun&& fun, int iters, std::true_type) { + TimingOf measure_one(Fun&& fun, int iters, std::true_type) { Detail::ChronometerModel meter; auto&& result = Detail::complete_invoke(fun, Chronometer(meter, iters)); @@ -43,8 +43,8 @@ namespace Catch { void throw_optimized_away_error(); template - TimingOf> - run_for_at_least(ClockDuration how_long, + TimingOf> + run_for_at_least(IDuration how_long, const int initial_iterations, Fun&& fun) { auto iters = initial_iterations; diff --git a/src/catch2/benchmark/detail/catch_stats.cpp b/src/catch2/benchmark/detail/catch_stats.cpp index caad0167..fe64b1c5 100644 --- a/src/catch2/benchmark/detail/catch_stats.cpp +++ b/src/catch2/benchmark/detail/catch_stats.cpp @@ -11,7 +11,9 @@ #include +#include #include +#include #include #include #include @@ -257,6 +259,27 @@ namespace Catch { return sum / static_cast(count); } + sample jackknife( double ( *estimator )( double const*, + double const* ), + double* first, + double* last ) { + auto n = static_cast( last - first ); + auto second = first; + ++second; + sample results; + results.reserve( n ); + + for ( auto it = first; it != last; ++it ) { + std::iter_swap( it, first ); + results.push_back( estimator( second, last ) ); + } + + return results; + } + + double normal_cdf( double x ) { + return std::erfc( -x / std::sqrt( 2.0 ) ) / 2.0; + } double erfc_inv(double x) { return erf_inv(1.0 - x); @@ -278,6 +301,64 @@ namespace Catch { return result; } + Estimate + bootstrap( double confidence_level, + double* first, + double* last, + sample const& resample, + double ( *estimator )( double const*, double const* ) ) { + auto n_samples = last - first; + + double point = estimator( first, last ); + // Degenerate case with a single sample + if ( n_samples == 1 ) + return { point, point, point, confidence_level }; + + sample jack = jackknife( estimator, first, last ); + double jack_mean = + mean( jack.data(), jack.data() + jack.size() ); + double sum_squares = 0, sum_cubes = 0; + for ( double x : jack ) { + auto difference = jack_mean - x; + auto square = difference * difference; + auto cube = square * difference; + sum_squares += square; + sum_cubes += cube; + } + + double accel = sum_cubes / ( 6 * std::pow( sum_squares, 1.5 ) ); + long n = static_cast( resample.size() ); + double prob_n = + std::count_if( resample.begin(), + resample.end(), + [point]( double x ) { return x < point; } ) / + static_cast( n ); + // degenerate case with uniform samples + if ( directCompare( prob_n, 0. ) ) { + return { point, point, point, confidence_level }; + } + + double bias = normal_quantile( prob_n ); + double z1 = normal_quantile( ( 1. - confidence_level ) / 2. ); + + auto cumn = [n]( double x ) -> long { + return std::lround( normal_cdf( x ) * + static_cast( n ) ); + }; + auto a = [bias, accel]( double b ) { + return bias + b / ( 1. - accel * b ); + }; + double b1 = bias + z1; + double b2 = bias - z1; + double a1 = a( b1 ); + double a2 = a( b2 ); + auto lo = static_cast( (std::max)( cumn( a1 ), 0l ) ); + auto hi = + static_cast( (std::min)( cumn( a2 ), n - 1 ) ); + + return { point, resample[lo], resample[hi], confidence_level }; + } + bootstrap_analysis analyse_samples(double confidence_level, unsigned int n_resamples, double* first, diff --git a/src/catch2/benchmark/detail/catch_stats.hpp b/src/catch2/benchmark/detail/catch_stats.hpp index 54621670..9450406b 100644 --- a/src/catch2/benchmark/detail/catch_stats.hpp +++ b/src/catch2/benchmark/detail/catch_stats.hpp @@ -13,9 +13,7 @@ #include #include -#include #include -#include namespace Catch { namespace Benchmark { @@ -36,78 +34,23 @@ namespace Catch { double mean( double const* first, double const* last ); - template - sample jackknife(Estimator&& estimator, - double* first, - double* last) { - auto n = static_cast(last - first); - auto second = first; - ++second; - sample results; - results.reserve(n); + sample jackknife( double ( *estimator )( double const*, + double const* ), + double* first, + double* last ); - for (auto it = first; it != last; ++it) { - std::iter_swap(it, first); - results.push_back(estimator(second, last)); - } - - return results; - } - - inline double normal_cdf(double x) { - return std::erfc(-x / std::sqrt(2.0)) / 2.0; - } + double normal_cdf( double x ); double erfc_inv(double x); double normal_quantile(double p); - template - Estimate bootstrap( double confidence_level, - double* first, - double* last, - sample const& resample, - Estimator&& estimator ) { - auto n_samples = last - first; - - double point = estimator(first, last); - // Degenerate case with a single sample - if (n_samples == 1) return { point, point, point, confidence_level }; - - sample jack = jackknife(estimator, first, last); - double jack_mean = mean(jack.data(), jack.data() + jack.size()); - double sum_squares = 0, sum_cubes = 0; - for (double x : jack) { - auto difference = jack_mean - x; - auto square = difference * difference; - auto cube = square * difference; - sum_squares += square; sum_cubes += cube; - } - - double accel = sum_cubes / (6 * std::pow(sum_squares, 1.5)); - long n = static_cast(resample.size()); - double prob_n = std::count_if(resample.begin(), resample.end(), [point](double x) { return x < point; }) / static_cast(n); - // degenerate case with uniform samples - if ( directCompare( prob_n, 0. ) ) { - return { point, point, point, confidence_level }; - } - - double bias = normal_quantile(prob_n); - double z1 = normal_quantile((1. - confidence_level) / 2.); - - auto cumn = [n]( double x ) -> long { - return std::lround( normal_cdf( x ) * static_cast(n) ); - }; - auto a = [bias, accel](double b) { return bias + b / (1. - accel * b); }; - double b1 = bias + z1; - double b2 = bias - z1; - double a1 = a(b1); - double a2 = a(b2); - auto lo = static_cast((std::max)(cumn(a1), 0l)); - auto hi = static_cast((std::min)(cumn(a2), n - 1)); - - return { point, resample[lo], resample[hi], confidence_level }; - } + Estimate + bootstrap( double confidence_level, + double* first, + double* last, + sample const& resample, + double ( *estimator )( double const*, double const* ) ); struct bootstrap_analysis { Estimate mean; diff --git a/src/catch2/benchmark/detail/catch_timing.hpp b/src/catch2/benchmark/detail/catch_timing.hpp index f5c25571..da567190 100644 --- a/src/catch2/benchmark/detail/catch_timing.hpp +++ b/src/catch2/benchmark/detail/catch_timing.hpp @@ -17,14 +17,14 @@ namespace Catch { namespace Benchmark { - template + template struct Timing { - Duration elapsed; + IDuration elapsed; Result result; int iterations; }; - template - using TimingOf = Timing, Detail::CompleteType_t>>; + template + using TimingOf = Timing>>; } // namespace Benchmark } // namespace Catch diff --git a/src/catch2/meson.build b/src/catch2/meson.build index 2e9469d8..f1c32219 100644 --- a/src/catch2/meson.build +++ b/src/catch2/meson.build @@ -45,6 +45,7 @@ benchmark_headers = [ benchmark_sources = files( 'benchmark/catch_chronometer.cpp', + 'benchmark/detail/catch_analyse.cpp', 'benchmark/detail/catch_benchmark_function.cpp', 'benchmark/detail/catch_run_for_at_least.cpp', 'benchmark/detail/catch_stats.cpp', diff --git a/tests/SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp b/tests/SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp index 34ed8814..bc8d715b 100644 --- a/tests/SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp +++ b/tests/SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp @@ -292,15 +292,13 @@ TEST_CASE("analyse", "[approvals][benchmark]") { data.benchmarkSamples = 99; Catch::Config config{data}; - using Duration = Catch::Benchmark::FloatDuration; - - Catch::Benchmark::Environment env; - std::vector samples(99); + using FDuration = Catch::Benchmark::FDuration; + std::vector samples(99); for (size_t i = 0; i < samples.size(); ++i) { - samples[i] = Duration(23 + (i % 3 - 1)); + samples[i] = FDuration(23 + (i % 3 - 1)); } - auto analysis = Catch::Benchmark::Detail::analyse(config, env, samples.begin(), samples.end()); + auto analysis = Catch::Benchmark::Detail::analyse(config, samples.data(), samples.data() + samples.size()); CHECK( analysis.mean.point.count() == 23 ); CHECK( analysis.mean.lower_bound.count() < 23 ); CHECK(analysis.mean.lower_bound.count() > 22); @@ -333,15 +331,13 @@ TEST_CASE("analyse no analysis", "[benchmark]") { data.benchmarkSamples = 99; Catch::Config config{ data }; - using Duration = Catch::Benchmark::FloatDuration; - - Catch::Benchmark::Environment env; - std::vector samples(99); + using FDuration = Catch::Benchmark::FDuration; + std::vector samples(99); for (size_t i = 0; i < samples.size(); ++i) { - samples[i] = Duration(23 + (i % 3 - 1)); + samples[i] = FDuration(23 + (i % 3 - 1)); } - auto analysis = Catch::Benchmark::Detail::analyse(config, env, samples.begin(), samples.end()); + auto analysis = Catch::Benchmark::Detail::analyse(config, samples.data(), samples.data() + samples.size()); CHECK(analysis.mean.point.count() == 23); CHECK(analysis.mean.lower_bound.count() == 23); CHECK(analysis.mean.upper_bound.count() == 23);