diff --git a/projects/ExtraTests/CMakeLists.txt b/projects/ExtraTests/CMakeLists.txt index c0dd82d8..5a8dd79b 100644 --- a/projects/ExtraTests/CMakeLists.txt +++ b/projects/ExtraTests/CMakeLists.txt @@ -116,6 +116,17 @@ set_tests_properties( ) +add_executable(BenchmarkingMacros ${TESTS_DIR}/X20-BenchmarkingMacros.cpp) +target_compile_definitions( BenchmarkingMacros PRIVATE CATCH_CONFIG_ENABLE_BENCHMARKING ) + +add_test(NAME BenchmarkingMacros COMMAND BenchmarkingMacros -r console -s) +set_tests_properties( + BenchmarkingMacros + PROPERTIES + PASS_REGULAR_EXPRESSION "benchmark name samples iterations estimated" +) + + set( EXTRA_TEST_BINARIES PrefixedMacros DisabledMacros @@ -123,6 +134,7 @@ set( EXTRA_TEST_BINARIES DisabledExceptions-CustomHandler FallbackStringifier DisableStringification + BenchmarkingMacros ) # Shared config diff --git a/projects/ExtraTests/X20-BenchmarkingMacros.cpp b/projects/ExtraTests/X20-BenchmarkingMacros.cpp new file mode 100644 index 00000000..e76af0c7 --- /dev/null +++ b/projects/ExtraTests/X20-BenchmarkingMacros.cpp @@ -0,0 +1,133 @@ +// X20-BenchmarkingMacros.cpp +// Test that the benchmarking support macros compile properly with the single header + +#define CATCH_CONFIG_MAIN +#include + +namespace { +std::uint64_t factorial(std::uint64_t number) { + if (number < 2) { + return 1; + } + return number * factorial(number - 1); +} +} + +TEST_CASE("Benchmark factorial", "[benchmark]") { + CHECK(factorial(0) == 1); + // some more asserts.. + CHECK(factorial(10) == 3628800); + + BENCHMARK("factorial 10") { + return factorial(10); + }; + + CHECK(factorial(14) == 87178291200ull); + BENCHMARK("factorial 14") { + return factorial(14); + }; +// +// BENCHMARK("factorial 20") { +// return factorial(20); +// }; +// +// BENCHMARK("factorial 35") { +// return factorial(35); +// }; +} + +TEST_CASE("Benchmark containers", "[.][benchmark]") { + static const int size = 100; + + std::vector v; + std::map m; + + SECTION("without generator") { + BENCHMARK("Load up a vector") { + v = std::vector(); + for (int i = 0; i < size; ++i) + v.push_back(i); + }; + REQUIRE(v.size() == size); + + // test optimizer control + BENCHMARK("Add up a vector's content") { + uint64_t add = 0; + for (int i = 0; i < size; ++i) + add += v[i]; + return add; + }; + + BENCHMARK("Load up a map") { + m = std::map(); + for (int i = 0; i < size; ++i) + m.insert({ i, i + 1 }); + }; + REQUIRE(m.size() == size); + + BENCHMARK("Reserved vector") { + v = std::vector(); + v.reserve(size); + for (int i = 0; i < size; ++i) + v.push_back(i); + }; + REQUIRE(v.size() == size); + + BENCHMARK("Resized vector") { + v = std::vector(); + v.resize(size); + for (int i = 0; i < size; ++i) + v[i] = i; + }; + REQUIRE(v.size() == size); + + int array[size]; + BENCHMARK("A fixed size array that should require no allocations") { + for (int i = 0; i < size; ++i) + array[i] = i; + }; + int sum = 0; + for (int i = 0; i < size; ++i) + sum += array[i]; + REQUIRE(sum > size); + + SECTION("XYZ") { + + BENCHMARK_ADVANCED("Load up vector with chronometer")(Catch::Benchmark::Chronometer meter) { + std::vector k; + meter.measure([&](int idx) { + k = std::vector(); + for (int i = 0; i < size; ++i) + k.push_back(idx); + }); + REQUIRE(k.size() == size); + }; + + int runs = 0; + BENCHMARK("Fill vector indexed", benchmarkIndex) { + v = std::vector(); + v.resize(size); + for (int i = 0; i < size; ++i) + v[i] = benchmarkIndex; + runs = benchmarkIndex; + }; + + for (size_t i = 0; i < v.size(); ++i) { + REQUIRE(v[i] == runs); + } + } + } + + SECTION("with generator") { + auto generated = GENERATE(range(0, 10)); + BENCHMARK("Fill vector generated") { + v = std::vector(); + v.resize(size); + for (int i = 0; i < size; ++i) + v[i] = generated; + }; + for (size_t i = 0; i < v.size(); ++i) { + REQUIRE(v[i] == generated); + } + } +} diff --git a/projects/SelfTest/Baselines/compact.sw.approved.txt b/projects/SelfTest/Baselines/compact.sw.approved.txt index 89ebdfb1..eb30151f 100644 --- a/projects/SelfTest/Baselines/compact.sw.approved.txt +++ b/projects/SelfTest/Baselines/compact.sw.approved.txt @@ -944,6 +944,14 @@ CmdLine.tests.cpp:: passed: cli.parse({"test", "--use-colour", "no" CmdLine.tests.cpp:: passed: config.useColour == UseColour::No for: 2 == 2 CmdLine.tests.cpp:: passed: !result for: true CmdLine.tests.cpp:: passed: result.errorMessage(), Contains( "colour mode must be one of" ) for: "colour mode must be one of: auto, yes or no. 'wrong' not recognised" contains: "colour mode must be one of" +CmdLine.tests.cpp:: passed: cli.parse({ "test", "--benchmark-samples=200" }) for: {?} +CmdLine.tests.cpp:: passed: config.benchmarkSamples == 200 for: 200 == 200 +CmdLine.tests.cpp:: passed: cli.parse({ "test", "--benchmark-resamples=20000" }) for: {?} +CmdLine.tests.cpp:: passed: config.benchmarkResamples == 20000 for: 20000 (0x) == 20000 (0x) +CmdLine.tests.cpp:: passed: cli.parse({ "test", "--benchmark-confidence-interval=0.99" }) for: {?} +CmdLine.tests.cpp:: passed: config.benchmarkConfidenceInterval == Catch::Detail::Approx(0.99) for: 0.99 == Approx( 0.99 ) +CmdLine.tests.cpp:: passed: cli.parse({ "test", "--benchmark-no-analysis" }) for: {?} +CmdLine.tests.cpp:: passed: config.benchmarkNoAnalysis for: true Misc.tests.cpp:: passed: std::tuple_size::value >= 1 for: 3 >= 1 Misc.tests.cpp:: passed: std::tuple_size::value >= 1 for: 2 >= 1 Misc.tests.cpp:: passed: std::tuple_size::value >= 1 for: 1 >= 1 diff --git a/projects/SelfTest/Baselines/console.std.approved.txt b/projects/SelfTest/Baselines/console.std.approved.txt index 5f8e0362..ebbef313 100644 --- a/projects/SelfTest/Baselines/console.std.approved.txt +++ b/projects/SelfTest/Baselines/console.std.approved.txt @@ -1381,5 +1381,5 @@ due to unexpected exception with message: =============================================================================== test cases: 289 | 215 passed | 70 failed | 4 failed as expected -assertions: 1539 | 1387 passed | 131 failed | 21 failed as expected +assertions: 1547 | 1395 passed | 131 failed | 21 failed as expected diff --git a/projects/SelfTest/Baselines/console.sw.approved.txt b/projects/SelfTest/Baselines/console.sw.approved.txt index cd65bc17..3ff1e189 100644 --- a/projects/SelfTest/Baselines/console.sw.approved.txt +++ b/projects/SelfTest/Baselines/console.sw.approved.txt @@ -6954,6 +6954,78 @@ with expansion: "colour mode must be one of: auto, yes or no. 'wrong' not recognised" contains: "colour mode must be one of" +------------------------------------------------------------------------------- +Process can be configured on command line + Benchmark options + samples +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + CHECK( cli.parse({ "test", "--benchmark-samples=200" }) ) +with expansion: + {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.benchmarkSamples == 200 ) +with expansion: + 200 == 200 + +------------------------------------------------------------------------------- +Process can be configured on command line + Benchmark options + resamples +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + CHECK( cli.parse({ "test", "--benchmark-resamples=20000" }) ) +with expansion: + {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.benchmarkResamples == 20000 ) +with expansion: + 20000 (0x) == 20000 (0x) + +------------------------------------------------------------------------------- +Process can be configured on command line + Benchmark options + resamples +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + CHECK( cli.parse({ "test", "--benchmark-confidence-interval=0.99" }) ) +with expansion: + {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.benchmarkConfidenceInterval == Catch::Detail::Approx(0.99) ) +with expansion: + 0.99 == Approx( 0.99 ) + +------------------------------------------------------------------------------- +Process can be configured on command line + Benchmark options + resamples +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + CHECK( cli.parse({ "test", "--benchmark-no-analysis" }) ) +with expansion: + {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.benchmarkNoAnalysis ) +with expansion: + true + ------------------------------------------------------------------------------- Product with differing arities - std::tuple ------------------------------------------------------------------------------- @@ -12219,5 +12291,5 @@ Misc.tests.cpp:: PASSED: =============================================================================== test cases: 289 | 199 passed | 86 failed | 4 failed as expected -assertions: 1556 | 1387 passed | 148 failed | 21 failed as expected +assertions: 1564 | 1395 passed | 148 failed | 21 failed as expected diff --git a/projects/SelfTest/Baselines/junit.sw.approved.txt b/projects/SelfTest/Baselines/junit.sw.approved.txt index 94f01385..4592a90c 100644 --- a/projects/SelfTest/Baselines/junit.sw.approved.txt +++ b/projects/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ - + @@ -648,6 +648,10 @@ Message.tests.cpp: + + + + diff --git a/projects/SelfTest/Baselines/xml.sw.approved.txt b/projects/SelfTest/Baselines/xml.sw.approved.txt index dd2c3f4a..0d306b2e 100644 --- a/projects/SelfTest/Baselines/xml.sw.approved.txt +++ b/projects/SelfTest/Baselines/xml.sw.approved.txt @@ -8669,6 +8669,94 @@ Nor would this +
+
+ + + cli.parse({ "test", "--benchmark-samples=200" }) + + + {?} + + + + + config.benchmarkSamples == 200 + + + 200 == 200 + + + +
+ +
+
+
+ + + cli.parse({ "test", "--benchmark-resamples=20000" }) + + + {?} + + + + + config.benchmarkResamples == 20000 + + + 20000 (0x) == 20000 (0x) + + + +
+ +
+
+
+ + + cli.parse({ "test", "--benchmark-confidence-interval=0.99" }) + + + {?} + + + + + config.benchmarkConfidenceInterval == Catch::Detail::Approx(0.99) + + + 0.99 == Approx( 0.99 ) + + + +
+ +
+
+
+ + + cli.parse({ "test", "--benchmark-no-analysis" }) + + + {?} + + + + + config.benchmarkNoAnalysis + + + true + + + +
+ +
@@ -14586,7 +14674,7 @@ loose text artifact - + - + diff --git a/projects/SelfTest/UsageTests/Benchmark.tests.cpp b/projects/SelfTest/UsageTests/Benchmark.tests.cpp index 3d3cb621..24fda013 100644 --- a/projects/SelfTest/UsageTests/Benchmark.tests.cpp +++ b/projects/SelfTest/UsageTests/Benchmark.tests.cpp @@ -3,10 +3,10 @@ #include #if defined(CATCH_CONFIG_ENABLE_BENCHMARKING) -std::uint64_t Fibonacci(std::uint64_t number); - -std::uint64_t Fibonacci(std::uint64_t number) { - return number < 2 ? 1 : Fibonacci(number - 1) + Fibonacci(number - 2); +namespace { + std::uint64_t Fibonacci(std::uint64_t number) { + return number < 2 ? 1 : Fibonacci(number - 1) + Fibonacci(number - 2); + } } TEST_CASE("Benchmark Fibonacci", "[!benchmark]") {