From a1be19aa1bfcabcbd28e442c0878c1a9468e83e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20Ho=C5=99e=C5=88ovsk=C3=BD?=
 <martin.horenovsky@gmail.com>
Date: Tue, 18 Feb 2020 13:48:34 +0100
Subject: [PATCH] Remove the separately compiled test for Benchmarking Macros

Thanks to the changes to compilation model, the tests for benchmarking
macros have been made part of the normal test run. This means that
the only purpose these separately compiled tests served was to waste
CI time.
---
 tests/CMakeLists.txt                        |   1 +
 tests/ExtraTests/CMakeLists.txt             |  13 --
 tests/ExtraTests/X20-BenchmarkingMacros.cpp | 127 --------------------
 3 files changed, 1 insertion(+), 140 deletions(-)
 delete mode 100644 tests/ExtraTests/X20-BenchmarkingMacros.cpp

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 981e5519..7dcb819f 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -123,6 +123,7 @@ endif()
 add_test(NAME RunTests COMMAND $<TARGET_FILE:SelfTest>)
 set_tests_properties(RunTests PROPERTIES
     FAIL_REGULAR_EXPRESSION "Filters:"
+    COST 60
 )
 
 # Because CTest does not allow us to check both return code _and_ expected
diff --git a/tests/ExtraTests/CMakeLists.txt b/tests/ExtraTests/CMakeLists.txt
index 36895168..b89af55e 100644
--- a/tests/ExtraTests/CMakeLists.txt
+++ b/tests/ExtraTests/CMakeLists.txt
@@ -119,18 +119,6 @@ set_tests_properties(
 )
 
 
-add_executable(BenchmarkingMacros ${TESTS_DIR}/X20-BenchmarkingMacros.cpp)
-target_link_libraries( BenchmarkingMacros Catch2_buildall_interface )
-
-add_test(NAME BenchmarkingMacros COMMAND BenchmarkingMacros -r console -s)
-set_tests_properties(
-    BenchmarkingMacros
-  PROPERTIES
-    PASS_REGULAR_EXPRESSION "benchmark name                                  samples       iterations    estimated"
-    COST 80 # We know that this is either the most, or second most,
-   # expensive test in the test suite, so we give it high estimate for CI runs
-)
-
 # This test touches windows.h, so it should only be compiled under msvc
 if (MSVC)
     # This test fails if it does not compile and succeeds otherwise
@@ -158,7 +146,6 @@ set( EXTRA_TEST_BINARIES
     DisabledExceptions-CustomHandler
     FallbackStringifier
     DisableStringification
-    BenchmarkingMacros
 #    DebugBreakMacros
 )
 
diff --git a/tests/ExtraTests/X20-BenchmarkingMacros.cpp b/tests/ExtraTests/X20-BenchmarkingMacros.cpp
deleted file mode 100644
index 56e0a05a..00000000
--- a/tests/ExtraTests/X20-BenchmarkingMacros.cpp
+++ /dev/null
@@ -1,127 +0,0 @@
-// X20-BenchmarkingMacros.cpp
-// Test that the benchmarking support macros compile properly with the single header
-
-#include <catch2/catch_default_main.hpp>
-#include <catch2/catch_test_macros.hpp>
-#include <catch2/benchmark/catch_benchmark.hpp>
-#include <catch2/catch_generators_specific.hpp>
-
-namespace {
-std::uint64_t factorial(std::uint64_t number) {
-    if (number < 2) {
-        return 1;
-    }
-    return number * factorial(number - 1);
-}
-}
-
-TEST_CASE("Benchmark factorial", "[benchmark]") {
-    CHECK(factorial(0) == 1);
-    // some more asserts..
-    CHECK(factorial(10) == 3628800);
-
-    BENCHMARK("factorial 10") {
-        return factorial(10);
-    };
-
-    CHECK(factorial(14) == 87178291200ull);
-    BENCHMARK("factorial 14") {
-        return factorial(14);
-    };
-}
-
-TEST_CASE("Benchmark containers", "[.][benchmark]") {
-    static const int size = 100;
-
-    std::vector<int> v;
-    std::map<int, int> m;
-
-    SECTION("without generator") {
-        BENCHMARK("Load up a vector") {
-            v = std::vector<int>();
-            for (int i = 0; i < size; ++i)
-                v.push_back(i);
-        };
-        REQUIRE(v.size() == size);
-
-        // test optimizer control
-        BENCHMARK("Add up a vector's content") {
-            uint64_t add = 0;
-            for (int i = 0; i < size; ++i)
-                add += v[i];
-            return add;
-        };
-
-        BENCHMARK("Load up a map") {
-            m = std::map<int, int>();
-            for (int i = 0; i < size; ++i)
-                m.insert({ i, i + 1 });
-        };
-        REQUIRE(m.size() == size);
-
-        BENCHMARK("Reserved vector") {
-            v = std::vector<int>();
-            v.reserve(size);
-            for (int i = 0; i < size; ++i)
-                v.push_back(i);
-        };
-        REQUIRE(v.size() == size);
-
-        BENCHMARK("Resized vector") {
-            v = std::vector<int>();
-            v.resize(size);
-            for (int i = 0; i < size; ++i)
-                v[i] = i;
-        };
-        REQUIRE(v.size() == size);
-
-        int array[size];
-        BENCHMARK("A fixed size array that should require no allocations") {
-            for (int i = 0; i < size; ++i)
-                array[i] = i;
-        };
-        int sum = 0;
-        for (int i = 0; i < size; ++i)
-            sum += array[i];
-        REQUIRE(sum > size);
-
-        SECTION("XYZ") {
-
-            BENCHMARK_ADVANCED("Load up vector with chronometer")(Catch::Benchmark::Chronometer meter) {
-                std::vector<int> k;
-                meter.measure([&](int idx) {
-                    k = std::vector<int>();
-                    for (int i = 0; i < size; ++i)
-                        k.push_back(idx);
-                });
-                REQUIRE(k.size() == size);
-            };
-
-            int runs = 0;
-            BENCHMARK("Fill vector indexed", benchmarkIndex) {
-                v = std::vector<int>();
-                v.resize(size);
-                for (int i = 0; i < size; ++i)
-                    v[i] = benchmarkIndex;
-                runs = benchmarkIndex;
-            };
-
-            for (size_t i = 0; i < v.size(); ++i) {
-                REQUIRE(v[i] == runs);
-            }
-        }
-    }
-
-    SECTION("with generator") {
-        auto generated = GENERATE(range(0, 10));
-        BENCHMARK("Fill vector generated") {
-            v = std::vector<int>();
-            v.resize(size);
-            for (int i = 0; i < size; ++i)
-                v[i] = generated;
-        };
-        for (size_t i = 0; i < v.size(); ++i) {
-            REQUIRE(v[i] == generated);
-        }
-    }
-}