Added --skip-benchmarks command-line option. (#2408)

Closes #2392

Co-authored-by: Martin Hořeňovský <martin.horenovsky@gmail.com>

tests/CMakeLists.txt

@@ -359,6 +359,18 @@ add_test(NAME CheckConvenienceHeaders
     ${PYTHON_EXECUTABLE} ${CATCH_DIR}/tools/scripts/checkConvenienceHeaders.py
 )
 
+add_test(NAME "Benchmarking::SkipBenchmarkMacros"
+  COMMAND
+    $<TARGET_FILE:SelfTest> "Skip benchmark macros"
+      --reporter console
+      --skip-benchmarks
+)
+set_tests_properties("Benchmarking::SkipBenchmarkMacros"
+  PROPERTIES
+    PASS_REGULAR_EXPRESSION "All tests passed \\(2 assertions in 1 test case\\)"
+    FAIL_REGULAR_EXPRESSION "benchmark name"
+)
+
 
 add_test(NAME "Benchmarking::FailureReporting::OptimizedOut"
   COMMAND

tests/SelfTest/UsageTests/Benchmark.tests.cpp

@@ -152,3 +152,20 @@ TEST_CASE("Benchmark containers", "[!benchmark]") {
         };
     }
 }
+
+TEST_CASE("Skip benchmark macros", "[!benchmark]") {
+    std::vector<int> v;
+    BENCHMARK("fill vector") {
+        v.emplace_back(1);
+        v.emplace_back(2);
+        v.emplace_back(3);
+    };
+    REQUIRE(v.size() == 0);
+
+    std::size_t counter{0};
+    BENCHMARK_ADVANCED("construct vector")(Catch::Benchmark::Chronometer meter) {
+        std::vector<Catch::Benchmark::storage_for<std::string>> storage(meter.runs());
+        meter.measure([&](int i) { storage[i].construct("thing"); counter++; });
+    };
+    REQUIRE(counter == 0);
+}