Exit with non-0 return code if no tests were run

A new flag, `--allow-running-no-tests` was added to override this
behaviour if exit code of 0 was desired.

This change also made `-w NoTests` obsolete, and so it has been
removed.

tests/CMakeLists.txt

@@ -207,15 +207,29 @@ set_tests_properties(List::Reporters::XmlOutput PROPERTIES
 add_test(NAME NoAssertions COMMAND $<TARGET_FILE:SelfTest> -w NoAssertions "An empty test with no assertions")
 set_tests_properties(NoAssertions PROPERTIES PASS_REGULAR_EXPRESSION "No assertions in test case")
 
-add_test(NAME NoTest COMMAND $<TARGET_FILE:SelfTest> Tracker, "___nonexistent_test___")
-set_tests_properties(NoTest PROPERTIES
+# We cannot combine a regular expression on output with return code check
+# in one test, so we register two instead of making a checking script because
+# the runtime overhead is small enough.
+add_test(NAME TestSpecs::CombiningMatchingAndNonMatchingIsOk-1 COMMAND $<TARGET_FILE:SelfTest> Tracker, "___nonexistent_test___")
+
+add_test(NAME TestSpecs::CombiningMatchingAndNonMatchingIsOk-2 COMMAND $<TARGET_FILE:SelfTest> Tracker, "___nonexistent_test___")
+set_tests_properties(TestSpecs::CombiningMatchingAndNonMatchingIsOk-2 PROPERTIES
     PASS_REGULAR_EXPRESSION "No test cases matched '___nonexistent_test___'"
     FAIL_REGULAR_EXPRESSION "No tests ran"
 )
 
-add_test(NAME WarnAboutNoTests COMMAND ${CMAKE_COMMAND} -P ${SELF_TEST_DIR}/WarnAboutNoTests.cmake $<TARGET_FILE:SelfTest>)
+add_test(NAME TestSpecs::NoMatchedTestsFail
+  COMMAND $<TARGET_FILE:SelfTest> "___nonexistent_test___"
+)
+set_tests_properties(TestSpecs::NoMatchedTestsFail
+  PROPERTIES
+    WILL_FAIL ON
+)
+add_test(NAME TestSpecs::OverrideFailureWithNoMatchedTests
+  COMMAND $<TARGET_FILE:SelfTest> "___nonexistent_test___" --allow-running-no-tests
+)
 
-add_test(NAME UnmatchedOutputFilter COMMAND $<TARGET_FILE:SelfTest> [this-tag-does-not-exist] -w NoTests)
+add_test(NAME UnmatchedOutputFilter COMMAND $<TARGET_FILE:SelfTest> [this-tag-does-not-exist])
 set_tests_properties(UnmatchedOutputFilter
   PROPERTIES
     PASS_REGULAR_EXPRESSION "No test cases matched '\\[this-tag-does-not-exist\\]'"

tests/ExtraTests/CMakeLists.txt

@@ -299,6 +299,22 @@ set_tests_properties(
 #    PASS_REGULAR_EXPRESSION "Pretty please, break into debugger"
 #)
 
+add_executable(NoTests ${TESTS_DIR}/X92-NoTests.cpp)
+target_link_libraries(NoTests PRIVATE Catch2::Catch2WithMain)
+
+add_test(
+  NAME TestSpecs::EmptySpecWithNoTestsFails
+  COMMAND $<TARGET_FILE:NoTests>
+)
+set_tests_properties(TestSpecs::EmptySpecWithNoTestsFails
+  PROPERTIES
+    WILL_FAIL ON
+)
+add_test(
+  NAME TestSpecs::OverrideFailureWithEmptySpec
+  COMMAND $<TARGET_FILE:NoTests> --allow-running-no-tests
+)
+
 set( EXTRA_TEST_BINARIES
     PrefixedMacros
     DisabledMacros
@@ -310,6 +326,7 @@ set( EXTRA_TEST_BINARIES
     DuplicatedTestCases-SameNameAndTags
     DuplicatedTestCases-SameNameDifferentTags
     DuplicatedTestCases-DuplicatedTestCaseMethods
+    NoTests
 #    DebugBreakMacros
 )
 

tests/ExtraTests/X92-NoTests.cpp

@@ -0,0 +1,11 @@
+
+//              Copyright Catch2 Authors
+// Distributed under the Boost Software License, Version 1.0.
+//   (See accompanying file LICENSE_1_0.txt or copy at
+//        https://www.boost.org/LICENSE_1_0.txt)
+
+// SPDX-License-Identifier: BSL-1.0
+
+/**\file
+ * Links into executable with no tests which should fail when run
+ */

tests/SelfTest/WarnAboutNoTests.cmake

@@ -1,19 +0,0 @@
-# Workaround for a peculiarity where CTest disregards the return code from a
-# test command if a PASS_REGULAR_EXPRESSION is also set
-execute_process(
-    COMMAND ${CMAKE_ARGV3} -w NoTests "___nonexistent_test___"
-    RESULT_VARIABLE ret
-    OUTPUT_VARIABLE out
-)
-
-message("${out}")
-
-if(NOT ${ret} MATCHES "^[0-9]+$")
-    message(FATAL_ERROR "${ret}")
-endif()
-
-if(${ret} EQUAL 0)
-    message(FATAL_ERROR "Expected nonzero return code")
-elseif(${out} MATCHES "Helper failed with")
-    message(FATAL_ERROR "Helper failed")
-endif()