diff --git a/docs/command-line.md b/docs/command-line.md
index 0154f075..40195eb4 100644
--- a/docs/command-line.md
+++ b/docs/command-line.md
@@ -30,6 +30,7 @@
[Filenames as tags](#filenames-as-tags)
[Override output colouring](#override-output-colouring)
[Test Sharding](#test-sharding)
+[Allow running the binary without tests](#no-tests-override)
Catch works quite nicely without any command line options at all - but for those times when you want greater control the following options are available.
Click one of the following links to take you straight to that option - or scroll on to browse the available options.
@@ -70,6 +71,7 @@ Click one of the following links to take you straight to that option - or scroll
` --use-colour`
` --shard-count`
` --shard-index`
+ ` --allow-running-no-tests`
@@ -210,14 +212,12 @@ This option transforms tabs and newline characters into ```\t``` and ```\n``` re
## Warnings
-w, --warn <warning name>
-Enables reporting of suspicious test states. There are currently two
-available warnings
+Enables reporting of suspicious test runs. There is currently only one
+available warning.
```
NoAssertions // Fail test case / leaf section if no assertions
// (e.g. `REQUIRE`) is encountered.
- NoTests // Return non-zero exit code when no test cases were run
- // Also calls reporter's noMatchingTestCases method
```
@@ -441,6 +441,17 @@ the number of shards.
This is useful when you want to split test execution across multiple processes, as is done with [Bazel test sharding](https://docs.bazel.build/versions/main/test-encyclopedia.html#test-sharding).
+
+## Allow running the binary without tests
+--allow-running-no-tests
+
+> Introduced in Catch2 X.Y.Z.
+
+By default, Catch2 test binaries return non-0 exit code if no tests were
+run, e.g. if the binary was compiled with no tests, or the provided test
+spec matched no tests. This flag overrides that, so a test run with no
+tests still returns 0.
+
---
diff --git a/docs/release-notes.md b/docs/release-notes.md
index ca14092a..3eecf8f8 100644
--- a/docs/release-notes.md
+++ b/docs/release-notes.md
@@ -198,6 +198,9 @@ new design.
* `catch2` is the statically compiled implementation by itself
* `catch2-with-main` also links in the default main
* Passing invalid test specifications passed to Catch2 are now reported before tests are run, and are a hard error.
+* Running 0 tests (e.g. due to empty binary, or test spec not matching anything) returns non-0 exit code
+ * Flag `--allow-running-no-tests` overrides this behaviour.
+ * `NoTests` warning has been removed because it is fully subsumed by this change.
diff --git a/src/catch2/catch_config.cpp b/src/catch2/catch_config.cpp
index e2569de9..33951dbd 100644
--- a/src/catch2/catch_config.cpp
+++ b/src/catch2/catch_config.cpp
@@ -67,8 +67,10 @@ namespace Catch {
std::ostream& Config::stream() const { return m_stream->stream(); }
StringRef Config::name() const { return m_data.name.empty() ? m_data.processName : m_data.name; }
bool Config::includeSuccessfulResults() const { return m_data.showSuccessfulTests; }
- bool Config::warnAboutMissingAssertions() const { return !!(m_data.warnings & WarnAbout::NoAssertions); }
- bool Config::warnAboutNoTests() const { return !!(m_data.warnings & WarnAbout::NoTests); }
+ bool Config::warnAboutMissingAssertions() const {
+ return !!( m_data.warnings & WarnAbout::NoAssertions );
+ }
+ bool Config::zeroTestsCountAsSuccess() const { return m_data.allowZeroTests; }
ShowDurations Config::showDurations() const { return m_data.showDurations; }
double Config::minDuration() const { return m_data.minDuration; }
TestRunOrder Config::runOrder() const { return m_data.runOrder; }
diff --git a/src/catch2/catch_config.hpp b/src/catch2/catch_config.hpp
index dbb9bf0e..8f249c09 100644
--- a/src/catch2/catch_config.hpp
+++ b/src/catch2/catch_config.hpp
@@ -33,6 +33,7 @@ namespace Catch {
bool showInvisibles = false;
bool filenamesAsTags = false;
bool libIdentify = false;
+ bool allowZeroTests = false;
int abortAfter = -1;
uint32_t rngSeed = generateRandomSeed(GenerateFrom::Default);
@@ -97,7 +98,7 @@ namespace Catch {
StringRef name() const override;
bool includeSuccessfulResults() const override;
bool warnAboutMissingAssertions() const override;
- bool warnAboutNoTests() const override;
+ bool zeroTestsCountAsSuccess() const override;
ShowDurations showDurations() const override;
double minDuration() const override;
TestRunOrder runOrder() const override;
diff --git a/src/catch2/catch_session.cpp b/src/catch2/catch_session.cpp
index ff5e4cd8..647dcfa3 100644
--- a/src/catch2/catch_session.cpp
+++ b/src/catch2/catch_session.cpp
@@ -105,12 +105,10 @@ namespace Catch {
}
private:
- using Tests = std::set;
-
IStreamingReporter* m_reporter;
Config const* m_config;
RunContext m_context;
- Tests m_tests;
+ std::set m_tests;
TestSpec::Matches m_matches;
};
@@ -303,8 +301,10 @@ namespace Catch {
TestGroup tests { CATCH_MOVE(reporter), m_config.get() };
auto const totals = tests.execute();
- if( m_config->warnAboutNoTests() && totals.error == -1 )
+ if ( totals.testCases.total() == 0
+ && !m_config->zeroTestsCountAsSuccess() ) {
return 2;
+ }
// Note that on unices only the lower 8 bits are usually used, clamping
// the return value to 255 prevents false negative when some multiple
diff --git a/src/catch2/interfaces/catch_interfaces_config.hpp b/src/catch2/interfaces/catch_interfaces_config.hpp
index 4f466b22..8d93c964 100644
--- a/src/catch2/interfaces/catch_interfaces_config.hpp
+++ b/src/catch2/interfaces/catch_interfaces_config.hpp
@@ -27,7 +27,6 @@ namespace Catch {
struct WarnAbout { enum What {
Nothing = 0x00,
NoAssertions = 0x01,
- NoTests = 0x02
}; };
enum class ShowDurations {
@@ -64,7 +63,7 @@ namespace Catch {
virtual bool includeSuccessfulResults() const = 0;
virtual bool shouldDebugBreak() const = 0;
virtual bool warnAboutMissingAssertions() const = 0;
- virtual bool warnAboutNoTests() const = 0;
+ virtual bool zeroTestsCountAsSuccess() const = 0;
virtual int abortAfter() const = 0;
virtual bool showInvisibles() const = 0;
virtual ShowDurations showDurations() const = 0;
diff --git a/src/catch2/internal/catch_commandline.cpp b/src/catch2/internal/catch_commandline.cpp
index 02056da5..8576e808 100644
--- a/src/catch2/internal/catch_commandline.cpp
+++ b/src/catch2/internal/catch_commandline.cpp
@@ -24,21 +24,14 @@ namespace Catch {
using namespace Clara;
auto const setWarning = [&]( std::string const& warning ) {
- auto warningSet = [&]() {
- if( warning == "NoAssertions" )
- return WarnAbout::NoAssertions;
-
- if ( warning == "NoTests" )
- return WarnAbout::NoTests;
-
- return WarnAbout::Nothing;
- }();
-
- if (warningSet == WarnAbout::Nothing)
- return ParserResult::runtimeError( "Unrecognised warning: '" + warning + '\'' );
- config.warnings = static_cast( config.warnings | warningSet );
+ if ( warning == "NoAssertions" ) {
+ config.warnings = WarnAbout::NoAssertions;
return ParserResult::ok( ParseResultType::Matched );
- };
+ }
+
+ return ParserResult ::runtimeError(
+ "Unrecognised warning option: '" + warning + '\'' );
+ };
auto const loadTestNamesFromFile = [&]( std::string const& filename ) {
std::ifstream f( filename.c_str() );
if( !f.is_open() )
@@ -280,7 +273,10 @@ namespace Catch {
( "split the tests to execute into this many groups" )
| Opt( setShardIndex, "shard index" )
["--shard-index"]
- ( "index of the group of tests to execute (see --shard-count)" )
+ ( "index of the group of tests to execute (see --shard-count)" ) |
+ Opt( config.allowZeroTests )
+ ["--allow-running-no-tests"]
+ ( "Treat 'No tests run' as a success" )
| Arg( config.testsOrTags, "test name|pattern|tags" )
( "which test or tests to use" );
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 6a7563ec..4a84ad8e 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -207,15 +207,29 @@ set_tests_properties(List::Reporters::XmlOutput PROPERTIES
add_test(NAME NoAssertions COMMAND $ -w NoAssertions "An empty test with no assertions")
set_tests_properties(NoAssertions PROPERTIES PASS_REGULAR_EXPRESSION "No assertions in test case")
-add_test(NAME NoTest COMMAND $ Tracker, "___nonexistent_test___")
-set_tests_properties(NoTest PROPERTIES
+# We cannot combine a regular expression on output with return code check
+# in one test, so we register two instead of making a checking script because
+# the runtime overhead is small enough.
+add_test(NAME TestSpecs::CombiningMatchingAndNonMatchingIsOk-1 COMMAND $ Tracker, "___nonexistent_test___")
+
+add_test(NAME TestSpecs::CombiningMatchingAndNonMatchingIsOk-2 COMMAND $ Tracker, "___nonexistent_test___")
+set_tests_properties(TestSpecs::CombiningMatchingAndNonMatchingIsOk-2 PROPERTIES
PASS_REGULAR_EXPRESSION "No test cases matched '___nonexistent_test___'"
FAIL_REGULAR_EXPRESSION "No tests ran"
)
-add_test(NAME WarnAboutNoTests COMMAND ${CMAKE_COMMAND} -P ${SELF_TEST_DIR}/WarnAboutNoTests.cmake $)
+add_test(NAME TestSpecs::NoMatchedTestsFail
+ COMMAND $ "___nonexistent_test___"
+)
+set_tests_properties(TestSpecs::NoMatchedTestsFail
+ PROPERTIES
+ WILL_FAIL ON
+)
+add_test(NAME TestSpecs::OverrideFailureWithNoMatchedTests
+ COMMAND $ "___nonexistent_test___" --allow-running-no-tests
+)
-add_test(NAME UnmatchedOutputFilter COMMAND $ [this-tag-does-not-exist] -w NoTests)
+add_test(NAME UnmatchedOutputFilter COMMAND $ [this-tag-does-not-exist])
set_tests_properties(UnmatchedOutputFilter
PROPERTIES
PASS_REGULAR_EXPRESSION "No test cases matched '\\[this-tag-does-not-exist\\]'"
diff --git a/tests/ExtraTests/CMakeLists.txt b/tests/ExtraTests/CMakeLists.txt
index f7e9ec18..b3b8dbec 100644
--- a/tests/ExtraTests/CMakeLists.txt
+++ b/tests/ExtraTests/CMakeLists.txt
@@ -299,6 +299,22 @@ set_tests_properties(
# PASS_REGULAR_EXPRESSION "Pretty please, break into debugger"
#)
+add_executable(NoTests ${TESTS_DIR}/X92-NoTests.cpp)
+target_link_libraries(NoTests PRIVATE Catch2::Catch2WithMain)
+
+add_test(
+ NAME TestSpecs::EmptySpecWithNoTestsFails
+ COMMAND $
+)
+set_tests_properties(TestSpecs::EmptySpecWithNoTestsFails
+ PROPERTIES
+ WILL_FAIL ON
+)
+add_test(
+ NAME TestSpecs::OverrideFailureWithEmptySpec
+ COMMAND $ --allow-running-no-tests
+)
+
set( EXTRA_TEST_BINARIES
PrefixedMacros
DisabledMacros
@@ -310,6 +326,7 @@ set( EXTRA_TEST_BINARIES
DuplicatedTestCases-SameNameAndTags
DuplicatedTestCases-SameNameDifferentTags
DuplicatedTestCases-DuplicatedTestCaseMethods
+ NoTests
# DebugBreakMacros
)
diff --git a/tests/ExtraTests/X92-NoTests.cpp b/tests/ExtraTests/X92-NoTests.cpp
new file mode 100644
index 00000000..fc6bf597
--- /dev/null
+++ b/tests/ExtraTests/X92-NoTests.cpp
@@ -0,0 +1,11 @@
+
+// Copyright Catch2 Authors
+// Distributed under the Boost Software License, Version 1.0.
+// (See accompanying file LICENSE_1_0.txt or copy at
+// https://www.boost.org/LICENSE_1_0.txt)
+
+// SPDX-License-Identifier: BSL-1.0
+
+/**\file
+ * Links into executable with no tests which should fail when run
+ */
diff --git a/tests/SelfTest/WarnAboutNoTests.cmake b/tests/SelfTest/WarnAboutNoTests.cmake
deleted file mode 100644
index 4637e3f3..00000000
--- a/tests/SelfTest/WarnAboutNoTests.cmake
+++ /dev/null
@@ -1,19 +0,0 @@
-# Workaround for a peculiarity where CTest disregards the return code from a
-# test command if a PASS_REGULAR_EXPRESSION is also set
-execute_process(
- COMMAND ${CMAKE_ARGV3} -w NoTests "___nonexistent_test___"
- RESULT_VARIABLE ret
- OUTPUT_VARIABLE out
-)
-
-message("${out}")
-
-if(NOT ${ret} MATCHES "^[0-9]+$")
- message(FATAL_ERROR "${ret}")
-endif()
-
-if(${ret} EQUAL 0)
- message(FATAL_ERROR "Expected nonzero return code")
-elseif(${out} MATCHES "Helper failed with")
- message(FATAL_ERROR "Helper failed")
-endif()