diff --git a/docs/command-line.md b/docs/command-line.md index 40195eb4..37199c44 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -212,14 +212,22 @@ This option transforms tabs and newline characters into ```\t``` and ```\n``` re ## Warnings
-w, --warn <warning name>
-Enables reporting of suspicious test runs. There is currently only one -available warning. +You can think of Catch2's warnings as the equivalent of `-Werror` (`/WX`) +flag for C++ compilers. It turns some suspicious occurences, like a section +without assertions, into errors. Because these might be intended, warnings +are not enabled by default, but user can opt in. + +There are currently two warnings implemented: ``` - NoAssertions // Fail test case / leaf section if no assertions - // (e.g. `REQUIRE`) is encountered. + NoAssertions // Fail test case / leaf section if no assertions + // (e.g. `REQUIRE`) is encountered. + UnmatchedTestSpec // Fail test run if any of the CLI test specs did + // not match any tests. ``` +> `UnmatchedTestSpec` was introduced in Catch2 X.Y.Z. + ## Reporting timings diff --git a/docs/release-notes.md b/docs/release-notes.md index 3eecf8f8..5b3e5c65 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -173,6 +173,7 @@ new design. * When deferred tu runtime, it behaves like `CHECK`, and not like `REQUIRE`. * You can have multiple tests with the same name, as long as other parts of the test identity differ (#1915, #1999, #2175) * Test identity includes test's name, test's tags and and test's class name if applicable. +* Added new warning, `UnmatchedTestSpec`, to error on test specs with no matching tests ### Fixes diff --git a/src/catch2/catch_config.cpp b/src/catch2/catch_config.cpp index 33951dbd..da9457da 100644 --- a/src/catch2/catch_config.cpp +++ b/src/catch2/catch_config.cpp @@ -70,6 +70,9 @@ namespace Catch { bool Config::warnAboutMissingAssertions() const { return !!( m_data.warnings & WarnAbout::NoAssertions ); } + bool Config::warnAboutUnmatchedTestSpecs() const { + return !!( m_data.warnings & WarnAbout::UnmatchedTestSpec ); + } bool Config::zeroTestsCountAsSuccess() const { return m_data.allowZeroTests; } ShowDurations Config::showDurations() const { return m_data.showDurations; } double Config::minDuration() const { return m_data.minDuration; } diff --git a/src/catch2/catch_config.hpp b/src/catch2/catch_config.hpp index 8f249c09..2b881f6b 100644 --- a/src/catch2/catch_config.hpp +++ b/src/catch2/catch_config.hpp @@ -98,6 +98,7 @@ namespace Catch { StringRef name() const override; bool includeSuccessfulResults() const override; bool warnAboutMissingAssertions() const override; + bool warnAboutUnmatchedTestSpecs() const override; bool zeroTestsCountAsSuccess() const override; ShowDurations showDurations() const override; double minDuration() const override; diff --git a/src/catch2/catch_session.cpp b/src/catch2/catch_session.cpp index 647dcfa3..38111fe8 100644 --- a/src/catch2/catch_session.cpp +++ b/src/catch2/catch_session.cpp @@ -96,20 +96,26 @@ namespace Catch { for (auto const& match : m_matches) { if (match.tests.empty()) { - m_reporter->noMatchingTestCases(match.name); - totals.error = -1; + m_unmatchedTestSpecs = true; + m_reporter->noMatchingTestCases( match.name ); } } return totals; } + bool hadUnmatchedTestSpecs() const { + return m_unmatchedTestSpecs; + } + + private: IStreamingReporter* m_reporter; Config const* m_config; RunContext m_context; std::set m_tests; TestSpec::Matches m_matches; + bool m_unmatchedTestSpecs = false; }; void applyFilenamesAsTags() { @@ -301,6 +307,11 @@ namespace Catch { TestGroup tests { CATCH_MOVE(reporter), m_config.get() }; auto const totals = tests.execute(); + if ( tests.hadUnmatchedTestSpecs() + && m_config->warnAboutUnmatchedTestSpecs() ) { + return 3; + } + if ( totals.testCases.total() == 0 && !m_config->zeroTestsCountAsSuccess() ) { return 2; diff --git a/src/catch2/interfaces/catch_interfaces_config.hpp b/src/catch2/interfaces/catch_interfaces_config.hpp index 8d93c964..b8da0df0 100644 --- a/src/catch2/interfaces/catch_interfaces_config.hpp +++ b/src/catch2/interfaces/catch_interfaces_config.hpp @@ -26,7 +26,10 @@ namespace Catch { struct WarnAbout { enum What { Nothing = 0x00, + //! A test case or leaf section did not run any assertions NoAssertions = 0x01, + //! A command line test spec matched no test cases + UnmatchedTestSpec = 0x02, }; }; enum class ShowDurations { @@ -63,6 +66,7 @@ namespace Catch { virtual bool includeSuccessfulResults() const = 0; virtual bool shouldDebugBreak() const = 0; virtual bool warnAboutMissingAssertions() const = 0; + virtual bool warnAboutUnmatchedTestSpecs() const = 0; virtual bool zeroTestsCountAsSuccess() const = 0; virtual int abortAfter() const = 0; virtual bool showInvisibles() const = 0; diff --git a/src/catch2/internal/catch_commandline.cpp b/src/catch2/internal/catch_commandline.cpp index 8576e808..71550945 100644 --- a/src/catch2/internal/catch_commandline.cpp +++ b/src/catch2/internal/catch_commandline.cpp @@ -27,6 +27,9 @@ namespace Catch { if ( warning == "NoAssertions" ) { config.warnings = WarnAbout::NoAssertions; return ParserResult::ok( ParseResultType::Matched ); + } else if ( warning == "UnmatchedTestSpec" ) { + config.warnings = WarnAbout::UnmatchedTestSpec; + return ParserResult::ok( ParseResultType::Matched ); } return ParserResult ::runtimeError( diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4a84ad8e..838227aa 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -229,6 +229,23 @@ add_test(NAME TestSpecs::OverrideFailureWithNoMatchedTests COMMAND $ "___nonexistent_test___" --allow-running-no-tests ) +add_test(NAME Warnings::UnmatchedTestSpecIsAccepted + COMMAND $ Tracker --warn UnmatchedTestSpec +) +set_tests_properties(Warnings::UnmatchedTestSpecIsAccepted + PROPERTIES + FAIL_REGULAR_EXPRESSION "Unrecognised warning option: " +) + +add_test(NAME TestSpecs::WarnUnmatchedTestSpecFailsWithUnmatchedTestSpec + COMMAND + $ Tracker, "___nonexistent_test___" --warn UnmatchedTestSpec +) +set_tests_properties(TestSpecs::WarnUnmatchedTestSpecFailsWithUnmatchedTestSpec + PROPERTIES + WILL_FAIL ON +) + add_test(NAME UnmatchedOutputFilter COMMAND $ [this-tag-does-not-exist]) set_tests_properties(UnmatchedOutputFilter PROPERTIES