diff --git a/CMakeLists.txt b/CMakeLists.txt index dca4c7bf..c07bc470 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -368,7 +368,11 @@ if (BUILD_TESTING AND NOT_SUBPROJECT) add_test(NAME ListTestNamesOnly COMMAND $ --list-test-names-only) set_tests_properties(ListTestNamesOnly PROPERTIES PASS_REGULAR_EXPRESSION "Regex string matcher") + add_test(NAME NoAssertions COMMAND $ -w NoAssertions) + set_tests_properties(NoAssertions PROPERTIES PASS_REGULAR_EXPRESSION "No assertions in test case") + add_test(NAME NoTest COMMAND $ -w NoTests "___nonexistent_test___") + set_tests_properties(NoTest PROPERTIES PASS_REGULAR_EXPRESSION "No test cases matched") # AppVeyor has a Python 2.7 in path, but doesn't have .py files as autorunnable add_test(NAME ApprovalTests COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/scripts/approvalTests.py $) diff --git a/docs/command-line.md b/docs/command-line.md index 8afcb7d4..0c24c3b1 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -192,9 +192,16 @@ This option transforms tabs and newline characters into ```\t``` and ```\n``` re ## Warnings 
-w, --warn <warning name>
-Enables reporting of warnings (only one, at time of this writing). If a warning is issued it fails the test. +Enables reporting of suspicious test states. There are currently two +available warnings + +``` + NoAssertions // Fail test case / leaf section if no assertions + // (e.g. `REQUIRE`) is encountered. + NoTests // Return non-zero exit code when no test cases were run + // Also calls reporter's noMatchingTestCases method +``` -The ony available warning, presently, is ```NoAssertions```. This warning fails a test case, or (leaf) section if no assertions (```REQUIRE```/ ```CHECK``` etc) are encountered. ## Reporting timings diff --git a/include/internal/catch_commandline.cpp b/include/internal/catch_commandline.cpp index c0231d1a..85af1085 100644 --- a/include/internal/catch_commandline.cpp +++ b/include/internal/catch_commandline.cpp @@ -20,9 +20,19 @@ namespace Catch { using namespace clara; auto const setWarning = [&]( std::string const& warning ) { - if( warning != "NoAssertions" ) + auto warningSet = [&]() { + if( warning == "NoAssertions" ) + return WarnAbout::NoAssertions; + + if ( warning == "NoTests" ) + return WarnAbout::NoTests; + + return WarnAbout::Nothing; + }(); + + if (warningSet == WarnAbout::Nothing) return ParserResult::runtimeError( "Unrecognised warning: '" + warning + "'" ); - config.warnings = static_cast( config.warnings | WarnAbout::NoAssertions ); + config.warnings = static_cast( config.warnings | warningSet ); return ParserResult::ok( ParseResultType::Matched ); }; auto const loadTestNamesFromFile = [&]( std::string const& filename ) { diff --git a/include/internal/catch_config.cpp b/include/internal/catch_config.cpp index 67d2b270..5fda44ff 100644 --- a/include/internal/catch_config.cpp +++ b/include/internal/catch_config.cpp @@ -35,6 +35,7 @@ namespace Catch { std::string Config::getProcessName() const { return m_data.processName; } std::vector const& Config::getReporterNames() const { return m_data.reporterNames; } + std::vector const& Config::getTestsOrTags() const { return m_data.testsOrTags; } std::vector const& Config::getSectionsToRun() const { return m_data.sectionsToRun; } TestSpec const& Config::testSpec() const { return m_testSpec; } @@ -46,7 +47,8 @@ namespace Catch { std::ostream& Config::stream() const { return m_stream->stream(); } std::string Config::name() const { return m_data.name.empty() ? m_data.processName : m_data.name; } bool Config::includeSuccessfulResults() const { return m_data.showSuccessfulTests; } - bool Config::warnAboutMissingAssertions() const { return m_data.warnings & WarnAbout::NoAssertions; } + bool Config::warnAboutMissingAssertions() const { return !!(m_data.warnings & WarnAbout::NoAssertions); } + bool Config::warnAboutNoTests() const { return !!(m_data.warnings & WarnAbout::NoTests); } ShowDurations::OrNot Config::showDurations() const { return m_data.showDurations; } RunTests::InWhatOrder Config::runOrder() const { return m_data.runOrder; } unsigned int Config::rngSeed() const { return m_data.rngSeed; } diff --git a/include/internal/catch_config.hpp b/include/internal/catch_config.hpp index fcaa0500..21e04069 100644 --- a/include/internal/catch_config.hpp +++ b/include/internal/catch_config.hpp @@ -78,6 +78,7 @@ namespace Catch { std::string getProcessName() const; std::vector const& getReporterNames() const; + std::vector const& getTestsOrTags() const; std::vector const& getSectionsToRun() const override; virtual TestSpec const& testSpec() const override; @@ -90,6 +91,7 @@ namespace Catch { std::string name() const override; bool includeSuccessfulResults() const override; bool warnAboutMissingAssertions() const override; + bool warnAboutNoTests() const override; ShowDurations::OrNot showDurations() const override; RunTests::InWhatOrder runOrder() const override; unsigned int rngSeed() const override; diff --git a/include/internal/catch_interfaces_config.h b/include/internal/catch_interfaces_config.h index 2584ccc0..35ec82cf 100644 --- a/include/internal/catch_interfaces_config.h +++ b/include/internal/catch_interfaces_config.h @@ -25,7 +25,8 @@ namespace Catch { struct WarnAbout { enum What { Nothing = 0x00, - NoAssertions = 0x01 + NoAssertions = 0x01, + NoTests = 0x02 }; }; struct ShowDurations { enum OrNot { @@ -62,6 +63,7 @@ namespace Catch { virtual bool includeSuccessfulResults() const = 0; virtual bool shouldDebugBreak() const = 0; virtual bool warnAboutMissingAssertions() const = 0; + virtual bool warnAboutNoTests() const = 0; virtual int abortAfter() const = 0; virtual bool showInvisibles() const = 0; virtual ShowDurations::OrNot showDurations() const = 0; diff --git a/include/internal/catch_session.cpp b/include/internal/catch_session.cpp index 1d3ce06e..e170f7ca 100644 --- a/include/internal/catch_session.cpp +++ b/include/internal/catch_session.cpp @@ -18,6 +18,7 @@ #include "catch_random_number_generator.h" #include "catch_startup_exception_registry.h" #include "catch_text.h" +#include "catch_stream.h" #include #include @@ -80,6 +81,20 @@ namespace Catch { context.reporter().skipTest(testCase); } + if (config->warnAboutNoTests() && totals.testCases.total() == 0) { + ReusableStringStream testConfig; + + bool first = true; + for (const auto& input : config->getTestsOrTags()) { + if (!first) { testConfig << ' '; } + first = false; + testConfig << input; + } + + context.reporter().noMatchingTestCases(testConfig.str()); + totals.error = -1; + } + context.testGroupEnded(config->name(), totals, 1, 1); return totals; } @@ -259,10 +274,11 @@ namespace Catch { if( Option listed = list( config() ) ) return static_cast( *listed ); + auto totals = runTests( m_config ); // Note that on unices only the lower 8 bits are usually used, clamping // the return value to 255 prevents false negative when some multiple // of 256 tests has failed - return (std::min)( MaxExitCode, static_cast( runTests( m_config ).assertions.failed ) ); + return (std::min)( { MaxExitCode, totals.error, static_cast( totals.assertions.failed ) } ); } catch( std::exception& ex ) { Catch::cerr() << ex.what() << std::endl; diff --git a/include/internal/catch_totals.h b/include/internal/catch_totals.h index 9507582d..56927283 100644 --- a/include/internal/catch_totals.h +++ b/include/internal/catch_totals.h @@ -32,7 +32,7 @@ namespace Catch { Totals delta( Totals const& prevTotals ) const; - + int error = 0; Counts assertions; Counts testCases; };