Add option to warn when no tests ran

Closes #1158
This commit is contained in:
dvirtz 2018-02-09 01:31:19 +02:00 committed by Martin Hořeňovský
parent 7cbd0b587a
commit 355b3f9952
8 changed files with 51 additions and 8 deletions

View File

@ -368,7 +368,11 @@ if (BUILD_TESTING AND NOT_SUBPROJECT)
add_test(NAME ListTestNamesOnly COMMAND $<TARGET_FILE:SelfTest> --list-test-names-only) add_test(NAME ListTestNamesOnly COMMAND $<TARGET_FILE:SelfTest> --list-test-names-only)
set_tests_properties(ListTestNamesOnly PROPERTIES PASS_REGULAR_EXPRESSION "Regex string matcher") set_tests_properties(ListTestNamesOnly PROPERTIES PASS_REGULAR_EXPRESSION "Regex string matcher")
add_test(NAME NoAssertions COMMAND $<TARGET_FILE:SelfTest> -w NoAssertions)
set_tests_properties(NoAssertions PROPERTIES PASS_REGULAR_EXPRESSION "No assertions in test case")
add_test(NAME NoTest COMMAND $<TARGET_FILE:SelfTest> -w NoTests "___nonexistent_test___")
set_tests_properties(NoTest PROPERTIES PASS_REGULAR_EXPRESSION "No test cases matched")
# AppVeyor has a Python 2.7 in path, but doesn't have .py files as autorunnable # AppVeyor has a Python 2.7 in path, but doesn't have .py files as autorunnable
add_test(NAME ApprovalTests COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/scripts/approvalTests.py $<TARGET_FILE:SelfTest>) add_test(NAME ApprovalTests COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/scripts/approvalTests.py $<TARGET_FILE:SelfTest>)

View File

@ -192,9 +192,16 @@ This option transforms tabs and newline characters into ```\t``` and ```\n``` re
## Warnings ## Warnings
<pre>-w, --warn &lt;warning name></pre> <pre>-w, --warn &lt;warning name></pre>
Enables reporting of warnings (only one, at time of this writing). If a warning is issued it fails the test. Enables reporting of suspicious test states. There are currently two
available warnings
```
NoAssertions // Fail test case / leaf section if no assertions
// (e.g. `REQUIRE`) is encountered.
NoTests // Return non-zero exit code when no test cases were run
// Also calls reporter's noMatchingTestCases method
```
The ony available warning, presently, is ```NoAssertions```. This warning fails a test case, or (leaf) section if no assertions (```REQUIRE```/ ```CHECK``` etc) are encountered.
<a id="reporting-timings"></a> <a id="reporting-timings"></a>
## Reporting timings ## Reporting timings

View File

@ -20,9 +20,19 @@ namespace Catch {
using namespace clara; using namespace clara;
auto const setWarning = [&]( std::string const& warning ) { auto const setWarning = [&]( std::string const& warning ) {
if( warning != "NoAssertions" ) auto warningSet = [&]() {
if( warning == "NoAssertions" )
return WarnAbout::NoAssertions;
if ( warning == "NoTests" )
return WarnAbout::NoTests;
return WarnAbout::Nothing;
}();
if (warningSet == WarnAbout::Nothing)
return ParserResult::runtimeError( "Unrecognised warning: '" + warning + "'" ); return ParserResult::runtimeError( "Unrecognised warning: '" + warning + "'" );
config.warnings = static_cast<WarnAbout::What>( config.warnings | WarnAbout::NoAssertions ); config.warnings = static_cast<WarnAbout::What>( config.warnings | warningSet );
return ParserResult::ok( ParseResultType::Matched ); return ParserResult::ok( ParseResultType::Matched );
}; };
auto const loadTestNamesFromFile = [&]( std::string const& filename ) { auto const loadTestNamesFromFile = [&]( std::string const& filename ) {

View File

@ -35,6 +35,7 @@ namespace Catch {
std::string Config::getProcessName() const { return m_data.processName; } std::string Config::getProcessName() const { return m_data.processName; }
std::vector<std::string> const& Config::getReporterNames() const { return m_data.reporterNames; } std::vector<std::string> const& Config::getReporterNames() const { return m_data.reporterNames; }
std::vector<std::string> const& Config::getTestsOrTags() const { return m_data.testsOrTags; }
std::vector<std::string> const& Config::getSectionsToRun() const { return m_data.sectionsToRun; } std::vector<std::string> const& Config::getSectionsToRun() const { return m_data.sectionsToRun; }
TestSpec const& Config::testSpec() const { return m_testSpec; } TestSpec const& Config::testSpec() const { return m_testSpec; }
@ -46,7 +47,8 @@ namespace Catch {
std::ostream& Config::stream() const { return m_stream->stream(); } std::ostream& Config::stream() const { return m_stream->stream(); }
std::string Config::name() const { return m_data.name.empty() ? m_data.processName : m_data.name; } std::string Config::name() const { return m_data.name.empty() ? m_data.processName : m_data.name; }
bool Config::includeSuccessfulResults() const { return m_data.showSuccessfulTests; } bool Config::includeSuccessfulResults() const { return m_data.showSuccessfulTests; }
bool Config::warnAboutMissingAssertions() const { return m_data.warnings & WarnAbout::NoAssertions; } bool Config::warnAboutMissingAssertions() const { return !!(m_data.warnings & WarnAbout::NoAssertions); }
bool Config::warnAboutNoTests() const { return !!(m_data.warnings & WarnAbout::NoTests); }
ShowDurations::OrNot Config::showDurations() const { return m_data.showDurations; } ShowDurations::OrNot Config::showDurations() const { return m_data.showDurations; }
RunTests::InWhatOrder Config::runOrder() const { return m_data.runOrder; } RunTests::InWhatOrder Config::runOrder() const { return m_data.runOrder; }
unsigned int Config::rngSeed() const { return m_data.rngSeed; } unsigned int Config::rngSeed() const { return m_data.rngSeed; }

View File

@ -78,6 +78,7 @@ namespace Catch {
std::string getProcessName() const; std::string getProcessName() const;
std::vector<std::string> const& getReporterNames() const; std::vector<std::string> const& getReporterNames() const;
std::vector<std::string> const& getTestsOrTags() const;
std::vector<std::string> const& getSectionsToRun() const override; std::vector<std::string> const& getSectionsToRun() const override;
virtual TestSpec const& testSpec() const override; virtual TestSpec const& testSpec() const override;
@ -90,6 +91,7 @@ namespace Catch {
std::string name() const override; std::string name() const override;
bool includeSuccessfulResults() const override; bool includeSuccessfulResults() const override;
bool warnAboutMissingAssertions() const override; bool warnAboutMissingAssertions() const override;
bool warnAboutNoTests() const override;
ShowDurations::OrNot showDurations() const override; ShowDurations::OrNot showDurations() const override;
RunTests::InWhatOrder runOrder() const override; RunTests::InWhatOrder runOrder() const override;
unsigned int rngSeed() const override; unsigned int rngSeed() const override;

View File

@ -25,7 +25,8 @@ namespace Catch {
struct WarnAbout { enum What { struct WarnAbout { enum What {
Nothing = 0x00, Nothing = 0x00,
NoAssertions = 0x01 NoAssertions = 0x01,
NoTests = 0x02
}; }; }; };
struct ShowDurations { enum OrNot { struct ShowDurations { enum OrNot {
@ -62,6 +63,7 @@ namespace Catch {
virtual bool includeSuccessfulResults() const = 0; virtual bool includeSuccessfulResults() const = 0;
virtual bool shouldDebugBreak() const = 0; virtual bool shouldDebugBreak() const = 0;
virtual bool warnAboutMissingAssertions() const = 0; virtual bool warnAboutMissingAssertions() const = 0;
virtual bool warnAboutNoTests() const = 0;
virtual int abortAfter() const = 0; virtual int abortAfter() const = 0;
virtual bool showInvisibles() const = 0; virtual bool showInvisibles() const = 0;
virtual ShowDurations::OrNot showDurations() const = 0; virtual ShowDurations::OrNot showDurations() const = 0;

View File

@ -18,6 +18,7 @@
#include "catch_random_number_generator.h" #include "catch_random_number_generator.h"
#include "catch_startup_exception_registry.h" #include "catch_startup_exception_registry.h"
#include "catch_text.h" #include "catch_text.h"
#include "catch_stream.h"
#include <cstdlib> #include <cstdlib>
#include <iomanip> #include <iomanip>
@ -80,6 +81,20 @@ namespace Catch {
context.reporter().skipTest(testCase); context.reporter().skipTest(testCase);
} }
if (config->warnAboutNoTests() && totals.testCases.total() == 0) {
ReusableStringStream testConfig;
bool first = true;
for (const auto& input : config->getTestsOrTags()) {
if (!first) { testConfig << ' '; }
first = false;
testConfig << input;
}
context.reporter().noMatchingTestCases(testConfig.str());
totals.error = -1;
}
context.testGroupEnded(config->name(), totals, 1, 1); context.testGroupEnded(config->name(), totals, 1, 1);
return totals; return totals;
} }
@ -259,10 +274,11 @@ namespace Catch {
if( Option<std::size_t> listed = list( config() ) ) if( Option<std::size_t> listed = list( config() ) )
return static_cast<int>( *listed ); return static_cast<int>( *listed );
auto totals = runTests( m_config );
// Note that on unices only the lower 8 bits are usually used, clamping // Note that on unices only the lower 8 bits are usually used, clamping
// the return value to 255 prevents false negative when some multiple // the return value to 255 prevents false negative when some multiple
// of 256 tests has failed // of 256 tests has failed
return (std::min)( MaxExitCode, static_cast<int>( runTests( m_config ).assertions.failed ) ); return (std::min)( { MaxExitCode, totals.error, static_cast<int>( totals.assertions.failed ) } );
} }
catch( std::exception& ex ) { catch( std::exception& ex ) {
Catch::cerr() << ex.what() << std::endl; Catch::cerr() << ex.what() << std::endl;

View File

@ -32,7 +32,7 @@ namespace Catch {
Totals delta( Totals const& prevTotals ) const; Totals delta( Totals const& prevTotals ) const;
int error = 0;
Counts assertions; Counts assertions;
Counts testCases; Counts testCases;
}; };