Improve reporting of unmatched filters (#1684)

This PR ultimately does 3 things
* Separately tracks matched tests per each filter part (that is, a set of filters separated by an OR (`,`)), which allows Catch2 to report each of the alternative filters that don't match any tests.
* Fixes `-w NoTests` to return non-zero in the process
* Adds tests for `-w NoTests`.
This commit is contained in:
Steven Franzen 2019-08-06 20:51:19 +02:00 committed by Martin Hořeňovský
parent cf55cfd76f
commit 6070745cab
10 changed files with 278 additions and 109 deletions

View File

@ -28,6 +28,7 @@ namespace Catch {
virtual std::vector<TestCase> const& getAllTestsSorted( IConfig const& config ) const = 0; virtual std::vector<TestCase> const& getAllTestsSorted( IConfig const& config ) const = 0;
}; };
bool isThrowSafe( TestCase const& testCase, IConfig const& config );
bool matchTest( TestCase const& testCase, TestSpec const& testSpec, IConfig const& config ); bool matchTest( TestCase const& testCase, TestSpec const& testSpec, IConfig const& config );
std::vector<TestCase> filterTests( std::vector<TestCase> const& testCases, TestSpec const& testSpec, IConfig const& config ); std::vector<TestCase> filterTests( std::vector<TestCase> const& testCases, TestSpec const& testSpec, IConfig const& config );
std::vector<TestCase> const& getAllTestCasesSorted( IConfig const& config ); std::vector<TestCase> const& getAllTestCasesSorted( IConfig const& config );

View File

@ -25,6 +25,8 @@
#include <cstdlib> #include <cstdlib>
#include <iomanip> #include <iomanip>
#include <set>
#include <iterator>
namespace Catch { namespace Catch {
@ -58,46 +60,53 @@ namespace Catch {
return ret; return ret;
} }
class TestGroup {
public:
explicit TestGroup(std::shared_ptr<Config> const& config)
: m_config{config}
, m_context{config, makeReporter(config)}
{
auto const& allTestCases = getAllTestCasesSorted(*m_config);
m_matches = m_config->testSpec().matchesByFilter(allTestCases, *m_config);
Catch::Totals runTests(std::shared_ptr<Config> const& config) { if (m_matches.empty()) {
auto reporter = makeReporter(config); for (auto const& test : allTestCases)
if (!test.isHidden())
RunContext context(config, std::move(reporter)); m_tests.emplace(&test);
} else {
Totals totals; for (auto const& match : m_matches)
m_tests.insert(match.tests.begin(), match.tests.end());
context.testGroupStarting(config->name(), 1, 1); }
TestSpec testSpec = config->testSpec();
auto const& allTestCases = getAllTestCasesSorted(*config);
for (auto const& testCase : allTestCases) {
bool matching = (!testSpec.hasFilters() && !testCase.isHidden()) ||
(testSpec.hasFilters() && matchTest(testCase, testSpec, *config));
if (!context.aborting() && matching)
totals += context.runTest(testCase);
else
context.reporter().skipTest(testCase);
} }
if (config->warnAboutNoTests() && totals.testCases.total() == 0) { Totals execute() {
ReusableStringStream testConfig; Totals totals;
m_context.testGroupStarting(m_config->name(), 1, 1);
bool first = true; for (auto const& testCase : m_tests) {
for (const auto& input : config->getTestsOrTags()) { if (!m_context.aborting())
if (!first) { testConfig << ' '; } totals += m_context.runTest(*testCase);
first = false; else
testConfig << input; m_context.reporter().skipTest(*testCase);
} }
context.reporter().noMatchingTestCases(testConfig.str()); for (auto const& match : m_matches) {
totals.error = -1; if (match.tests.empty()) {
m_context.reporter().noMatchingTestCases(match.name);
totals.error = -1;
}
}
m_context.testGroupEnded(m_config->name(), totals, 1, 1);
return totals;
} }
context.testGroupEnded(config->name(), totals, 1, 1); private:
return totals; using Tests = std::set<TestCase const*>;
}
std::shared_ptr<Config> m_config;
RunContext m_context;
Tests m_tests;
TestSpec::Matches m_matches;
};
void applyFilenamesAsTags(Catch::IConfig const& config) { void applyFilenamesAsTags(Catch::IConfig const& config) {
auto& tests = const_cast<std::vector<TestCase>&>(getAllTestCasesSorted(config)); auto& tests = const_cast<std::vector<TestCase>&>(getAllTestCasesSorted(config));
@ -274,7 +283,12 @@ namespace Catch {
if( Option<std::size_t> listed = list( m_config ) ) if( Option<std::size_t> listed = list( m_config ) )
return static_cast<int>( *listed ); return static_cast<int>( *listed );
auto totals = runTests( m_config ); TestGroup tests { m_config };
auto const totals = tests.execute();
if( m_config->warnAboutNoTests() && totals.error == -1 )
return 2;
// Note that on unices only the lower 8 bits are usually used, clamping // Note that on unices only the lower 8 bits are usually used, clamping
// the return value to 255 prevents false negative when some multiple // the return value to 255 prevents false negative when some multiple
// of 256 tests has failed // of 256 tests has failed

View File

@ -36,8 +36,13 @@ namespace Catch {
} }
return sorted; return sorted;
} }
bool isThrowSafe( TestCase const& testCase, IConfig const& config ) {
return !testCase.throws() || config.allowThrows();
}
bool matchTest( TestCase const& testCase, TestSpec const& testSpec, IConfig const& config ) { bool matchTest( TestCase const& testCase, TestSpec const& testSpec, IConfig const& config ) {
return testSpec.matches( testCase ) && ( config.allowThrows() || !testCase.throws() ); return testSpec.matches( testCase ) && isThrowSafe( testCase, config );
} }
void enforceNoDuplicateTestCases( std::vector<TestCase> const& functions ) { void enforceNoDuplicateTestCases( std::vector<TestCase> const& functions ) {

View File

@ -23,6 +23,8 @@ namespace Catch {
struct IConfig; struct IConfig;
std::vector<TestCase> sortTests( IConfig const& config, std::vector<TestCase> const& unsortedTestCases ); std::vector<TestCase> sortTests( IConfig const& config, std::vector<TestCase> const& unsortedTestCases );
bool isThrowSafe( TestCase const& testCase, IConfig const& config );
bool matchTest( TestCase const& testCase, TestSpec const& testSpec, IConfig const& config ); bool matchTest( TestCase const& testCase, TestSpec const& testSpec, IConfig const& config );
void enforceNoDuplicateTestCases( std::vector<TestCase> const& functions ); void enforceNoDuplicateTestCases( std::vector<TestCase> const& functions );

View File

@ -7,6 +7,7 @@
#include "catch_test_spec.h" #include "catch_test_spec.h"
#include "catch_string_manip.h" #include "catch_string_manip.h"
#include "catch_interfaces_config.h"
#include <algorithm> #include <algorithm>
#include <string> #include <string>
@ -15,45 +16,80 @@
namespace Catch { namespace Catch {
TestSpec::Pattern::~Pattern() = default; TestSpec::Pattern::Pattern( std::string const& name )
TestSpec::NamePattern::~NamePattern() = default; : m_name( name )
TestSpec::TagPattern::~TagPattern() = default;
TestSpec::ExcludedPattern::~ExcludedPattern() = default;
TestSpec::NamePattern::NamePattern( std::string const& name )
: m_wildcardPattern( toLower( name ), CaseSensitive::No )
{} {}
TestSpec::Pattern::~Pattern() = default;
std::string const& TestSpec::Pattern::name() const {
return m_name;
}
TestSpec::NamePattern::NamePattern( std::string const& name, std::string const& filterString )
: Pattern( filterString )
, m_wildcardPattern( toLower( name ), CaseSensitive::No )
{}
bool TestSpec::NamePattern::matches( TestCaseInfo const& testCase ) const { bool TestSpec::NamePattern::matches( TestCaseInfo const& testCase ) const {
return m_wildcardPattern.matches( toLower( testCase.name ) ); return m_wildcardPattern.matches( toLower( testCase.name ) );
} }
TestSpec::TagPattern::TagPattern( std::string const& tag ) : m_tag( toLower( tag ) ) {}
TestSpec::TagPattern::TagPattern( std::string const& tag, std::string const& filterString )
: Pattern( filterString )
, m_tag( toLower( tag ) )
{}
bool TestSpec::TagPattern::matches( TestCaseInfo const& testCase ) const { bool TestSpec::TagPattern::matches( TestCaseInfo const& testCase ) const {
return std::find(begin(testCase.lcaseTags), return std::find(begin(testCase.lcaseTags),
end(testCase.lcaseTags), end(testCase.lcaseTags),
m_tag) != end(testCase.lcaseTags); m_tag) != end(testCase.lcaseTags);
} }
TestSpec::ExcludedPattern::ExcludedPattern( PatternPtr const& underlyingPattern ) : m_underlyingPattern( underlyingPattern ) {}
bool TestSpec::ExcludedPattern::matches( TestCaseInfo const& testCase ) const { return !m_underlyingPattern->matches( testCase ); } TestSpec::ExcludedPattern::ExcludedPattern( PatternPtr const& underlyingPattern )
: Pattern( underlyingPattern->name() )
, m_underlyingPattern( underlyingPattern )
{}
bool TestSpec::ExcludedPattern::matches( TestCaseInfo const& testCase ) const {
return !m_underlyingPattern->matches( testCase );
}
bool TestSpec::Filter::matches( TestCaseInfo const& testCase ) const { bool TestSpec::Filter::matches( TestCaseInfo const& testCase ) const {
// All patterns in a filter must match for the filter to be a match return std::all_of( m_patterns.begin(), m_patterns.end(), [&]( PatternPtr const& p ){ return p->matches( testCase ); } );
for( auto const& pattern : m_patterns ) {
if( !pattern->matches( testCase ) )
return false;
}
return true;
} }
std::string TestSpec::Filter::name() const {
std::string name;
for( auto const& p : m_patterns )
name += p->name();
return name;
}
bool TestSpec::hasFilters() const { bool TestSpec::hasFilters() const {
return !m_filters.empty(); return !m_filters.empty();
} }
bool TestSpec::matches( TestCaseInfo const& testCase ) const { bool TestSpec::matches( TestCaseInfo const& testCase ) const {
// A TestSpec matches if any filter matches return std::any_of( m_filters.begin(), m_filters.end(), [&]( Filter const& f ){ return f.matches( testCase ); } );
for( auto const& filter : m_filters )
if( filter.matches( testCase ) )
return true;
return false;
} }
TestSpec::Matches TestSpec::matchesByFilter( std::vector<TestCase> const& testCases, IConfig const& config ) const
{
Matches matches( m_filters.size() );
std::transform( m_filters.begin(), m_filters.end(), matches.begin(), [&]( Filter const& filter ){
std::vector<TestCase const*> currentMatches;
for( auto const& test : testCases )
if( isThrowSafe( test, config ) && filter.matches( test ) )
currentMatches.emplace_back( &test );
return FilterMatch{ filter.name(), currentMatches };
} );
return matches;
}
} }

View File

@ -22,17 +22,23 @@
namespace Catch { namespace Catch {
struct IConfig;
class TestSpec { class TestSpec {
struct Pattern { class Pattern {
public:
explicit Pattern( std::string const& name );
virtual ~Pattern(); virtual ~Pattern();
virtual bool matches( TestCaseInfo const& testCase ) const = 0; virtual bool matches( TestCaseInfo const& testCase ) const = 0;
std::string const& name() const;
private:
std::string const m_name;
}; };
using PatternPtr = std::shared_ptr<Pattern>; using PatternPtr = std::shared_ptr<Pattern>;
class NamePattern : public Pattern { class NamePattern : public Pattern {
public: public:
NamePattern( std::string const& name ); explicit NamePattern( std::string const& name, std::string const& filterString );
virtual ~NamePattern();
bool matches( TestCaseInfo const& testCase ) const override; bool matches( TestCaseInfo const& testCase ) const override;
private: private:
WildcardPattern m_wildcardPattern; WildcardPattern m_wildcardPattern;
@ -40,8 +46,7 @@ namespace Catch {
class TagPattern : public Pattern { class TagPattern : public Pattern {
public: public:
TagPattern( std::string const& tag ); explicit TagPattern( std::string const& tag, std::string const& filterString );
virtual ~TagPattern();
bool matches( TestCaseInfo const& testCase ) const override; bool matches( TestCaseInfo const& testCase ) const override;
private: private:
std::string m_tag; std::string m_tag;
@ -49,8 +54,7 @@ namespace Catch {
class ExcludedPattern : public Pattern { class ExcludedPattern : public Pattern {
public: public:
ExcludedPattern( PatternPtr const& underlyingPattern ); explicit ExcludedPattern( PatternPtr const& underlyingPattern );
virtual ~ExcludedPattern();
bool matches( TestCaseInfo const& testCase ) const override; bool matches( TestCaseInfo const& testCase ) const override;
private: private:
PatternPtr m_underlyingPattern; PatternPtr m_underlyingPattern;
@ -60,11 +64,19 @@ namespace Catch {
std::vector<PatternPtr> m_patterns; std::vector<PatternPtr> m_patterns;
bool matches( TestCaseInfo const& testCase ) const; bool matches( TestCaseInfo const& testCase ) const;
std::string name() const;
}; };
public: public:
struct FilterMatch {
std::string name;
std::vector<TestCase const*> tests;
};
using Matches = std::vector<FilterMatch>;
bool hasFilters() const; bool hasFilters() const;
bool matches( TestCaseInfo const& testCase ) const; bool matches( TestCaseInfo const& testCase ) const;
Matches matchesByFilter( std::vector<TestCase> const& testCases, IConfig const& config ) const;
private: private:
std::vector<Filter> m_filters; std::vector<Filter> m_filters;

View File

@ -14,64 +14,125 @@ namespace Catch {
TestSpecParser& TestSpecParser::parse( std::string const& arg ) { TestSpecParser& TestSpecParser::parse( std::string const& arg ) {
m_mode = None; m_mode = None;
m_exclusion = false; m_exclusion = false;
m_start = std::string::npos;
m_arg = m_tagAliases->expandAliases( arg ); m_arg = m_tagAliases->expandAliases( arg );
m_escapeChars.clear(); m_escapeChars.clear();
m_substring.reserve(m_arg.size());
m_patternName.reserve(m_arg.size());
for( m_pos = 0; m_pos < m_arg.size(); ++m_pos ) for( m_pos = 0; m_pos < m_arg.size(); ++m_pos )
visitChar( m_arg[m_pos] ); visitChar( m_arg[m_pos] );
if( m_mode == Name ) endMode();
addPattern<TestSpec::NamePattern>();
return *this; return *this;
} }
TestSpec TestSpecParser::testSpec() { TestSpec TestSpecParser::testSpec() {
addFilter(); addFilter();
return m_testSpec; return m_testSpec;
} }
void TestSpecParser::visitChar( char c ) { void TestSpecParser::visitChar( char c ) {
if( m_mode == None ) { if( c == ',' ) {
switch( c ) { endMode();
case ' ': return; addFilter();
case '~': m_exclusion = true; return; return;
case '[': return startNewMode( Tag, ++m_pos );
case '"': return startNewMode( QuotedName, ++m_pos );
case '\\': return escape();
default: startNewMode( Name, m_pos ); break;
}
} }
if( m_mode == Name ) {
if( c == ',' ) { switch( m_mode ) {
addPattern<TestSpec::NamePattern>(); case None:
addFilter(); if( processNoneChar( c ) )
} return;
else if( c == '[' ) { break;
if( subString() == "exclude:" ) case Name:
m_exclusion = true; processNameChar( c );
else break;
addPattern<TestSpec::NamePattern>(); case EscapedName:
startNewMode( Tag, ++m_pos ); endMode();
} break;
else if( c == '\\' ) default:
escape(); case Tag:
case QuotedName:
if( processOtherChar( c ) )
return;
break;
} }
else if( m_mode == EscapedName )
m_mode = Name; m_substring += c;
else if( m_mode == QuotedName && c == '"' ) if( !isControlChar( c ) )
addPattern<TestSpec::NamePattern>(); m_patternName += c;
else if( m_mode == Tag && c == ']' )
addPattern<TestSpec::TagPattern>();
} }
void TestSpecParser::startNewMode( Mode mode, std::size_t start ) { // Two of the processing methods return true to signal the caller to return
// without adding the given character to the current pattern strings
bool TestSpecParser::processNoneChar( char c ) {
switch( c ) {
case ' ':
return true;
case '~':
m_exclusion = true;
return false;
case '[':
startNewMode( Tag );
return false;
case '"':
startNewMode( QuotedName );
return false;
case '\\':
escape();
return true;
default:
startNewMode( Name );
return false;
}
}
void TestSpecParser::processNameChar( char c ) {
if( c == '[' ) {
if( m_substring == "exclude:" )
m_exclusion = true;
else
endMode();
startNewMode( Tag );
}
}
bool TestSpecParser::processOtherChar( char c ) {
if( !isControlChar( c ) )
return false;
m_substring += c;
endMode();
return true;
}
void TestSpecParser::startNewMode( Mode mode ) {
m_mode = mode; m_mode = mode;
m_start = start; }
void TestSpecParser::endMode() {
switch( m_mode ) {
case Name:
case QuotedName:
return addPattern<TestSpec::NamePattern>();
case Tag:
return addPattern<TestSpec::TagPattern>();
case EscapedName:
return startNewMode( Name );
case None:
default:
return startNewMode( None );
}
} }
void TestSpecParser::escape() { void TestSpecParser::escape() {
if( m_mode == None )
m_start = m_pos;
m_mode = EscapedName; m_mode = EscapedName;
m_escapeChars.push_back( m_pos ); m_escapeChars.push_back( m_pos );
} }
std::string TestSpecParser::subString() const { return m_arg.substr( m_start, m_pos - m_start ); } bool TestSpecParser::isControlChar( char c ) const {
switch( m_mode ) {
default:
return false;
case None:
return c == '~';
case Name:
return c == '[';
case EscapedName:
return true;
case QuotedName:
return c == '"';
case Tag:
return c == '[' || c == ']';
}
}
void TestSpecParser::addFilter() { void TestSpecParser::addFilter() {
if( !m_currentFilter.m_patterns.empty() ) { if( !m_currentFilter.m_patterns.empty() ) {

View File

@ -23,8 +23,10 @@ namespace Catch {
enum Mode{ None, Name, QuotedName, Tag, EscapedName }; enum Mode{ None, Name, QuotedName, Tag, EscapedName };
Mode m_mode = None; Mode m_mode = None;
bool m_exclusion = false; bool m_exclusion = false;
std::size_t m_start = std::string::npos, m_pos = 0; std::size_t m_pos = 0;
std::string m_arg; std::string m_arg;
std::string m_substring;
std::string m_patternName;
std::vector<std::size_t> m_escapeChars; std::vector<std::size_t> m_escapeChars;
TestSpec::Filter m_currentFilter; TestSpec::Filter m_currentFilter;
TestSpec m_testSpec; TestSpec m_testSpec;
@ -38,26 +40,32 @@ namespace Catch {
private: private:
void visitChar( char c ); void visitChar( char c );
void startNewMode( Mode mode, std::size_t start ); void startNewMode( Mode mode );
bool processNoneChar( char c );
void processNameChar( char c );
bool processOtherChar( char c );
void endMode();
void escape(); void escape();
std::string subString() const; bool isControlChar( char c ) const;
template<typename T> template<typename T>
void addPattern() { void addPattern() {
std::string token = subString(); std::string token = m_patternName;
for( std::size_t i = 0; i < m_escapeChars.size(); ++i ) for( std::size_t i = 0; i < m_escapeChars.size(); ++i )
token = token.substr( 0, m_escapeChars[i]-m_start-i ) + token.substr( m_escapeChars[i]-m_start-i+1 ); token = token.substr( 0, m_escapeChars[i] - i ) + token.substr( m_escapeChars[i] -i +1 );
m_escapeChars.clear(); m_escapeChars.clear();
if( startsWith( token, "exclude:" ) ) { if( startsWith( token, "exclude:" ) ) {
m_exclusion = true; m_exclusion = true;
token = token.substr( 8 ); token = token.substr( 8 );
} }
if( !token.empty() ) { if( !token.empty() ) {
TestSpec::PatternPtr pattern = std::make_shared<T>( token ); TestSpec::PatternPtr pattern = std::make_shared<T>( token, m_substring );
if( m_exclusion ) if( m_exclusion )
pattern = std::make_shared<TestSpec::ExcludedPattern>( pattern ); pattern = std::make_shared<TestSpec::ExcludedPattern>( pattern );
m_currentFilter.m_patterns.push_back( pattern ); m_currentFilter.m_patterns.push_back( pattern );
} }
m_substring.clear();
m_patternName.clear();
m_exclusion = false; m_exclusion = false;
m_mode = None; m_mode = None;
} }

View File

@ -393,8 +393,19 @@ set_tests_properties(ListTestNamesOnly PROPERTIES
add_test(NAME NoAssertions COMMAND $<TARGET_FILE:SelfTest> -w NoAssertions) add_test(NAME NoAssertions COMMAND $<TARGET_FILE:SelfTest> -w NoAssertions)
set_tests_properties(NoAssertions PROPERTIES PASS_REGULAR_EXPRESSION "No assertions in test case") set_tests_properties(NoAssertions PROPERTIES PASS_REGULAR_EXPRESSION "No assertions in test case")
add_test(NAME NoTest COMMAND $<TARGET_FILE:SelfTest> -w NoTests "___nonexistent_test___") add_test(NAME NoTest COMMAND $<TARGET_FILE:SelfTest> Tracker, "___nonexistent_test___")
set_tests_properties(NoTest PROPERTIES PASS_REGULAR_EXPRESSION "No test cases matched") set_tests_properties(NoTest PROPERTIES
PASS_REGULAR_EXPRESSION "No test cases matched '___nonexistent_test___'"
FAIL_REGULAR_EXPRESSION "No tests ran"
)
add_test(NAME WarnAboutNoTests COMMAND ${CMAKE_COMMAND} -P ${CATCH_DIR}/projects/SelfTest/WarnAboutNoTests.cmake $<TARGET_FILE:SelfTest>)
add_test(NAME UnmatchedOutputFilter COMMAND $<TARGET_FILE:SelfTest> [this-tag-does-not-exist] -w NoTests)
set_tests_properties(UnmatchedOutputFilter
PROPERTIES
PASS_REGULAR_EXPRESSION "No test cases matched '\\[this-tag-does-not-exist\\]'"
)
add_test(NAME FilteredSection-1 COMMAND $<TARGET_FILE:SelfTest> \#1394 -c RunSection) add_test(NAME FilteredSection-1 COMMAND $<TARGET_FILE:SelfTest> \#1394 -c RunSection)
set_tests_properties(FilteredSection-1 PROPERTIES FAIL_REGULAR_EXPRESSION "No tests ran") set_tests_properties(FilteredSection-1 PROPERTIES FAIL_REGULAR_EXPRESSION "No tests ran")

View File

@ -0,0 +1,19 @@
# Workaround for a peculiarity where CTest disregards the return code from a
# test command if a PASS_REGULAR_EXPRESSION is also set
execute_process(
COMMAND ${CMAKE_ARGV3} -w NoTests "___nonexistent_test___"
RESULT_VARIABLE ret
OUTPUT_VARIABLE out
)
message("${out}")
if(NOT ${ret} MATCHES "^[0-9]+$")
message(FATAL_ERROR "${ret}")
endif()
if(${ret} EQUAL 0)
message(FATAL_ERROR "Expected nonzero return code")
elseif(${out} MATCHES "Helper failed with")
message(FATAL_ERROR "Helper failed")
endif()