Hidden tests now require positive filter match to be selected

This also required some refactoring of how the pattern matching
works. This means that the concepts of include and exclude patterns
are no longer unified, with exclusion patterns working as just
negation of an inclusion patterns (which led to including hidden
tags by default, as they did not match the exclusion), but rather
both include and exclude patterns are handled separately.

The new logic is that given a filter and a test case, the test
case must match _all_ include patterns and _no_ exclude patterns
to be included by the filter. Furthermore, if the test case is
hidden, then the filter must have at least one include pattern
for the test case to be used.

Closes #1184
This commit is contained in:
Martin Hořeňovský 2019-06-22 20:26:03 +02:00
parent 2bcf1b3db6
commit 4f47d1c6c1
No known key found for this signature in database
GPG Key ID: DE48307B8B0D381A
15 changed files with 63 additions and 69 deletions

View File

@ -27,20 +27,6 @@ will be added), which means that their failure will not fail the test,
making the `else` actually useful. making the `else` actually useful.
### Change semantics of `[.]` and tag exclusion
Currently, given these 2 tests
```cpp
TEST_CASE("A", "[.][foo]") {}
TEST_CASE("B", "[.][bar]") {}
```
specifying `[foo]` as the testspec will run test "A" and specifying
`~[foo]` will run test "B", even though it is hidden. Also, specifying
`~[baz]` will run both tests. This behaviour is often surprising and will
be changed so that hidden tests are included in a run only if they
positively match a testspec.
### Console Colour API ### Console Colour API
The API for Catch2's console colour will be changed to take an extra The API for Catch2's console colour will be changed to take an extra

View File

@ -42,6 +42,10 @@
* XmlReporter outputs a machine-parseable XML * XmlReporter outputs a machine-parseable XML
* `TEST_CASE` description support has been removed * `TEST_CASE` description support has been removed
* If the second argument has text outside tags, the text will be ignored. * If the second argument has text outside tags, the text will be ignored.
* Hidden test cases are no longer included just because they don't match an exclusion tag
* Previously, a `TEST_CASE("A", "[.foo]")` would be included by asking for `~[bar]`.
### Fixes ### Fixes
* The `INFO` macro no longer contains superfluous semicolon (#1456) * The `INFO` macro no longer contains superfluous semicolon (#1456)

View File

@ -48,25 +48,30 @@ namespace Catch {
m_tag) != end(testCase.lcaseTags); m_tag) != end(testCase.lcaseTags);
} }
TestSpec::ExcludedPattern::ExcludedPattern( PatternPtr const& underlyingPattern )
: Pattern( underlyingPattern->name() )
, m_underlyingPattern( underlyingPattern )
{}
bool TestSpec::ExcludedPattern::matches( TestCaseInfo const& testCase ) const {
return !m_underlyingPattern->matches( testCase );
}
bool TestSpec::Filter::matches( TestCaseInfo const& testCase ) const { bool TestSpec::Filter::matches( TestCaseInfo const& testCase ) const {
return std::all_of( m_patterns.begin(), m_patterns.end(), [&]( PatternPtr const& p ){ return p->matches( testCase ); } ); bool should_use = !testCase.isHidden();
for (auto const& pattern : m_required) {
should_use = true;
if (!pattern->matches(testCase)) {
return false;
}
}
for (auto const& pattern : m_forbidden) {
if (pattern->matches(testCase)) {
return false;
}
}
return should_use;
} }
std::string TestSpec::Filter::name() const { std::string TestSpec::Filter::name() const {
std::string name; std::string name;
for( auto const& p : m_patterns ) for (auto const& p : m_required) {
name += p->name(); name += p->name();
}
for (auto const& p : m_forbidden) {
name += p->name();
}
return name; return name;
} }

View File

@ -52,16 +52,9 @@ namespace Catch {
std::string m_tag; std::string m_tag;
}; };
class ExcludedPattern : public Pattern {
public:
explicit ExcludedPattern( PatternPtr const& underlyingPattern );
bool matches( TestCaseInfo const& testCase ) const override;
private:
PatternPtr m_underlyingPattern;
};
struct Filter { struct Filter {
std::vector<PatternPtr> m_patterns; std::vector<PatternPtr> m_required;
std::vector<PatternPtr> m_forbidden;
bool matches( TestCaseInfo const& testCase ) const; bool matches( TestCaseInfo const& testCase ) const;
std::string name() const; std::string name() const;

View File

@ -147,7 +147,7 @@ namespace Catch {
} }
void TestSpecParser::addFilter() { void TestSpecParser::addFilter() {
if( !m_currentFilter.m_patterns.empty() ) { if( !m_currentFilter.m_required.empty() || !m_currentFilter.m_forbidden.empty() ) {
m_testSpec.m_filters.push_back( m_currentFilter ); m_testSpec.m_filters.push_back( m_currentFilter );
m_currentFilter = TestSpec::Filter(); m_currentFilter = TestSpec::Filter();
} }

View File

@ -66,9 +66,11 @@ namespace Catch {
} }
if( !token.empty() ) { if( !token.empty() ) {
TestSpec::PatternPtr pattern = std::make_shared<T>( token, m_substring ); TestSpec::PatternPtr pattern = std::make_shared<T>( token, m_substring );
if( m_exclusion ) if (m_exclusion) {
pattern = std::make_shared<TestSpec::ExcludedPattern>( pattern ); m_currentFilter.m_forbidden.push_back(pattern);
m_currentFilter.m_patterns.push_back( pattern ); } else {
m_currentFilter.m_required.push_back(pattern);
}
} }
m_substring.clear(); m_substring.clear();
m_patternName.clear(); m_patternName.clear();

View File

@ -455,6 +455,9 @@ set_tests_properties(FilenameAsTagsTest PROPERTIES PASS_REGULAR_EXPRESSION "\\[#
add_test(NAME EscapeSpecialCharactersInTestNames COMMAND $<TARGET_FILE:SelfTest> "Test with special\\, characters \"in name") add_test(NAME EscapeSpecialCharactersInTestNames COMMAND $<TARGET_FILE:SelfTest> "Test with special\\, characters \"in name")
set_tests_properties(EscapeSpecialCharactersInTestNames PROPERTIES PASS_REGULAR_EXPRESSION "1 assertion in 1 test case") set_tests_properties(EscapeSpecialCharactersInTestNames PROPERTIES PASS_REGULAR_EXPRESSION "1 assertion in 1 test case")
add_test(NAME NegativeSpecNoHiddenTests COMMAND $<TARGET_FILE:SelfTest> --list-tests ~[approval])
set_tests_properties(NegativeSpecNoHiddenTests PROPERTIES FAIL_REGULAR_EXPRESSION "\\[\\.\\]")
add_test(NAME TestsInFile::SimpleSpecs COMMAND $<TARGET_FILE:SelfTest> "-f ${CATCH_DIR}/projects/SelfTest/Misc/plain-old-tests.input") add_test(NAME TestsInFile::SimpleSpecs COMMAND $<TARGET_FILE:SelfTest> "-f ${CATCH_DIR}/projects/SelfTest/Misc/plain-old-tests.input")
set_tests_properties(TestsInFile::SimpleSpecs PROPERTIES PASS_REGULAR_EXPRESSION "6 assertions in 2 test cases") set_tests_properties(TestsInFile::SimpleSpecs PROPERTIES PASS_REGULAR_EXPRESSION "6 assertions in 2 test cases")

View File

@ -962,7 +962,7 @@ CmdLine.tests.cpp:<line number>: passed: spec.matches( tcD ) == false for: false
CmdLine.tests.cpp:<line number>: passed: spec.hasFilters() == true for: true == true CmdLine.tests.cpp:<line number>: passed: spec.hasFilters() == true for: true == true
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcA ) == true for: true == true CmdLine.tests.cpp:<line number>: passed: spec.matches( tcA ) == true for: true == true
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcB ) == false for: false == false CmdLine.tests.cpp:<line number>: passed: spec.matches( tcB ) == false for: false == false
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcC ) == true for: true == true CmdLine.tests.cpp:<line number>: passed: spec.matches( tcC ) == false for: false == false
CmdLine.tests.cpp:<line number>: passed: spec.hasFilters() == true for: true == true CmdLine.tests.cpp:<line number>: passed: spec.hasFilters() == true for: true == true
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcA ) == false for: false == false CmdLine.tests.cpp:<line number>: passed: spec.matches( tcA ) == false for: false == false
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcB ) == true for: true == true CmdLine.tests.cpp:<line number>: passed: spec.matches( tcB ) == true for: true == true
@ -980,7 +980,7 @@ CmdLine.tests.cpp:<line number>: passed: spec.matches( tcD ) == true for: true =
CmdLine.tests.cpp:<line number>: passed: spec.hasFilters() == true for: true == true CmdLine.tests.cpp:<line number>: passed: spec.hasFilters() == true for: true == true
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcA ) == true for: true == true CmdLine.tests.cpp:<line number>: passed: spec.matches( tcA ) == true for: true == true
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcB ) == false for: false == false CmdLine.tests.cpp:<line number>: passed: spec.matches( tcB ) == false for: false == false
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcC ) == true for: true == true CmdLine.tests.cpp:<line number>: passed: spec.matches( tcC ) == false for: false == false
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcD ) == true for: true == true CmdLine.tests.cpp:<line number>: passed: spec.matches( tcD ) == true for: true == true
CmdLine.tests.cpp:<line number>: passed: spec.hasFilters() == true for: true == true CmdLine.tests.cpp:<line number>: passed: spec.hasFilters() == true for: true == true
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcA ) == true for: true == true CmdLine.tests.cpp:<line number>: passed: spec.matches( tcA ) == true for: true == true

View File

@ -1,4 +1,4 @@
Filters: ~[!nonportable]~[!benchmark]~[approvals] Filters: ~[!nonportable]~[!benchmark]~[approvals] *
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
<exe-name> is a <version> host application. <exe-name> is a <version> host application.

View File

@ -1,4 +1,4 @@
Filters: ~[!nonportable]~[!benchmark]~[approvals] Filters: ~[!nonportable]~[!benchmark]~[approvals] *
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
<exe-name> is a <version> host application. <exe-name> is a <version> host application.
@ -6946,9 +6946,9 @@ with expansion:
false == false false == false
CmdLine.tests.cpp:<line number>: PASSED: CmdLine.tests.cpp:<line number>: PASSED:
CHECK( spec.matches( tcC ) == true ) CHECK( spec.matches( tcC ) == false )
with expansion: with expansion:
true == true false == false
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Parse test names and tags Parse test names and tags
@ -7064,9 +7064,9 @@ with expansion:
false == false false == false
CmdLine.tests.cpp:<line number>: PASSED: CmdLine.tests.cpp:<line number>: PASSED:
CHECK( spec.matches( tcC ) == true ) CHECK( spec.matches( tcC ) == false )
with expansion: with expansion:
true == true false == false
CmdLine.tests.cpp:<line number>: PASSED: CmdLine.tests.cpp:<line number>: PASSED:
CHECK( spec.matches( tcD ) == true ) CHECK( spec.matches( tcD ) == true )

View File

@ -1,4 +1,4 @@
Filters: ~[!nonportable]~[!benchmark]~[approvals] Filters: ~[!nonportable]~[!benchmark]~[approvals] *
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
<exe-name> is a <version> host application. <exe-name> is a <version> host application.

View File

@ -3,7 +3,7 @@
> >
<testsuite name="<exe-name>" errors="17" failures="132" tests="1713" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}"> <testsuite name="<exe-name>" errors="17" failures="132" tests="1713" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<properties> <properties>
<property name="filters" value="~[!nonportable]~[!benchmark]~[approvals]"/> <property name="filters" value="~[!nonportable]~[!benchmark]~[approvals] *"/>
<property name="random-seed" value="1"/> <property name="random-seed" value="1"/>
</properties> </properties>
<testcase classname="<exe-name>.global" name="# A test name that starts with a #" time="{duration}"/> <testcase classname="<exe-name>.global" name="# A test name that starts with a #" time="{duration}"/>

View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<Catch name="<exe-name>" filters="~[!nonportable]~[!benchmark]~[approvals]"> <Catch name="<exe-name>" filters="~[!nonportable]~[!benchmark]~[approvals] *">
<Randomness seed="1"/> <Randomness seed="1"/>
<Group name="<exe-name>"> <Group name="<exe-name>">
<TestCase name="# A test name that starts with a #" filename="projects/<exe-name>/UsageTests/Misc.tests.cpp" > <TestCase name="# A test name that starts with a #" filename="projects/<exe-name>/UsageTests/Misc.tests.cpp" >
@ -8720,10 +8720,10 @@ Nor would this
</Expression> </Expression>
<Expression success="true" type="CHECK" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" > <Expression success="true" type="CHECK" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original> <Original>
spec.matches( tcC ) == true spec.matches( tcC ) == false
</Original> </Original>
<Expanded> <Expanded>
true == true false == false
</Expanded> </Expanded>
</Expression> </Expression>
<OverallResults successes="4" failures="0" expectedFailures="0"/> <OverallResults successes="4" failures="0" expectedFailures="0"/>
@ -8876,10 +8876,10 @@ Nor would this
</Expression> </Expression>
<Expression success="true" type="CHECK" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" > <Expression success="true" type="CHECK" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original> <Original>
spec.matches( tcC ) == true spec.matches( tcC ) == false
</Original> </Original>
<Expanded> <Expanded>
true == true false == false
</Expanded> </Expanded>
</Expression> </Expression>
<Expression success="true" type="CHECK" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" > <Expression success="true" type="CHECK" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >

View File

@ -173,7 +173,7 @@ TEST_CASE( "Parse test names and tags" ) {
CHECK( spec.hasFilters() == true ); CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == true ); CHECK( spec.matches( tcA ) == true );
CHECK( spec.matches( tcB ) == false ); CHECK( spec.matches( tcB ) == false );
CHECK( spec.matches( tcC ) == true ); CHECK( spec.matches( tcC ) == false );
} }
SECTION( "One tag exclusion and one tag inclusion" ) { SECTION( "One tag exclusion and one tag inclusion" ) {
TestSpec spec = parseTestSpec( "~[two][x]" ); TestSpec spec = parseTestSpec( "~[two][x]" );
@ -203,7 +203,7 @@ TEST_CASE( "Parse test names and tags" ) {
CHECK( spec.hasFilters() == true ); CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == true ); CHECK( spec.matches( tcA ) == true );
CHECK( spec.matches( tcB ) == false ); CHECK( spec.matches( tcB ) == false );
CHECK( spec.matches( tcC ) == true ); CHECK( spec.matches( tcC ) == false );
CHECK( spec.matches( tcD ) == true ); CHECK( spec.matches( tcD ) == true );
} }
SECTION( "wildcarded name exclusion" ) { SECTION( "wildcarded name exclusion" ) {

View File

@ -195,19 +195,20 @@ print(" " + cmdPath)
# ## Keep default reporters here ## # ## Keep default reporters here ##
# Standard console reporter # Standard console reporter
approve("console.std", ["~[!nonportable]~[!benchmark]~[approvals]", "--order", "lex", "--rng-seed", "1"]) approve("console.std", ["~[!nonportable]~[!benchmark]~[approvals] *", "--order", "lex", "--rng-seed", "1"])
# console reporter, include passes, warn about No Assertions # console reporter, include passes, warn about No Assertions
approve("console.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "--order", "lex", "--rng-seed", "1"]) approve("console.sw", ["~[!nonportable]~[!benchmark]~[approvals] *", "-s", "-w", "NoAssertions", "--order", "lex", "--rng-seed", "1"])
# console reporter, include passes, warn about No Assertions, limit failures to first 4 # console reporter, include passes, warn about No Assertions, limit failures to first 4
approve("console.swa4", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex", "--rng-seed", "1"]) approve("console.swa4", ["~[!nonportable]~[!benchmark]~[approvals] *", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex", "--rng-seed", "1"])
# junit reporter, include passes, warn about No Assertions # junit reporter, include passes, warn about No Assertions
approve("junit.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex", "--rng-seed", "1"]) approve("junit.sw", ["~[!nonportable]~[!benchmark]~[approvals] *", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex", "--rng-seed", "1"])
# xml reporter, include passes, warn about No Assertions # xml reporter, include passes, warn about No Assertions
approve("xml.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex", "--rng-seed", "1"]) approve("xml.sw", ["~[!nonportable]~[!benchmark]~[approvals] *", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex", "--rng-seed", "1"])
# compact reporter, include passes, warn about No Assertions # compact reporter, include passes, warn about No Assertions
approve('compact.sw', ['~[!nonportable]~[!benchmark]~[approvals]', '-s', '-w', 'NoAssertions', '-r', 'compact', '--order', 'lex', "--rng-seed", "1"]) approve('compact.sw', ['~[!nonportable]~[!benchmark]~[approvals] *', '-s', '-w', 'NoAssertions', '-r', 'compact', '--order', 'lex', "--rng-seed", "1"])
# sonarqube reporter, include passes, warn about No Assertions # sonarqube reporter, include passes, warn about No Assertions
approve("sonarqube.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-r", "sonarqube", "--order", "lex", "--rng-seed", "1"]) approve("sonarqube.sw", ["~[!nonportable]~[!benchmark]~[approvals] *", "-s", "-w", "NoAssertions", "-r", "sonarqube", "--order", "lex", "--rng-seed", "1"])
if overallResult != 0: if overallResult != 0:
print("If these differences are expected, run approve.py to approve new baselines.") print("If these differences are expected, run approve.py to approve new baselines.")