Hidden tests now require positive filter match to be selected

This also required some refactoring of how the pattern matching
works. This means that the concepts of include and exclude patterns
are no longer unified, with exclusion patterns working as just
negation of an inclusion patterns (which led to including hidden
tags by default, as they did not match the exclusion), but rather
both include and exclude patterns are handled separately.

The new logic is that given a filter and a test case, the test
case must match _all_ include patterns and _no_ exclude patterns
to be included by the filter. Furthermore, if the test case is
hidden, then the filter must have at least one include pattern
for the test case to be used.

Closes #1184
This commit is contained in:
Martin Hořeňovský
2019-06-22 20:26:03 +02:00
parent 2bcf1b3db6
commit 4f47d1c6c1
15 changed files with 63 additions and 69 deletions

View File

@@ -962,7 +962,7 @@ CmdLine.tests.cpp:<line number>: passed: spec.matches( tcD ) == false for: false
CmdLine.tests.cpp:<line number>: passed: spec.hasFilters() == true for: true == true
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcA ) == true for: true == true
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcB ) == false for: false == false
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcC ) == true for: true == true
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcC ) == false for: false == false
CmdLine.tests.cpp:<line number>: passed: spec.hasFilters() == true for: true == true
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcA ) == false for: false == false
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcB ) == true for: true == true
@@ -980,7 +980,7 @@ CmdLine.tests.cpp:<line number>: passed: spec.matches( tcD ) == true for: true =
CmdLine.tests.cpp:<line number>: passed: spec.hasFilters() == true for: true == true
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcA ) == true for: true == true
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcB ) == false for: false == false
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcC ) == true for: true == true
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcC ) == false for: false == false
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcD ) == true for: true == true
CmdLine.tests.cpp:<line number>: passed: spec.hasFilters() == true for: true == true
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcA ) == true for: true == true

View File

@@ -1,4 +1,4 @@
Filters: ~[!nonportable]~[!benchmark]~[approvals]
Filters: ~[!nonportable]~[!benchmark]~[approvals] *
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
<exe-name> is a <version> host application.

View File

@@ -1,4 +1,4 @@
Filters: ~[!nonportable]~[!benchmark]~[approvals]
Filters: ~[!nonportable]~[!benchmark]~[approvals] *
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
<exe-name> is a <version> host application.
@@ -6946,9 +6946,9 @@ with expansion:
false == false
CmdLine.tests.cpp:<line number>: PASSED:
CHECK( spec.matches( tcC ) == true )
CHECK( spec.matches( tcC ) == false )
with expansion:
true == true
false == false
-------------------------------------------------------------------------------
Parse test names and tags
@@ -7064,9 +7064,9 @@ with expansion:
false == false
CmdLine.tests.cpp:<line number>: PASSED:
CHECK( spec.matches( tcC ) == true )
CHECK( spec.matches( tcC ) == false )
with expansion:
true == true
false == false
CmdLine.tests.cpp:<line number>: PASSED:
CHECK( spec.matches( tcD ) == true )

View File

@@ -1,4 +1,4 @@
Filters: ~[!nonportable]~[!benchmark]~[approvals]
Filters: ~[!nonportable]~[!benchmark]~[approvals] *
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
<exe-name> is a <version> host application.

View File

@@ -3,7 +3,7 @@
>
<testsuite name="<exe-name>" errors="17" failures="132" tests="1713" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<properties>
<property name="filters" value="~[!nonportable]~[!benchmark]~[approvals]"/>
<property name="filters" value="~[!nonportable]~[!benchmark]~[approvals] *"/>
<property name="random-seed" value="1"/>
</properties>
<testcase classname="<exe-name>.global" name="# A test name that starts with a #" time="{duration}"/>

View File

@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<Catch name="<exe-name>" filters="~[!nonportable]~[!benchmark]~[approvals]">
<Catch name="<exe-name>" filters="~[!nonportable]~[!benchmark]~[approvals] *">
<Randomness seed="1"/>
<Group name="<exe-name>">
<TestCase name="# A test name that starts with a #" filename="projects/<exe-name>/UsageTests/Misc.tests.cpp" >
@@ -8720,10 +8720,10 @@ Nor would this
</Expression>
<Expression success="true" type="CHECK" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
spec.matches( tcC ) == true
spec.matches( tcC ) == false
</Original>
<Expanded>
true == true
false == false
</Expanded>
</Expression>
<OverallResults successes="4" failures="0" expectedFailures="0"/>
@@ -8876,10 +8876,10 @@ Nor would this
</Expression>
<Expression success="true" type="CHECK" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
spec.matches( tcC ) == true
spec.matches( tcC ) == false
</Original>
<Expanded>
true == true
false == false
</Expanded>
</Expression>
<Expression success="true" type="CHECK" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >

View File

@@ -173,7 +173,7 @@ TEST_CASE( "Parse test names and tags" ) {
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == true );
CHECK( spec.matches( tcB ) == false );
CHECK( spec.matches( tcC ) == true );
CHECK( spec.matches( tcC ) == false );
}
SECTION( "One tag exclusion and one tag inclusion" ) {
TestSpec spec = parseTestSpec( "~[two][x]" );
@@ -203,7 +203,7 @@ TEST_CASE( "Parse test names and tags" ) {
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == true );
CHECK( spec.matches( tcB ) == false );
CHECK( spec.matches( tcC ) == true );
CHECK( spec.matches( tcC ) == false );
CHECK( spec.matches( tcD ) == true );
}
SECTION( "wildcarded name exclusion" ) {
@@ -486,7 +486,7 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]"
REQUIRE(config.benchmarkSamples == 200);
}
SECTION("resamples") {
CHECK(cli.parse({ "test", "--benchmark-resamples=20000" }));