This commit is contained in:
dvirtz 2018-02-09 00:18:32 +02:00 committed by Martin Hořeňovský
parent 355b3f9952
commit ca8470fbad
14 changed files with 134 additions and 31 deletions

View File

@ -357,16 +357,23 @@ if (BUILD_TESTING AND NOT_SUBPROJECT)
add_test(NAME RunTests COMMAND $<TARGET_FILE:SelfTest>)
add_test(NAME ListTests COMMAND $<TARGET_FILE:SelfTest> --list-tests --verbosity high)
set_tests_properties(ListTests PROPERTIES PASS_REGULAR_EXPRESSION "[0-9]+ test cases")
set_tests_properties(ListTests PROPERTIES
PASS_REGULAR_EXPRESSION "[0-9]+ test cases"
FAIL_REGULAR_EXPRESSION "Hidden Test"
)
add_test(NAME ListTags COMMAND $<TARGET_FILE:SelfTest> --list-tags)
set_tests_properties(ListTags PROPERTIES PASS_REGULAR_EXPRESSION "[0-9]+ tags")
set_tests_properties(ListTags PROPERTIES
PASS_REGULAR_EXPRESSION "[0-9]+ tags"
FAIL_REGULAR_EXPRESSION "[.]")
add_test(NAME ListReporters COMMAND $<TARGET_FILE:SelfTest> --list-reporters)
set_tests_properties(ListReporters PROPERTIES PASS_REGULAR_EXPRESSION "Available reporters:")
add_test(NAME ListTestNamesOnly COMMAND $<TARGET_FILE:SelfTest> --list-test-names-only)
set_tests_properties(ListTestNamesOnly PROPERTIES PASS_REGULAR_EXPRESSION "Regex string matcher")
set_tests_properties(ListTestNamesOnly PROPERTIES
PASS_REGULAR_EXPRESSION "Regex string matcher"
FAIL_REGULAR_EXPRESSION "Hidden Test")
add_test(NAME NoAssertions COMMAND $<TARGET_FILE:SelfTest> -w NoAssertions)
set_tests_properties(NoAssertions PROPERTIES PASS_REGULAR_EXPRESSION "No assertions in test case")

View File

@ -15,12 +15,16 @@ namespace Catch {
: m_data( data ),
m_stream( openStream() )
{
if( !data.testsOrTags.empty() ) {
TestSpecParser parser( ITagAliasRegistry::get() );
TestSpecParser parser(ITagAliasRegistry::get());
if (data.testsOrTags.empty()) {
parser.parse("~[.]"); // All not hidden tests
}
else {
m_hasTestFilters = true;
for( auto const& testOrTags : data.testsOrTags )
parser.parse( testOrTags );
m_testSpec = parser.testSpec();
}
m_testSpec = parser.testSpec();
}
std::string const& Config::getFilename() const {
@ -39,6 +43,7 @@ namespace Catch {
std::vector<std::string> const& Config::getSectionsToRun() const { return m_data.sectionsToRun; }
TestSpec const& Config::testSpec() const { return m_testSpec; }
bool Config::hasTestFilters() const { return m_hasTestFilters; }
bool Config::showHelp() const { return m_data.showHelp; }

View File

@ -82,6 +82,7 @@ namespace Catch {
std::vector<std::string> const& getSectionsToRun() const override;
virtual TestSpec const& testSpec() const override;
bool hasTestFilters() const override;
bool showHelp() const;
@ -109,6 +110,7 @@ namespace Catch {
std::unique_ptr<IStream const> m_stream;
TestSpec m_testSpec;
bool m_hasTestFilters = false;
};
} // end namespace Catch

View File

@ -68,6 +68,7 @@ namespace Catch {
virtual bool showInvisibles() const = 0;
virtual ShowDurations::OrNot showDurations() const = 0;
virtual TestSpec const& testSpec() const = 0;
virtual bool hasTestFilters() const = 0;
virtual RunTests::InWhatOrder runOrder() const = 0;
virtual unsigned int rngSeed() const = 0;
virtual int benchmarkResolutionMultiple() const = 0;

View File

@ -28,11 +28,10 @@ namespace Catch {
std::size_t listTests( Config const& config ) {
TestSpec testSpec = config.testSpec();
if( config.testSpec().hasFilters() )
if( config.hasTestFilters() )
Catch::cout() << "Matching test cases:\n";
else {
Catch::cout() << "All available test cases:\n";
testSpec = TestSpecParser( ITagAliasRegistry::get() ).parse( "*" ).testSpec();
}
auto matchedTestCases = filterTests( getAllTestCasesSorted( config ), testSpec, config );
@ -54,7 +53,7 @@ namespace Catch {
Catch::cout() << Column( testCaseInfo.tagsAsString() ).indent( 6 ) << "\n";
}
if( !config.testSpec().hasFilters() )
if( !config.hasTestFilters() )
Catch::cout() << pluralise( matchedTestCases.size(), "test case" ) << '\n' << std::endl;
else
Catch::cout() << pluralise( matchedTestCases.size(), "matching test case" ) << '\n' << std::endl;
@ -63,8 +62,6 @@ namespace Catch {
std::size_t listTestsNamesOnly( Config const& config ) {
TestSpec testSpec = config.testSpec();
if( !config.testSpec().hasFilters() )
testSpec = TestSpecParser( ITagAliasRegistry::get() ).parse( "*" ).testSpec();
std::size_t matchedTests = 0;
std::vector<TestCase> matchedTestCases = filterTests( getAllTestCasesSorted( config ), testSpec, config );
for( auto const& testCaseInfo : matchedTestCases ) {
@ -94,11 +91,10 @@ namespace Catch {
std::size_t listTags( Config const& config ) {
TestSpec testSpec = config.testSpec();
if( config.testSpec().hasFilters() )
if( config.hasTestFilters() )
Catch::cout() << "Tags for matching test cases:\n";
else {
Catch::cout() << "All available tags:\n";
testSpec = TestSpecParser( ITagAliasRegistry::get() ).parse( "*" ).testSpec();
}
std::map<std::string, TagInfo> tagCounts;

View File

@ -70,8 +70,6 @@ namespace Catch {
context.testGroupStarting(config->name(), 1, 1);
TestSpec testSpec = config->testSpec();
if (!testSpec.hasFilters())
testSpec = TestSpecParser(ITagAliasRegistry::get()).parse("~[.]").testSpec(); // All not hidden tests
auto const& allTestCases = getAllTestCasesSorted(*config);
for (auto const& testCase : allTestCases) {

View File

@ -9,6 +9,7 @@ Compilation.tests.cpp:<line number>: passed: t1 < t2 for: {?} < {?}
Compilation.tests.cpp:<line number>: passed: t1 > t2 for: {?} > {?}
Compilation.tests.cpp:<line number>: passed: t1 <= t2 for: {?} <= {?}
Compilation.tests.cpp:<line number>: passed: t1 >= t2 for: {?} >= {?}
Misc.tests.cpp:<line number>: passed:
Exception.tests.cpp:<line number>: failed: unexpected exception with message: 'answer := 42' with 1 message: 'expected exception'
Exception.tests.cpp:<line number>: failed: unexpected exception with message: 'answer := 42'; expression was: thisThrows() with 1 message: 'expected exception'
Exception.tests.cpp:<line number>: passed: thisThrows() with 1 message: 'answer := 42'
@ -514,13 +515,17 @@ CmdLine.tests.cpp:<line number>: passed: config.shouldDebugBreak == false for: f
CmdLine.tests.cpp:<line number>: passed: config.abortAfter == -1 for: -1 == -1
CmdLine.tests.cpp:<line number>: passed: config.noThrow == false for: false == false
CmdLine.tests.cpp:<line number>: passed: config.reporterNames.empty() for: true
CmdLine.tests.cpp:<line number>: passed: !(cfg.hasTestFilters()) for: !false
CmdLine.tests.cpp:<line number>: passed: result for: {?}
CmdLine.tests.cpp:<line number>: passed: cfg.hasTestFilters() for: true
CmdLine.tests.cpp:<line number>: passed: cfg.testSpec().matches(fakeTestCase("notIncluded")) == false for: false == false
CmdLine.tests.cpp:<line number>: passed: cfg.testSpec().matches(fakeTestCase("test1")) for: true
CmdLine.tests.cpp:<line number>: passed: result for: {?}
CmdLine.tests.cpp:<line number>: passed: cfg.hasTestFilters() for: true
CmdLine.tests.cpp:<line number>: passed: cfg.testSpec().matches(fakeTestCase("test1")) == false for: false == false
CmdLine.tests.cpp:<line number>: passed: cfg.testSpec().matches(fakeTestCase("alwaysIncluded")) for: true
CmdLine.tests.cpp:<line number>: passed: result for: {?}
CmdLine.tests.cpp:<line number>: passed: cfg.hasTestFilters() for: true
CmdLine.tests.cpp:<line number>: passed: cfg.testSpec().matches(fakeTestCase("test1")) == false for: false == false
CmdLine.tests.cpp:<line number>: passed: cfg.testSpec().matches(fakeTestCase("alwaysIncluded")) for: true
CmdLine.tests.cpp:<line number>: passed: cli.parse({"test", "-r", "console"}) for: {?}

View File

@ -1064,6 +1064,6 @@ with expansion:
"first" == "second"
===============================================================================
test cases: 198 | 147 passed | 47 failed | 4 failed as expected
assertions: 999 | 873 passed | 105 failed | 21 failed as expected
test cases: 199 | 148 passed | 47 failed | 4 failed as expected
assertions: 1004 | 878 passed | 105 failed | 21 failed as expected

View File

@ -93,6 +93,15 @@ PASSED:
with expansion:
{?} >= {?}
-------------------------------------------------------------------------------
#1175 - Hidden Test
-------------------------------------------------------------------------------
Misc.tests.cpp:<line number>
...............................................................................
Misc.tests.cpp:<line number>:
PASSED:
-------------------------------------------------------------------------------
#748 - captures with unexpected exceptions
outside assertions
@ -3965,6 +3974,12 @@ PASSED:
with expansion:
true
CmdLine.tests.cpp:<line number>:
PASSED:
CHECK_FALSE( cfg.hasTestFilters() )
with expansion:
!false
-------------------------------------------------------------------------------
Process can be configured on command line
test lists
@ -3979,6 +3994,12 @@ PASSED:
with expansion:
{?}
CmdLine.tests.cpp:<line number>:
PASSED:
REQUIRE( cfg.hasTestFilters() )
with expansion:
true
CmdLine.tests.cpp:<line number>:
PASSED:
REQUIRE( cfg.testSpec().matches(fakeTestCase("notIncluded")) == false )
@ -4005,6 +4026,12 @@ PASSED:
with expansion:
{?}
CmdLine.tests.cpp:<line number>:
PASSED:
REQUIRE( cfg.hasTestFilters() )
with expansion:
true
CmdLine.tests.cpp:<line number>:
PASSED:
REQUIRE( cfg.testSpec().matches(fakeTestCase("test1")) == false )
@ -4031,6 +4058,12 @@ PASSED:
with expansion:
{?}
CmdLine.tests.cpp:<line number>:
PASSED:
REQUIRE( cfg.hasTestFilters() )
with expansion:
true
CmdLine.tests.cpp:<line number>:
PASSED:
REQUIRE( cfg.testSpec().matches(fakeTestCase("test1")) == false )
@ -8466,6 +8499,6 @@ Misc.tests.cpp:<line number>:
PASSED:
===============================================================================
test cases: 198 | 133 passed | 61 failed | 4 failed as expected
assertions: 1010 | 869 passed | 120 failed | 21 failed as expected
test cases: 199 | 134 passed | 61 failed | 4 failed as expected
assertions: 1015 | 874 passed | 120 failed | 21 failed as expected

View File

@ -93,6 +93,15 @@ PASSED:
with expansion:
{?} >= {?}
-------------------------------------------------------------------------------
#1175 - Hidden Test
-------------------------------------------------------------------------------
Misc.tests.cpp:<line number>
...............................................................................
Misc.tests.cpp:<line number>:
PASSED:
-------------------------------------------------------------------------------
#748 - captures with unexpected exceptions
outside assertions
@ -299,6 +308,6 @@ with expansion:
!true
===============================================================================
test cases: 11 | 8 passed | 1 failed | 2 failed as expected
assertions: 34 | 27 passed | 4 failed | 3 failed as expected
test cases: 12 | 9 passed | 1 failed | 2 failed as expected
assertions: 35 | 28 passed | 4 failed | 3 failed as expected

View File

@ -1,11 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuitesloose text artifact
>
<testsuite name="<exe-name>" errors="15" failures="106" tests="1011" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<testsuite name="<exe-name>" errors="15" failures="106" tests="1016" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<testcase classname="<exe-name>.global" name="# A test name that starts with a #" time="{duration}"/>
<testcase classname="<exe-name>.global" name="#1005: Comparing pointer to int and long (NULL can be either on various systems)" time="{duration}"/>
<testcase classname="<exe-name>.global" name="#1027" time="{duration}"/>
<testcase classname="<exe-name>.global" name="#1147" time="{duration}"/>
<testcase classname="<exe-name>.global" name="#1175 - Hidden Test" time="{duration}"/>
<testcase classname="<exe-name>.global" name="#748 - captures with unexpected exceptions/outside assertions" time="{duration}">
<error type="TEST_CASE">
expected exception

View File

@ -93,6 +93,9 @@
</Expression>
<OverallResult success="true"/>
</TestCase>
<TestCase name="#1175 - Hidden Test" tags="[.]" filename="projects/<exe-name>/UsageTests/Misc.tests.cpp" >
<OverallResult success="true"/>
</TestCase>
<TestCase name="#748 - captures with unexpected exceptions" tags="[!shouldfail][!throws][.][failing]" filename="projects/<exe-name>/UsageTests/Exception.tests.cpp" >
<Section name="outside assertions" filename="projects/<exe-name>/UsageTests/Exception.tests.cpp" >
<Info>
@ -4624,7 +4627,15 @@
true
</Expanded>
</Expression>
<OverallResults successes="6" failures="0" expectedFailures="0"/>
<Expression success="true" type="CHECK_FALSE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
!(cfg.hasTestFilters())
</Original>
<Expanded>
!false
</Expanded>
</Expression>
<OverallResults successes="7" failures="0" expectedFailures="0"/>
</Section>
<Section name="test lists" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Section name="1 test" description="Specify one test case using" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
@ -4636,6 +4647,14 @@
{?}
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
cfg.hasTestFilters()
</Original>
<Expanded>
true
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
cfg.testSpec().matches(fakeTestCase("notIncluded")) == false
@ -4652,9 +4671,9 @@
true
</Expanded>
</Expression>
<OverallResults successes="3" failures="0" expectedFailures="0"/>
<OverallResults successes="4" failures="0" expectedFailures="0"/>
</Section>
<OverallResults successes="3" failures="0" expectedFailures="0"/>
<OverallResults successes="4" failures="0" expectedFailures="0"/>
</Section>
<Section name="test lists" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Section name="Specify one test case exclusion using exclude:" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
@ -4666,6 +4685,14 @@
{?}
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
cfg.hasTestFilters()
</Original>
<Expanded>
true
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
cfg.testSpec().matches(fakeTestCase("test1")) == false
@ -4682,9 +4709,9 @@
true
</Expanded>
</Expression>
<OverallResults successes="3" failures="0" expectedFailures="0"/>
<OverallResults successes="4" failures="0" expectedFailures="0"/>
</Section>
<OverallResults successes="3" failures="0" expectedFailures="0"/>
<OverallResults successes="4" failures="0" expectedFailures="0"/>
</Section>
<Section name="test lists" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Section name="Specify one test case exclusion using ~" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
@ -4696,6 +4723,14 @@
{?}
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
cfg.hasTestFilters()
</Original>
<Expanded>
true
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
cfg.testSpec().matches(fakeTestCase("test1")) == false
@ -4712,9 +4747,9 @@
true
</Expanded>
</Expression>
<OverallResults successes="3" failures="0" expectedFailures="0"/>
<OverallResults successes="4" failures="0" expectedFailures="0"/>
</Section>
<OverallResults successes="3" failures="0" expectedFailures="0"/>
<OverallResults successes="4" failures="0" expectedFailures="0"/>
</Section>
<Section name="reporter" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Section name="-r/console" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
@ -9323,7 +9358,7 @@ loose text artifact
</Section>
<OverallResult success="true"/>
</TestCase>
<OverallResults successes="869" failures="121" expectedFailures="21"/>
<OverallResults successes="874" failures="121" expectedFailures="21"/>
</Group>
<OverallResults successes="869" failures="120" expectedFailures="21"/>
<OverallResults successes="874" failures="120" expectedFailures="21"/>
</Catch>

View File

@ -289,6 +289,9 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]"
CHECK(config.abortAfter == -1);
CHECK(config.noThrow == false);
CHECK(config.reporterNames.empty());
Catch::Config cfg(config);
CHECK_FALSE(cfg.hasTestFilters());
}
SECTION("test lists") {
@ -297,6 +300,7 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]"
CHECK(result);
Catch::Config cfg(config);
REQUIRE(cfg.hasTestFilters());
REQUIRE(cfg.testSpec().matches(fakeTestCase("notIncluded")) == false);
REQUIRE(cfg.testSpec().matches(fakeTestCase("test1")));
}
@ -305,6 +309,7 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]"
CHECK(result);
Catch::Config cfg(config);
REQUIRE(cfg.hasTestFilters());
REQUIRE(cfg.testSpec().matches(fakeTestCase("test1")) == false);
REQUIRE(cfg.testSpec().matches(fakeTestCase("alwaysIncluded")));
}
@ -314,6 +319,7 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]"
CHECK(result);
Catch::Config cfg(config);
REQUIRE(cfg.hasTestFilters());
REQUIRE(cfg.testSpec().matches(fakeTestCase("test1")) == false);
REQUIRE(cfg.testSpec().matches(fakeTestCase("alwaysIncluded")));
}

View File

@ -348,4 +348,9 @@ TEST_CASE( "#961 -- Dynamically created sections should all be reported", "[.]"
}
}
TEST_CASE( "#1175 - Hidden Test", "[.]" ) {
// Just for checking that hidden test is not listed by default
SUCCEED();
}
}} // namespace MiscTests