3rd-party
/
catch2
mirror of https://github.com/catchorg/Catch2.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

https://github.com/catchorg/Catch2/issues/1175 - don't list hidden tests by default

This commit is contained in:
dvirtz 2018-02-09 00:18:32 +02:00 committed by Martin Hořeňovský
parent 355b3f9952
commit ca8470fbad
14 changed files with 134 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
CMakeLists.txt
View File

@ -357,16 +357,23 @@ if (BUILD_TESTING AND NOT_SUBPROJECT)
add_test(NAME RunTests COMMAND $<TARGET_FILE:SelfTest>)
add_test(NAME ListTests COMMAND $<TARGET_FILE:SelfTest> --list-tests --verbosity high)
set_tests_properties(ListTests PROPERTIES PASS_REGULAR_EXPRESSION "[0-9]+ test cases")
set_tests_properties(ListTests PROPERTIES
PASS_REGULAR_EXPRESSION "[0-9]+ test cases"
FAIL_REGULAR_EXPRESSION "Hidden Test"
)
add_test(NAME ListTags COMMAND $<TARGET_FILE:SelfTest> --list-tags)
set_tests_properties(ListTags PROPERTIES PASS_REGULAR_EXPRESSION "[0-9]+ tags")
set_tests_properties(ListTags PROPERTIES
PASS_REGULAR_EXPRESSION "[0-9]+ tags"
FAIL_REGULAR_EXPRESSION "[.]")
add_test(NAME ListReporters COMMAND $<TARGET_FILE:SelfTest> --list-reporters)
set_tests_properties(ListReporters PROPERTIES PASS_REGULAR_EXPRESSION "Available reporters:")
add_test(NAME ListTestNamesOnly COMMAND $<TARGET_FILE:SelfTest> --list-test-names-only)
set_tests_properties(ListTestNamesOnly PROPERTIES PASS_REGULAR_EXPRESSION "Regex string matcher")
set_tests_properties(ListTestNamesOnly PROPERTIES
PASS_REGULAR_EXPRESSION "Regex string matcher"
FAIL_REGULAR_EXPRESSION "Hidden Test")
add_test(NAME NoAssertions COMMAND $<TARGET_FILE:SelfTest> -w NoAssertions)
set_tests_properties(NoAssertions PROPERTIES PASS_REGULAR_EXPRESSION "No assertions in test case")

9
include/internal/catch_config.cpp
View File

@ -15,12 +15,16 @@ namespace Catch {
: m_data( data ),
m_stream( openStream() )
{
if( !data.testsOrTags.empty() ) {
TestSpecParser parser(ITagAliasRegistry::get());
if (data.testsOrTags.empty()) {
parser.parse("~[.]"); // All not hidden tests
}
else {
m_hasTestFilters = true;
for( auto const& testOrTags : data.testsOrTags )
parser.parse( testOrTags );
m_testSpec = parser.testSpec();
}
m_testSpec = parser.testSpec();
}
std::string const& Config::getFilename() const {
@ -39,6 +43,7 @@ namespace Catch {
std::vector<std::string> const& Config::getSectionsToRun() const { return m_data.sectionsToRun; }
TestSpec const& Config::testSpec() const { return m_testSpec; }
bool Config::hasTestFilters() const { return m_hasTestFilters; }
bool Config::showHelp() const { return m_data.showHelp; }

2
include/internal/catch_config.hpp
View File

@ -82,6 +82,7 @@ namespace Catch {
std::vector<std::string> const& getSectionsToRun() const override;
virtual TestSpec const& testSpec() const override;
bool hasTestFilters() const override;
bool showHelp() const;
@ -109,6 +110,7 @@ namespace Catch {
std::unique_ptr<IStream const> m_stream;
TestSpec m_testSpec;
bool m_hasTestFilters = false;
};
} // end namespace Catch

1
include/internal/catch_interfaces_config.h
View File

@ -68,6 +68,7 @@ namespace Catch {
virtual bool showInvisibles() const = 0;
virtual ShowDurations::OrNot showDurations() const = 0;
virtual TestSpec const& testSpec() const = 0;
virtual bool hasTestFilters() const = 0;
virtual RunTests::InWhatOrder runOrder() const = 0;
virtual unsigned int rngSeed() const = 0;
virtual int benchmarkResolutionMultiple() const = 0;

10
include/internal/catch_list.cpp
View File

@ -28,11 +28,10 @@ namespace Catch {
std::size_t listTests( Config const& config ) {
TestSpec testSpec = config.testSpec();
if( config.testSpec().hasFilters() )
if( config.hasTestFilters() )
Catch::cout() << "Matching test cases:\n";
else {
Catch::cout() << "All available test cases:\n";
testSpec = TestSpecParser( ITagAliasRegistry::get() ).parse( "*" ).testSpec();
}
auto matchedTestCases = filterTests( getAllTestCasesSorted( config ), testSpec, config );
@ -54,7 +53,7 @@ namespace Catch {
Catch::cout() << Column( testCaseInfo.tagsAsString() ).indent( 6 ) << "\n";
}
if( !config.testSpec().hasFilters() )
if( !config.hasTestFilters() )
Catch::cout() << pluralise( matchedTestCases.size(), "test case" ) << '\n' << std::endl;
else
Catch::cout() << pluralise( matchedTestCases.size(), "matching test case" ) << '\n' << std::endl;
@ -63,8 +62,6 @@ namespace Catch {
std::size_t listTestsNamesOnly( Config const& config ) {
TestSpec testSpec = config.testSpec();
if( !config.testSpec().hasFilters() )
testSpec = TestSpecParser( ITagAliasRegistry::get() ).parse( "*" ).testSpec();
std::size_t matchedTests = 0;
std::vector<TestCase> matchedTestCases = filterTests( getAllTestCasesSorted( config ), testSpec, config );
for( auto const& testCaseInfo : matchedTestCases ) {
@ -94,11 +91,10 @@ namespace Catch {
std::size_t listTags( Config const& config ) {
TestSpec testSpec = config.testSpec();
if( config.testSpec().hasFilters() )
if( config.hasTestFilters() )
Catch::cout() << "Tags for matching test cases:\n";
else {
Catch::cout() << "All available tags:\n";
testSpec = TestSpecParser( ITagAliasRegistry::get() ).parse( "*" ).testSpec();
}
std::map<std::string, TagInfo> tagCounts;

2
include/internal/catch_session.cpp
View File

@ -70,8 +70,6 @@ namespace Catch {
context.testGroupStarting(config->name(), 1, 1);
TestSpec testSpec = config->testSpec();
if (!testSpec.hasFilters())
testSpec = TestSpecParser(ITagAliasRegistry::get()).parse("~[.]").testSpec(); // All not hidden tests
auto const& allTestCases = getAllTestCasesSorted(*config);
for (auto const& testCase : allTestCases) {

5
projects/SelfTest/Baselines/compact.sw.approved.txt
View File

@ -9,6 +9,7 @@ Compilation.tests.cpp:<line number>: passed: t1 < t2 for: {?} < {?}
Compilation.tests.cpp:<line number>: passed: t1 > t2 for: {?} > {?}
Compilation.tests.cpp:<line number>: passed: t1 <= t2 for: {?} <= {?}
Compilation.tests.cpp:<line number>: passed: t1 >= t2 for: {?} >= {?}
Misc.tests.cpp:<line number>: passed:
Exception.tests.cpp:<line number>: failed: unexpected exception with message: 'answer := 42' with 1 message: 'expected exception'
Exception.tests.cpp:<line number>: failed: unexpected exception with message: 'answer := 42'; expression was: thisThrows() with 1 message: 'expected exception'
Exception.tests.cpp:<line number>: passed: thisThrows() with 1 message: 'answer := 42'
@ -514,13 +515,17 @@ CmdLine.tests.cpp:<line number>: passed: config.shouldDebugBreak == false for: f
CmdLine.tests.cpp:<line number>: passed: config.abortAfter == -1 for: -1 == -1
CmdLine.tests.cpp:<line number>: passed: config.noThrow == false for: false == false
CmdLine.tests.cpp:<line number>: passed: config.reporterNames.empty() for: true
CmdLine.tests.cpp:<line number>: passed: !(cfg.hasTestFilters()) for: !false
CmdLine.tests.cpp:<line number>: passed: result for: {?}
CmdLine.tests.cpp:<line number>: passed: cfg.hasTestFilters() for: true
CmdLine.tests.cpp:<line number>: passed: cfg.testSpec().matches(fakeTestCase("notIncluded")) == false for: false == false
CmdLine.tests.cpp:<line number>: passed: cfg.testSpec().matches(fakeTestCase("test1")) for: true
CmdLine.tests.cpp:<line number>: passed: result for: {?}
CmdLine.tests.cpp:<line number>: passed: cfg.hasTestFilters() for: true
CmdLine.tests.cpp:<line number>: passed: cfg.testSpec().matches(fakeTestCase("test1")) == false for: false == false
CmdLine.tests.cpp:<line number>: passed: cfg.testSpec().matches(fakeTestCase("alwaysIncluded")) for: true
CmdLine.tests.cpp:<line number>: passed: result for: {?}
CmdLine.tests.cpp:<line number>: passed: cfg.hasTestFilters() for: true
CmdLine.tests.cpp:<line number>: passed: cfg.testSpec().matches(fakeTestCase("test1")) == false for: false == false
CmdLine.tests.cpp:<line number>: passed: cfg.testSpec().matches(fakeTestCase("alwaysIncluded")) for: true
CmdLine.tests.cpp:<line number>: passed: cli.parse({"test", "-r", "console"}) for: {?}

4
projects/SelfTest/Baselines/console.std.approved.txt
View File

@ -1064,6 +1064,6 @@ with expansion:
"first" == "second"
===============================================================================
test cases: 198 | 147 passed | 47 failed | 4 failed as expected
assertions: 999 | 873 passed | 105 failed | 21 failed as expected
test cases: 199 | 148 passed | 47 failed | 4 failed as expected
assertions: 1004 | 878 passed | 105 failed | 21 failed as expected

37
projects/SelfTest/Baselines/console.sw.approved.txt
View File

@ -93,6 +93,15 @@ PASSED:
with expansion:
{?} >= {?}
-------------------------------------------------------------------------------
#1175 - Hidden Test
-------------------------------------------------------------------------------
Misc.tests.cpp:<line number>
...............................................................................
Misc.tests.cpp:<line number>:
PASSED:
-------------------------------------------------------------------------------
#748 - captures with unexpected exceptions
outside assertions
@ -3965,6 +3974,12 @@ PASSED:
with expansion:
true
CmdLine.tests.cpp:<line number>:
PASSED:
CHECK_FALSE( cfg.hasTestFilters() )
with expansion:
!false
-------------------------------------------------------------------------------
Process can be configured on command line
test lists
@ -3979,6 +3994,12 @@ PASSED:
with expansion:
{?}
CmdLine.tests.cpp:<line number>:
PASSED:
REQUIRE( cfg.hasTestFilters() )
with expansion:
true
CmdLine.tests.cpp:<line number>:
PASSED:
REQUIRE( cfg.testSpec().matches(fakeTestCase("notIncluded")) == false )
@ -4005,6 +4026,12 @@ PASSED:
with expansion:
{?}
CmdLine.tests.cpp:<line number>:
PASSED:
REQUIRE( cfg.hasTestFilters() )
with expansion:
true
CmdLine.tests.cpp:<line number>:
PASSED:
REQUIRE( cfg.testSpec().matches(fakeTestCase("test1")) == false )
@ -4031,6 +4058,12 @@ PASSED:
with expansion:
{?}
CmdLine.tests.cpp:<line number>:
PASSED:
REQUIRE( cfg.hasTestFilters() )
with expansion:
true
CmdLine.tests.cpp:<line number>:
PASSED:
REQUIRE( cfg.testSpec().matches(fakeTestCase("test1")) == false )
@ -8466,6 +8499,6 @@ Misc.tests.cpp:<line number>:
PASSED:
===============================================================================
test cases: 198 | 133 passed | 61 failed | 4 failed as expected
assertions: 1010 | 869 passed | 120 failed | 21 failed as expected
test cases: 199 | 134 passed | 61 failed | 4 failed as expected
assertions: 1015 | 874 passed | 120 failed | 21 failed as expected

13
projects/SelfTest/Baselines/console.swa4.approved.txt
View File

@ -93,6 +93,15 @@ PASSED:
with expansion:
{?} >= {?}
-------------------------------------------------------------------------------
#1175 - Hidden Test
-------------------------------------------------------------------------------
Misc.tests.cpp:<line number>
...............................................................................
Misc.tests.cpp:<line number>:
PASSED:
-------------------------------------------------------------------------------
#748 - captures with unexpected exceptions
outside assertions
@ -299,6 +308,6 @@ with expansion:
!true
===============================================================================
test cases: 11 | 8 passed | 1 failed | 2 failed as expected
assertions: 34 | 27 passed | 4 failed | 3 failed as expected
test cases: 12 | 9 passed | 1 failed | 2 failed as expected
assertions: 35 | 28 passed | 4 failed | 3 failed as expected

3
projects/SelfTest/Baselines/junit.sw.approved.txt
View File

@ -1,11 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuitesloose text artifact
>
<testsuite name="<exe-name>" errors="15" failures="106" tests="1011" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<testsuite name="<exe-name>" errors="15" failures="106" tests="1016" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<testcase classname="<exe-name>.global" name="# A test name that starts with a #" time="{duration}"/>
<testcase classname="<exe-name>.global" name="#1005: Comparing pointer to int and long (NULL can be either on various systems)" time="{duration}"/>
<testcase classname="<exe-name>.global" name="#1027" time="{duration}"/>
<testcase classname="<exe-name>.global" name="#1147" time="{duration}"/>
<testcase classname="<exe-name>.global" name="#1175 - Hidden Test" time="{duration}"/>
<testcase classname="<exe-name>.global" name="#748 - captures with unexpected exceptions/outside assertions" time="{duration}">
<error type="TEST_CASE">
expected exception

53
projects/SelfTest/Baselines/xml.sw.approved.txt
View File

@ -93,6 +93,9 @@
</Expression>
<OverallResult success="true"/>
</TestCase>
<TestCase name="#1175 - Hidden Test" tags="[.]" filename="projects/<exe-name>/UsageTests/Misc.tests.cpp" >
<OverallResult success="true"/>
</TestCase>
<TestCase name="#748 - captures with unexpected exceptions" tags="[!shouldfail][!throws][.][failing]" filename="projects/<exe-name>/UsageTests/Exception.tests.cpp" >
<Section name="outside assertions" filename="projects/<exe-name>/UsageTests/Exception.tests.cpp" >
<Info>
@ -4624,7 +4627,15 @@
true
</Expanded>
</Expression>
<OverallResults successes="6" failures="0" expectedFailures="0"/>
<Expression success="true" type="CHECK_FALSE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
!(cfg.hasTestFilters())
</Original>
<Expanded>
!false
</Expanded>
</Expression>
<OverallResults successes="7" failures="0" expectedFailures="0"/>
</Section>
<Section name="test lists" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Section name="1 test" description="Specify one test case using" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
@ -4636,6 +4647,14 @@
{?}
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
cfg.hasTestFilters()
</Original>
<Expanded>
true
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
cfg.testSpec().matches(fakeTestCase("notIncluded")) == false
@ -4652,9 +4671,9 @@
true
</Expanded>
</Expression>
<OverallResults successes="3" failures="0" expectedFailures="0"/>
<OverallResults successes="4" failures="0" expectedFailures="0"/>
</Section>
<OverallResults successes="3" failures="0" expectedFailures="0"/>
<OverallResults successes="4" failures="0" expectedFailures="0"/>
</Section>
<Section name="test lists" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Section name="Specify one test case exclusion using exclude:" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
@ -4666,6 +4685,14 @@
{?}
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
cfg.hasTestFilters()
</Original>
<Expanded>
true
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
cfg.testSpec().matches(fakeTestCase("test1")) == false
@ -4682,9 +4709,9 @@
true
</Expanded>
</Expression>
<OverallResults successes="3" failures="0" expectedFailures="0"/>
<OverallResults successes="4" failures="0" expectedFailures="0"/>
</Section>
<OverallResults successes="3" failures="0" expectedFailures="0"/>
<OverallResults successes="4" failures="0" expectedFailures="0"/>
</Section>
<Section name="test lists" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Section name="Specify one test case exclusion using ~" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
@ -4696,6 +4723,14 @@
{?}
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
cfg.hasTestFilters()
</Original>
<Expanded>
true
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
cfg.testSpec().matches(fakeTestCase("test1")) == false
@ -4712,9 +4747,9 @@
true
</Expanded>
</Expression>
<OverallResults successes="3" failures="0" expectedFailures="0"/>
<OverallResults successes="4" failures="0" expectedFailures="0"/>
</Section>
<OverallResults successes="3" failures="0" expectedFailures="0"/>
<OverallResults successes="4" failures="0" expectedFailures="0"/>
</Section>
<Section name="reporter" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Section name="-r/console" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
@ -9323,7 +9358,7 @@ loose text artifact
</Section>
<OverallResult success="true"/>
</TestCase>
<OverallResults successes="869" failures="121" expectedFailures="21"/>
<OverallResults successes="874" failures="121" expectedFailures="21"/>
</Group>
<OverallResults successes="869" failures="120" expectedFailures="21"/>
<OverallResults successes="874" failures="120" expectedFailures="21"/>
</Catch>

6
projects/SelfTest/IntrospectiveTests/CmdLine.tests.cpp
View File

@ -289,6 +289,9 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]"
CHECK(config.abortAfter == -1);
CHECK(config.noThrow == false);
CHECK(config.reporterNames.empty());
Catch::Config cfg(config);
CHECK_FALSE(cfg.hasTestFilters());
}
SECTION("test lists") {
@ -297,6 +300,7 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]"
CHECK(result);
Catch::Config cfg(config);
REQUIRE(cfg.hasTestFilters());
REQUIRE(cfg.testSpec().matches(fakeTestCase("notIncluded")) == false);
REQUIRE(cfg.testSpec().matches(fakeTestCase("test1")));
}
@ -305,6 +309,7 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]"
CHECK(result);
Catch::Config cfg(config);
REQUIRE(cfg.hasTestFilters());
REQUIRE(cfg.testSpec().matches(fakeTestCase("test1")) == false);
REQUIRE(cfg.testSpec().matches(fakeTestCase("alwaysIncluded")));
}
@ -314,6 +319,7 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]"
CHECK(result);
Catch::Config cfg(config);
REQUIRE(cfg.hasTestFilters());
REQUIRE(cfg.testSpec().matches(fakeTestCase("test1")) == false);
REQUIRE(cfg.testSpec().matches(fakeTestCase("alwaysIncluded")));
}

5
projects/SelfTest/UsageTests/Misc.tests.cpp
View File

@ -348,4 +348,9 @@ TEST_CASE( "#961 -- Dynamically created sections should all be reported", "[.]"
}
}
TEST_CASE( "#1175 - Hidden Test", "[.]" ) {
// Just for checking that hidden test is not listed by default
SUCCEED();
}
}} // namespace MiscTests