From ca8470fbad3447e16b738fb6f93606dfee03f424 Mon Sep 17 00:00:00 2001 From: dvirtz Date: Fri, 9 Feb 2018 00:18:32 +0200 Subject: [PATCH] https://github.com/catchorg/Catch2/issues/1175 - don't list hidden tests by default --- CMakeLists.txt | 13 +++-- include/internal/catch_config.cpp | 11 ++-- include/internal/catch_config.hpp | 2 + include/internal/catch_interfaces_config.h | 1 + include/internal/catch_list.cpp | 10 ++-- include/internal/catch_session.cpp | 2 - .../Baselines/compact.sw.approved.txt | 5 ++ .../Baselines/console.std.approved.txt | 4 +- .../Baselines/console.sw.approved.txt | 37 ++++++++++++- .../Baselines/console.swa4.approved.txt | 13 ++++- .../SelfTest/Baselines/junit.sw.approved.txt | 3 +- .../SelfTest/Baselines/xml.sw.approved.txt | 53 +++++++++++++++---- .../IntrospectiveTests/CmdLine.tests.cpp | 6 +++ projects/SelfTest/UsageTests/Misc.tests.cpp | 5 ++ 14 files changed, 134 insertions(+), 31 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c07bc470..cf7e5965 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -357,16 +357,23 @@ if (BUILD_TESTING AND NOT_SUBPROJECT) add_test(NAME RunTests COMMAND $) add_test(NAME ListTests COMMAND $ --list-tests --verbosity high) - set_tests_properties(ListTests PROPERTIES PASS_REGULAR_EXPRESSION "[0-9]+ test cases") + set_tests_properties(ListTests PROPERTIES + PASS_REGULAR_EXPRESSION "[0-9]+ test cases" + FAIL_REGULAR_EXPRESSION "Hidden Test" + ) add_test(NAME ListTags COMMAND $ --list-tags) - set_tests_properties(ListTags PROPERTIES PASS_REGULAR_EXPRESSION "[0-9]+ tags") + set_tests_properties(ListTags PROPERTIES + PASS_REGULAR_EXPRESSION "[0-9]+ tags" + FAIL_REGULAR_EXPRESSION "[.]") add_test(NAME ListReporters COMMAND $ --list-reporters) set_tests_properties(ListReporters PROPERTIES PASS_REGULAR_EXPRESSION "Available reporters:") add_test(NAME ListTestNamesOnly COMMAND $ --list-test-names-only) - set_tests_properties(ListTestNamesOnly PROPERTIES PASS_REGULAR_EXPRESSION "Regex string matcher") + set_tests_properties(ListTestNamesOnly PROPERTIES + PASS_REGULAR_EXPRESSION "Regex string matcher" + FAIL_REGULAR_EXPRESSION "Hidden Test") add_test(NAME NoAssertions COMMAND $ -w NoAssertions) set_tests_properties(NoAssertions PROPERTIES PASS_REGULAR_EXPRESSION "No assertions in test case") diff --git a/include/internal/catch_config.cpp b/include/internal/catch_config.cpp index 5fda44ff..1db94119 100644 --- a/include/internal/catch_config.cpp +++ b/include/internal/catch_config.cpp @@ -15,12 +15,16 @@ namespace Catch { : m_data( data ), m_stream( openStream() ) { - if( !data.testsOrTags.empty() ) { - TestSpecParser parser( ITagAliasRegistry::get() ); + TestSpecParser parser(ITagAliasRegistry::get()); + if (data.testsOrTags.empty()) { + parser.parse("~[.]"); // All not hidden tests + } + else { + m_hasTestFilters = true; for( auto const& testOrTags : data.testsOrTags ) parser.parse( testOrTags ); - m_testSpec = parser.testSpec(); } + m_testSpec = parser.testSpec(); } std::string const& Config::getFilename() const { @@ -39,6 +43,7 @@ namespace Catch { std::vector const& Config::getSectionsToRun() const { return m_data.sectionsToRun; } TestSpec const& Config::testSpec() const { return m_testSpec; } + bool Config::hasTestFilters() const { return m_hasTestFilters; } bool Config::showHelp() const { return m_data.showHelp; } diff --git a/include/internal/catch_config.hpp b/include/internal/catch_config.hpp index 21e04069..3afdff5e 100644 --- a/include/internal/catch_config.hpp +++ b/include/internal/catch_config.hpp @@ -82,6 +82,7 @@ namespace Catch { std::vector const& getSectionsToRun() const override; virtual TestSpec const& testSpec() const override; + bool hasTestFilters() const override; bool showHelp() const; @@ -109,6 +110,7 @@ namespace Catch { std::unique_ptr m_stream; TestSpec m_testSpec; + bool m_hasTestFilters = false; }; } // end namespace Catch diff --git a/include/internal/catch_interfaces_config.h b/include/internal/catch_interfaces_config.h index 35ec82cf..f509c7ed 100644 --- a/include/internal/catch_interfaces_config.h +++ b/include/internal/catch_interfaces_config.h @@ -68,6 +68,7 @@ namespace Catch { virtual bool showInvisibles() const = 0; virtual ShowDurations::OrNot showDurations() const = 0; virtual TestSpec const& testSpec() const = 0; + virtual bool hasTestFilters() const = 0; virtual RunTests::InWhatOrder runOrder() const = 0; virtual unsigned int rngSeed() const = 0; virtual int benchmarkResolutionMultiple() const = 0; diff --git a/include/internal/catch_list.cpp b/include/internal/catch_list.cpp index 3efad0b7..3b0e33f1 100644 --- a/include/internal/catch_list.cpp +++ b/include/internal/catch_list.cpp @@ -28,11 +28,10 @@ namespace Catch { std::size_t listTests( Config const& config ) { TestSpec testSpec = config.testSpec(); - if( config.testSpec().hasFilters() ) + if( config.hasTestFilters() ) Catch::cout() << "Matching test cases:\n"; else { Catch::cout() << "All available test cases:\n"; - testSpec = TestSpecParser( ITagAliasRegistry::get() ).parse( "*" ).testSpec(); } auto matchedTestCases = filterTests( getAllTestCasesSorted( config ), testSpec, config ); @@ -54,7 +53,7 @@ namespace Catch { Catch::cout() << Column( testCaseInfo.tagsAsString() ).indent( 6 ) << "\n"; } - if( !config.testSpec().hasFilters() ) + if( !config.hasTestFilters() ) Catch::cout() << pluralise( matchedTestCases.size(), "test case" ) << '\n' << std::endl; else Catch::cout() << pluralise( matchedTestCases.size(), "matching test case" ) << '\n' << std::endl; @@ -63,8 +62,6 @@ namespace Catch { std::size_t listTestsNamesOnly( Config const& config ) { TestSpec testSpec = config.testSpec(); - if( !config.testSpec().hasFilters() ) - testSpec = TestSpecParser( ITagAliasRegistry::get() ).parse( "*" ).testSpec(); std::size_t matchedTests = 0; std::vector matchedTestCases = filterTests( getAllTestCasesSorted( config ), testSpec, config ); for( auto const& testCaseInfo : matchedTestCases ) { @@ -94,11 +91,10 @@ namespace Catch { std::size_t listTags( Config const& config ) { TestSpec testSpec = config.testSpec(); - if( config.testSpec().hasFilters() ) + if( config.hasTestFilters() ) Catch::cout() << "Tags for matching test cases:\n"; else { Catch::cout() << "All available tags:\n"; - testSpec = TestSpecParser( ITagAliasRegistry::get() ).parse( "*" ).testSpec(); } std::map tagCounts; diff --git a/include/internal/catch_session.cpp b/include/internal/catch_session.cpp index e170f7ca..71317eba 100644 --- a/include/internal/catch_session.cpp +++ b/include/internal/catch_session.cpp @@ -70,8 +70,6 @@ namespace Catch { context.testGroupStarting(config->name(), 1, 1); TestSpec testSpec = config->testSpec(); - if (!testSpec.hasFilters()) - testSpec = TestSpecParser(ITagAliasRegistry::get()).parse("~[.]").testSpec(); // All not hidden tests auto const& allTestCases = getAllTestCasesSorted(*config); for (auto const& testCase : allTestCases) { diff --git a/projects/SelfTest/Baselines/compact.sw.approved.txt b/projects/SelfTest/Baselines/compact.sw.approved.txt index 57afb2fd..39ca4347 100644 --- a/projects/SelfTest/Baselines/compact.sw.approved.txt +++ b/projects/SelfTest/Baselines/compact.sw.approved.txt @@ -9,6 +9,7 @@ Compilation.tests.cpp:: passed: t1 < t2 for: {?} < {?} Compilation.tests.cpp:: passed: t1 > t2 for: {?} > {?} Compilation.tests.cpp:: passed: t1 <= t2 for: {?} <= {?} Compilation.tests.cpp:: passed: t1 >= t2 for: {?} >= {?} +Misc.tests.cpp:: passed: Exception.tests.cpp:: failed: unexpected exception with message: 'answer := 42' with 1 message: 'expected exception' Exception.tests.cpp:: failed: unexpected exception with message: 'answer := 42'; expression was: thisThrows() with 1 message: 'expected exception' Exception.tests.cpp:: passed: thisThrows() with 1 message: 'answer := 42' @@ -514,13 +515,17 @@ CmdLine.tests.cpp:: passed: config.shouldDebugBreak == false for: f CmdLine.tests.cpp:: passed: config.abortAfter == -1 for: -1 == -1 CmdLine.tests.cpp:: passed: config.noThrow == false for: false == false CmdLine.tests.cpp:: passed: config.reporterNames.empty() for: true +CmdLine.tests.cpp:: passed: !(cfg.hasTestFilters()) for: !false CmdLine.tests.cpp:: passed: result for: {?} +CmdLine.tests.cpp:: passed: cfg.hasTestFilters() for: true CmdLine.tests.cpp:: passed: cfg.testSpec().matches(fakeTestCase("notIncluded")) == false for: false == false CmdLine.tests.cpp:: passed: cfg.testSpec().matches(fakeTestCase("test1")) for: true CmdLine.tests.cpp:: passed: result for: {?} +CmdLine.tests.cpp:: passed: cfg.hasTestFilters() for: true CmdLine.tests.cpp:: passed: cfg.testSpec().matches(fakeTestCase("test1")) == false for: false == false CmdLine.tests.cpp:: passed: cfg.testSpec().matches(fakeTestCase("alwaysIncluded")) for: true CmdLine.tests.cpp:: passed: result for: {?} +CmdLine.tests.cpp:: passed: cfg.hasTestFilters() for: true CmdLine.tests.cpp:: passed: cfg.testSpec().matches(fakeTestCase("test1")) == false for: false == false CmdLine.tests.cpp:: passed: cfg.testSpec().matches(fakeTestCase("alwaysIncluded")) for: true CmdLine.tests.cpp:: passed: cli.parse({"test", "-r", "console"}) for: {?} diff --git a/projects/SelfTest/Baselines/console.std.approved.txt b/projects/SelfTest/Baselines/console.std.approved.txt index 1e9d2c31..cbea6267 100644 --- a/projects/SelfTest/Baselines/console.std.approved.txt +++ b/projects/SelfTest/Baselines/console.std.approved.txt @@ -1064,6 +1064,6 @@ with expansion: "first" == "second" =============================================================================== -test cases: 198 | 147 passed | 47 failed | 4 failed as expected -assertions: 999 | 873 passed | 105 failed | 21 failed as expected +test cases: 199 | 148 passed | 47 failed | 4 failed as expected +assertions: 1004 | 878 passed | 105 failed | 21 failed as expected diff --git a/projects/SelfTest/Baselines/console.sw.approved.txt b/projects/SelfTest/Baselines/console.sw.approved.txt index fd9dfb6a..db544f8c 100644 --- a/projects/SelfTest/Baselines/console.sw.approved.txt +++ b/projects/SelfTest/Baselines/console.sw.approved.txt @@ -93,6 +93,15 @@ PASSED: with expansion: {?} >= {?} +------------------------------------------------------------------------------- +#1175 - Hidden Test +------------------------------------------------------------------------------- +Misc.tests.cpp: +............................................................................... + +Misc.tests.cpp:: +PASSED: + ------------------------------------------------------------------------------- #748 - captures with unexpected exceptions outside assertions @@ -3965,6 +3974,12 @@ PASSED: with expansion: true +CmdLine.tests.cpp:: +PASSED: + CHECK_FALSE( cfg.hasTestFilters() ) +with expansion: + !false + ------------------------------------------------------------------------------- Process can be configured on command line test lists @@ -3979,6 +3994,12 @@ PASSED: with expansion: {?} +CmdLine.tests.cpp:: +PASSED: + REQUIRE( cfg.hasTestFilters() ) +with expansion: + true + CmdLine.tests.cpp:: PASSED: REQUIRE( cfg.testSpec().matches(fakeTestCase("notIncluded")) == false ) @@ -4005,6 +4026,12 @@ PASSED: with expansion: {?} +CmdLine.tests.cpp:: +PASSED: + REQUIRE( cfg.hasTestFilters() ) +with expansion: + true + CmdLine.tests.cpp:: PASSED: REQUIRE( cfg.testSpec().matches(fakeTestCase("test1")) == false ) @@ -4031,6 +4058,12 @@ PASSED: with expansion: {?} +CmdLine.tests.cpp:: +PASSED: + REQUIRE( cfg.hasTestFilters() ) +with expansion: + true + CmdLine.tests.cpp:: PASSED: REQUIRE( cfg.testSpec().matches(fakeTestCase("test1")) == false ) @@ -8466,6 +8499,6 @@ Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 198 | 133 passed | 61 failed | 4 failed as expected -assertions: 1010 | 869 passed | 120 failed | 21 failed as expected +test cases: 199 | 134 passed | 61 failed | 4 failed as expected +assertions: 1015 | 874 passed | 120 failed | 21 failed as expected diff --git a/projects/SelfTest/Baselines/console.swa4.approved.txt b/projects/SelfTest/Baselines/console.swa4.approved.txt index 6232f8d9..9b41d398 100644 --- a/projects/SelfTest/Baselines/console.swa4.approved.txt +++ b/projects/SelfTest/Baselines/console.swa4.approved.txt @@ -93,6 +93,15 @@ PASSED: with expansion: {?} >= {?} +------------------------------------------------------------------------------- +#1175 - Hidden Test +------------------------------------------------------------------------------- +Misc.tests.cpp: +............................................................................... + +Misc.tests.cpp:: +PASSED: + ------------------------------------------------------------------------------- #748 - captures with unexpected exceptions outside assertions @@ -299,6 +308,6 @@ with expansion: !true =============================================================================== -test cases: 11 | 8 passed | 1 failed | 2 failed as expected -assertions: 34 | 27 passed | 4 failed | 3 failed as expected +test cases: 12 | 9 passed | 1 failed | 2 failed as expected +assertions: 35 | 28 passed | 4 failed | 3 failed as expected diff --git a/projects/SelfTest/Baselines/junit.sw.approved.txt b/projects/SelfTest/Baselines/junit.sw.approved.txt index 7c0c67f6..2c6a6fc7 100644 --- a/projects/SelfTest/Baselines/junit.sw.approved.txt +++ b/projects/SelfTest/Baselines/junit.sw.approved.txt @@ -1,11 +1,12 @@ - + + expected exception diff --git a/projects/SelfTest/Baselines/xml.sw.approved.txt b/projects/SelfTest/Baselines/xml.sw.approved.txt index c08b575a..af22a1bc 100644 --- a/projects/SelfTest/Baselines/xml.sw.approved.txt +++ b/projects/SelfTest/Baselines/xml.sw.approved.txt @@ -93,6 +93,9 @@ + + +
@@ -4624,7 +4627,15 @@ true - + + + !(cfg.hasTestFilters()) + + + !false + + +
@@ -4636,6 +4647,14 @@ {?} + + + cfg.hasTestFilters() + + + true + + cfg.testSpec().matches(fakeTestCase("notIncluded")) == false @@ -4652,9 +4671,9 @@ true - +
- +
@@ -4666,6 +4685,14 @@ {?} + + + cfg.hasTestFilters() + + + true + + cfg.testSpec().matches(fakeTestCase("test1")) == false @@ -4682,9 +4709,9 @@ true - +
- +
@@ -4696,6 +4723,14 @@ {?} + + + cfg.hasTestFilters() + + + true + + cfg.testSpec().matches(fakeTestCase("test1")) == false @@ -4712,9 +4747,9 @@ true - +
- +
@@ -9323,7 +9358,7 @@ loose text artifact
- + - + diff --git a/projects/SelfTest/IntrospectiveTests/CmdLine.tests.cpp b/projects/SelfTest/IntrospectiveTests/CmdLine.tests.cpp index a0735852..075e7d34 100644 --- a/projects/SelfTest/IntrospectiveTests/CmdLine.tests.cpp +++ b/projects/SelfTest/IntrospectiveTests/CmdLine.tests.cpp @@ -289,6 +289,9 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]" CHECK(config.abortAfter == -1); CHECK(config.noThrow == false); CHECK(config.reporterNames.empty()); + + Catch::Config cfg(config); + CHECK_FALSE(cfg.hasTestFilters()); } SECTION("test lists") { @@ -297,6 +300,7 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]" CHECK(result); Catch::Config cfg(config); + REQUIRE(cfg.hasTestFilters()); REQUIRE(cfg.testSpec().matches(fakeTestCase("notIncluded")) == false); REQUIRE(cfg.testSpec().matches(fakeTestCase("test1"))); } @@ -305,6 +309,7 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]" CHECK(result); Catch::Config cfg(config); + REQUIRE(cfg.hasTestFilters()); REQUIRE(cfg.testSpec().matches(fakeTestCase("test1")) == false); REQUIRE(cfg.testSpec().matches(fakeTestCase("alwaysIncluded"))); } @@ -314,6 +319,7 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]" CHECK(result); Catch::Config cfg(config); + REQUIRE(cfg.hasTestFilters()); REQUIRE(cfg.testSpec().matches(fakeTestCase("test1")) == false); REQUIRE(cfg.testSpec().matches(fakeTestCase("alwaysIncluded"))); } diff --git a/projects/SelfTest/UsageTests/Misc.tests.cpp b/projects/SelfTest/UsageTests/Misc.tests.cpp index 757a99b5..683b8391 100644 --- a/projects/SelfTest/UsageTests/Misc.tests.cpp +++ b/projects/SelfTest/UsageTests/Misc.tests.cpp @@ -348,4 +348,9 @@ TEST_CASE( "#961 -- Dynamically created sections should all be reported", "[.]" } } +TEST_CASE( "#1175 - Hidden Test", "[.]" ) { + // Just for checking that hidden test is not listed by default + SUCCEED(); +} + }} // namespace MiscTests