diff --git a/docs/generators.md b/docs/generators.md index 37d7424a..09799752 100644 --- a/docs/generators.md +++ b/docs/generators.md @@ -221,3 +221,21 @@ For full example of implementing your own generator, look into Catch2's examples, specifically [Generators: Create your own generator](../examples/300-Gen-OwnGenerator.cpp). + +### Handling empty generators + +The generator interface assumes that a generator always has at least one +element. This is not always true, e.g. if the generator depends on an external +datafile, the file might be missing. + +There are two ways to handle this, depending on whether you want this +to be an error or not. + + * If empty generator **is** an error, throw an exception in constructor. + * If empty generator **is not** an error, use the [`SKIP`](skipping-passing-failing.md#skipping-test-cases-at-runtime) in constructor. + + + +--- + +[Home](Readme.md#top) diff --git a/docs/skipping-passing-failing.md b/docs/skipping-passing-failing.md index 4300d9d3..d866b418 100644 --- a/docs/skipping-passing-failing.md +++ b/docs/skipping-passing-failing.md @@ -84,6 +84,12 @@ exit code, same as it does if no test cases have run. This behaviour can be overridden using the [--allow-running-no-tests](command-line.md#no-tests-override) flag. +### `SKIP` inside generators + +You can also use the `SKIP` macro inside generator's constructor to handle +cases where the generator is empty, but you do not want to fail the test +case. + ## Passing and failing test cases diff --git a/tests/SelfTest/Baselines/automake.sw.approved.txt b/tests/SelfTest/Baselines/automake.sw.approved.txt index 7c2836e8..6b5938a6 100644 --- a/tests/SelfTest/Baselines/automake.sw.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.approved.txt @@ -130,6 +130,7 @@ Nor would this :test-result: FAIL Custom std-exceptions can be custom translated :test-result: PASS Default scale is invisible to comparison :test-result: PASS Directly creating an EnumInfo +:test-result: SKIP Empty generators can SKIP in constructor :test-result: PASS Empty stream name opens cout stream :test-result: FAIL EndsWith string matcher :test-result: PASS Enums can quickly have stringification enabled using REGISTER_ENUM diff --git a/tests/SelfTest/Baselines/automake.sw.multi.approved.txt b/tests/SelfTest/Baselines/automake.sw.multi.approved.txt index 75569816..cd56e648 100644 --- a/tests/SelfTest/Baselines/automake.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.multi.approved.txt @@ -128,6 +128,7 @@ :test-result: FAIL Custom std-exceptions can be custom translated :test-result: PASS Default scale is invisible to comparison :test-result: PASS Directly creating an EnumInfo +:test-result: SKIP Empty generators can SKIP in constructor :test-result: PASS Empty stream name opens cout stream :test-result: FAIL EndsWith string matcher :test-result: PASS Enums can quickly have stringification enabled using REGISTER_ENUM diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt index c72f140d..be7a4120 100644 --- a/tests/SelfTest/Baselines/compact.sw.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.approved.txt @@ -520,6 +520,7 @@ ToString.tests.cpp:: passed: enumInfo->lookup(1) == "Value2" for: V ToString.tests.cpp:: passed: enumInfo->lookup(3) == "{** unexpected enum value **}" for: {** unexpected enum value **} == "{** unexpected enum value **}" +Skip.tests.cpp:: skipped: 'This generator is empty' Stream.tests.cpp:: passed: Catch::makeStream( "" )->isConsole() for: true Matchers.tests.cpp:: failed: testStringForMatching(), EndsWith( "Substring" ) for: "this string contains 'abc' as a substring" ends with: "Substring" Matchers.tests.cpp:: failed: testStringForMatching(), EndsWith( "this", Catch::CaseSensitive::No ) for: "this string contains 'abc' as a substring" ends with: "this" (case insensitive) @@ -2537,7 +2538,7 @@ InternalBenchmark.tests.cpp:: passed: med == 18. for: 18.0 == 18.0 InternalBenchmark.tests.cpp:: passed: q3 == 23. for: 23.0 == 23.0 Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: -test cases: 408 | 308 passed | 84 failed | 5 skipped | 11 failed as expected +test cases: 409 | 308 passed | 84 failed | 6 skipped | 11 failed as expected assertions: 2225 | 2048 passed | 145 failed | 32 failed as expected diff --git a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt index 7970ca44..6c48ab91 100644 --- a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt @@ -518,6 +518,7 @@ ToString.tests.cpp:: passed: enumInfo->lookup(1) == "Value2" for: V ToString.tests.cpp:: passed: enumInfo->lookup(3) == "{** unexpected enum value **}" for: {** unexpected enum value **} == "{** unexpected enum value **}" +Skip.tests.cpp:: skipped: 'This generator is empty' Stream.tests.cpp:: passed: Catch::makeStream( "" )->isConsole() for: true Matchers.tests.cpp:: failed: testStringForMatching(), EndsWith( "Substring" ) for: "this string contains 'abc' as a substring" ends with: "Substring" Matchers.tests.cpp:: failed: testStringForMatching(), EndsWith( "this", Catch::CaseSensitive::No ) for: "this string contains 'abc' as a substring" ends with: "this" (case insensitive) @@ -2526,7 +2527,7 @@ InternalBenchmark.tests.cpp:: passed: med == 18. for: 18.0 == 18.0 InternalBenchmark.tests.cpp:: passed: q3 == 23. for: 23.0 == 23.0 Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: -test cases: 408 | 308 passed | 84 failed | 5 skipped | 11 failed as expected +test cases: 409 | 308 passed | 84 failed | 6 skipped | 11 failed as expected assertions: 2225 | 2048 passed | 145 failed | 32 failed as expected diff --git a/tests/SelfTest/Baselines/console.std.approved.txt b/tests/SelfTest/Baselines/console.std.approved.txt index 31ca04f0..0945f0df 100644 --- a/tests/SelfTest/Baselines/console.std.approved.txt +++ b/tests/SelfTest/Baselines/console.std.approved.txt @@ -383,6 +383,16 @@ Exception.tests.cpp:: FAILED: due to unexpected exception with message: custom std exception +------------------------------------------------------------------------------- +Empty generators can SKIP in constructor +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: +explicitly with message: + This generator is empty + ------------------------------------------------------------------------------- EndsWith string matcher ------------------------------------------------------------------------------- @@ -1533,6 +1543,6 @@ due to unexpected exception with message: Why would you throw a std::string? =============================================================================== -test cases: 408 | 322 passed | 69 failed | 6 skipped | 11 failed as expected +test cases: 409 | 322 passed | 69 failed | 7 skipped | 11 failed as expected assertions: 2208 | 2048 passed | 128 failed | 32 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt index 68bda6cd..150980e8 100644 --- a/tests/SelfTest/Baselines/console.sw.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.approved.txt @@ -3956,6 +3956,16 @@ with expansion: == "{** unexpected enum value **}" +------------------------------------------------------------------------------- +Empty generators can SKIP in constructor +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: +explicitly with message: + This generator is empty + ------------------------------------------------------------------------------- Empty stream name opens cout stream ------------------------------------------------------------------------------- @@ -18222,6 +18232,6 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 408 | 308 passed | 84 failed | 5 skipped | 11 failed as expected +test cases: 409 | 308 passed | 84 failed | 6 skipped | 11 failed as expected assertions: 2225 | 2048 passed | 145 failed | 32 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.multi.approved.txt b/tests/SelfTest/Baselines/console.sw.multi.approved.txt index 3f5e91d1..4cc942dd 100644 --- a/tests/SelfTest/Baselines/console.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.multi.approved.txt @@ -3954,6 +3954,16 @@ with expansion: == "{** unexpected enum value **}" +------------------------------------------------------------------------------- +Empty generators can SKIP in constructor +------------------------------------------------------------------------------- +Skip.tests.cpp: +............................................................................... + +Skip.tests.cpp:: SKIPPED: +explicitly with message: + This generator is empty + ------------------------------------------------------------------------------- Empty stream name opens cout stream ------------------------------------------------------------------------------- @@ -18211,6 +18221,6 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 408 | 308 passed | 84 failed | 5 skipped | 11 failed as expected +test cases: 409 | 308 passed | 84 failed | 6 skipped | 11 failed as expected assertions: 2225 | 2048 passed | 145 failed | 32 failed as expected diff --git a/tests/SelfTest/Baselines/junit.sw.approved.txt b/tests/SelfTest/Baselines/junit.sw.approved.txt index 74e08986..c992154c 100644 --- a/tests/SelfTest/Baselines/junit.sw.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ - + @@ -462,6 +462,13 @@ at Exception.tests.cpp: + + +SKIPPED +This generator is empty +at Skip.tests.cpp: + + diff --git a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt index 73f37422..79c32365 100644 --- a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt @@ -1,6 +1,6 @@ - + @@ -461,6 +461,13 @@ at Exception.tests.cpp: + + +SKIPPED +This generator is empty +at Skip.tests.cpp: + + diff --git a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt index eeb8d17b..592887f9 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt @@ -1870,6 +1870,13 @@ at Misc.tests.cpp: + + +SKIPPED +This generator is empty +at Skip.tests.cpp: + + SKIPPED diff --git a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt index a804cb6b..3509287f 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt @@ -1869,6 +1869,13 @@ at Misc.tests.cpp: + + +SKIPPED +This generator is empty +at Skip.tests.cpp: + + SKIPPED diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt index ba7d2dfa..acd0a1c1 100644 --- a/tests/SelfTest/Baselines/tap.sw.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.approved.txt @@ -984,6 +984,8 @@ ok {test-number} - enumInfo->lookup(0) == "Value1" for: Value1 == "Value1" ok {test-number} - enumInfo->lookup(1) == "Value2" for: Value2 == "Value2" # Directly creating an EnumInfo ok {test-number} - enumInfo->lookup(3) == "{** unexpected enum value **}" for: {** unexpected enum value **} == "{** unexpected enum value **}" +# Empty generators can SKIP in constructor +ok {test-number} - # SKIP 'This generator is empty' # Empty stream name opens cout stream ok {test-number} - Catch::makeStream( "" )->isConsole() for: true # EndsWith string matcher @@ -4475,5 +4477,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2236 +1..2237 diff --git a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt index 07014c2e..03329049 100644 --- a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt @@ -982,6 +982,8 @@ ok {test-number} - enumInfo->lookup(0) == "Value1" for: Value1 == "Value1" ok {test-number} - enumInfo->lookup(1) == "Value2" for: Value2 == "Value2" # Directly creating an EnumInfo ok {test-number} - enumInfo->lookup(3) == "{** unexpected enum value **}" for: {** unexpected enum value **} == "{** unexpected enum value **}" +# Empty generators can SKIP in constructor +ok {test-number} - # SKIP 'This generator is empty' # Empty stream name opens cout stream ok {test-number} - Catch::makeStream( "" )->isConsole() for: true # EndsWith string matcher @@ -4464,5 +4466,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2236 +1..2237 diff --git a/tests/SelfTest/Baselines/teamcity.sw.approved.txt b/tests/SelfTest/Baselines/teamcity.sw.approved.txt index aff95cf6..a298633a 100644 --- a/tests/SelfTest/Baselines/teamcity.sw.approved.txt +++ b/tests/SelfTest/Baselines/teamcity.sw.approved.txt @@ -299,6 +299,9 @@ ##teamcity[testFinished name='Default scale is invisible to comparison' duration="{duration}"] ##teamcity[testStarted name='Directly creating an EnumInfo'] ##teamcity[testFinished name='Directly creating an EnumInfo' duration="{duration}"] +##teamcity[testStarted name='Empty generators can SKIP in constructor'] +##teamcity[testIgnored name='Empty generators can SKIP in constructor' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit skip with message:|n "This generator is empty"'] +##teamcity[testFinished name='Empty generators can SKIP in constructor' duration="{duration}"] ##teamcity[testStarted name='Empty stream name opens cout stream'] ##teamcity[testFinished name='Empty stream name opens cout stream' duration="{duration}"] ##teamcity[testStarted name='EndsWith string matcher'] diff --git a/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt b/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt index f16bc135..861d6471 100644 --- a/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt @@ -299,6 +299,9 @@ ##teamcity[testFinished name='Default scale is invisible to comparison' duration="{duration}"] ##teamcity[testStarted name='Directly creating an EnumInfo'] ##teamcity[testFinished name='Directly creating an EnumInfo' duration="{duration}"] +##teamcity[testStarted name='Empty generators can SKIP in constructor'] +##teamcity[testIgnored name='Empty generators can SKIP in constructor' message='Skip.tests.cpp:|n...............................................................................|n|nSkip.tests.cpp:|nexplicit skip with message:|n "This generator is empty"'] +##teamcity[testFinished name='Empty generators can SKIP in constructor' duration="{duration}"] ##teamcity[testStarted name='Empty stream name opens cout stream'] ##teamcity[testFinished name='Empty stream name opens cout stream' duration="{duration}"] ##teamcity[testStarted name='EndsWith string matcher'] diff --git a/tests/SelfTest/Baselines/xml.sw.approved.txt b/tests/SelfTest/Baselines/xml.sw.approved.txt index 8509e370..bf9cf205 100644 --- a/tests/SelfTest/Baselines/xml.sw.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.approved.txt @@ -4364,6 +4364,12 @@ C + + + This generator is empty + + + @@ -21192,6 +21198,6 @@ b1! - - + + diff --git a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt index adf83986..41dc8cb3 100644 --- a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt @@ -4364,6 +4364,12 @@ C + + + This generator is empty + + + @@ -21191,6 +21197,6 @@ b1! - - + + diff --git a/tests/SelfTest/UsageTests/Generators.tests.cpp b/tests/SelfTest/UsageTests/Generators.tests.cpp index 274c63b8..8e2c387a 100644 --- a/tests/SelfTest/UsageTests/Generators.tests.cpp +++ b/tests/SelfTest/UsageTests/Generators.tests.cpp @@ -261,6 +261,10 @@ TEST_CASE("Copy and then generate a range", "[generators]") { } } +#if defined( __clang__ ) +# pragma clang diagnostic pop +#endif + TEST_CASE("#1913 - GENERATE inside a for loop should not keep recreating the generator", "[regression][generators]") { static int counter = 0; for (int i = 0; i < 3; ++i) { @@ -305,9 +309,5 @@ TEST_CASE( "#2615 - Throwing in constructor generator fails test case but does n // this should fail the test case, but not abort the application auto sample = GENERATE( make_test_generator() ); // this assertion shouldn't trigger - REQUIRE( sample == 0U ); + REQUIRE( sample == 0 ); } - -#if defined( __clang__ ) -# pragma clang diagnostic pop -#endif diff --git a/tests/SelfTest/UsageTests/Skip.tests.cpp b/tests/SelfTest/UsageTests/Skip.tests.cpp index 6bd4189b..661795e1 100644 --- a/tests/SelfTest/UsageTests/Skip.tests.cpp +++ b/tests/SelfTest/UsageTests/Skip.tests.cpp @@ -71,3 +71,30 @@ TEST_CASE( "failing for some generator values causes entire test case to fail", FAIL(); } } + +namespace { + class test_skip_generator : public Catch::Generators::IGenerator { + public: + explicit test_skip_generator() { SKIP( "This generator is empty" ); } + + auto get() const -> int const& override { + static constexpr int value = 1; + return value; + } + + auto next() -> bool override { return false; } + }; + + static auto make_test_skip_generator() + -> Catch::Generators::GeneratorWrapper { + return { new test_skip_generator() }; + } + +} // namespace + +TEST_CASE( "Empty generators can SKIP in constructor", "[skipping]" ) { + // The generator signals emptiness with `SKIP` + auto sample = GENERATE( make_test_skip_generator() ); + // This assertion would fail, but shouldn't trigger + REQUIRE( sample == 0 ); +}