Test & document SKIP in generator constructor

Closes #1593

docs/generators.md

@@ -221,3 +221,21 @@ For full example of implementing your own generator, look into Catch2's
 examples, specifically
 [Generators: Create your own generator](../examples/300-Gen-OwnGenerator.cpp).
 
+
+### Handling empty generators
+
+The generator interface assumes that a generator always has at least one
+element. This is not always true, e.g. if the generator depends on an external
+datafile, the file might be missing.
+
+There are two ways to handle this, depending on whether you want this
+to be an error or not.
+
+ * If empty generator **is** an error, throw an exception in constructor.
+ * If empty generator **is not** an error, use the [`SKIP`](skipping-passing-failing.md#skipping-test-cases-at-runtime) in constructor.
+
+
+
+---
+
+[Home](Readme.md#top)

docs/skipping-passing-failing.md

@@ -84,6 +84,12 @@ exit code, same as it does if no test cases have run. This behaviour can
 be overridden using the [--allow-running-no-tests](command-line.md#no-tests-override)
 flag.
 
+### `SKIP` inside generators
+
+You can also use the `SKIP` macro inside generator's constructor to handle
+cases where the generator is empty, but you do not want to fail the test
+case.
+
 
 ## Passing and failing test cases
 

tests/SelfTest/Baselines/automake.sw.approved.txt

@@ -130,6 +130,7 @@ Nor would this
 :test-result: FAIL Custom std-exceptions can be custom translated
 :test-result: PASS Default scale is invisible to comparison
 :test-result: PASS Directly creating an EnumInfo
+:test-result: SKIP Empty generators can SKIP in constructor
 :test-result: PASS Empty stream name opens cout stream
 :test-result: FAIL EndsWith string matcher
 :test-result: PASS Enums can quickly have stringification enabled using REGISTER_ENUM

tests/SelfTest/Baselines/automake.sw.multi.approved.txt

@@ -128,6 +128,7 @@
 :test-result: FAIL Custom std-exceptions can be custom translated
 :test-result: PASS Default scale is invisible to comparison
 :test-result: PASS Directly creating an EnumInfo
+:test-result: SKIP Empty generators can SKIP in constructor
 :test-result: PASS Empty stream name opens cout stream
 :test-result: FAIL EndsWith string matcher
 :test-result: PASS Enums can quickly have stringification enabled using REGISTER_ENUM

tests/SelfTest/Baselines/compact.sw.approved.txt

@@ -520,6 +520,7 @@ ToString.tests.cpp:<line number>: passed: enumInfo->lookup(1) == "Value2" for: V
 ToString.tests.cpp:<line number>: passed: enumInfo->lookup(3) == "{** unexpected enum value **}" for: {** unexpected enum value **}
 ==
 "{** unexpected enum value **}"
+Skip.tests.cpp:<line number>: skipped: 'This generator is empty'
 Stream.tests.cpp:<line number>: passed: Catch::makeStream( "" )->isConsole() for: true
 Matchers.tests.cpp:<line number>: failed: testStringForMatching(), EndsWith( "Substring" ) for: "this string contains 'abc' as a substring" ends with: "Substring"
 Matchers.tests.cpp:<line number>: failed: testStringForMatching(), EndsWith( "this", Catch::CaseSensitive::No ) for: "this string contains 'abc' as a substring" ends with: "this" (case insensitive)
@@ -2537,7 +2538,7 @@ InternalBenchmark.tests.cpp:<line number>: passed: med == 18. for: 18.0 == 18.0
 InternalBenchmark.tests.cpp:<line number>: passed: q3 == 23. for: 23.0 == 23.0
 Misc.tests.cpp:<line number>: passed:
 Misc.tests.cpp:<line number>: passed:
-test cases:  408 |  308 passed |  84 failed | 5 skipped | 11 failed as expected
+test cases:  409 |  308 passed |  84 failed | 6 skipped | 11 failed as expected
 assertions: 2225 | 2048 passed | 145 failed | 32 failed as expected
 
 

tests/SelfTest/Baselines/compact.sw.multi.approved.txt

@@ -518,6 +518,7 @@ ToString.tests.cpp:<line number>: passed: enumInfo->lookup(1) == "Value2" for: V
 ToString.tests.cpp:<line number>: passed: enumInfo->lookup(3) == "{** unexpected enum value **}" for: {** unexpected enum value **}
 ==
 "{** unexpected enum value **}"
+Skip.tests.cpp:<line number>: skipped: 'This generator is empty'
 Stream.tests.cpp:<line number>: passed: Catch::makeStream( "" )->isConsole() for: true
 Matchers.tests.cpp:<line number>: failed: testStringForMatching(), EndsWith( "Substring" ) for: "this string contains 'abc' as a substring" ends with: "Substring"
 Matchers.tests.cpp:<line number>: failed: testStringForMatching(), EndsWith( "this", Catch::CaseSensitive::No ) for: "this string contains 'abc' as a substring" ends with: "this" (case insensitive)
@@ -2526,7 +2527,7 @@ InternalBenchmark.tests.cpp:<line number>: passed: med == 18. for: 18.0 == 18.0
 InternalBenchmark.tests.cpp:<line number>: passed: q3 == 23. for: 23.0 == 23.0
 Misc.tests.cpp:<line number>: passed:
 Misc.tests.cpp:<line number>: passed:
-test cases:  408 |  308 passed |  84 failed | 5 skipped | 11 failed as expected
+test cases:  409 |  308 passed |  84 failed | 6 skipped | 11 failed as expected
 assertions: 2225 | 2048 passed | 145 failed | 32 failed as expected
 
 

tests/SelfTest/Baselines/console.std.approved.txt

@@ -383,6 +383,16 @@ Exception.tests.cpp:<line number>: FAILED:
 due to unexpected exception with message:
   custom std exception
 
+-------------------------------------------------------------------------------
+Empty generators can SKIP in constructor
+-------------------------------------------------------------------------------
+Skip.tests.cpp:<line number>
+...............................................................................
+
+Skip.tests.cpp:<line number>: SKIPPED:
+explicitly with message:
+  This generator is empty
+
 -------------------------------------------------------------------------------
 EndsWith string matcher
 -------------------------------------------------------------------------------
@@ -1533,6 +1543,6 @@ due to unexpected exception with message:
   Why would you throw a std::string?
 
 ===============================================================================
-test cases:  408 |  322 passed |  69 failed | 6 skipped | 11 failed as expected
+test cases:  409 |  322 passed |  69 failed | 7 skipped | 11 failed as expected
 assertions: 2208 | 2048 passed | 128 failed | 32 failed as expected
 

tests/SelfTest/Baselines/console.sw.approved.txt

@@ -3956,6 +3956,16 @@ with expansion:
   ==
   "{** unexpected enum value **}"
 
+-------------------------------------------------------------------------------
+Empty generators can SKIP in constructor
+-------------------------------------------------------------------------------
+Skip.tests.cpp:<line number>
+...............................................................................
+
+Skip.tests.cpp:<line number>: SKIPPED:
+explicitly with message:
+  This generator is empty
+
 -------------------------------------------------------------------------------
 Empty stream name opens cout stream
 -------------------------------------------------------------------------------
@@ -18222,6 +18232,6 @@ Misc.tests.cpp:<line number>
 Misc.tests.cpp:<line number>: PASSED:
 
 ===============================================================================
-test cases:  408 |  308 passed |  84 failed | 5 skipped | 11 failed as expected
+test cases:  409 |  308 passed |  84 failed | 6 skipped | 11 failed as expected
 assertions: 2225 | 2048 passed | 145 failed | 32 failed as expected
 

tests/SelfTest/Baselines/console.sw.multi.approved.txt

@@ -3954,6 +3954,16 @@ with expansion:
   ==
   "{** unexpected enum value **}"
 
+-------------------------------------------------------------------------------
+Empty generators can SKIP in constructor
+-------------------------------------------------------------------------------
+Skip.tests.cpp:<line number>
+...............................................................................
+
+Skip.tests.cpp:<line number>: SKIPPED:
+explicitly with message:
+  This generator is empty
+
 -------------------------------------------------------------------------------
 Empty stream name opens cout stream
 -------------------------------------------------------------------------------
@@ -18211,6 +18221,6 @@ Misc.tests.cpp:<line number>
 Misc.tests.cpp:<line number>: PASSED:
 
 ===============================================================================
-test cases:  408 |  308 passed |  84 failed | 5 skipped | 11 failed as expected
+test cases:  409 |  308 passed |  84 failed | 6 skipped | 11 failed as expected
 assertions: 2225 | 2048 passed | 145 failed | 32 failed as expected
 

tests/SelfTest/Baselines/junit.sw.approved.txt

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <testsuitesloose text artifact
 >
-  <testsuite name="<exe-name>" errors="17" failures="128" skipped="11" tests="2236" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+  <testsuite name="<exe-name>" errors="17" failures="128" skipped="12" tests="2237" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
     <properties>
       <property name="random-seed" value="1"/>
       <property name="filters" value="&quot;*&quot; ~[!nonportable] ~[!benchmark] ~[approvals]"/>
@@ -462,6 +462,13 @@ at Exception.tests.cpp:<line number>
     </testcase>
     <testcase classname="<exe-name>.global" name="Default scale is invisible to comparison" time="{duration}" status="run"/>
     <testcase classname="<exe-name>.global" name="Directly creating an EnumInfo" time="{duration}" status="run"/>
+    <testcase classname="<exe-name>.global" name="Empty generators can SKIP in constructor" time="{duration}" status="run">
+      <skipped type="SKIP">
+SKIPPED
+This generator is empty
+at Skip.tests.cpp:<line number>
+      </skipped>
+    </testcase>
     <testcase classname="<exe-name>.global" name="Empty stream name opens cout stream" time="{duration}" status="run"/>
     <testcase classname="<exe-name>.global" name="EndsWith string matcher" time="{duration}" status="run">
       <failure message="testStringForMatching(), EndsWith( &quot;Substring&quot; )" type="CHECK_THAT">

tests/SelfTest/Baselines/junit.sw.multi.approved.txt

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <testsuites>
-  <testsuite name="<exe-name>" errors="17" failures="128" skipped="11" tests="2236" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+  <testsuite name="<exe-name>" errors="17" failures="128" skipped="12" tests="2237" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
     <properties>
       <property name="random-seed" value="1"/>
       <property name="filters" value="&quot;*&quot; ~[!nonportable] ~[!benchmark] ~[approvals]"/>
@@ -461,6 +461,13 @@ at Exception.tests.cpp:<line number>
     </testcase>
     <testcase classname="<exe-name>.global" name="Default scale is invisible to comparison" time="{duration}" status="run"/>
     <testcase classname="<exe-name>.global" name="Directly creating an EnumInfo" time="{duration}" status="run"/>
+    <testcase classname="<exe-name>.global" name="Empty generators can SKIP in constructor" time="{duration}" status="run">
+      <skipped type="SKIP">
+SKIPPED
+This generator is empty
+at Skip.tests.cpp:<line number>
+      </skipped>
+    </testcase>
     <testcase classname="<exe-name>.global" name="Empty stream name opens cout stream" time="{duration}" status="run"/>
     <testcase classname="<exe-name>.global" name="EndsWith string matcher" time="{duration}" status="run">
       <failure message="testStringForMatching(), EndsWith( &quot;Substring&quot; )" type="CHECK_THAT">

tests/SelfTest/Baselines/sonarqube.sw.approved.txt

@@ -1870,6 +1870,13 @@ at Misc.tests.cpp:<line number>
     <testCase name="xmlentitycheck/encoded chars: these should all be encoded: &amp;&amp;&amp;&quot;&quot;&quot;&lt;&lt;&lt;&amp;&quot;&lt;&lt;&amp;&quot;" duration="{duration}"/>
   </file>
   <file path="tests/<exe-name>/UsageTests/Skip.tests.cpp">
+    <testCase name="Empty generators can SKIP in constructor" duration="{duration}">
+      <skipped message="SKIP()">
+SKIPPED
+This generator is empty
+at Skip.tests.cpp:<line number>
+      </skipped>
+    </testCase>
     <testCase name="a succeeding test can still be skipped" duration="{duration}">
       <skipped message="SKIP()">
 SKIPPED

tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt

@@ -1869,6 +1869,13 @@ at Misc.tests.cpp:<line number>
     <testCase name="xmlentitycheck/encoded chars: these should all be encoded: &amp;&amp;&amp;&quot;&quot;&quot;&lt;&lt;&lt;&amp;&quot;&lt;&lt;&amp;&quot;" duration="{duration}"/>
   </file>
   <file path="tests/<exe-name>/UsageTests/Skip.tests.cpp">
+    <testCase name="Empty generators can SKIP in constructor" duration="{duration}">
+      <skipped message="SKIP()">
+SKIPPED
+This generator is empty
+at Skip.tests.cpp:<line number>
+      </skipped>
+    </testCase>
     <testCase name="a succeeding test can still be skipped" duration="{duration}">
       <skipped message="SKIP()">
 SKIPPED

tests/SelfTest/Baselines/tap.sw.approved.txt

@@ -984,6 +984,8 @@ ok {test-number} - enumInfo->lookup(0) == "Value1" for: Value1 == "Value1"
 ok {test-number} - enumInfo->lookup(1) == "Value2" for: Value2 == "Value2"
 # Directly creating an EnumInfo
 ok {test-number} - enumInfo->lookup(3) == "{** unexpected enum value **}" for: {** unexpected enum value **} == "{** unexpected enum value **}"
+# Empty generators can SKIP in constructor
+ok {test-number} -  # SKIP 'This generator is empty'
 # Empty stream name opens cout stream
 ok {test-number} - Catch::makeStream( "" )->isConsole() for: true
 # EndsWith string matcher
@@ -4475,5 +4477,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0
 ok {test-number} -
 # xmlentitycheck
 ok {test-number} -
-1..2236
+1..2237
 

tests/SelfTest/Baselines/tap.sw.multi.approved.txt

@@ -982,6 +982,8 @@ ok {test-number} - enumInfo->lookup(0) == "Value1" for: Value1 == "Value1"
 ok {test-number} - enumInfo->lookup(1) == "Value2" for: Value2 == "Value2"
 # Directly creating an EnumInfo
 ok {test-number} - enumInfo->lookup(3) == "{** unexpected enum value **}" for: {** unexpected enum value **} == "{** unexpected enum value **}"
+# Empty generators can SKIP in constructor
+ok {test-number} -  # SKIP 'This generator is empty'
 # Empty stream name opens cout stream
 ok {test-number} - Catch::makeStream( "" )->isConsole() for: true
 # EndsWith string matcher
@@ -4464,5 +4466,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0
 ok {test-number} -
 # xmlentitycheck
 ok {test-number} -
-1..2236
+1..2237
 

tests/SelfTest/Baselines/teamcity.sw.approved.txt

@@ -299,6 +299,9 @@
 ##teamcity[testFinished name='Default scale is invisible to comparison' duration="{duration}"]
 ##teamcity[testStarted name='Directly creating an EnumInfo']
 ##teamcity[testFinished name='Directly creating an EnumInfo' duration="{duration}"]
+##teamcity[testStarted name='Empty generators can SKIP in constructor']
+##teamcity[testIgnored name='Empty generators can SKIP in constructor' message='Skip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit skip with message:|n  "This generator is empty"']
+##teamcity[testFinished name='Empty generators can SKIP in constructor' duration="{duration}"]
 ##teamcity[testStarted name='Empty stream name opens cout stream']
 ##teamcity[testFinished name='Empty stream name opens cout stream' duration="{duration}"]
 ##teamcity[testStarted name='EndsWith string matcher']

tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt

@@ -299,6 +299,9 @@
 ##teamcity[testFinished name='Default scale is invisible to comparison' duration="{duration}"]
 ##teamcity[testStarted name='Directly creating an EnumInfo']
 ##teamcity[testFinished name='Directly creating an EnumInfo' duration="{duration}"]
+##teamcity[testStarted name='Empty generators can SKIP in constructor']
+##teamcity[testIgnored name='Empty generators can SKIP in constructor' message='Skip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit skip with message:|n  "This generator is empty"']
+##teamcity[testFinished name='Empty generators can SKIP in constructor' duration="{duration}"]
 ##teamcity[testStarted name='Empty stream name opens cout stream']
 ##teamcity[testFinished name='Empty stream name opens cout stream' duration="{duration}"]
 ##teamcity[testStarted name='EndsWith string matcher']

tests/SelfTest/Baselines/xml.sw.approved.txt

@@ -4364,6 +4364,12 @@ C
     </Expression>
     <OverallResult success="true" skips="0"/>
   </TestCase>
+  <TestCase name="Empty generators can SKIP in constructor" tags="[skipping]" filename="tests/<exe-name>/UsageTests/Skip.tests.cpp" >
+    <Skip filename="tests/<exe-name>/UsageTests/Skip.tests.cpp" >
+      This generator is empty
+    </Skip>
+    <OverallResult success="true" skips="1"/>
+  </TestCase>
   <TestCase name="Empty stream name opens cout stream" tags="[streams]" filename="tests/<exe-name>/IntrospectiveTests/Stream.tests.cpp" >
     <Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Stream.tests.cpp" >
       <Original>
@@ -21192,6 +21198,6 @@ b1!
     </Section>
     <OverallResult success="true" skips="0"/>
   </TestCase>
-  <OverallResults successes="2048" failures="145" expectedFailures="32" skips="11"/>
+  <OverallResults successes="2048" failures="145" expectedFailures="32" skips="12"/>
-  <OverallResultsCases successes="308" failures="84" expectedFailures="11" skips="5"/>
+  <OverallResultsCases successes="308" failures="84" expectedFailures="11" skips="6"/>
 </Catch2TestRun>

tests/SelfTest/Baselines/xml.sw.multi.approved.txt

@@ -4364,6 +4364,12 @@ C
     </Expression>
     <OverallResult success="true" skips="0"/>
   </TestCase>
+  <TestCase name="Empty generators can SKIP in constructor" tags="[skipping]" filename="tests/<exe-name>/UsageTests/Skip.tests.cpp" >
+    <Skip filename="tests/<exe-name>/UsageTests/Skip.tests.cpp" >
+      This generator is empty
+    </Skip>
+    <OverallResult success="true" skips="1"/>
+  </TestCase>
   <TestCase name="Empty stream name opens cout stream" tags="[streams]" filename="tests/<exe-name>/IntrospectiveTests/Stream.tests.cpp" >
     <Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Stream.tests.cpp" >
       <Original>
@@ -21191,6 +21197,6 @@ b1!
     </Section>
     <OverallResult success="true" skips="0"/>
   </TestCase>
-  <OverallResults successes="2048" failures="145" expectedFailures="32" skips="11"/>
+  <OverallResults successes="2048" failures="145" expectedFailures="32" skips="12"/>
-  <OverallResultsCases successes="308" failures="84" expectedFailures="11" skips="5"/>
+  <OverallResultsCases successes="308" failures="84" expectedFailures="11" skips="6"/>
 </Catch2TestRun>

tests/SelfTest/UsageTests/Generators.tests.cpp

@@ -261,6 +261,10 @@ TEST_CASE("Copy and then generate a range", "[generators]") {
     }
 }
 
+#if defined( __clang__ )
+#    pragma clang diagnostic pop
+#endif
+
 TEST_CASE("#1913 - GENERATE inside a for loop should not keep recreating the generator", "[regression][generators]") {
     static int counter = 0;
     for (int i = 0; i < 3; ++i) {
@@ -305,9 +309,5 @@ TEST_CASE( "#2615 - Throwing in constructor generator fails test case but does n
     // this should fail the test case, but not abort the application
     auto sample = GENERATE( make_test_generator() );
     // this assertion shouldn't trigger
-    REQUIRE( sample == 0U );
+    REQUIRE( sample == 0 );
 }
-
-#if defined( __clang__ )
-#    pragma clang diagnostic pop
-#endif

tests/SelfTest/UsageTests/Skip.tests.cpp

@@ -71,3 +71,30 @@ TEST_CASE( "failing for some generator values causes entire test case to fail",
         FAIL();
     }
 }
+
+namespace {
+    class test_skip_generator : public Catch::Generators::IGenerator<int> {
+    public:
+        explicit test_skip_generator() { SKIP( "This generator is empty" ); }
+
+        auto get() const -> int const& override {
+            static constexpr int value = 1;
+            return value;
+        }
+
+        auto next() -> bool override { return false; }
+    };
+
+    static auto make_test_skip_generator()
+        -> Catch::Generators::GeneratorWrapper<int> {
+        return { new test_skip_generator() };
+    }
+
+} // namespace
+
+TEST_CASE( "Empty generators can SKIP in constructor", "[skipping]" ) {
+    // The generator signals emptiness with `SKIP`
+    auto sample = GENERATE( make_test_skip_generator() );
+    // This assertion would fail, but shouldn't trigger
+    REQUIRE( sample == 0 );
+}