3rd-party/catch2
1
0
Fork 0
mirror of https://github.com/catchorg/Catch2.git synced 2025-08-01 21:05:39 +02:00
Code Issues Packages Projects Releases Wiki Activity

Test & document SKIP in generator constructor

Browse Source 
Closes #1593
This commit is contained in:
Martin Hořeňovský
2023-05-31 15:12:23 +02:00
parent dff7513b28
commit 0631b607ee
21 changed files with 153 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tests/SelfTest/Baselines/automake.sw.approved.txt
View File

@@ -130,6 +130,7 @@ Nor would this
:test-result: FAIL Custom std-exceptions can be custom translated
:test-result: PASS Default scale is invisible to comparison
:test-result: PASS Directly creating an EnumInfo
:test-result: SKIP Empty generators can SKIP in constructor
:test-result: PASS Empty stream name opens cout stream
:test-result: FAIL EndsWith string matcher
:test-result: PASS Enums can quickly have stringification enabled using REGISTER_ENUM

1
tests/SelfTest/Baselines/automake.sw.multi.approved.txt
View File

@@ -128,6 +128,7 @@
:test-result: FAIL Custom std-exceptions can be custom translated
:test-result: PASS Default scale is invisible to comparison
:test-result: PASS Directly creating an EnumInfo
:test-result: SKIP Empty generators can SKIP in constructor
:test-result: PASS Empty stream name opens cout stream
:test-result: FAIL EndsWith string matcher
:test-result: PASS Enums can quickly have stringification enabled using REGISTER_ENUM

3
tests/SelfTest/Baselines/compact.sw.approved.txt
View File

@@ -520,6 +520,7 @@ ToString.tests.cpp:<line number>: passed: enumInfo->lookup(1) == "Value2" for: V
ToString.tests.cpp:<line number>: passed: enumInfo->lookup(3) == "{** unexpected enum value **}" for: {** unexpected enum value **}
==
"{** unexpected enum value **}"
Skip.tests.cpp:<line number>: skipped: 'This generator is empty'
Stream.tests.cpp:<line number>: passed: Catch::makeStream( "" )->isConsole() for: true
Matchers.tests.cpp:<line number>: failed: testStringForMatching(), EndsWith( "Substring" ) for: "this string contains 'abc' as a substring" ends with: "Substring"
Matchers.tests.cpp:<line number>: failed: testStringForMatching(), EndsWith( "this", Catch::CaseSensitive::No ) for: "this string contains 'abc' as a substring" ends with: "this" (case insensitive)
@@ -2537,7 +2538,7 @@ InternalBenchmark.tests.cpp:<line number>: passed: med == 18. for: 18.0 == 18.0
InternalBenchmark.tests.cpp:<line number>: passed: q3 == 23. for: 23.0 == 23.0
Misc.tests.cpp:<line number>: passed:
Misc.tests.cpp:<line number>: passed:
test cases: 408 | 308 passed | 84 failed | 5 skipped | 11 failed as expected
test cases: 409 | 308 passed | 84 failed | 6 skipped | 11 failed as expected
assertions: 2225 | 2048 passed | 145 failed | 32 failed as expected

3
tests/SelfTest/Baselines/compact.sw.multi.approved.txt
View File

@@ -518,6 +518,7 @@ ToString.tests.cpp:<line number>: passed: enumInfo->lookup(1) == "Value2" for: V
ToString.tests.cpp:<line number>: passed: enumInfo->lookup(3) == "{** unexpected enum value **}" for: {** unexpected enum value **}
==
"{** unexpected enum value **}"
Skip.tests.cpp:<line number>: skipped: 'This generator is empty'
Stream.tests.cpp:<line number>: passed: Catch::makeStream( "" )->isConsole() for: true
Matchers.tests.cpp:<line number>: failed: testStringForMatching(), EndsWith( "Substring" ) for: "this string contains 'abc' as a substring" ends with: "Substring"
Matchers.tests.cpp:<line number>: failed: testStringForMatching(), EndsWith( "this", Catch::CaseSensitive::No ) for: "this string contains 'abc' as a substring" ends with: "this" (case insensitive)
@@ -2526,7 +2527,7 @@ InternalBenchmark.tests.cpp:<line number>: passed: med == 18. for: 18.0 == 18.0
InternalBenchmark.tests.cpp:<line number>: passed: q3 == 23. for: 23.0 == 23.0
Misc.tests.cpp:<line number>: passed:
Misc.tests.cpp:<line number>: passed:
test cases: 408 | 308 passed | 84 failed | 5 skipped | 11 failed as expected
test cases: 409 | 308 passed | 84 failed | 6 skipped | 11 failed as expected
assertions: 2225 | 2048 passed | 145 failed | 32 failed as expected

12
tests/SelfTest/Baselines/console.std.approved.txt
View File

@@ -383,6 +383,16 @@ Exception.tests.cpp:<line number>: FAILED:
due to unexpected exception with message:
custom std exception
-------------------------------------------------------------------------------
Empty generators can SKIP in constructor
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
explicitly with message:
This generator is empty
-------------------------------------------------------------------------------
EndsWith string matcher
-------------------------------------------------------------------------------
@@ -1533,6 +1543,6 @@ due to unexpected exception with message:
Why would you throw a std::string?
===============================================================================
test cases: 408 | 322 passed | 69 failed | 6 skipped | 11 failed as expected
test cases: 409 | 322 passed | 69 failed | 7 skipped | 11 failed as expected
assertions: 2208 | 2048 passed | 128 failed | 32 failed as expected

12
tests/SelfTest/Baselines/console.sw.approved.txt
View File

@@ -3956,6 +3956,16 @@ with expansion:
==
"{** unexpected enum value **}"
-------------------------------------------------------------------------------
Empty generators can SKIP in constructor
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
explicitly with message:
This generator is empty
-------------------------------------------------------------------------------
Empty stream name opens cout stream
-------------------------------------------------------------------------------
@@ -18222,6 +18232,6 @@ Misc.tests.cpp:<line number>
Misc.tests.cpp:<line number>: PASSED:
===============================================================================
test cases: 408 | 308 passed | 84 failed | 5 skipped | 11 failed as expected
test cases: 409 | 308 passed | 84 failed | 6 skipped | 11 failed as expected
assertions: 2225 | 2048 passed | 145 failed | 32 failed as expected

12
tests/SelfTest/Baselines/console.sw.multi.approved.txt
View File

@@ -3954,6 +3954,16 @@ with expansion:
==
"{** unexpected enum value **}"
-------------------------------------------------------------------------------
Empty generators can SKIP in constructor
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
explicitly with message:
This generator is empty
-------------------------------------------------------------------------------
Empty stream name opens cout stream
-------------------------------------------------------------------------------
@@ -18211,6 +18221,6 @@ Misc.tests.cpp:<line number>
Misc.tests.cpp:<line number>: PASSED:
===============================================================================
test cases: 408 | 308 passed | 84 failed | 5 skipped | 11 failed as expected
test cases: 409 | 308 passed | 84 failed | 6 skipped | 11 failed as expected
assertions: 2225 | 2048 passed | 145 failed | 32 failed as expected

9
tests/SelfTest/Baselines/junit.sw.approved.txt
View File

@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuitesloose text artifact
>
<testsuite name="<exe-name>" errors="17" failures="128" skipped="11" tests="2236" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<testsuite name="<exe-name>" errors="17" failures="128" skipped="12" tests="2237" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<properties>
<property name="random-seed" value="1"/>
<property name="filters" value="&quot;*&quot; ~[!nonportable] ~[!benchmark] ~[approvals]"/>
@@ -462,6 +462,13 @@ at Exception.tests.cpp:<line number>
</testcase>
<testcase classname="<exe-name>.global" name="Default scale is invisible to comparison" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="Directly creating an EnumInfo" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="Empty generators can SKIP in constructor" time="{duration}" status="run">
<skipped type="SKIP">
SKIPPED
This generator is empty
at Skip.tests.cpp:<line number>
</skipped>
</testcase>
<testcase classname="<exe-name>.global" name="Empty stream name opens cout stream" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="EndsWith string matcher" time="{duration}" status="run">
<failure message="testStringForMatching(), EndsWith( &quot;Substring&quot; )" type="CHECK_THAT">

9
tests/SelfTest/Baselines/junit.sw.multi.approved.txt
View File

@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuites>
<testsuite name="<exe-name>" errors="17" failures="128" skipped="11" tests="2236" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<testsuite name="<exe-name>" errors="17" failures="128" skipped="12" tests="2237" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<properties>
<property name="random-seed" value="1"/>
<property name="filters" value="&quot;*&quot; ~[!nonportable] ~[!benchmark] ~[approvals]"/>
@@ -461,6 +461,13 @@ at Exception.tests.cpp:<line number>
</testcase>
<testcase classname="<exe-name>.global" name="Default scale is invisible to comparison" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="Directly creating an EnumInfo" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="Empty generators can SKIP in constructor" time="{duration}" status="run">
<skipped type="SKIP">
SKIPPED
This generator is empty
at Skip.tests.cpp:<line number>
</skipped>
</testcase>
<testcase classname="<exe-name>.global" name="Empty stream name opens cout stream" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="EndsWith string matcher" time="{duration}" status="run">
<failure message="testStringForMatching(), EndsWith( &quot;Substring&quot; )" type="CHECK_THAT">

7
tests/SelfTest/Baselines/sonarqube.sw.approved.txt
View File

@@ -1870,6 +1870,13 @@ at Misc.tests.cpp:<line number>
<testCase name="xmlentitycheck/encoded chars: these should all be encoded: &amp;&amp;&amp;&quot;&quot;&quot;&lt;&lt;&lt;&amp;&quot;&lt;&lt;&amp;&quot;" duration="{duration}"/>
</file>
<file path="tests/<exe-name>/UsageTests/Skip.tests.cpp">
<testCase name="Empty generators can SKIP in constructor" duration="{duration}">
<skipped message="SKIP()">
SKIPPED
This generator is empty
at Skip.tests.cpp:<line number>
</skipped>
</testCase>
<testCase name="a succeeding test can still be skipped" duration="{duration}">
<skipped message="SKIP()">
SKIPPED

7
tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt
View File

@@ -1869,6 +1869,13 @@ at Misc.tests.cpp:<line number>
<testCase name="xmlentitycheck/encoded chars: these should all be encoded: &amp;&amp;&amp;&quot;&quot;&quot;&lt;&lt;&lt;&amp;&quot;&lt;&lt;&amp;&quot;" duration="{duration}"/>
</file>
<file path="tests/<exe-name>/UsageTests/Skip.tests.cpp">
<testCase name="Empty generators can SKIP in constructor" duration="{duration}">
<skipped message="SKIP()">
SKIPPED
This generator is empty
at Skip.tests.cpp:<line number>
</skipped>
</testCase>
<testCase name="a succeeding test can still be skipped" duration="{duration}">
<skipped message="SKIP()">
SKIPPED

4
tests/SelfTest/Baselines/tap.sw.approved.txt
View File

@@ -984,6 +984,8 @@ ok {test-number} - enumInfo->lookup(0) == "Value1" for: Value1 == "Value1"
ok {test-number} - enumInfo->lookup(1) == "Value2" for: Value2 == "Value2"
# Directly creating an EnumInfo
ok {test-number} - enumInfo->lookup(3) == "{** unexpected enum value **}" for: {** unexpected enum value **} == "{** unexpected enum value **}"
# Empty generators can SKIP in constructor
ok {test-number} - # SKIP 'This generator is empty'
# Empty stream name opens cout stream
ok {test-number} - Catch::makeStream( "" )->isConsole() for: true
# EndsWith string matcher
@@ -4475,5 +4477,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0
ok {test-number} -
# xmlentitycheck
ok {test-number} -
1..2236
1..2237

4
tests/SelfTest/Baselines/tap.sw.multi.approved.txt
View File

@@ -982,6 +982,8 @@ ok {test-number} - enumInfo->lookup(0) == "Value1" for: Value1 == "Value1"
ok {test-number} - enumInfo->lookup(1) == "Value2" for: Value2 == "Value2"
# Directly creating an EnumInfo
ok {test-number} - enumInfo->lookup(3) == "{** unexpected enum value **}" for: {** unexpected enum value **} == "{** unexpected enum value **}"
# Empty generators can SKIP in constructor
ok {test-number} - # SKIP 'This generator is empty'
# Empty stream name opens cout stream
ok {test-number} - Catch::makeStream( "" )->isConsole() for: true
# EndsWith string matcher
@@ -4464,5 +4466,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0
ok {test-number} -
# xmlentitycheck
ok {test-number} -
1..2236
1..2237

3
tests/SelfTest/Baselines/teamcity.sw.approved.txt
View File

@@ -299,6 +299,9 @@
##teamcity[testFinished name='Default scale is invisible to comparison' duration="{duration}"]
##teamcity[testStarted name='Directly creating an EnumInfo']
##teamcity[testFinished name='Directly creating an EnumInfo' duration="{duration}"]
##teamcity[testStarted name='Empty generators can SKIP in constructor']
##teamcity[testIgnored name='Empty generators can SKIP in constructor' message='Skip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit skip with message:|n "This generator is empty"']
##teamcity[testFinished name='Empty generators can SKIP in constructor' duration="{duration}"]
##teamcity[testStarted name='Empty stream name opens cout stream']
##teamcity[testFinished name='Empty stream name opens cout stream' duration="{duration}"]
##teamcity[testStarted name='EndsWith string matcher']

3
tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt
View File

@@ -299,6 +299,9 @@
##teamcity[testFinished name='Default scale is invisible to comparison' duration="{duration}"]
##teamcity[testStarted name='Directly creating an EnumInfo']
##teamcity[testFinished name='Directly creating an EnumInfo' duration="{duration}"]
##teamcity[testStarted name='Empty generators can SKIP in constructor']
##teamcity[testIgnored name='Empty generators can SKIP in constructor' message='Skip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit skip with message:|n "This generator is empty"']
##teamcity[testFinished name='Empty generators can SKIP in constructor' duration="{duration}"]
##teamcity[testStarted name='Empty stream name opens cout stream']
##teamcity[testFinished name='Empty stream name opens cout stream' duration="{duration}"]
##teamcity[testStarted name='EndsWith string matcher']

10
tests/SelfTest/Baselines/xml.sw.approved.txt
View File

@@ -4364,6 +4364,12 @@ C
</Expression>
<OverallResult success="true" skips="0"/>
</TestCase>
<TestCase name="Empty generators can SKIP in constructor" tags="[skipping]" filename="tests/<exe-name>/UsageTests/Skip.tests.cpp" >
<Skip filename="tests/<exe-name>/UsageTests/Skip.tests.cpp" >
This generator is empty
</Skip>
<OverallResult success="true" skips="1"/>
</TestCase>
<TestCase name="Empty stream name opens cout stream" tags="[streams]" filename="tests/<exe-name>/IntrospectiveTests/Stream.tests.cpp" >
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Stream.tests.cpp" >
<Original>
@@ -21192,6 +21198,6 @@ b1!
</Section>
<OverallResult success="true" skips="0"/>
</TestCase>
<OverallResults successes="2048" failures="145" expectedFailures="32" skips="11"/>
<OverallResultsCases successes="308" failures="84" expectedFailures="11" skips="5"/>
<OverallResults successes="2048" failures="145" expectedFailures="32" skips="12"/>
<OverallResultsCases successes="308" failures="84" expectedFailures="11" skips="6"/>
</Catch2TestRun>

10
tests/SelfTest/Baselines/xml.sw.multi.approved.txt
View File

@@ -4364,6 +4364,12 @@ C
</Expression>
<OverallResult success="true" skips="0"/>
</TestCase>
<TestCase name="Empty generators can SKIP in constructor" tags="[skipping]" filename="tests/<exe-name>/UsageTests/Skip.tests.cpp" >
<Skip filename="tests/<exe-name>/UsageTests/Skip.tests.cpp" >
This generator is empty
</Skip>
<OverallResult success="true" skips="1"/>
</TestCase>
<TestCase name="Empty stream name opens cout stream" tags="[streams]" filename="tests/<exe-name>/IntrospectiveTests/Stream.tests.cpp" >
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Stream.tests.cpp" >
<Original>
@@ -21191,6 +21197,6 @@ b1!
</Section>
<OverallResult success="true" skips="0"/>
</TestCase>
<OverallResults successes="2048" failures="145" expectedFailures="32" skips="11"/>
<OverallResultsCases successes="308" failures="84" expectedFailures="11" skips="5"/>
<OverallResults successes="2048" failures="145" expectedFailures="32" skips="12"/>
<OverallResultsCases successes="308" failures="84" expectedFailures="11" skips="6"/>
</Catch2TestRun>

10
tests/SelfTest/UsageTests/Generators.tests.cpp
View File

@@ -261,6 +261,10 @@ TEST_CASE("Copy and then generate a range", "[generators]") {
}
}
#if defined( __clang__ )
# pragma clang diagnostic pop
#endif
TEST_CASE("#1913 - GENERATE inside a for loop should not keep recreating the generator", "[regression][generators]") {
static int counter = 0;
for (int i = 0; i < 3; ++i) {
@@ -305,9 +309,5 @@ TEST_CASE( "#2615 - Throwing in constructor generator fails test case but does n
// this should fail the test case, but not abort the application
auto sample = GENERATE( make_test_generator() );
// this assertion shouldn't trigger
REQUIRE( sample == 0U );
REQUIRE( sample == 0 );
}
#if defined( __clang__ )
# pragma clang diagnostic pop
#endif

27
tests/SelfTest/UsageTests/Skip.tests.cpp
View File

@@ -71,3 +71,30 @@ TEST_CASE( "failing for some generator values causes entire test case to fail",
FAIL();
}
}
namespace {
class test_skip_generator : public Catch::Generators::IGenerator<int> {
public:
explicit test_skip_generator() { SKIP( "This generator is empty" ); }
auto get() const -> int const& override {
static constexpr int value = 1;
return value;
}
auto next() -> bool override { return false; }
};
static auto make_test_skip_generator()
-> Catch::Generators::GeneratorWrapper<int> {
return { new test_skip_generator() };
}
} // namespace
TEST_CASE( "Empty generators can SKIP in constructor", "[skipping]" ) {
// The generator signals emptiness with `SKIP`
auto sample = GENERATE( make_test_skip_generator() );
// This assertion would fail, but shouldn't trigger
REQUIRE( sample == 0 );
}