Improved generator tracking

* Successive executions of the same `GENERATE` macro (e.g. because
of a for loop) no longer lead to multiple nested generators.
* The same line can now contain multiple `GENERATE` macros without
issues.

Fixes #1913
This commit is contained in:
Martin Hořeňovský 2020-06-01 19:04:23 +02:00
parent 3ceaad7d66
commit 480f3f418b
No known key found for this signature in database
GPG Key ID: DE48307B8B0D381A
18 changed files with 283 additions and 26 deletions

View File

@ -29,7 +29,7 @@ TEST_CASE("Generate random doubles across different ranges",
// This will take r1 by reference and r2 by value.
// Note that there are no advantages for doing so in this example,
// it is done only for expository purposes.
auto number = Catch::Generators::generate( CATCH_INTERNAL_LINEINFO,
auto number = Catch::Generators::generate( "custom capture generator", CATCH_INTERNAL_LINEINFO,
[&r1, r2]{
using namespace Catch::Generators;
return makeGenerators(take(50, random(std::get<0>(r1), std::get<1>(r2))));

View File

@ -9,6 +9,7 @@
#include <catch2/interfaces/catch_interfaces_generatortracker.hpp>
#include <catch2/internal/catch_common.hpp>
#include <catch2/internal/catch_stringref.hpp>
#include <vector>
#include <tuple>
@ -179,16 +180,16 @@ namespace Detail {
return makeGenerators( value( T( std::forward<U>( val ) ) ), std::forward<Gs>( moreGenerators )... );
}
auto acquireGeneratorTracker( SourceLineInfo const& lineInfo ) -> IGeneratorTracker&;
auto acquireGeneratorTracker( StringRef generatorName, SourceLineInfo const& lineInfo ) -> IGeneratorTracker&;
template<typename L>
// Note: The type after -> is weird, because VS2015 cannot parse
// the expression used in the typedef inside, when it is in
// return type. Yeah.
auto generate( SourceLineInfo const& lineInfo, L const& generatorExpression ) -> decltype(std::declval<decltype(generatorExpression())>().get()) {
auto generate( StringRef generatorName, SourceLineInfo const& lineInfo, L const& generatorExpression ) -> decltype(std::declval<decltype(generatorExpression())>().get()) {
using UnderlyingType = typename decltype(generatorExpression())::type;
IGeneratorTracker& tracker = acquireGeneratorTracker( lineInfo );
IGeneratorTracker& tracker = acquireGeneratorTracker( generatorName, lineInfo );
if (!tracker.hasGenerator()) {
tracker.setGenerator(Catch::Detail::make_unique<Generators<UnderlyingType>>(generatorExpression()));
}
@ -201,10 +202,16 @@ namespace Detail {
} // namespace Catch
#define GENERATE( ... ) \
Catch::Generators::generate( CATCH_INTERNAL_LINEINFO, [ ]{ using namespace Catch::Generators; return makeGenerators( __VA_ARGS__ ); } ) //NOLINT(google-build-using-namespace)
Catch::Generators::generate( INTERNAL_CATCH_STRINGIZE(INTERNAL_CATCH_UNIQUE_NAME(generator)), \
CATCH_INTERNAL_LINEINFO, \
[ ]{ using namespace Catch::Generators; return makeGenerators( __VA_ARGS__ ); } ) //NOLINT(google-build-using-namespace)
#define GENERATE_COPY( ... ) \
Catch::Generators::generate( CATCH_INTERNAL_LINEINFO, [=]{ using namespace Catch::Generators; return makeGenerators( __VA_ARGS__ ); } ) //NOLINT(google-build-using-namespace)
Catch::Generators::generate( INTERNAL_CATCH_STRINGIZE(INTERNAL_CATCH_UNIQUE_NAME(generator)), \
CATCH_INTERNAL_LINEINFO, \
[=]{ using namespace Catch::Generators; return makeGenerators( __VA_ARGS__ ); } ) //NOLINT(google-build-using-namespace)
#define GENERATE_REF( ... ) \
Catch::Generators::generate( CATCH_INTERNAL_LINEINFO, [&]{ using namespace Catch::Generators; return makeGenerators( __VA_ARGS__ ); } ) //NOLINT(google-build-using-namespace)
Catch::Generators::generate( INTERNAL_CATCH_STRINGIZE(INTERNAL_CATCH_UNIQUE_NAME(generator)), \
CATCH_INTERNAL_LINEINFO, \
[&]{ using namespace Catch::Generators; return makeGenerators( __VA_ARGS__ ); } ) //NOLINT(google-build-using-namespace)
#endif // TWOBLUECUBES_CATCH_GENERATORS_HPP_INCLUDED

View File

@ -50,8 +50,8 @@ namespace Detail {
GeneratorUntypedBase::~GeneratorUntypedBase() = default;
auto acquireGeneratorTracker( SourceLineInfo const& lineInfo ) -> IGeneratorTracker& {
return getResultCapture().acquireGeneratorTracker( lineInfo );
auto acquireGeneratorTracker(StringRef generatorName, SourceLineInfo const& lineInfo ) -> IGeneratorTracker& {
return getResultCapture().acquireGeneratorTracker( generatorName, lineInfo );
}
} // namespace Generators

View File

@ -42,7 +42,7 @@ namespace Catch {
virtual void sectionEnded( SectionEndInfo const& endInfo ) = 0;
virtual void sectionEndedEarly( SectionEndInfo const& endInfo ) = 0;
virtual auto acquireGeneratorTracker( SourceLineInfo const& lineInfo ) -> IGeneratorTracker& = 0;
virtual auto acquireGeneratorTracker( StringRef generatorName, SourceLineInfo const& lineInfo ) -> IGeneratorTracker& = 0;
virtual void benchmarkPreparing( std::string const& name ) = 0;
virtual void benchmarkStarting( BenchmarkInfo const& info ) = 0;

View File

@ -27,12 +27,27 @@ namespace Catch {
std::shared_ptr<GeneratorTracker> tracker;
ITracker& currentTracker = ctx.currentTracker();
if( TestCaseTracking::ITrackerPtr childTracker = currentTracker.findChild( nameAndLocation ) ) {
// Under specific circumstances, the generator we want
// to acquire is also the current tracker. If this is
// the case, we have to avoid looking through current
// tracker's children, and instead return the current
// tracker.
// A case where this check is important is e.g.
// for (int i = 0; i < 5; ++i) {
// int n = GENERATE(1, 2);
// }
//
// without it, the code above creates 5 nested generators.
if (currentTracker.nameAndLocation() == nameAndLocation) {
auto thisTracker = currentTracker.parent().findChild(nameAndLocation);
assert(thisTracker);
assert(thisTracker->isGeneratorTracker());
tracker = std::static_pointer_cast<GeneratorTracker>(thisTracker);
} else if ( TestCaseTracking::ITrackerPtr childTracker = currentTracker.findChild( nameAndLocation ) ) {
assert( childTracker );
assert( childTracker->isGeneratorTracker() );
tracker = std::static_pointer_cast<GeneratorTracker>( childTracker );
}
else {
} else {
tracker = std::make_shared<GeneratorTracker>( nameAndLocation, ctx, &currentTracker );
currentTracker.addChild( tracker );
}
@ -181,9 +196,10 @@ namespace Catch {
return true;
}
auto RunContext::acquireGeneratorTracker( SourceLineInfo const& lineInfo ) -> IGeneratorTracker& {
auto RunContext::acquireGeneratorTracker( StringRef generatorName, SourceLineInfo const& lineInfo ) -> IGeneratorTracker& {
using namespace Generators;
GeneratorTracker& tracker = GeneratorTracker::acquire( m_trackerContext, TestCaseTracking::NameAndLocation( "generator", lineInfo ) );
GeneratorTracker& tracker = GeneratorTracker::acquire(m_trackerContext,
TestCaseTracking::NameAndLocation( static_cast<std::string>(generatorName), lineInfo ) );
assert( tracker.isOpen() );
m_lastAssertionInfo.lineInfo = lineInfo;
return tracker;

View File

@ -76,7 +76,7 @@ namespace Catch {
void sectionEnded( SectionEndInfo const& endInfo ) override;
void sectionEndedEarly( SectionEndInfo const& endInfo ) override;
auto acquireGeneratorTracker( SourceLineInfo const& lineInfo ) -> IGeneratorTracker& override;
auto acquireGeneratorTracker( StringRef generatorName, SourceLineInfo const& lineInfo ) -> IGeneratorTracker& override;
void benchmarkPreparing( std::string const& name ) override;
void benchmarkStarting( BenchmarkInfo const& info ) override;

View File

@ -23,6 +23,10 @@ namespace TestCaseTracking {
SourceLineInfo location;
NameAndLocation( std::string const& _name, SourceLineInfo const& _location );
friend bool operator==(NameAndLocation const& lhs, NameAndLocation const& rhs) {
return lhs.name == rhs.name
&& lhs.location == rhs.location;
}
};
struct ITracker;

View File

@ -14,6 +14,8 @@ Nor would this
:test-result: PASS #1548
:test-result: PASS #1905 -- test spec parser properly clears internal state between compound tests
:test-result: PASS #1912 -- test spec parser handles escaping
:test-result: PASS #1913 - GENERATE inside a for loop should not keep recreating the generator
:test-result: PASS #1913 - GENERATEs can share a line
:test-result: XFAIL #748 - captures with unexpected exceptions
:test-result: PASS #809
:test-result: PASS #833

View File

@ -29,6 +29,12 @@ CmdLine.tests.cpp:<line number>: passed: spec.matches(*fakeTestCase(R"(spec {a}
CmdLine.tests.cpp:<line number>: passed: spec.matches(*fakeTestCase(R"(spec [a] char)")) for: true
CmdLine.tests.cpp:<line number>: passed: !(spec.matches(*fakeTestCase("differs but has similar tag", "[a]"))) for: !false
CmdLine.tests.cpp:<line number>: passed: spec.matches(*fakeTestCase(R"(spec \ char)")) for: true
Generators.tests.cpp:<line number>: passed: counter < 7 for: 3 < 7
Generators.tests.cpp:<line number>: passed: counter < 7 for: 6 < 7
Generators.tests.cpp:<line number>: passed: i != j for: 1 != 3
Generators.tests.cpp:<line number>: passed: i != j for: 1 != 4
Generators.tests.cpp:<line number>: passed: i != j for: 2 != 3
Generators.tests.cpp:<line number>: passed: i != j for: 2 != 4
Exception.tests.cpp:<line number>: failed: unexpected exception with message: 'answer := 42' with 1 message: 'expected exception'
Exception.tests.cpp:<line number>: failed: unexpected exception with message: 'answer := 42'; expression was: thisThrows() with 1 message: 'expected exception'
Exception.tests.cpp:<line number>: passed: thisThrows() with 1 message: 'answer := 42'

View File

@ -1380,6 +1380,6 @@ due to unexpected exception with message:
Why would you throw a std::string?
===============================================================================
test cases: 337 | 263 passed | 70 failed | 4 failed as expected
assertions: 1926 | 1774 passed | 131 failed | 21 failed as expected
test cases: 339 | 265 passed | 70 failed | 4 failed as expected
assertions: 1932 | 1780 passed | 131 failed | 21 failed as expected

View File

@ -236,6 +236,72 @@ CmdLine.tests.cpp:<line number>: PASSED:
with expansion:
true
-------------------------------------------------------------------------------
#1913 - GENERATE inside a for loop should not keep recreating the generator
-------------------------------------------------------------------------------
Generators.tests.cpp:<line number>
...............................................................................
Generators.tests.cpp:<line number>: PASSED:
REQUIRE( counter < 7 )
with expansion:
3 < 7
-------------------------------------------------------------------------------
#1913 - GENERATE inside a for loop should not keep recreating the generator
-------------------------------------------------------------------------------
Generators.tests.cpp:<line number>
...............................................................................
Generators.tests.cpp:<line number>: PASSED:
REQUIRE( counter < 7 )
with expansion:
6 < 7
-------------------------------------------------------------------------------
#1913 - GENERATEs can share a line
-------------------------------------------------------------------------------
Generators.tests.cpp:<line number>
...............................................................................
Generators.tests.cpp:<line number>: PASSED:
REQUIRE( i != j )
with expansion:
1 != 3
-------------------------------------------------------------------------------
#1913 - GENERATEs can share a line
-------------------------------------------------------------------------------
Generators.tests.cpp:<line number>
...............................................................................
Generators.tests.cpp:<line number>: PASSED:
REQUIRE( i != j )
with expansion:
1 != 4
-------------------------------------------------------------------------------
#1913 - GENERATEs can share a line
-------------------------------------------------------------------------------
Generators.tests.cpp:<line number>
...............................................................................
Generators.tests.cpp:<line number>: PASSED:
REQUIRE( i != j )
with expansion:
2 != 3
-------------------------------------------------------------------------------
#1913 - GENERATEs can share a line
-------------------------------------------------------------------------------
Generators.tests.cpp:<line number>
...............................................................................
Generators.tests.cpp:<line number>: PASSED:
REQUIRE( i != j )
with expansion:
2 != 4
-------------------------------------------------------------------------------
#748 - captures with unexpected exceptions
outside assertions
@ -15058,6 +15124,6 @@ Misc.tests.cpp:<line number>
Misc.tests.cpp:<line number>: PASSED:
===============================================================================
test cases: 337 | 247 passed | 86 failed | 4 failed as expected
assertions: 1943 | 1774 passed | 148 failed | 21 failed as expected
test cases: 339 | 249 passed | 86 failed | 4 failed as expected
assertions: 1949 | 1780 passed | 148 failed | 21 failed as expected

View File

@ -236,6 +236,72 @@ CmdLine.tests.cpp:<line number>: PASSED:
with expansion:
true
-------------------------------------------------------------------------------
#1913 - GENERATE inside a for loop should not keep recreating the generator
-------------------------------------------------------------------------------
Generators.tests.cpp:<line number>
...............................................................................
Generators.tests.cpp:<line number>: PASSED:
REQUIRE( counter < 7 )
with expansion:
3 < 7
-------------------------------------------------------------------------------
#1913 - GENERATE inside a for loop should not keep recreating the generator
-------------------------------------------------------------------------------
Generators.tests.cpp:<line number>
...............................................................................
Generators.tests.cpp:<line number>: PASSED:
REQUIRE( counter < 7 )
with expansion:
6 < 7
-------------------------------------------------------------------------------
#1913 - GENERATEs can share a line
-------------------------------------------------------------------------------
Generators.tests.cpp:<line number>
...............................................................................
Generators.tests.cpp:<line number>: PASSED:
REQUIRE( i != j )
with expansion:
1 != 3
-------------------------------------------------------------------------------
#1913 - GENERATEs can share a line
-------------------------------------------------------------------------------
Generators.tests.cpp:<line number>
...............................................................................
Generators.tests.cpp:<line number>: PASSED:
REQUIRE( i != j )
with expansion:
1 != 4
-------------------------------------------------------------------------------
#1913 - GENERATEs can share a line
-------------------------------------------------------------------------------
Generators.tests.cpp:<line number>
...............................................................................
Generators.tests.cpp:<line number>: PASSED:
REQUIRE( i != j )
with expansion:
2 != 3
-------------------------------------------------------------------------------
#1913 - GENERATEs can share a line
-------------------------------------------------------------------------------
Generators.tests.cpp:<line number>
...............................................................................
Generators.tests.cpp:<line number>: PASSED:
REQUIRE( i != j )
with expansion:
2 != 4
-------------------------------------------------------------------------------
#748 - captures with unexpected exceptions
outside assertions
@ -416,6 +482,6 @@ Condition.tests.cpp:<line number>: FAILED:
CHECK( true != true )
===============================================================================
test cases: 21 | 16 passed | 3 failed | 2 failed as expected
assertions: 48 | 41 passed | 4 failed | 3 failed as expected
test cases: 23 | 18 passed | 3 failed | 2 failed as expected
assertions: 54 | 47 passed | 4 failed | 3 failed as expected

View File

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuitesloose text artifact
>
<testsuite name="<exe-name>" errors="17" failures="132" tests="1944" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<testsuite name="<exe-name>" errors="17" failures="132" tests="1950" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<properties>
<property name="filters" value="~[!nonportable]~[!benchmark]~[approvals] *"/>
<property name="random-seed" value="1"/>
@ -33,6 +33,8 @@ Nor would this
<testcase classname="<exe-name>.global" name="#1905 -- test spec parser properly clears internal state between compound tests" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="#1912 -- test spec parser handles escaping/Various parentheses" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="#1912 -- test spec parser handles escaping/backslash in test name" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="#1913 - GENERATE inside a for loop should not keep recreating the generator" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="#1913 - GENERATEs can share a line" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="#748 - captures with unexpected exceptions/outside assertions" time="{duration}" status="run">
<error type="TEST_CASE">
FAILED:

View File

@ -915,6 +915,8 @@ Exception.tests.cpp:<line number>
</testCase>
</file>
<file path="tests/<exe-name>/UsageTests/Generators.tests.cpp">
<testCase name="#1913 - GENERATE inside a for loop should not keep recreating the generator" duration="{duration}"/>
<testCase name="#1913 - GENERATEs can share a line" duration="{duration}"/>
<testCase name="3x3x3 ints" duration="{duration}"/>
<testCase name="Copy and then generate a range/from var and iterators" duration="{duration}"/>
<testCase name="Copy and then generate a range/From a temporary container" duration="{duration}"/>

View File

@ -56,6 +56,18 @@ ok {test-number} - spec.matches(*fakeTestCase(R"(spec [a] char)")) for: true
ok {test-number} - !(spec.matches(*fakeTestCase("differs but has similar tag", "[a]"))) for: !false
# #1912 -- test spec parser handles escaping
ok {test-number} - spec.matches(*fakeTestCase(R"(spec \ char)")) for: true
# #1913 - GENERATE inside a for loop should not keep recreating the generator
ok {test-number} - counter < 7 for: 3 < 7
# #1913 - GENERATE inside a for loop should not keep recreating the generator
ok {test-number} - counter < 7 for: 6 < 7
# #1913 - GENERATEs can share a line
ok {test-number} - i != j for: 1 != 3
# #1913 - GENERATEs can share a line
ok {test-number} - i != j for: 1 != 4
# #1913 - GENERATEs can share a line
ok {test-number} - i != j for: 2 != 3
# #1913 - GENERATEs can share a line
ok {test-number} - i != j for: 2 != 4
# #748 - captures with unexpected exceptions
not ok {test-number} - unexpected exception with message: 'answer := 42' with 1 message: 'expected exception'
# #748 - captures with unexpected exceptions
@ -3878,5 +3890,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0
ok {test-number} -
# xmlentitycheck
ok {test-number} -
1..1935
1..1941

View File

@ -30,6 +30,10 @@ Tricky.tests.cpp:<line number>|nexplicit failure with message:|n "1514"']
##teamcity[testFinished name='#1905 -- test spec parser properly clears internal state between compound tests' duration="{duration}"]
##teamcity[testStarted name='#1912 -- test spec parser handles escaping']
##teamcity[testFinished name='#1912 -- test spec parser handles escaping' duration="{duration}"]
##teamcity[testStarted name='#1913 - GENERATE inside a for loop should not keep recreating the generator']
##teamcity[testFinished name='#1913 - GENERATE inside a for loop should not keep recreating the generator' duration="{duration}"]
##teamcity[testStarted name='#1913 - GENERATEs can share a line']
##teamcity[testFinished name='#1913 - GENERATEs can share a line' duration="{duration}"]
##teamcity[testStarted name='#748 - captures with unexpected exceptions']
Exception.tests.cpp:<line number>|nunexpected exception with messages:|n "answer := 42"|n "expected exception"- failure ignore as test marked as |'ok to fail|'|n']
Exception.tests.cpp:<line number>|nunexpected exception with messages:|n "answer := 42"|n "expected exception"|n REQUIRE_NOTHROW( thisThrows() )|nwith expansion:|n thisThrows()|n- failure ignore as test marked as |'ok to fail|'|n']

View File

@ -251,6 +251,60 @@ Nor would this
</Section>
<OverallResult success="true"/>
</TestCase>
<TestCase name="#1913 - GENERATE inside a for loop should not keep recreating the generator" tags="[generators][regression]" filename="tests/<exe-name>/UsageTests/Generators.tests.cpp" >
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/UsageTests/Generators.tests.cpp" >
<Original>
counter &lt; 7
</Original>
<Expanded>
3 &lt; 7
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/UsageTests/Generators.tests.cpp" >
<Original>
counter &lt; 7
</Original>
<Expanded>
6 &lt; 7
</Expanded>
</Expression>
<OverallResult success="true"/>
</TestCase>
<TestCase name="#1913 - GENERATEs can share a line" tags="[generators][regression]" filename="tests/<exe-name>/UsageTests/Generators.tests.cpp" >
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/UsageTests/Generators.tests.cpp" >
<Original>
i != j
</Original>
<Expanded>
1 != 3
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/UsageTests/Generators.tests.cpp" >
<Original>
i != j
</Original>
<Expanded>
1 != 4
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/UsageTests/Generators.tests.cpp" >
<Original>
i != j
</Original>
<Expanded>
2 != 3
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/UsageTests/Generators.tests.cpp" >
<Original>
i != j
</Original>
<Expanded>
2 != 4
</Expanded>
</Expression>
<OverallResult success="true"/>
</TestCase>
<TestCase name="#748 - captures with unexpected exceptions" tags="[!shouldfail][!throws][.][failing]" filename="tests/<exe-name>/UsageTests/Exception.tests.cpp" >
<Section name="outside assertions" filename="tests/<exe-name>/UsageTests/Exception.tests.cpp" >
<Info>
@ -18055,7 +18109,7 @@ loose text artifact
</Section>
<OverallResult success="true"/>
</TestCase>
<OverallResults successes="1774" failures="149" expectedFailures="21"/>
<OverallResults successes="1780" failures="149" expectedFailures="21"/>
</Group>
<OverallResults successes="1774" failures="148" expectedFailures="21"/>
<OverallResults successes="1780" failures="148" expectedFailures="21"/>
</Catch>

View File

@ -255,6 +255,22 @@ TEST_CASE("Copy and then generate a range", "[generators]") {
}
}
TEST_CASE("#1913 - GENERATE inside a for loop should not keep recreating the generator", "[regression][generators]") {
static int counter = 0;
for (int i = 0; i < 3; ++i) {
int _ = GENERATE(1, 2);
(void)_;
++counter;
}
// There should be at most 6 (3 * 2) counter increments
REQUIRE(counter < 7);
}
TEST_CASE("#1913 - GENERATEs can share a line", "[regression][generators]") {
int i = GENERATE(1, 2); int j = GENERATE(3, 4);
REQUIRE(i != j);
}
#if defined(__clang__)
#pragma clang diagnostic pop
#endif