Improved generator tracking

* Successive executions of the same `GENERATE` macro (e.g. because
of a for loop) no longer lead to multiple nested generators.
* The same line can now contain multiple `GENERATE` macros without
issues.

Fixes #1913
This commit is contained in:
Martin Hořeňovský 2020-06-01 19:04:23 +02:00
parent 6c6ebe374a
commit 9500ded83b
No known key found for this signature in database
GPG Key ID: DE48307B8B0D381A
14 changed files with 263 additions and 24 deletions

View File

@ -24,8 +24,8 @@ namespace Generators {
GeneratorUntypedBase::~GeneratorUntypedBase() {}
auto acquireGeneratorTracker( SourceLineInfo const& lineInfo ) -> IGeneratorTracker& {
return getResultCapture().acquireGeneratorTracker( lineInfo );
auto acquireGeneratorTracker( StringRef generatorName, SourceLineInfo const& lineInfo ) -> IGeneratorTracker& {
return getResultCapture().acquireGeneratorTracker( generatorName, lineInfo );
}
} // namespace Generators

View File

@ -10,6 +10,7 @@
#include "catch_interfaces_generatortracker.h"
#include "catch_common.h"
#include "catch_enforce.h"
#include "catch_stringref.h"
#include <memory>
#include <vector>
@ -181,16 +182,16 @@ namespace Generators {
return makeGenerators( value( T( std::forward<U>( val ) ) ), std::forward<Gs>( moreGenerators )... );
}
auto acquireGeneratorTracker( SourceLineInfo const& lineInfo ) -> IGeneratorTracker&;
auto acquireGeneratorTracker( StringRef generatorName, SourceLineInfo const& lineInfo ) -> IGeneratorTracker&;
template<typename L>
// Note: The type after -> is weird, because VS2015 cannot parse
// the expression used in the typedef inside, when it is in
// return type. Yeah.
auto generate( SourceLineInfo const& lineInfo, L const& generatorExpression ) -> decltype(std::declval<decltype(generatorExpression())>().get()) {
auto generate( StringRef generatorName, SourceLineInfo const& lineInfo, L const& generatorExpression ) -> decltype(std::declval<decltype(generatorExpression())>().get()) {
using UnderlyingType = typename decltype(generatorExpression())::type;
IGeneratorTracker& tracker = acquireGeneratorTracker( lineInfo );
IGeneratorTracker& tracker = acquireGeneratorTracker( generatorName, lineInfo );
if (!tracker.hasGenerator()) {
tracker.setGenerator(pf::make_unique<Generators<UnderlyingType>>(generatorExpression()));
}
@ -203,10 +204,16 @@ namespace Generators {
} // namespace Catch
#define GENERATE( ... ) \
Catch::Generators::generate( CATCH_INTERNAL_LINEINFO, [ ]{ using namespace Catch::Generators; return makeGenerators( __VA_ARGS__ ); } ) //NOLINT(google-build-using-namespace)
Catch::Generators::generate( INTERNAL_CATCH_STRINGIZE(INTERNAL_CATCH_UNIQUE_NAME(generator)), \
CATCH_INTERNAL_LINEINFO, \
[ ]{ using namespace Catch::Generators; return makeGenerators( __VA_ARGS__ ); } ) //NOLINT(google-build-using-namespace)
#define GENERATE_COPY( ... ) \
Catch::Generators::generate( CATCH_INTERNAL_LINEINFO, [=]{ using namespace Catch::Generators; return makeGenerators( __VA_ARGS__ ); } ) //NOLINT(google-build-using-namespace)
Catch::Generators::generate( INTERNAL_CATCH_STRINGIZE(INTERNAL_CATCH_UNIQUE_NAME(generator)), \
CATCH_INTERNAL_LINEINFO, \
[=]{ using namespace Catch::Generators; return makeGenerators( __VA_ARGS__ ); } ) //NOLINT(google-build-using-namespace)
#define GENERATE_REF( ... ) \
Catch::Generators::generate( CATCH_INTERNAL_LINEINFO, [&]{ using namespace Catch::Generators; return makeGenerators( __VA_ARGS__ ); } ) //NOLINT(google-build-using-namespace)
Catch::Generators::generate( INTERNAL_CATCH_STRINGIZE(INTERNAL_CATCH_UNIQUE_NAME(generator)), \
CATCH_INTERNAL_LINEINFO, \
[&]{ using namespace Catch::Generators; return makeGenerators( __VA_ARGS__ ); } ) //NOLINT(google-build-using-namespace)
#endif // TWOBLUECUBES_CATCH_GENERATORS_HPP_INCLUDED

View File

@ -44,7 +44,7 @@ namespace Catch {
virtual void sectionEnded( SectionEndInfo const& endInfo ) = 0;
virtual void sectionEndedEarly( SectionEndInfo const& endInfo ) = 0;
virtual auto acquireGeneratorTracker( SourceLineInfo const& lineInfo ) -> IGeneratorTracker& = 0;
virtual auto acquireGeneratorTracker( StringRef generatorName, SourceLineInfo const& lineInfo ) -> IGeneratorTracker& = 0;
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
virtual void benchmarkPreparing( std::string const& name ) = 0;

View File

@ -25,12 +25,27 @@ namespace Catch {
std::shared_ptr<GeneratorTracker> tracker;
ITracker& currentTracker = ctx.currentTracker();
if( TestCaseTracking::ITrackerPtr childTracker = currentTracker.findChild( nameAndLocation ) ) {
// Under specific circumstances, the generator we want
// to acquire is also the current tracker. If this is
// the case, we have to avoid looking through current
// tracker's children, and instead return the current
// tracker.
// A case where this check is important is e.g.
// for (int i = 0; i < 5; ++i) {
// int n = GENERATE(1, 2);
// }
//
// without it, the code above creates 5 nested generators.
if (currentTracker.nameAndLocation() == nameAndLocation) {
auto thisTracker = currentTracker.parent().findChild(nameAndLocation);
assert(thisTracker);
assert(thisTracker->isGeneratorTracker());
tracker = std::static_pointer_cast<GeneratorTracker>(thisTracker);
} else if ( TestCaseTracking::ITrackerPtr childTracker = currentTracker.findChild( nameAndLocation ) ) {
assert( childTracker );
assert( childTracker->isGeneratorTracker() );
tracker = std::static_pointer_cast<GeneratorTracker>( childTracker );
}
else {
} else {
tracker = std::make_shared<GeneratorTracker>( nameAndLocation, ctx, &currentTracker );
currentTracker.addChild( tracker );
}
@ -187,9 +202,10 @@ namespace Catch {
return true;
}
auto RunContext::acquireGeneratorTracker( SourceLineInfo const& lineInfo ) -> IGeneratorTracker& {
auto RunContext::acquireGeneratorTracker( StringRef generatorName, SourceLineInfo const& lineInfo ) -> IGeneratorTracker& {
using namespace Generators;
GeneratorTracker& tracker = GeneratorTracker::acquire( m_trackerContext, TestCaseTracking::NameAndLocation( "generator", lineInfo ) );
GeneratorTracker& tracker = GeneratorTracker::acquire(m_trackerContext,
TestCaseTracking::NameAndLocation( static_cast<std::string>(generatorName), lineInfo ) );
assert( tracker.isOpen() );
m_lastAssertionInfo.lineInfo = lineInfo;
return tracker;

View File

@ -80,7 +80,7 @@ namespace Catch {
void sectionEnded( SectionEndInfo const& endInfo ) override;
void sectionEndedEarly( SectionEndInfo const& endInfo ) override;
auto acquireGeneratorTracker( SourceLineInfo const& lineInfo ) -> IGeneratorTracker& override;
auto acquireGeneratorTracker( StringRef generatorName, SourceLineInfo const& lineInfo ) -> IGeneratorTracker& override;
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
void benchmarkPreparing( std::string const& name ) override;

View File

@ -23,6 +23,10 @@ namespace TestCaseTracking {
SourceLineInfo location;
NameAndLocation( std::string const& _name, SourceLineInfo const& _location );
friend bool operator==(NameAndLocation const& lhs, NameAndLocation const& rhs) {
return lhs.name == rhs.name
&& lhs.location == rhs.location;
}
};
struct ITracker;

View File

@ -30,6 +30,12 @@ CmdLine.tests.cpp:<line number>: passed: spec.matches(fakeTestCase(R"(spec {a} c
CmdLine.tests.cpp:<line number>: passed: spec.matches(fakeTestCase(R"(spec [a] char)")) for: true
CmdLine.tests.cpp:<line number>: passed: !(spec.matches(fakeTestCase("differs but has similar tag", "[a]"))) for: !false
CmdLine.tests.cpp:<line number>: passed: spec.matches(fakeTestCase(R"(spec \ char)")) for: true
Generators.tests.cpp:<line number>: passed: counter < 7 for: 3 < 7
Generators.tests.cpp:<line number>: passed: counter < 7 for: 6 < 7
Generators.tests.cpp:<line number>: passed: i != j for: 1 != 3
Generators.tests.cpp:<line number>: passed: i != j for: 1 != 4
Generators.tests.cpp:<line number>: passed: i != j for: 2 != 3
Generators.tests.cpp:<line number>: passed: i != j for: 2 != 4
Exception.tests.cpp:<line number>: failed: unexpected exception with message: 'answer := 42' with 1 message: 'expected exception'
Exception.tests.cpp:<line number>: failed: unexpected exception with message: 'answer := 42'; expression was: thisThrows() with 1 message: 'expected exception'
Exception.tests.cpp:<line number>: passed: thisThrows() with 1 message: 'answer := 42'

View File

@ -1380,6 +1380,6 @@ due to unexpected exception with message:
Why would you throw a std::string?
===============================================================================
test cases: 310 | 236 passed | 70 failed | 4 failed as expected
assertions: 1701 | 1549 passed | 131 failed | 21 failed as expected
test cases: 312 | 238 passed | 70 failed | 4 failed as expected
assertions: 1707 | 1555 passed | 131 failed | 21 failed as expected

View File

@ -243,6 +243,72 @@ CmdLine.tests.cpp:<line number>: PASSED:
with expansion:
true
-------------------------------------------------------------------------------
#1913 - GENERATE inside a for loop should not keep recreating the generator
-------------------------------------------------------------------------------
Generators.tests.cpp:<line number>
...............................................................................
Generators.tests.cpp:<line number>: PASSED:
REQUIRE( counter < 7 )
with expansion:
3 < 7
-------------------------------------------------------------------------------
#1913 - GENERATE inside a for loop should not keep recreating the generator
-------------------------------------------------------------------------------
Generators.tests.cpp:<line number>
...............................................................................
Generators.tests.cpp:<line number>: PASSED:
REQUIRE( counter < 7 )
with expansion:
6 < 7
-------------------------------------------------------------------------------
#1913 - GENERATEs can share a line
-------------------------------------------------------------------------------
Generators.tests.cpp:<line number>
...............................................................................
Generators.tests.cpp:<line number>: PASSED:
REQUIRE( i != j )
with expansion:
1 != 3
-------------------------------------------------------------------------------
#1913 - GENERATEs can share a line
-------------------------------------------------------------------------------
Generators.tests.cpp:<line number>
...............................................................................
Generators.tests.cpp:<line number>: PASSED:
REQUIRE( i != j )
with expansion:
1 != 4
-------------------------------------------------------------------------------
#1913 - GENERATEs can share a line
-------------------------------------------------------------------------------
Generators.tests.cpp:<line number>
...............................................................................
Generators.tests.cpp:<line number>: PASSED:
REQUIRE( i != j )
with expansion:
2 != 3
-------------------------------------------------------------------------------
#1913 - GENERATEs can share a line
-------------------------------------------------------------------------------
Generators.tests.cpp:<line number>
...............................................................................
Generators.tests.cpp:<line number>: PASSED:
REQUIRE( i != j )
with expansion:
2 != 4
-------------------------------------------------------------------------------
#748 - captures with unexpected exceptions
outside assertions
@ -13573,6 +13639,6 @@ Misc.tests.cpp:<line number>
Misc.tests.cpp:<line number>: PASSED:
===============================================================================
test cases: 310 | 220 passed | 86 failed | 4 failed as expected
assertions: 1718 | 1549 passed | 148 failed | 21 failed as expected
test cases: 312 | 222 passed | 86 failed | 4 failed as expected
assertions: 1724 | 1555 passed | 148 failed | 21 failed as expected

View File

@ -243,6 +243,72 @@ CmdLine.tests.cpp:<line number>: PASSED:
with expansion:
true
-------------------------------------------------------------------------------
#1913 - GENERATE inside a for loop should not keep recreating the generator
-------------------------------------------------------------------------------
Generators.tests.cpp:<line number>
...............................................................................
Generators.tests.cpp:<line number>: PASSED:
REQUIRE( counter < 7 )
with expansion:
3 < 7
-------------------------------------------------------------------------------
#1913 - GENERATE inside a for loop should not keep recreating the generator
-------------------------------------------------------------------------------
Generators.tests.cpp:<line number>
...............................................................................
Generators.tests.cpp:<line number>: PASSED:
REQUIRE( counter < 7 )
with expansion:
6 < 7
-------------------------------------------------------------------------------
#1913 - GENERATEs can share a line
-------------------------------------------------------------------------------
Generators.tests.cpp:<line number>
...............................................................................
Generators.tests.cpp:<line number>: PASSED:
REQUIRE( i != j )
with expansion:
1 != 3
-------------------------------------------------------------------------------
#1913 - GENERATEs can share a line
-------------------------------------------------------------------------------
Generators.tests.cpp:<line number>
...............................................................................
Generators.tests.cpp:<line number>: PASSED:
REQUIRE( i != j )
with expansion:
1 != 4
-------------------------------------------------------------------------------
#1913 - GENERATEs can share a line
-------------------------------------------------------------------------------
Generators.tests.cpp:<line number>
...............................................................................
Generators.tests.cpp:<line number>: PASSED:
REQUIRE( i != j )
with expansion:
2 != 3
-------------------------------------------------------------------------------
#1913 - GENERATEs can share a line
-------------------------------------------------------------------------------
Generators.tests.cpp:<line number>
...............................................................................
Generators.tests.cpp:<line number>: PASSED:
REQUIRE( i != j )
with expansion:
2 != 4
-------------------------------------------------------------------------------
#748 - captures with unexpected exceptions
outside assertions
@ -423,6 +489,6 @@ Condition.tests.cpp:<line number>: FAILED:
CHECK( true != true )
===============================================================================
test cases: 21 | 16 passed | 3 failed | 2 failed as expected
assertions: 49 | 42 passed | 4 failed | 3 failed as expected
test cases: 23 | 18 passed | 3 failed | 2 failed as expected
assertions: 55 | 48 passed | 4 failed | 3 failed as expected

View File

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuitesloose text artifact
>
<testsuite name="<exe-name>" errors="17" failures="132" tests="1719" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<testsuite name="<exe-name>" errors="17" failures="132" tests="1725" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<properties>
<property name="filters" value="~[!nonportable]~[!benchmark]~[approvals]"/>
<property name="random-seed" value="1"/>
@ -33,6 +33,8 @@ Nor would this
<testcase classname="<exe-name>.global" name="#1905 -- test spec parser properly clears internal state between compound tests" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="#1912 -- test spec parser handles escaping/Various parentheses" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="#1912 -- test spec parser handles escaping/backslash in test name" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="#1913 - GENERATE inside a for loop should not keep recreating the generator" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="#1913 - GENERATEs can share a line" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="#748 - captures with unexpected exceptions/outside assertions" time="{duration}" status="run">
<error type="TEST_CASE">
FAILED:

View File

@ -875,6 +875,8 @@ Exception.tests.cpp:<line number>
</testCase>
</file>
<file path="projects/<exe-name>/UsageTests/Generators.tests.cpp">
<testCase name="#1913 - GENERATE inside a for loop should not keep recreating the generator" duration="{duration}"/>
<testCase name="#1913 - GENERATEs can share a line" duration="{duration}"/>
<testCase name="3x3x3 ints" duration="{duration}"/>
<testCase name="Copy and then generate a range/from var and iterators" duration="{duration}"/>
<testCase name="Copy and then generate a range/From a temporary container" duration="{duration}"/>

View File

@ -264,6 +264,60 @@ Nor would this
</Section>
<OverallResult success="true"/>
</TestCase>
<TestCase name="#1913 - GENERATE inside a for loop should not keep recreating the generator" tags="[generators][regression]" filename="projects/<exe-name>/UsageTests/Generators.tests.cpp" >
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/UsageTests/Generators.tests.cpp" >
<Original>
counter &lt; 7
</Original>
<Expanded>
3 &lt; 7
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/UsageTests/Generators.tests.cpp" >
<Original>
counter &lt; 7
</Original>
<Expanded>
6 &lt; 7
</Expanded>
</Expression>
<OverallResult success="true"/>
</TestCase>
<TestCase name="#1913 - GENERATEs can share a line" tags="[generators][regression]" filename="projects/<exe-name>/UsageTests/Generators.tests.cpp" >
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/UsageTests/Generators.tests.cpp" >
<Original>
i != j
</Original>
<Expanded>
1 != 3
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/UsageTests/Generators.tests.cpp" >
<Original>
i != j
</Original>
<Expanded>
1 != 4
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/UsageTests/Generators.tests.cpp" >
<Original>
i != j
</Original>
<Expanded>
2 != 3
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/UsageTests/Generators.tests.cpp" >
<Original>
i != j
</Original>
<Expanded>
2 != 4
</Expanded>
</Expression>
<OverallResult success="true"/>
</TestCase>
<TestCase name="#748 - captures with unexpected exceptions" tags="[!hide][!shouldfail][!throws][.][failing]" filename="projects/<exe-name>/UsageTests/Exception.tests.cpp" >
<Section name="outside assertions" filename="projects/<exe-name>/UsageTests/Exception.tests.cpp" >
<Info>
@ -16255,7 +16309,7 @@ loose text artifact
</Section>
<OverallResult success="true"/>
</TestCase>
<OverallResults successes="1549" failures="149" expectedFailures="21"/>
<OverallResults successes="1555" failures="149" expectedFailures="21"/>
</Group>
<OverallResults successes="1549" failures="148" expectedFailures="21"/>
<OverallResults successes="1555" failures="148" expectedFailures="21"/>
</Catch>

View File

@ -251,6 +251,22 @@ TEST_CASE("Copy and then generate a range", "[generators]") {
}
}
TEST_CASE("#1913 - GENERATE inside a for loop should not keep recreating the generator", "[regression][generators]") {
static int counter = 0;
for (int i = 0; i < 3; ++i) {
int _ = GENERATE(1, 2);
(void)_;
++counter;
}
// There should be at most 6 (3 * 2) counter increments
REQUIRE(counter < 7);
}
TEST_CASE("#1913 - GENERATEs can share a line", "[regression][generators]") {
int i = GENERATE(1, 2); int j = GENERATE(3, 4);
REQUIRE(i != j);
}
#if defined(__clang__)
#pragma clang diagnostic pop
#endif