3rd-party/catch2
1
0
Fork 0
mirror of https://github.com/catchorg/Catch2.git synced 2025-08-01 21:05:39 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add an extra test for benchmarking macros

Browse Source 
Also updated baselines
This commit is contained in:
Martin Hořeňovský
2019-06-02 13:19:09 +02:00
parent e340ab8db6
commit 10067a47da
8 changed files with 326 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
projects/SelfTest/Baselines/compact.sw.approved.txt
View File

@@ -944,6 +944,14 @@ CmdLine.tests.cpp:<line number>: passed: cli.parse({"test", "--use-colour", "no"
CmdLine.tests.cpp:<line number>: passed: config.useColour == UseColour::No for: 2 == 2
CmdLine.tests.cpp:<line number>: passed: !result for: true
CmdLine.tests.cpp:<line number>: passed: result.errorMessage(), Contains( "colour mode must be one of" ) for: "colour mode must be one of: auto, yes or no. 'wrong' not recognised" contains: "colour mode must be one of"
CmdLine.tests.cpp:<line number>: passed: cli.parse({ "test", "--benchmark-samples=200" }) for: {?}
CmdLine.tests.cpp:<line number>: passed: config.benchmarkSamples == 200 for: 200 == 200
CmdLine.tests.cpp:<line number>: passed: cli.parse({ "test", "--benchmark-resamples=20000" }) for: {?}
CmdLine.tests.cpp:<line number>: passed: config.benchmarkResamples == 20000 for: 20000 (0x<hex digits>) == 20000 (0x<hex digits>)
CmdLine.tests.cpp:<line number>: passed: cli.parse({ "test", "--benchmark-confidence-interval=0.99" }) for: {?}
CmdLine.tests.cpp:<line number>: passed: config.benchmarkConfidenceInterval == Catch::Detail::Approx(0.99) for: 0.99 == Approx( 0.99 )
CmdLine.tests.cpp:<line number>: passed: cli.parse({ "test", "--benchmark-no-analysis" }) for: {?}
CmdLine.tests.cpp:<line number>: passed: config.benchmarkNoAnalysis for: true
Misc.tests.cpp:<line number>: passed: std::tuple_size<TestType>::value >= 1 for: 3 >= 1
Misc.tests.cpp:<line number>: passed: std::tuple_size<TestType>::value >= 1 for: 2 >= 1
Misc.tests.cpp:<line number>: passed: std::tuple_size<TestType>::value >= 1 for: 1 >= 1

2
projects/SelfTest/Baselines/console.std.approved.txt
View File

@@ -1381,5 +1381,5 @@ due to unexpected exception with message:
===============================================================================
test cases: 289 | 215 passed | 70 failed | 4 failed as expected
assertions: 1539 | 1387 passed | 131 failed | 21 failed as expected
assertions: 1547 | 1395 passed | 131 failed | 21 failed as expected

74
projects/SelfTest/Baselines/console.sw.approved.txt
View File

@@ -6954,6 +6954,78 @@ with expansion:
"colour mode must be one of: auto, yes or no. 'wrong' not recognised"
contains: "colour mode must be one of"
-------------------------------------------------------------------------------
Process can be configured on command line
Benchmark options
samples
-------------------------------------------------------------------------------
CmdLine.tests.cpp:<line number>
...............................................................................
CmdLine.tests.cpp:<line number>: PASSED:
CHECK( cli.parse({ "test", "--benchmark-samples=200" }) )
with expansion:
{?}
CmdLine.tests.cpp:<line number>: PASSED:
REQUIRE( config.benchmarkSamples == 200 )
with expansion:
200 == 200
-------------------------------------------------------------------------------
Process can be configured on command line
Benchmark options
resamples
-------------------------------------------------------------------------------
CmdLine.tests.cpp:<line number>
...............................................................................
CmdLine.tests.cpp:<line number>: PASSED:
CHECK( cli.parse({ "test", "--benchmark-resamples=20000" }) )
with expansion:
{?}
CmdLine.tests.cpp:<line number>: PASSED:
REQUIRE( config.benchmarkResamples == 20000 )
with expansion:
20000 (0x<hex digits>) == 20000 (0x<hex digits>)
-------------------------------------------------------------------------------
Process can be configured on command line
Benchmark options
resamples
-------------------------------------------------------------------------------
CmdLine.tests.cpp:<line number>
...............................................................................
CmdLine.tests.cpp:<line number>: PASSED:
CHECK( cli.parse({ "test", "--benchmark-confidence-interval=0.99" }) )
with expansion:
{?}
CmdLine.tests.cpp:<line number>: PASSED:
REQUIRE( config.benchmarkConfidenceInterval == Catch::Detail::Approx(0.99) )
with expansion:
0.99 == Approx( 0.99 )
-------------------------------------------------------------------------------
Process can be configured on command line
Benchmark options
resamples
-------------------------------------------------------------------------------
CmdLine.tests.cpp:<line number>
...............................................................................
CmdLine.tests.cpp:<line number>: PASSED:
CHECK( cli.parse({ "test", "--benchmark-no-analysis" }) )
with expansion:
{?}
CmdLine.tests.cpp:<line number>: PASSED:
REQUIRE( config.benchmarkNoAnalysis )
with expansion:
true
-------------------------------------------------------------------------------
Product with differing arities - std::tuple<int, double, float>
-------------------------------------------------------------------------------
@@ -12219,5 +12291,5 @@ Misc.tests.cpp:<line number>: PASSED:
===============================================================================
test cases: 289 | 199 passed | 86 failed | 4 failed as expected
assertions: 1556 | 1387 passed | 148 failed | 21 failed as expected
assertions: 1564 | 1395 passed | 148 failed | 21 failed as expected

6
projects/SelfTest/Baselines/junit.sw.approved.txt
View File

@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuitesloose text artifact
>
<testsuite name="<exe-name>" errors="17" failures="132" tests="1557" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<testsuite name="<exe-name>" errors="17" failures="132" tests="1565" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<properties>
<property name="filters" value="~[!nonportable]~[!benchmark]~[approvals]"/>
<property name="random-seed" value="1"/>
@@ -648,6 +648,10 @@ Message.tests.cpp:<line number>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/use-colour/yes" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/use-colour/no" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/use-colour/error" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/samples" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/resamples" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/resamples" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/resamples" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Product with differing arities - std::tuple&lt;int, double, float>" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Product with differing arities - std::tuple&lt;int, double>" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Product with differing arities - std::tuple&lt;int>" time="{duration}"/>

92
projects/SelfTest/Baselines/xml.sw.approved.txt
View File

@@ -8669,6 +8669,94 @@ Nor would this
</Section>
<OverallResults successes="2" failures="0" expectedFailures="0"/>
</Section>
<Section name="Benchmark options" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Section name="samples" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Expression success="true" type="CHECK" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
cli.parse({ "test", "--benchmark-samples=200" })
</Original>
<Expanded>
{?}
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
config.benchmarkSamples == 200
</Original>
<Expanded>
200 == 200
</Expanded>
</Expression>
<OverallResults successes="2" failures="0" expectedFailures="0"/>
</Section>
<OverallResults successes="2" failures="0" expectedFailures="0"/>
</Section>
<Section name="Benchmark options" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Section name="resamples" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Expression success="true" type="CHECK" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
cli.parse({ "test", "--benchmark-resamples=20000" })
</Original>
<Expanded>
{?}
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
config.benchmarkResamples == 20000
</Original>
<Expanded>
20000 (0x<hex digits>) == 20000 (0x<hex digits>)
</Expanded>
</Expression>
<OverallResults successes="2" failures="0" expectedFailures="0"/>
</Section>
<OverallResults successes="2" failures="0" expectedFailures="0"/>
</Section>
<Section name="Benchmark options" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Section name="resamples" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Expression success="true" type="CHECK" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
cli.parse({ "test", "--benchmark-confidence-interval=0.99" })
</Original>
<Expanded>
{?}
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
config.benchmarkConfidenceInterval == Catch::Detail::Approx(0.99)
</Original>
<Expanded>
0.99 == Approx( 0.99 )
</Expanded>
</Expression>
<OverallResults successes="2" failures="0" expectedFailures="0"/>
</Section>
<OverallResults successes="2" failures="0" expectedFailures="0"/>
</Section>
<Section name="Benchmark options" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Section name="resamples" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Expression success="true" type="CHECK" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
cli.parse({ "test", "--benchmark-no-analysis" })
</Original>
<Expanded>
{?}
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
config.benchmarkNoAnalysis
</Original>
<Expanded>
true
</Expanded>
</Expression>
<OverallResults successes="2" failures="0" expectedFailures="0"/>
</Section>
<OverallResults successes="2" failures="0" expectedFailures="0"/>
</Section>
<OverallResult success="true"/>
</TestCase>
<TestCase name="Product with differing arities - std::tuple&lt;int, double, float>" tags="[product][template]" filename="projects/<exe-name>/UsageTests/Misc.tests.cpp" >
@@ -14586,7 +14674,7 @@ loose text artifact
</Section>
<OverallResult success="true"/>
</TestCase>
<OverallResults successes="1387" failures="149" expectedFailures="21"/>
<OverallResults successes="1395" failures="149" expectedFailures="21"/>
</Group>
<OverallResults successes="1387" failures="148" expectedFailures="21"/>
<OverallResults successes="1395" failures="148" expectedFailures="21"/>
</Catch>

8
projects/SelfTest/UsageTests/Benchmark.tests.cpp
View File

@@ -3,10 +3,10 @@
#include <map>
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
std::uint64_t Fibonacci(std::uint64_t number);
std::uint64_t Fibonacci(std::uint64_t number) {
return number < 2 ? 1 : Fibonacci(number - 1) + Fibonacci(number - 2);
namespace {
std::uint64_t Fibonacci(std::uint64_t number) {
return number < 2 ? 1 : Fibonacci(number - 1) + Fibonacci(number - 2);
}
}
TEST_CASE("Benchmark Fibonacci", "[!benchmark]") {