3rd-party/catch2
1
0
Fork 0
mirror of https://github.com/catchorg/Catch2.git synced 2025-10-31 12:17:11 +01:00
Code Issues Packages Projects Releases Wiki Activity

Allow configuring of benchmark warmup time

Browse Source
This commit is contained in:
khyperia
2020-01-27 15:43:27 +01:00
committed by Martin Hořeňovský
parent e880da93bd
commit 3c7e737a7b
14 changed files with 94 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tests/SelfTest/Baselines/compact.sw.approved.txt
View File

@@ -1152,6 +1152,8 @@ CmdLine.tests.cpp:<line number>: passed: cli.parse({ "test", "--benchmark-confid
CmdLine.tests.cpp:<line number>: passed: config.benchmarkConfidenceInterval == Catch::Approx(0.99) for: 0.99 == Approx( 0.99 )
CmdLine.tests.cpp:<line number>: passed: cli.parse({ "test", "--benchmark-no-analysis" }) for: {?}
CmdLine.tests.cpp:<line number>: passed: config.benchmarkNoAnalysis for: true
CmdLine.tests.cpp:<line number>: passed: cli.parse({ "test", "--benchmark-warmup-time=10" }) for: {?}
CmdLine.tests.cpp:<line number>: passed: config.benchmarkWarmupTime == 10 for: 10 == 10
Misc.tests.cpp:<line number>: passed: std::tuple_size<TestType>::value >= 1 for: 3 >= 1
Misc.tests.cpp:<line number>: passed: std::tuple_size<TestType>::value >= 1 for: 2 >= 1
Misc.tests.cpp:<line number>: passed: std::tuple_size<TestType>::value >= 1 for: 1 >= 1

2
tests/SelfTest/Baselines/console.std.approved.txt
View File

@@ -1381,5 +1381,5 @@ due to unexpected exception with message:
===============================================================================
test cases: 328 | 254 passed | 70 failed | 4 failed as expected
assertions: 1827 | 1675 passed | 131 failed | 21 failed as expected
assertions: 1829 | 1677 passed | 131 failed | 21 failed as expected

24
tests/SelfTest/Baselines/console.sw.approved.txt
View File

@@ -8284,7 +8284,7 @@ with expansion:
-------------------------------------------------------------------------------
Process can be configured on command line
Benchmark options
resamples
confidence-interval
-------------------------------------------------------------------------------
CmdLine.tests.cpp:<line number>
...............................................................................
@@ -8302,7 +8302,7 @@ with expansion:
-------------------------------------------------------------------------------
Process can be configured on command line
Benchmark options
resamples
no-analysis
-------------------------------------------------------------------------------
CmdLine.tests.cpp:<line number>
...............................................................................
@@ -8317,6 +8317,24 @@ CmdLine.tests.cpp:<line number>: PASSED:
with expansion:
true
-------------------------------------------------------------------------------
Process can be configured on command line
Benchmark options
warmup-time
-------------------------------------------------------------------------------
CmdLine.tests.cpp:<line number>
...............................................................................
CmdLine.tests.cpp:<line number>: PASSED:
CHECK( cli.parse({ "test", "--benchmark-warmup-time=10" }) )
with expansion:
{?}
CmdLine.tests.cpp:<line number>: PASSED:
REQUIRE( config.benchmarkWarmupTime == 10 )
with expansion:
10 == 10
-------------------------------------------------------------------------------
Product with differing arities - std::tuple<int, double, float>
-------------------------------------------------------------------------------
@@ -14283,5 +14301,5 @@ Misc.tests.cpp:<line number>: PASSED:
===============================================================================
test cases: 328 | 238 passed | 86 failed | 4 failed as expected
assertions: 1844 | 1675 passed | 148 failed | 21 failed as expected
assertions: 1846 | 1677 passed | 148 failed | 21 failed as expected

7
tests/SelfTest/Baselines/junit.sw.approved.txt
View File

@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuitesloose text artifact
>
<testsuite name="<exe-name>" errors="17" failures="132" tests="1845" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<testsuite name="<exe-name>" errors="17" failures="132" tests="1847" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<properties>
<property name="filters" value="~[!nonportable]~[!benchmark]~[approvals] *"/>
<property name="random-seed" value="1"/>
@@ -1024,8 +1024,9 @@ Message.tests.cpp:<line number>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/use-colour/error" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/samples" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/resamples" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/resamples" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/resamples" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/confidence-interval" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/no-analysis" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/warmup-time" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Product with differing arities - std::tuple&lt;int, double, float>" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Product with differing arities - std::tuple&lt;int, double>" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Product with differing arities - std::tuple&lt;int>" time="{duration}"/>

5
tests/SelfTest/Baselines/sonarqube.sw.approved.txt
View File

@@ -64,8 +64,9 @@
<testCase name="Process can be configured on command line/use-colour/error" duration="{duration}"/>
<testCase name="Process can be configured on command line/Benchmark options/samples" duration="{duration}"/>
<testCase name="Process can be configured on command line/Benchmark options/resamples" duration="{duration}"/>
<testCase name="Process can be configured on command line/Benchmark options/resamples" duration="{duration}"/>
<testCase name="Process can be configured on command line/Benchmark options/resamples" duration="{duration}"/>
<testCase name="Process can be configured on command line/Benchmark options/confidence-interval" duration="{duration}"/>
<testCase name="Process can be configured on command line/Benchmark options/no-analysis" duration="{duration}"/>
<testCase name="Process can be configured on command line/Benchmark options/warmup-time" duration="{duration}"/>
<testCase name="Test with special, characters &quot;in name" duration="{duration}"/>
</file>
<file path="tests/<exe-name>/IntrospectiveTests/GeneratorsImpl.tests.cpp">

6
tests/SelfTest/Baselines/tap.sw.approved.txt
View File

@@ -2226,6 +2226,10 @@ ok {test-number} - config.benchmarkConfidenceInterval == Catch::Approx(0.99) for
ok {test-number} - cli.parse({ "test", "--benchmark-no-analysis" }) for: {?}
# Process can be configured on command line
ok {test-number} - config.benchmarkNoAnalysis for: true
# Process can be configured on command line
ok {test-number} - cli.parse({ "test", "--benchmark-warmup-time=10" }) for: {?}
# Process can be configured on command line
ok {test-number} - config.benchmarkWarmupTime == 10 for: 10 == 10
# Product with differing arities - std::tuple<int, double, float>
ok {test-number} - std::tuple_size<TestType>::value >= 1 for: 3 >= 1
# Product with differing arities - std::tuple<int, double>
@@ -3680,5 +3684,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0
ok {test-number} -
# xmlentitycheck
ok {test-number} -
1..1836
1..1838

30
tests/SelfTest/Baselines/xml.sw.approved.txt
View File

@@ -10354,7 +10354,7 @@ Nor would this
<OverallResults successes="2" failures="0" expectedFailures="0"/>
</Section>
<Section name="Benchmark options" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Section name="resamples" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Section name="confidence-interval" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
cli.parse({ "test", "--benchmark-confidence-interval=0.99" })
@@ -10376,7 +10376,7 @@ Nor would this
<OverallResults successes="2" failures="0" expectedFailures="0"/>
</Section>
<Section name="Benchmark options" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Section name="resamples" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Section name="no-analysis" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
cli.parse({ "test", "--benchmark-no-analysis" })
@@ -10397,6 +10397,28 @@ Nor would this
</Section>
<OverallResults successes="2" failures="0" expectedFailures="0"/>
</Section>
<Section name="Benchmark options" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Section name="warmup-time" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
cli.parse({ "test", "--benchmark-warmup-time=10" })
</Original>
<Expanded>
{?}
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
config.benchmarkWarmupTime == 10
</Original>
<Expanded>
10 == 10
</Expanded>
</Expression>
<OverallResults successes="2" failures="0" expectedFailures="0"/>
</Section>
<OverallResults successes="2" failures="0" expectedFailures="0"/>
</Section>
<OverallResult success="true"/>
</TestCase>
<TestCase name="Product with differing arities - std::tuple&lt;int, double, float>" tags="[product][template]" filename="tests/<exe-name>/UsageTests/Misc.tests.cpp" >
@@ -17117,7 +17139,7 @@ loose text artifact
</Section>
<OverallResult success="true"/>
</TestCase>
<OverallResults successes="1675" failures="149" expectedFailures="21"/>
<OverallResults successes="1677" failures="149" expectedFailures="21"/>
</Group>
<OverallResults successes="1675" failures="148" expectedFailures="21"/>
<OverallResults successes="1677" failures="148" expectedFailures="21"/>
</Catch>

10
tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp
View File

@@ -497,17 +497,23 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]"
REQUIRE(config.benchmarkResamples == 20000);
}
SECTION("resamples") {
SECTION("confidence-interval") {
CHECK(cli.parse({ "test", "--benchmark-confidence-interval=0.99" }));
REQUIRE(config.benchmarkConfidenceInterval == Catch::Approx(0.99));
}
SECTION("resamples") {
SECTION("no-analysis") {
CHECK(cli.parse({ "test", "--benchmark-no-analysis" }));
REQUIRE(config.benchmarkNoAnalysis);
}
SECTION("warmup-time") {
CHECK(cli.parse({ "test", "--benchmark-warmup-time=10" }));
REQUIRE(config.benchmarkWarmupTime == 10);
}
}
}