3rd-party/catch2
1
0
Fork 0
mirror of https://github.com/catchorg/Catch2.git synced 2025-04-11 04:24:47 +02:00
Code Issues Packages Projects Releases Wiki Activity

Make CATCH_BREAK_INTO_DEBUGGER be user-configurable

Browse Source
This commit is contained in:
khyperia 2020-01-27 15:43:27 +01:00 committed by Martin Hořeňovský
parent 4a5bc0f39a
commit ccb1f70629
13 changed files with 89 additions and 21 deletions

10
docs/command-line.md

@ -24,6 +24,7 @@
[Specify the number of resamples for bootstrapping](#specify-the-number-of-resamples-for-bootstrapping)<br>
[Specify the confidence-interval for bootstrapping](#specify-the-confidence-interval-for-bootstrapping)<br>
[Disable statistical analysis of collected benchmark samples](#disable-statistical-analysis-of-collected-benchmark-samples)<br>
[Specify the amount of time in milliseconds spent on warming up each test](#specify-the-amount-of-time-in-milliseconds-spent-on-warming-up-each-test)<br>
[Usage](#usage)<br>
[Specify the section to run](#specify-the-section-to-run)<br>
[Filenames as tags](#filenames-as-tags)<br>
@ -64,6 +65,7 @@ Click one of the following links to take you straight to that option - or scroll
<a href="#benchmark-resamples"> ` --benchmark-resamples`</a><br />
<a href="#benchmark-confidence-interval"> ` --benchmark-confidence-interval`</a><br />
<a href="#benchmark-no-analysis"> ` --benchmark-no-analysis`</a><br />
<a href="#benchmark-warmup-time"> ` --benchmark-warmup-time`</a><br />
<a href="#use-colour"> ` --use-colour`</a><br />
</br>
@ -317,6 +319,14 @@ Must be between 0 and 1 and defaults to 0.95.
When this flag is specified no bootstrapping or any other statistical analysis is performed.
Instead the user code is only measured and the plain mean from the samples is reported.
<a id="benchmark-warmup-time"></a>
## Specify the amount of time in milliseconds spent on warming up each test
<pre>--benchmark-warmup-time</pre>
> [Introduced](https://github.com/catchorg/Catch2/pull/1844) in Catch X.Y.Z.
Configure the amount of time spent warming up each test.
<a id="usage"></a>
## Usage
<pre>-h, -?, --help</pre>

4
include/internal/benchmark/catch_benchmark.hpp

@ -44,10 +44,10 @@ namespace Catch {
template <typename Clock>
ExecutionPlan<FloatDuration<Clock>> prepare(const IConfig &cfg, Environment<FloatDuration<Clock>> env) const {
auto min_time = env.clock_resolution.mean * Detail::minimum_ticks;
auto run_time = std::max(min_time, std::chrono::duration_cast<decltype(min_time)>(Detail::warmup_time));
auto run_time = std::max(min_time, std::chrono::duration_cast<decltype(min_time)>(cfg.benchmarkWarmupTime()));
auto&& test = Detail::run_for_at_least<Clock>(std::chrono::duration_cast<ClockDuration<Clock>>(run_time), 1, fun);
int new_iters = static_cast<int>(std::ceil(min_time * test.iterations / test.elapsed));
return { new_iters, test.elapsed / test.iterations * new_iters * cfg.benchmarkSamples(), fun, std::chrono::duration_cast<FloatDuration<Clock>>(Detail::warmup_time), Detail::warmup_iterations };
return { new_iters, test.elapsed / test.iterations * new_iters * cfg.benchmarkSamples(), fun, std::chrono::duration_cast<FloatDuration<Clock>>(cfg.benchmarkWarmupTime()), Detail::warmup_iterations };
}
template <typename Clock = default_clock>

3
include/internal/catch_commandline.cpp

@ -213,6 +213,9 @@ namespace Catch {
| Opt( config.benchmarkNoAnalysis )
["--benchmark-no-analysis"]
( "perform only measurements; do not perform any analysis" )
| Opt( config.benchmarkWarmupTime, "benchmarkWarmupTime" )
["--benchmark-warmup-time"]
( "amount of time in milliseconds spent on warming up each test (default: 100)" )
| Arg( config.testsOrTags, "test name|pattern|tags" )
( "which test or tests to use" );

1
include/internal/catch_config.cpp

@ -76,6 +76,7 @@ namespace Catch {
int Config::benchmarkSamples() const { return m_data.benchmarkSamples; }
double Config::benchmarkConfidenceInterval() const { return m_data.benchmarkConfidenceInterval; }
unsigned int Config::benchmarkResamples() const { return m_data.benchmarkResamples; }
std::chrono::milliseconds Config::benchmarkWarmupTime() const { return std::chrono::milliseconds(m_data.benchmarkWarmupTime); }
IStream const* Config::openStream() {
return Catch::makeStream(m_data.outputFilename);

2
include/internal/catch_config.hpp

@ -47,6 +47,7 @@ namespace Catch {
unsigned int benchmarkSamples = 100;
double benchmarkConfidenceInterval = 0.95;
unsigned int benchmarkResamples = 100000;
std::chrono::milliseconds::rep benchmarkWarmupTime = 100;
Verbosity verbosity = Verbosity::Normal;
WarnAbout::What warnings = WarnAbout::Nothing;
@ -113,6 +114,7 @@ namespace Catch {
int benchmarkSamples() const override;
double benchmarkConfidenceInterval() const override;
unsigned int benchmarkResamples() const override;
std::chrono::milliseconds benchmarkWarmupTime() const override;
private:

2
include/internal/catch_interfaces_config.h

@ -11,6 +11,7 @@
#include "catch_common.h"
#include "catch_option.hpp"
#include <chrono>
#include <iosfwd>
#include <string>
#include <vector>
@ -81,6 +82,7 @@ namespace Catch {
virtual int benchmarkSamples() const = 0;
virtual double benchmarkConfidenceInterval() const = 0;
virtual unsigned int benchmarkResamples() const = 0;
virtual std::chrono::milliseconds benchmarkWarmupTime() const = 0;
};
using IConfigPtr = std::shared_ptr<IConfig const>;

2
projects/SelfTest/Baselines/compact.sw.approved.txt

@ -1116,6 +1116,8 @@ CmdLine.tests.cpp:<line number>: passed: cli.parse({ "test", "--benchmark-confid
CmdLine.tests.cpp:<line number>: passed: config.benchmarkConfidenceInterval == Catch::Detail::Approx(0.99) for: 0.99 == Approx( 0.99 )
CmdLine.tests.cpp:<line number>: passed: cli.parse({ "test", "--benchmark-no-analysis" }) for: {?}
CmdLine.tests.cpp:<line number>: passed: config.benchmarkNoAnalysis for: true
CmdLine.tests.cpp:<line number>: passed: cli.parse({ "test", "--benchmark-warmup-time=10" }) for: {?}
CmdLine.tests.cpp:<line number>: passed: config.benchmarkWarmupTime == 10 for: 10 == 10
Misc.tests.cpp:<line number>: passed: std::tuple_size<TestType>::value >= 1 for: 3 >= 1
Misc.tests.cpp:<line number>: passed: std::tuple_size<TestType>::value >= 1 for: 2 >= 1
Misc.tests.cpp:<line number>: passed: std::tuple_size<TestType>::value >= 1 for: 1 >= 1

2
projects/SelfTest/Baselines/console.std.approved.txt

@ -1381,5 +1381,5 @@ due to unexpected exception with message:
===============================================================================
test cases: 305 | 231 passed | 70 failed | 4 failed as expected
assertions: 1662 | 1510 passed | 131 failed | 21 failed as expected
assertions: 1664 | 1512 passed | 131 failed | 21 failed as expected

24
projects/SelfTest/Baselines/console.sw.approved.txt

@ -8046,7 +8046,7 @@ with expansion:
-------------------------------------------------------------------------------
Process can be configured on command line
Benchmark options
resamples
confidence-interval
-------------------------------------------------------------------------------
CmdLine.tests.cpp:<line number>
...............................................................................
@ -8064,7 +8064,7 @@ with expansion:
-------------------------------------------------------------------------------
Process can be configured on command line
Benchmark options
resamples
no-analysis
-------------------------------------------------------------------------------
CmdLine.tests.cpp:<line number>
...............................................................................
@ -8079,6 +8079,24 @@ CmdLine.tests.cpp:<line number>: PASSED:
with expansion:
true
-------------------------------------------------------------------------------
Process can be configured on command line
Benchmark options
warmup-time
-------------------------------------------------------------------------------
CmdLine.tests.cpp:<line number>
...............................................................................
CmdLine.tests.cpp:<line number>: PASSED:
CHECK( cli.parse({ "test", "--benchmark-warmup-time=10" }) )
with expansion:
{?}
CmdLine.tests.cpp:<line number>: PASSED:
REQUIRE( config.benchmarkWarmupTime == 10 )
with expansion:
10 == 10
-------------------------------------------------------------------------------
Product with differing arities - std::tuple<int, double, float>
-------------------------------------------------------------------------------
@ -13285,5 +13303,5 @@ Misc.tests.cpp:<line number>: PASSED:
===============================================================================
test cases: 305 | 215 passed | 86 failed | 4 failed as expected
assertions: 1679 | 1510 passed | 148 failed | 21 failed as expected
assertions: 1681 | 1512 passed | 148 failed | 21 failed as expected

7
projects/SelfTest/Baselines/junit.sw.approved.txt

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuitesloose text artifact
>
<testsuite name="<exe-name>" errors="17" failures="132" tests="1680" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<testsuite name="<exe-name>" errors="17" failures="132" tests="1682" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<properties>
<property name="filters" value="~[!nonportable]~[!benchmark]~[approvals]"/>
<property name="random-seed" value="1"/>
@ -1015,8 +1015,9 @@ Message.tests.cpp:<line number>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/use-colour/error" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/samples" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/resamples" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/resamples" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/resamples" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/confidence-interval" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/no-analysis" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/warmup-time" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Product with differing arities - std::tuple&lt;int, double, float>" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Product with differing arities - std::tuple&lt;int, double>" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Product with differing arities - std::tuple&lt;int>" time="{duration}"/>

5
projects/SelfTest/Baselines/sonarqube.sw.approved.txt

@ -64,8 +64,9 @@
<testCase name="Process can be configured on command line/use-colour/error" duration="{duration}"/>
<testCase name="Process can be configured on command line/Benchmark options/samples" duration="{duration}"/>
<testCase name="Process can be configured on command line/Benchmark options/resamples" duration="{duration}"/>
<testCase name="Process can be configured on command line/Benchmark options/resamples" duration="{duration}"/>
<testCase name="Process can be configured on command line/Benchmark options/resamples" duration="{duration}"/>
<testCase name="Process can be configured on command line/Benchmark options/confidence-interval" duration="{duration}"/>
<testCase name="Process can be configured on command line/Benchmark options/no-analysis" duration="{duration}"/>
<testCase name="Process can be configured on command line/Benchmark options/warmup-time" duration="{duration}"/>
<testCase name="Test with special, characters &quot;in name" duration="{duration}"/>
</file>
<file path="projects/<exe-name>/IntrospectiveTests/GeneratorsImpl.tests.cpp">

30
projects/SelfTest/Baselines/xml.sw.approved.txt

@ -10140,7 +10140,7 @@ Nor would this
<OverallResults successes="2" failures="0" expectedFailures="0"/>
</Section>
<Section name="Benchmark options" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Section name="resamples" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Section name="confidence-interval" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Expression success="true" type="CHECK" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
cli.parse({ "test", "--benchmark-confidence-interval=0.99" })
@ -10162,7 +10162,7 @@ Nor would this
<OverallResults successes="2" failures="0" expectedFailures="0"/>
</Section>
<Section name="Benchmark options" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Section name="resamples" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Section name="no-analysis" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Expression success="true" type="CHECK" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
cli.parse({ "test", "--benchmark-no-analysis" })
@ -10183,6 +10183,28 @@ Nor would this
</Section>
<OverallResults successes="2" failures="0" expectedFailures="0"/>
</Section>
<Section name="Benchmark options" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Section name="warmup-time" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Expression success="true" type="CHECK" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
cli.parse({ "test", "--benchmark-warmup-time=10" })
</Original>
<Expanded>
{?}
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
config.benchmarkWarmupTime == 10
</Original>
<Expanded>
10 == 10
</Expanded>
</Expression>
<OverallResults successes="2" failures="0" expectedFailures="0"/>
</Section>
<OverallResults successes="2" failures="0" expectedFailures="0"/>
</Section>
<OverallResult success="true"/>
</TestCase>
<TestCase name="Product with differing arities - std::tuple&lt;int, double, float>" tags="[product][template]" filename="projects/<exe-name>/UsageTests/Misc.tests.cpp" >
@ -15871,7 +15893,7 @@ loose text artifact
</Section>
<OverallResult success="true"/>
</TestCase>
<OverallResults successes="1510" failures="149" expectedFailures="21"/>
<OverallResults successes="1512" failures="149" expectedFailures="21"/>
</Group>
<OverallResults successes="1510" failures="148" expectedFailures="21"/>
<OverallResults successes="1512" failures="148" expectedFailures="21"/>
</Catch>

10
projects/SelfTest/IntrospectiveTests/CmdLine.tests.cpp

@ -503,17 +503,23 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]"
REQUIRE(config.benchmarkResamples == 20000);
}
SECTION("resamples") {
SECTION("confidence-interval") {
CHECK(cli.parse({ "test", "--benchmark-confidence-interval=0.99" }));
REQUIRE(config.benchmarkConfidenceInterval == Catch::Detail::Approx(0.99));
}
SECTION("resamples") {
SECTION("no-analysis") {
CHECK(cli.parse({ "test", "--benchmark-no-analysis" }));
REQUIRE(config.benchmarkNoAnalysis);
}
SECTION("warmup-time") {
CHECK(cli.parse({ "test", "--benchmark-warmup-time=10" }));
REQUIRE(config.benchmarkWarmupTime == 10);
}
}
}