3rd-party/catch2
1
0
Fork 0
mirror of https://github.com/catchorg/Catch2.git synced 2024-11-22 13:26:10 +01:00
Code Issues Packages Projects Releases Wiki Activity

Various improvements to the benchmarking support

Browse Source 
* Units from <ratio> are no longer redeclared in our own namespace
* The default clock is `steady_clock`, not `high_resolution_clock`,
because, as HH says "high_resolution_clock is useless. If you want
measure the passing of time, use steady_clock. If you want user
friendly time, use system_clock".
* Benchmarking support is opt-in, not opt-out, to avoid the large
(~10%) compile time penalty.
* Benchmarking-related options in CLI are always present, to decrease
the amount of code that is only compiled conditionally and making
the whole shebang more maintainble.
This commit is contained in:
Martin Hořeňovský 2019-05-19 20:54:44 +02:00
parent ce2560ca95
commit e340ab8db6
No known key found for this signature in database
GPG Key ID: DE48307B8B0D381A
20 changed files with 54 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/configuration.md
View File

@ -149,7 +149,7 @@ by using `_NO_` in the macro, e.g. `CATCH_CONFIG_NO_CPP17_UNCAUGHT_EXCEPTIONS`.
CATCH_CONFIG_DISABLE // Disables assertions and test case registration
CATCH_CONFIG_WCHAR // Enables use of wchart_t
CATCH_CONFIG_EXPERIMENTAL_REDIRECT // Enables the new (experimental) way of capturing stdout/stderr
CATCH_CONFIG_DISABLE_BENCHMARKING // Disables the compile-time heavy benchmarking features
CATCH_CONFIG_ENABLE_BENCHMARKING // Enables the integrated benchmarking features (has a significant effect on compilation speed)
Currently Catch enables `CATCH_CONFIG_WINDOWS_SEH` only when compiled with MSVC, because some versions of MinGW do not have the necessary Win32 API support.

19
include/catch.hpp
View File

@ -33,9 +33,6 @@
# if defined(CATCH_CONFIG_DISABLE_MATCHERS)
# undef CATCH_CONFIG_DISABLE_MATCHERS
# endif
# if defined(CATCH_CONFIG_DISABLE_BENCHMARKING)
# undef CATCH_CONFIG_DISABLE_BENCHMARKING
# endif
# if !defined(CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER)
# define CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER
# endif
@ -77,14 +74,14 @@
#include "internal/catch_objc.hpp"
#endif
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#include "internal/benchmark/catch_benchmark.hpp"
#endif
#ifdef CATCH_CONFIG_EXTERNAL_INTERFACES
#include "internal/catch_external_interfaces.h"
#endif
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
#include "internal/benchmark/catch_benchmark.hpp"
#endif
#endif // ! CATCH_CONFIG_IMPL_ONLY
#ifdef CATCH_IMPL
@ -195,12 +192,12 @@
#define CATCH_THEN( desc ) INTERNAL_CATCH_DYNAMIC_SECTION( " Then: " << desc )
#define CATCH_AND_THEN( desc ) INTERNAL_CATCH_DYNAMIC_SECTION( " And: " << desc )
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
#define CATCH_BENCHMARK(...) \
INTERNAL_CATCH_BENCHMARK(INTERNAL_CATCH_UNIQUE_NAME(____C_A_T_C_H____B_E_N_C_H____), INTERNAL_CATCH_GET_1_ARG(__VA_ARGS__,,), INTERNAL_CATCH_GET_2_ARG(__VA_ARGS__,,))
#define CATCH_BENCHMARK_ADVANCED(name) \
INTERNAL_CATCH_BENCHMARK_ADVANCED(INTERNAL_CATCH_UNIQUE_NAME(____C_A_T_C_H____B_E_N_C_H____), name)
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
// If CATCH_CONFIG_PREFIX_ALL is not defined then the CATCH_ prefix is not required
#else
@ -297,12 +294,12 @@
#define THEN( desc ) INTERNAL_CATCH_DYNAMIC_SECTION( " Then: " << desc )
#define AND_THEN( desc ) INTERNAL_CATCH_DYNAMIC_SECTION( " And: " << desc )
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
#define BENCHMARK(...) \
INTERNAL_CATCH_BENCHMARK(INTERNAL_CATCH_UNIQUE_NAME(____C_A_T_C_H____B_E_N_C_H____), INTERNAL_CATCH_GET_1_ARG(__VA_ARGS__,,), INTERNAL_CATCH_GET_2_ARG(__VA_ARGS__,,))
#define BENCHMARK_ADVANCED(name) \
INTERNAL_CATCH_BENCHMARK_ADVANCED(INTERNAL_CATCH_UNIQUE_NAME(____C_A_T_C_H____B_E_N_C_H____), name)
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
using Catch::Detail::Approx;

10
include/internal/benchmark/catch_clock.hpp
View File

@ -16,12 +16,6 @@
namespace Catch {
namespace Benchmark {
template <unsigned Num, unsigned Den = 1>
using ratio = std::ratio<Num, Den>;
using milli = ratio<1, 1000>;
using micro = ratio<1, 1000000>;
using nano = ratio<1, 1000000000>;
template <typename Clock>
using ClockDuration = typename Clock::duration;
template <typename Clock>
@ -30,7 +24,7 @@ namespace Catch {
template <typename Clock>
using TimePoint = typename Clock::time_point;
using default_clock = std::chrono::high_resolution_clock;
using default_clock = std::chrono::steady_clock;
template <typename Clock>
struct now {
@ -39,7 +33,7 @@ namespace Catch {
}
};
using fp_seconds = std::chrono::duration<double, ratio<1>>;
using fp_seconds = std::chrono::duration<double, std::ratio<1>>;
} // namespace Benchmark
} // namespace Catch

2
include/internal/catch_commandline.cpp
View File

@ -196,7 +196,6 @@ namespace Catch {
| Opt( setWaitForKeypress, "start|exit|both" )
["--wait-for-keypress"]
( "waits for a keypress before exiting" )
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
| Opt( config.benchmarkSamples, "samples" )
["--benchmark-samples"]
( "number of samples to collect (default: 100)" )
@ -209,7 +208,6 @@ namespace Catch {
| Opt( config.benchmarkNoAnalysis )
["--benchmark-no-analysis"]
( "perform only measurements; do not perform any analysis" )
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
| Arg( config.testsOrTags, "test name|pattern|tags" )
( "which test or tests to use" );

2
include/internal/catch_config.cpp
View File

@ -60,12 +60,10 @@ namespace Catch {
bool Config::showInvisibles() const { return m_data.showInvisibles; }
Verbosity Config::verbosity() const { return m_data.verbosity; }
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
bool Config::benchmarkNoAnalysis() const { return m_data.benchmarkNoAnalysis; }
int Config::benchmarkSamples() const { return m_data.benchmarkSamples; }
double Config::benchmarkConfidenceInterval() const { return m_data.benchmarkConfidenceInterval; }
unsigned int Config::benchmarkResamples() const { return m_data.benchmarkResamples; }
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
IStream const* Config::openStream() {
return Catch::makeStream(m_data.outputFilename);

4
include/internal/catch_config.hpp
View File

@ -43,12 +43,10 @@ namespace Catch {
int abortAfter = -1;
unsigned int rngSeed = 0;
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
bool benchmarkNoAnalysis = false;
unsigned int benchmarkSamples = 100;
double benchmarkConfidenceInterval = 0.95;
unsigned int benchmarkResamples = 100000;
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
Verbosity verbosity = Verbosity::Normal;
WarnAbout::What warnings = WarnAbout::Nothing;
@ -111,12 +109,10 @@ namespace Catch {
int abortAfter() const override;
bool showInvisibles() const override;
Verbosity verbosity() const override;
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
bool benchmarkNoAnalysis() const override;
int benchmarkSamples() const override;
double benchmarkConfidenceInterval() const override;
unsigned int benchmarkResamples() const override;
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
private:

8
include/internal/catch_interfaces_capture.h
View File

@ -29,11 +29,11 @@ namespace Catch {
struct ITransientExpression;
struct IGeneratorTracker;
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
struct BenchmarkInfo;
template <typename Duration = std::chrono::duration<double, std::nano>>
struct BenchmarkStats;
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
struct IResultCapture {
@ -46,12 +46,12 @@ namespace Catch {
virtual auto acquireGeneratorTracker( SourceLineInfo const& lineInfo ) -> IGeneratorTracker& = 0;
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
virtual void benchmarkPreparing( std::string const& name ) = 0;
virtual void benchmarkStarting( BenchmarkInfo const& info ) = 0;
virtual void benchmarkEnded( BenchmarkStats<> const& stats ) = 0;
virtual void benchmarkFailed( std::string const& error ) = 0;
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
virtual void pushScopedMessage( MessageInfo const& message ) = 0;
virtual void popScopedMessage( MessageInfo const& message ) = 0;

2
include/internal/catch_interfaces_config.h
View File

@ -77,12 +77,10 @@ namespace Catch {
virtual std::vector<std::string> const& getSectionsToRun() const = 0;
virtual Verbosity verbosity() const = 0;
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
virtual bool benchmarkNoAnalysis() const = 0;
virtual int benchmarkSamples() const = 0;
virtual double benchmarkConfidenceInterval() const = 0;
virtual unsigned int benchmarkResamples() const = 0;
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
};
using IConfigPtr = std::shared_ptr<IConfig const>;

12
include/internal/catch_interfaces_reporter.h
View File

@ -18,10 +18,10 @@
#include "catch_option.hpp"
#include "catch_stringref.h"
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
#include "benchmark/catch_estimate.hpp"
#include "benchmark/catch_outlier_classification.hpp"
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
#include <string>
@ -165,7 +165,7 @@ namespace Catch {
bool aborting;
};
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
struct BenchmarkInfo {
std::string name;
double estimatedDuration;
@ -201,7 +201,7 @@ namespace Catch {
};
}
};
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
struct IStreamingReporter {
virtual ~IStreamingReporter() = default;
@ -220,12 +220,12 @@ namespace Catch {
virtual void testCaseStarting( TestCaseInfo const& testInfo ) = 0;
virtual void sectionStarting( SectionInfo const& sectionInfo ) = 0;
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
virtual void benchmarkPreparing( std::string const& ) {}
virtual void benchmarkStarting( BenchmarkInfo const& ) {}
virtual void benchmarkEnded( BenchmarkStats<> const& ) {}
virtual void benchmarkFailed( std::string const& ) {}
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
virtual void assertionStarting( AssertionInfo const& assertionInfo ) = 0;

4
include/internal/catch_run_context.cpp
View File

@ -231,7 +231,7 @@ namespace Catch {
m_unfinishedSections.push_back(endInfo);
}
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
void RunContext::benchmarkPreparing(std::string const& name) {
m_reporter->benchmarkPreparing(name);
}
@ -244,7 +244,7 @@ namespace Catch {
void RunContext::benchmarkFailed(std::string const & error) {
m_reporter->benchmarkFailed(error);
}
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
void RunContext::pushScopedMessage(MessageInfo const & message) {
m_messages.push_back(message);

4
include/internal/catch_run_context.h
View File

@ -82,12 +82,12 @@ namespace Catch {
auto acquireGeneratorTracker( SourceLineInfo const& lineInfo ) -> IGeneratorTracker& override;
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
void benchmarkPreparing( std::string const& name ) override;
void benchmarkStarting( BenchmarkInfo const& info ) override;
void benchmarkEnded( BenchmarkStats<> const& stats ) override;
void benchmarkFailed( std::string const& error ) override;
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
void pushScopedMessage( MessageInfo const& message ) override;
void popScopedMessage( MessageInfo const& message ) override;

20
include/reporters/catch_reporter_console.cpp
View File

@ -20,10 +20,16 @@
#if defined(_MSC_VER)
#pragma warning(push)
#pragma warning(disable:4061) // Not all labels are EXPLICITLY handled in switch
// Note that 4062 (not all labels are handled
// and default is missing) is enabled
// Note that 4062 (not all labels are handled and default is missing) is enabled
#endif
#if defined(__clang__)
# pragma clang diagnostic push
// For simplicity, benchmarking-only helpers are always enabled
# pragma clang diagnostic ignored "-Wunused-function"
#endif
namespace Catch {
@ -287,7 +293,7 @@ public:
if (!m_isOpen) {
m_isOpen = true;
*this << RowBreak();
Columns headerCols;
Spacer spacer(2);
for (auto const& info : m_columnInfos) {
@ -408,7 +414,7 @@ void ConsoleReporter::sectionEnded(SectionStats const& _sectionStats) {
StreamingReporterBase::sectionEnded(_sectionStats);
}
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
void ConsoleReporter::benchmarkPreparing(std::string const& name) {
lazyPrintWithoutClosingBenchmarkTable();
@ -446,7 +452,7 @@ void ConsoleReporter::benchmarkFailed(std::string const& error) {
<< "Benchmark failed (" << error << ")"
<< ColumnBreak() << RowBreak();
}
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
void ConsoleReporter::testCaseEnded(TestCaseStats const& _testCaseStats) {
m_tablePrinter->close();
@ -665,3 +671,7 @@ CATCH_REGISTER_REPORTER("console", ConsoleReporter)
#if defined(_MSC_VER)
#pragma warning(pop)
#endif
#if defined(__clang__)
# pragma clang diagnostic pop
#endif

4
include/reporters/catch_reporter_console.h
View File

@ -39,12 +39,12 @@ namespace Catch {
void sectionStarting(SectionInfo const& _sectionInfo) override;
void sectionEnded(SectionStats const& _sectionStats) override;
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
void benchmarkPreparing(std::string const& name) override;
void benchmarkStarting(BenchmarkInfo const& info) override;
void benchmarkEnded(BenchmarkStats<> const& stats) override;
void benchmarkFailed(std::string const& error) override;
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
void testCaseEnded(TestCaseStats const& _testCaseStats) override;
void testGroupEnded(TestGroupStats const& _testGroupStats) override;

4
include/reporters/catch_reporter_listening.cpp
View File

@ -42,7 +42,7 @@ namespace Catch {
m_reporter->noMatchingTestCases( spec );
}
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
void ListeningReporter::benchmarkPreparing( std::string const& name ) {
for (auto const& listener : m_listeners) {
listener->benchmarkPreparing(name);
@ -68,7 +68,7 @@ namespace Catch {
}
m_reporter->benchmarkFailed(error);
}
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
void ListeningReporter::testRunStarting( TestRunInfo const& testRunInfo ) {
for ( auto const& listener : m_listeners ) {

4
include/reporters/catch_reporter_listening.h
View File

@ -31,12 +31,12 @@ namespace Catch {
static std::set<Verbosity> getSupportedVerbosities();
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
void benchmarkPreparing(std::string const& name) override;
void benchmarkStarting( BenchmarkInfo const& benchmarkInfo ) override;
void benchmarkEnded( BenchmarkStats<> const& benchmarkStats ) override;
void benchmarkFailed(std::string const&) override;
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
void testRunStarting( TestRunInfo const& testRunInfo ) override;
void testGroupStarting( GroupInfo const& groupInfo ) override;

4
include/reporters/catch_reporter_xml.cpp
View File

@ -219,7 +219,7 @@ namespace Catch {
m_xml.endElement();
}
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
void XmlReporter::benchmarkStarting(BenchmarkInfo const &info) {
m_xml.startElement("BenchmarkResults")
.writeAttribute("name", info.name)
@ -259,7 +259,7 @@ namespace Catch {
writeAttribute("message", error);
m_xml.endElement();
}
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
CATCH_REGISTER_REPORTER( "xml", XmlReporter )

4
include/reporters/catch_reporter_xml.h
View File

@ -50,11 +50,11 @@ namespace Catch {
void testRunEnded(TestRunStats const& testRunStats) override;
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
void benchmarkStarting(BenchmarkInfo const&) override;
void benchmarkEnded(BenchmarkStats<> const&) override;
void benchmarkFailed(std::string const&) override;
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
private:
Timer m_testCaseTimer;

2
projects/SelfTest/IntrospectiveTests/CmdLine.tests.cpp
View File

@ -463,7 +463,6 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]"
}
}
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
SECTION("Benchmark options") {
SECTION("samples") {
CHECK(cli.parse({ "test", "--benchmark-samples=200" }));
@ -489,5 +488,4 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]"
REQUIRE(config.benchmarkNoAnalysis);
}
}
#endif
}

4
projects/SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp
View File

@ -7,7 +7,7 @@
*/
#include "catch.hpp"
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
namespace {
struct manual_clock {
public:
@ -402,4 +402,4 @@ TEST_CASE("run benchmark", "[benchmark]") {
CHECK((end - start).count() == 2867251000);
}
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING

4
projects/SelfTest/UsageTests/Benchmark.tests.cpp
View File

@ -2,7 +2,7 @@
#include <map>
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
std::uint64_t Fibonacci(std::uint64_t number);
std::uint64_t Fibonacci(std::uint64_t number) {
@ -127,4 +127,4 @@ TEST_CASE("Benchmark containers", "[!benchmark]") {
}
}
}
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING