3rd-party/catch2
1
0
Fork 0
mirror of https://github.com/catchorg/Catch2.git synced 2025-10-31 12:17:11 +01:00
Code Issues Packages Projects Releases Wiki Activity

Various improvements to the benchmarking support

Browse Source 
* Units from <ratio> are no longer redeclared in our own namespace
* The default clock is `steady_clock`, not `high_resolution_clock`,
because, as HH says "high_resolution_clock is useless. If you want
measure the passing of time, use steady_clock. If you want user
friendly time, use system_clock".
* Benchmarking support is opt-in, not opt-out, to avoid the large
(~10%) compile time penalty.
* Benchmarking-related options in CLI are always present, to decrease
the amount of code that is only compiled conditionally and making
the whole shebang more maintainble.
This commit is contained in:
Martin Hořeňovský
2019-05-19 20:54:44 +02:00
parent ce2560ca95
commit e340ab8db6
20 changed files with 54 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
include/catch.hpp
View File

@@ -33,9 +33,6 @@
# if defined(CATCH_CONFIG_DISABLE_MATCHERS)
# undef CATCH_CONFIG_DISABLE_MATCHERS
# endif
# if defined(CATCH_CONFIG_DISABLE_BENCHMARKING)
# undef CATCH_CONFIG_DISABLE_BENCHMARKING
# endif
# if !defined(CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER)
# define CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER
# endif
@@ -77,14 +74,14 @@
#include "internal/catch_objc.hpp"
#endif
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#include "internal/benchmark/catch_benchmark.hpp"
#endif
#ifdef CATCH_CONFIG_EXTERNAL_INTERFACES
#include "internal/catch_external_interfaces.h"
#endif
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
#include "internal/benchmark/catch_benchmark.hpp"
#endif
#endif // ! CATCH_CONFIG_IMPL_ONLY
#ifdef CATCH_IMPL
@@ -195,12 +192,12 @@
#define CATCH_THEN( desc ) INTERNAL_CATCH_DYNAMIC_SECTION( " Then: " << desc )
#define CATCH_AND_THEN( desc ) INTERNAL_CATCH_DYNAMIC_SECTION( " And: " << desc )
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
#define CATCH_BENCHMARK(...) \
INTERNAL_CATCH_BENCHMARK(INTERNAL_CATCH_UNIQUE_NAME(____C_A_T_C_H____B_E_N_C_H____), INTERNAL_CATCH_GET_1_ARG(__VA_ARGS__,,), INTERNAL_CATCH_GET_2_ARG(__VA_ARGS__,,))
#define CATCH_BENCHMARK_ADVANCED(name) \
INTERNAL_CATCH_BENCHMARK_ADVANCED(INTERNAL_CATCH_UNIQUE_NAME(____C_A_T_C_H____B_E_N_C_H____), name)
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
// If CATCH_CONFIG_PREFIX_ALL is not defined then the CATCH_ prefix is not required
#else
@@ -297,12 +294,12 @@
#define THEN( desc ) INTERNAL_CATCH_DYNAMIC_SECTION( " Then: " << desc )
#define AND_THEN( desc ) INTERNAL_CATCH_DYNAMIC_SECTION( " And: " << desc )
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
#define BENCHMARK(...) \
INTERNAL_CATCH_BENCHMARK(INTERNAL_CATCH_UNIQUE_NAME(____C_A_T_C_H____B_E_N_C_H____), INTERNAL_CATCH_GET_1_ARG(__VA_ARGS__,,), INTERNAL_CATCH_GET_2_ARG(__VA_ARGS__,,))
#define BENCHMARK_ADVANCED(name) \
INTERNAL_CATCH_BENCHMARK_ADVANCED(INTERNAL_CATCH_UNIQUE_NAME(____C_A_T_C_H____B_E_N_C_H____), name)
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
using Catch::Detail::Approx;

10
include/internal/benchmark/catch_clock.hpp
View File

@@ -16,12 +16,6 @@
namespace Catch {
namespace Benchmark {
template <unsigned Num, unsigned Den = 1>
using ratio = std::ratio<Num, Den>;
using milli = ratio<1, 1000>;
using micro = ratio<1, 1000000>;
using nano = ratio<1, 1000000000>;
template <typename Clock>
using ClockDuration = typename Clock::duration;
template <typename Clock>
@@ -30,7 +24,7 @@ namespace Catch {
template <typename Clock>
using TimePoint = typename Clock::time_point;
using default_clock = std::chrono::high_resolution_clock;
using default_clock = std::chrono::steady_clock;
template <typename Clock>
struct now {
@@ -39,7 +33,7 @@ namespace Catch {
}
};
using fp_seconds = std::chrono::duration<double, ratio<1>>;
using fp_seconds = std::chrono::duration<double, std::ratio<1>>;
} // namespace Benchmark
} // namespace Catch

2
include/internal/catch_commandline.cpp
View File

@@ -196,7 +196,6 @@ namespace Catch {
| Opt( setWaitForKeypress, "start|exit|both" )
["--wait-for-keypress"]
( "waits for a keypress before exiting" )
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
| Opt( config.benchmarkSamples, "samples" )
["--benchmark-samples"]
( "number of samples to collect (default: 100)" )
@@ -209,7 +208,6 @@ namespace Catch {
| Opt( config.benchmarkNoAnalysis )
["--benchmark-no-analysis"]
( "perform only measurements; do not perform any analysis" )
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
| Arg( config.testsOrTags, "test name|pattern|tags" )
( "which test or tests to use" );

2
include/internal/catch_config.cpp
View File

@@ -60,12 +60,10 @@ namespace Catch {
bool Config::showInvisibles() const { return m_data.showInvisibles; }
Verbosity Config::verbosity() const { return m_data.verbosity; }
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
bool Config::benchmarkNoAnalysis() const { return m_data.benchmarkNoAnalysis; }
int Config::benchmarkSamples() const { return m_data.benchmarkSamples; }
double Config::benchmarkConfidenceInterval() const { return m_data.benchmarkConfidenceInterval; }
unsigned int Config::benchmarkResamples() const { return m_data.benchmarkResamples; }
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
IStream const* Config::openStream() {
return Catch::makeStream(m_data.outputFilename);

4
include/internal/catch_config.hpp
View File

@@ -43,12 +43,10 @@ namespace Catch {
int abortAfter = -1;
unsigned int rngSeed = 0;
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
bool benchmarkNoAnalysis = false;
unsigned int benchmarkSamples = 100;
double benchmarkConfidenceInterval = 0.95;
unsigned int benchmarkResamples = 100000;
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
Verbosity verbosity = Verbosity::Normal;
WarnAbout::What warnings = WarnAbout::Nothing;
@@ -111,12 +109,10 @@ namespace Catch {
int abortAfter() const override;
bool showInvisibles() const override;
Verbosity verbosity() const override;
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
bool benchmarkNoAnalysis() const override;
int benchmarkSamples() const override;
double benchmarkConfidenceInterval() const override;
unsigned int benchmarkResamples() const override;
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
private:

8
include/internal/catch_interfaces_capture.h
View File

@@ -29,11 +29,11 @@ namespace Catch {
struct ITransientExpression;
struct IGeneratorTracker;
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
struct BenchmarkInfo;
template <typename Duration = std::chrono::duration<double, std::nano>>
struct BenchmarkStats;
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
struct IResultCapture {
@@ -46,12 +46,12 @@ namespace Catch {
virtual auto acquireGeneratorTracker( SourceLineInfo const& lineInfo ) -> IGeneratorTracker& = 0;
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
virtual void benchmarkPreparing( std::string const& name ) = 0;
virtual void benchmarkStarting( BenchmarkInfo const& info ) = 0;
virtual void benchmarkEnded( BenchmarkStats<> const& stats ) = 0;
virtual void benchmarkFailed( std::string const& error ) = 0;
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
virtual void pushScopedMessage( MessageInfo const& message ) = 0;
virtual void popScopedMessage( MessageInfo const& message ) = 0;

2
include/internal/catch_interfaces_config.h
View File

@@ -77,12 +77,10 @@ namespace Catch {
virtual std::vector<std::string> const& getSectionsToRun() const = 0;
virtual Verbosity verbosity() const = 0;
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
virtual bool benchmarkNoAnalysis() const = 0;
virtual int benchmarkSamples() const = 0;
virtual double benchmarkConfidenceInterval() const = 0;
virtual unsigned int benchmarkResamples() const = 0;
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
};
using IConfigPtr = std::shared_ptr<IConfig const>;

12
include/internal/catch_interfaces_reporter.h
View File

@@ -18,10 +18,10 @@
#include "catch_option.hpp"
#include "catch_stringref.h"
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
#include "benchmark/catch_estimate.hpp"
#include "benchmark/catch_outlier_classification.hpp"
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
#include <string>
@@ -165,7 +165,7 @@ namespace Catch {
bool aborting;
};
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
struct BenchmarkInfo {
std::string name;
double estimatedDuration;
@@ -201,7 +201,7 @@ namespace Catch {
};
}
};
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
struct IStreamingReporter {
virtual ~IStreamingReporter() = default;
@@ -220,12 +220,12 @@ namespace Catch {
virtual void testCaseStarting( TestCaseInfo const& testInfo ) = 0;
virtual void sectionStarting( SectionInfo const& sectionInfo ) = 0;
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
virtual void benchmarkPreparing( std::string const& ) {}
virtual void benchmarkStarting( BenchmarkInfo const& ) {}
virtual void benchmarkEnded( BenchmarkStats<> const& ) {}
virtual void benchmarkFailed( std::string const& ) {}
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
virtual void assertionStarting( AssertionInfo const& assertionInfo ) = 0;

4
include/internal/catch_run_context.cpp
View File

@@ -231,7 +231,7 @@ namespace Catch {
m_unfinishedSections.push_back(endInfo);
}
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
void RunContext::benchmarkPreparing(std::string const& name) {
m_reporter->benchmarkPreparing(name);
}
@@ -244,7 +244,7 @@ namespace Catch {
void RunContext::benchmarkFailed(std::string const & error) {
m_reporter->benchmarkFailed(error);
}
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
void RunContext::pushScopedMessage(MessageInfo const & message) {
m_messages.push_back(message);

4
include/internal/catch_run_context.h
View File

@@ -82,12 +82,12 @@ namespace Catch {
auto acquireGeneratorTracker( SourceLineInfo const& lineInfo ) -> IGeneratorTracker& override;
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
void benchmarkPreparing( std::string const& name ) override;
void benchmarkStarting( BenchmarkInfo const& info ) override;
void benchmarkEnded( BenchmarkStats<> const& stats ) override;
void benchmarkFailed( std::string const& error ) override;
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
void pushScopedMessage( MessageInfo const& message ) override;
void popScopedMessage( MessageInfo const& message ) override;

20
include/reporters/catch_reporter_console.cpp
View File

@@ -20,10 +20,16 @@
#if defined(_MSC_VER)
#pragma warning(push)
#pragma warning(disable:4061) // Not all labels are EXPLICITLY handled in switch
// Note that 4062 (not all labels are handled
// and default is missing) is enabled
// Note that 4062 (not all labels are handled and default is missing) is enabled
#endif
#if defined(__clang__)
# pragma clang diagnostic push
// For simplicity, benchmarking-only helpers are always enabled
# pragma clang diagnostic ignored "-Wunused-function"
#endif
namespace Catch {
@@ -287,7 +293,7 @@ public:
if (!m_isOpen) {
m_isOpen = true;
*this << RowBreak();
Columns headerCols;
Spacer spacer(2);
for (auto const& info : m_columnInfos) {
@@ -408,7 +414,7 @@ void ConsoleReporter::sectionEnded(SectionStats const& _sectionStats) {
StreamingReporterBase::sectionEnded(_sectionStats);
}
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
void ConsoleReporter::benchmarkPreparing(std::string const& name) {
lazyPrintWithoutClosingBenchmarkTable();
@@ -446,7 +452,7 @@ void ConsoleReporter::benchmarkFailed(std::string const& error) {
<< "Benchmark failed (" << error << ")"
<< ColumnBreak() << RowBreak();
}
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
void ConsoleReporter::testCaseEnded(TestCaseStats const& _testCaseStats) {
m_tablePrinter->close();
@@ -665,3 +671,7 @@ CATCH_REGISTER_REPORTER("console", ConsoleReporter)
#if defined(_MSC_VER)
#pragma warning(pop)
#endif
#if defined(__clang__)
# pragma clang diagnostic pop
#endif

4
include/reporters/catch_reporter_console.h
View File

@@ -39,12 +39,12 @@ namespace Catch {
void sectionStarting(SectionInfo const& _sectionInfo) override;
void sectionEnded(SectionStats const& _sectionStats) override;
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
void benchmarkPreparing(std::string const& name) override;
void benchmarkStarting(BenchmarkInfo const& info) override;
void benchmarkEnded(BenchmarkStats<> const& stats) override;
void benchmarkFailed(std::string const& error) override;
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
void testCaseEnded(TestCaseStats const& _testCaseStats) override;
void testGroupEnded(TestGroupStats const& _testGroupStats) override;

4
include/reporters/catch_reporter_listening.cpp
View File

@@ -42,7 +42,7 @@ namespace Catch {
m_reporter->noMatchingTestCases( spec );
}
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
void ListeningReporter::benchmarkPreparing( std::string const& name ) {
for (auto const& listener : m_listeners) {
listener->benchmarkPreparing(name);
@@ -68,7 +68,7 @@ namespace Catch {
}
m_reporter->benchmarkFailed(error);
}
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
void ListeningReporter::testRunStarting( TestRunInfo const& testRunInfo ) {
for ( auto const& listener : m_listeners ) {

4
include/reporters/catch_reporter_listening.h
View File

@@ -31,12 +31,12 @@ namespace Catch {
static std::set<Verbosity> getSupportedVerbosities();
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
void benchmarkPreparing(std::string const& name) override;
void benchmarkStarting( BenchmarkInfo const& benchmarkInfo ) override;
void benchmarkEnded( BenchmarkStats<> const& benchmarkStats ) override;
void benchmarkFailed(std::string const&) override;
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
void testRunStarting( TestRunInfo const& testRunInfo ) override;
void testGroupStarting( GroupInfo const& groupInfo ) override;

4
include/reporters/catch_reporter_xml.cpp
View File

@@ -219,7 +219,7 @@ namespace Catch {
m_xml.endElement();
}
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
void XmlReporter::benchmarkStarting(BenchmarkInfo const &info) {
m_xml.startElement("BenchmarkResults")
.writeAttribute("name", info.name)
@@ -259,7 +259,7 @@ namespace Catch {
writeAttribute("message", error);
m_xml.endElement();
}
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
CATCH_REGISTER_REPORTER( "xml", XmlReporter )

4
include/reporters/catch_reporter_xml.h
View File

@@ -50,11 +50,11 @@ namespace Catch {
void testRunEnded(TestRunStats const& testRunStats) override;
#ifndef CATCH_CONFIG_DISABLE_BENCHMARKING
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
void benchmarkStarting(BenchmarkInfo const&) override;
void benchmarkEnded(BenchmarkStats<> const&) override;
void benchmarkFailed(std::string const&) override;
#endif // CATCH_CONFIG_DISABLE_BENCHMARKING
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
private:
Timer m_testCaseTimer;