2010-12-31 23:07:47 +01:00
|
|
|
/*
|
|
|
|
* Created by Phil on 31/12/2010.
|
|
|
|
* Copyright 2010 Two Blue Cubes Ltd. All rights reserved.
|
|
|
|
*
|
|
|
|
* Distributed under the Boost Software License, Version 1.0. (See accompanying
|
|
|
|
* file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
|
|
|
*/
|
2012-09-17 07:42:29 +02:00
|
|
|
#ifndef TWOBLUECUBES_CATCH_INTERFACES_REPORTER_H_INCLUDED
|
|
|
|
#define TWOBLUECUBES_CATCH_INTERFACES_REPORTER_H_INCLUDED
|
2010-12-31 23:07:47 +01:00
|
|
|
|
2013-12-03 19:52:41 +01:00
|
|
|
#include "catch_section_info.h"
|
2010-12-31 23:07:47 +01:00
|
|
|
#include "catch_common.h"
|
2012-08-28 09:20:18 +02:00
|
|
|
#include "catch_config.hpp"
|
2017-09-07 12:24:33 +02:00
|
|
|
#include "catch_totals.h"
|
2012-11-22 20:17:20 +01:00
|
|
|
#include "catch_test_case_info.h"
|
|
|
|
#include "catch_assertionresult.h"
|
2013-02-02 20:58:04 +01:00
|
|
|
#include "catch_message.h"
|
2012-12-05 09:40:53 +01:00
|
|
|
#include "catch_option.hpp"
|
2017-09-14 19:57:59 +02:00
|
|
|
#include "catch_stringref.h"
|
2010-12-31 23:07:47 +01:00
|
|
|
|
2019-05-19 20:54:44 +02:00
|
|
|
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
|
2019-04-23 23:41:13 +02:00
|
|
|
#include "benchmark/catch_estimate.hpp"
|
|
|
|
#include "benchmark/catch_outlier_classification.hpp"
|
2019-05-19 20:54:44 +02:00
|
|
|
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
|
2019-04-23 23:41:13 +02:00
|
|
|
|
2017-07-19 10:13:47 +02:00
|
|
|
|
2010-12-31 23:07:47 +01:00
|
|
|
#include <string>
|
2017-07-19 10:13:47 +02:00
|
|
|
#include <iosfwd>
|
2010-12-31 23:21:36 +01:00
|
|
|
#include <map>
|
2017-07-06 00:25:49 +02:00
|
|
|
#include <set>
|
2017-04-26 18:04:00 +02:00
|
|
|
#include <memory>
|
2019-04-23 23:41:13 +02:00
|
|
|
#include <algorithm>
|
2010-12-31 23:07:47 +01:00
|
|
|
|
2017-07-19 10:13:47 +02:00
|
|
|
namespace Catch {
|
|
|
|
|
2012-11-29 21:31:17 +01:00
|
|
|
struct ReporterConfig {
|
2017-07-19 10:13:47 +02:00
|
|
|
explicit ReporterConfig( IConfigPtr const& _fullConfig );
|
2013-05-28 19:39:32 +02:00
|
|
|
|
2017-07-19 10:13:47 +02:00
|
|
|
ReporterConfig( IConfigPtr const& _fullConfig, std::ostream& _stream );
|
2012-09-20 09:17:52 +02:00
|
|
|
|
2017-07-19 10:13:47 +02:00
|
|
|
std::ostream& stream() const;
|
|
|
|
IConfigPtr fullConfig() const;
|
2012-09-20 09:17:52 +02:00
|
|
|
|
|
|
|
private:
|
2012-11-27 00:28:00 +01:00
|
|
|
std::ostream* m_stream;
|
2017-04-25 21:18:02 +02:00
|
|
|
IConfigPtr m_fullConfig;
|
2012-08-28 09:20:18 +02:00
|
|
|
};
|
2012-11-22 20:17:20 +01:00
|
|
|
|
2012-11-29 21:31:17 +01:00
|
|
|
struct ReporterPreferences {
|
2017-07-19 10:13:47 +02:00
|
|
|
bool shouldRedirectStdOut = false;
|
2018-07-14 20:51:02 +02:00
|
|
|
bool shouldReportAllAssertions = false;
|
2012-11-29 10:05:51 +01:00
|
|
|
};
|
|
|
|
|
2013-08-08 09:24:37 +02:00
|
|
|
template<typename T>
|
|
|
|
struct LazyStat : Option<T> {
|
|
|
|
LazyStat& operator=( T const& _value ) {
|
|
|
|
Option<T>::operator=( _value );
|
|
|
|
used = false;
|
|
|
|
return *this;
|
|
|
|
}
|
|
|
|
void reset() {
|
|
|
|
Option<T>::reset();
|
|
|
|
used = false;
|
|
|
|
}
|
2017-07-19 10:13:47 +02:00
|
|
|
bool used = false;
|
2013-08-08 09:24:37 +02:00
|
|
|
};
|
|
|
|
|
2012-12-02 00:49:57 +01:00
|
|
|
struct TestRunInfo {
|
2017-07-19 10:13:47 +02:00
|
|
|
TestRunInfo( std::string const& _name );
|
2012-12-02 00:49:57 +01:00
|
|
|
std::string name;
|
|
|
|
};
|
|
|
|
struct GroupInfo {
|
2013-01-13 22:51:44 +01:00
|
|
|
GroupInfo( std::string const& _name,
|
|
|
|
std::size_t _groupIndex,
|
2017-07-19 10:13:47 +02:00
|
|
|
std::size_t _groupsCount );
|
2013-01-13 22:51:44 +01:00
|
|
|
|
2012-12-02 00:49:57 +01:00
|
|
|
std::string name;
|
2013-01-13 22:51:44 +01:00
|
|
|
std::size_t groupIndex;
|
|
|
|
std::size_t groupsCounts;
|
2012-12-02 00:49:57 +01:00
|
|
|
};
|
|
|
|
|
2013-01-03 10:04:46 +01:00
|
|
|
struct AssertionStats {
|
2012-11-29 21:31:17 +01:00
|
|
|
AssertionStats( AssertionResult const& _assertionResult,
|
2013-02-02 20:58:04 +01:00
|
|
|
std::vector<MessageInfo> const& _infoMessages,
|
2017-07-19 10:13:47 +02:00
|
|
|
Totals const& _totals );
|
2013-07-03 20:14:59 +02:00
|
|
|
|
2014-03-20 12:48:19 +01:00
|
|
|
AssertionStats( AssertionStats const& ) = default;
|
|
|
|
AssertionStats( AssertionStats && ) = default;
|
2019-02-12 21:57:00 +01:00
|
|
|
AssertionStats& operator = ( AssertionStats const& ) = delete;
|
|
|
|
AssertionStats& operator = ( AssertionStats && ) = delete;
|
2017-09-07 16:51:33 +02:00
|
|
|
virtual ~AssertionStats();
|
2014-03-20 12:48:19 +01:00
|
|
|
|
2012-11-22 20:17:20 +01:00
|
|
|
AssertionResult assertionResult;
|
2013-02-02 20:58:04 +01:00
|
|
|
std::vector<MessageInfo> infoMessages;
|
2012-11-22 20:17:20 +01:00
|
|
|
Totals totals;
|
|
|
|
};
|
|
|
|
|
2013-01-03 10:04:46 +01:00
|
|
|
struct SectionStats {
|
2012-11-30 09:58:46 +01:00
|
|
|
SectionStats( SectionInfo const& _sectionInfo,
|
|
|
|
Counts const& _assertions,
|
2013-08-07 19:56:35 +02:00
|
|
|
double _durationInSeconds,
|
2017-07-19 10:13:47 +02:00
|
|
|
bool _missingAssertions );
|
2014-03-20 12:48:19 +01:00
|
|
|
SectionStats( SectionStats const& ) = default;
|
|
|
|
SectionStats( SectionStats && ) = default;
|
|
|
|
SectionStats& operator = ( SectionStats const& ) = default;
|
|
|
|
SectionStats& operator = ( SectionStats && ) = default;
|
2017-09-07 16:51:33 +02:00
|
|
|
virtual ~SectionStats();
|
2012-11-30 19:54:06 +01:00
|
|
|
|
2012-11-29 21:31:17 +01:00
|
|
|
SectionInfo sectionInfo;
|
2012-11-30 09:58:46 +01:00
|
|
|
Counts assertions;
|
2013-08-07 19:56:35 +02:00
|
|
|
double durationInSeconds;
|
2012-11-30 09:58:46 +01:00
|
|
|
bool missingAssertions;
|
2012-11-29 21:31:17 +01:00
|
|
|
};
|
|
|
|
|
2013-01-03 10:04:46 +01:00
|
|
|
struct TestCaseStats {
|
2012-11-29 21:31:17 +01:00
|
|
|
TestCaseStats( TestCaseInfo const& _testInfo,
|
|
|
|
Totals const& _totals,
|
|
|
|
std::string const& _stdOut,
|
|
|
|
std::string const& _stdErr,
|
2017-07-19 10:13:47 +02:00
|
|
|
bool _aborting );
|
2012-11-25 22:43:36 +01:00
|
|
|
|
2014-03-20 12:48:19 +01:00
|
|
|
TestCaseStats( TestCaseStats const& ) = default;
|
|
|
|
TestCaseStats( TestCaseStats && ) = default;
|
|
|
|
TestCaseStats& operator = ( TestCaseStats const& ) = default;
|
|
|
|
TestCaseStats& operator = ( TestCaseStats && ) = default;
|
2017-09-07 16:51:33 +02:00
|
|
|
virtual ~TestCaseStats();
|
2014-03-20 12:48:19 +01:00
|
|
|
|
2012-11-25 22:43:36 +01:00
|
|
|
TestCaseInfo testInfo;
|
2012-11-22 20:17:20 +01:00
|
|
|
Totals totals;
|
|
|
|
std::string stdOut;
|
|
|
|
std::string stdErr;
|
|
|
|
bool aborting;
|
|
|
|
};
|
2013-07-03 20:14:59 +02:00
|
|
|
|
2013-01-03 10:04:46 +01:00
|
|
|
struct TestGroupStats {
|
2012-12-02 00:49:57 +01:00
|
|
|
TestGroupStats( GroupInfo const& _groupInfo,
|
2012-11-29 21:31:17 +01:00
|
|
|
Totals const& _totals,
|
2017-07-19 10:13:47 +02:00
|
|
|
bool _aborting );
|
|
|
|
TestGroupStats( GroupInfo const& _groupInfo );
|
2012-11-25 22:43:36 +01:00
|
|
|
|
2014-03-20 12:48:19 +01:00
|
|
|
TestGroupStats( TestGroupStats const& ) = default;
|
|
|
|
TestGroupStats( TestGroupStats && ) = default;
|
|
|
|
TestGroupStats& operator = ( TestGroupStats const& ) = default;
|
|
|
|
TestGroupStats& operator = ( TestGroupStats && ) = default;
|
2017-09-07 16:51:33 +02:00
|
|
|
virtual ~TestGroupStats();
|
2014-03-20 12:48:19 +01:00
|
|
|
|
2012-12-02 00:49:57 +01:00
|
|
|
GroupInfo groupInfo;
|
2012-11-22 20:17:20 +01:00
|
|
|
Totals totals;
|
|
|
|
bool aborting;
|
|
|
|
};
|
2013-07-03 20:14:59 +02:00
|
|
|
|
2013-01-03 10:04:46 +01:00
|
|
|
struct TestRunStats {
|
2012-12-02 00:49:57 +01:00
|
|
|
TestRunStats( TestRunInfo const& _runInfo,
|
2012-11-29 21:31:17 +01:00
|
|
|
Totals const& _totals,
|
2017-07-19 10:13:47 +02:00
|
|
|
bool _aborting );
|
2014-03-20 12:48:19 +01:00
|
|
|
|
|
|
|
TestRunStats( TestRunStats const& ) = default;
|
|
|
|
TestRunStats( TestRunStats && ) = default;
|
|
|
|
TestRunStats& operator = ( TestRunStats const& ) = default;
|
|
|
|
TestRunStats& operator = ( TestRunStats && ) = default;
|
2017-09-07 16:51:33 +02:00
|
|
|
virtual ~TestRunStats();
|
2013-01-03 10:04:46 +01:00
|
|
|
|
2012-12-02 00:49:57 +01:00
|
|
|
TestRunInfo runInfo;
|
2012-11-22 20:17:20 +01:00
|
|
|
Totals totals;
|
|
|
|
bool aborting;
|
|
|
|
};
|
|
|
|
|
2019-05-19 20:54:44 +02:00
|
|
|
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
|
2017-08-04 20:23:30 +02:00
|
|
|
struct BenchmarkInfo {
|
|
|
|
std::string name;
|
2019-04-23 23:41:13 +02:00
|
|
|
double estimatedDuration;
|
|
|
|
int iterations;
|
|
|
|
int samples;
|
|
|
|
unsigned int resamples;
|
|
|
|
double clockResolution;
|
|
|
|
double clockCost;
|
2017-08-04 20:23:30 +02:00
|
|
|
};
|
2019-04-23 23:41:13 +02:00
|
|
|
|
|
|
|
template <class Duration>
|
2017-08-04 20:23:30 +02:00
|
|
|
struct BenchmarkStats {
|
|
|
|
BenchmarkInfo info;
|
2019-04-23 23:41:13 +02:00
|
|
|
|
|
|
|
std::vector<Duration> samples;
|
|
|
|
Benchmark::Estimate<Duration> mean;
|
|
|
|
Benchmark::Estimate<Duration> standardDeviation;
|
|
|
|
Benchmark::OutlierClassification outliers;
|
|
|
|
double outlierVariance;
|
|
|
|
|
|
|
|
template <typename Duration2>
|
|
|
|
operator BenchmarkStats<Duration2>() const {
|
|
|
|
std::vector<Duration2> samples2;
|
|
|
|
samples2.reserve(samples.size());
|
|
|
|
std::transform(samples.begin(), samples.end(), std::back_inserter(samples2), [](Duration d) { return Duration2(d); });
|
|
|
|
return {
|
|
|
|
info,
|
|
|
|
std::move(samples2),
|
|
|
|
mean,
|
|
|
|
standardDeviation,
|
|
|
|
outliers,
|
|
|
|
outlierVariance,
|
|
|
|
};
|
|
|
|
}
|
2017-08-04 20:23:30 +02:00
|
|
|
};
|
2019-05-19 20:54:44 +02:00
|
|
|
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
|
2013-08-07 19:56:35 +02:00
|
|
|
|
2017-04-25 21:42:01 +02:00
|
|
|
struct IStreamingReporter {
|
2017-07-19 10:13:47 +02:00
|
|
|
virtual ~IStreamingReporter() = default;
|
2012-12-05 09:40:53 +01:00
|
|
|
|
2017-07-09 11:46:53 +02:00
|
|
|
// Implementing class must also provide the following static methods:
|
2012-12-05 09:40:53 +01:00
|
|
|
// static std::string getDescription();
|
|
|
|
|
2012-11-29 10:05:51 +01:00
|
|
|
virtual ReporterPreferences getPreferences() const = 0;
|
2013-07-03 20:14:59 +02:00
|
|
|
|
2013-03-12 20:06:40 +01:00
|
|
|
virtual void noMatchingTestCases( std::string const& spec ) = 0;
|
2012-11-29 10:05:51 +01:00
|
|
|
|
2019-10-19 15:50:46 +02:00
|
|
|
virtual void reportInvalidArguments(std::string const&) {}
|
|
|
|
|
2012-12-02 00:49:57 +01:00
|
|
|
virtual void testRunStarting( TestRunInfo const& testRunInfo ) = 0;
|
|
|
|
virtual void testGroupStarting( GroupInfo const& groupInfo ) = 0;
|
2012-11-22 20:17:20 +01:00
|
|
|
|
2012-11-29 21:31:17 +01:00
|
|
|
virtual void testCaseStarting( TestCaseInfo const& testInfo ) = 0;
|
2012-11-30 09:58:46 +01:00
|
|
|
virtual void sectionStarting( SectionInfo const& sectionInfo ) = 0;
|
|
|
|
|
2019-05-19 20:54:44 +02:00
|
|
|
#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
|
2019-04-23 23:41:13 +02:00
|
|
|
virtual void benchmarkPreparing( std::string const& ) {}
|
2017-08-04 20:23:30 +02:00
|
|
|
virtual void benchmarkStarting( BenchmarkInfo const& ) {}
|
2019-04-23 23:41:13 +02:00
|
|
|
virtual void benchmarkEnded( BenchmarkStats<> const& ) {}
|
|
|
|
virtual void benchmarkFailed( std::string const& ) {}
|
2019-05-19 20:54:44 +02:00
|
|
|
#endif // CATCH_CONFIG_ENABLE_BENCHMARKING
|
2017-08-04 20:23:30 +02:00
|
|
|
|
2012-11-29 21:31:17 +01:00
|
|
|
virtual void assertionStarting( AssertionInfo const& assertionInfo ) = 0;
|
2012-11-22 20:17:20 +01:00
|
|
|
|
2014-12-19 18:52:33 +01:00
|
|
|
// The return value indicates if the messages buffer should be cleared:
|
2013-06-28 17:25:49 +02:00
|
|
|
virtual bool assertionEnded( AssertionStats const& assertionStats ) = 0;
|
2015-11-04 19:01:28 +01:00
|
|
|
|
2013-01-03 10:04:46 +01:00
|
|
|
virtual void sectionEnded( SectionStats const& sectionStats ) = 0;
|
|
|
|
virtual void testCaseEnded( TestCaseStats const& testCaseStats ) = 0;
|
|
|
|
virtual void testGroupEnded( TestGroupStats const& testGroupStats ) = 0;
|
|
|
|
virtual void testRunEnded( TestRunStats const& testRunStats ) = 0;
|
2015-11-04 19:01:28 +01:00
|
|
|
|
2014-12-22 21:10:33 +01:00
|
|
|
virtual void skipTest( TestCaseInfo const& testInfo ) = 0;
|
2017-01-26 23:13:12 +01:00
|
|
|
|
2017-09-14 19:57:59 +02:00
|
|
|
// Default empty implementation provided
|
|
|
|
virtual void fatalErrorEncountered( StringRef name );
|
|
|
|
|
2017-07-19 10:13:47 +02:00
|
|
|
virtual bool isMulti() const;
|
2012-11-22 20:17:20 +01:00
|
|
|
};
|
2017-04-29 20:38:34 +02:00
|
|
|
using IStreamingReporterPtr = std::unique_ptr<IStreamingReporter>;
|
2013-07-03 20:14:59 +02:00
|
|
|
|
2017-04-25 21:28:53 +02:00
|
|
|
struct IReporterFactory {
|
2017-09-07 16:51:33 +02:00
|
|
|
virtual ~IReporterFactory();
|
2017-04-25 21:42:01 +02:00
|
|
|
virtual IStreamingReporterPtr create( ReporterConfig const& config ) const = 0;
|
2012-05-09 09:17:51 +02:00
|
|
|
virtual std::string getDescription() const = 0;
|
2010-12-31 23:07:47 +01:00
|
|
|
};
|
2017-04-25 21:28:53 +02:00
|
|
|
using IReporterFactoryPtr = std::shared_ptr<IReporterFactory>;
|
2010-12-31 23:07:47 +01:00
|
|
|
|
2012-05-09 09:17:51 +02:00
|
|
|
struct IReporterRegistry {
|
2017-04-25 21:28:53 +02:00
|
|
|
using FactoryMap = std::map<std::string, IReporterFactoryPtr>;
|
|
|
|
using Listeners = std::vector<IReporterFactoryPtr>;
|
2010-12-31 23:21:36 +01:00
|
|
|
|
2017-09-07 16:51:33 +02:00
|
|
|
virtual ~IReporterRegistry();
|
2017-04-25 21:42:01 +02:00
|
|
|
virtual IStreamingReporterPtr create( std::string const& name, IConfigPtr const& config ) const = 0;
|
2013-04-23 19:58:56 +02:00
|
|
|
virtual FactoryMap const& getFactories() const = 0;
|
2015-08-07 09:20:56 +02:00
|
|
|
virtual Listeners const& getListeners() const = 0;
|
2010-12-31 23:07:47 +01:00
|
|
|
};
|
2013-07-03 20:14:59 +02:00
|
|
|
|
2017-07-19 10:13:47 +02:00
|
|
|
} // end namespace Catch
|
2010-12-31 23:07:47 +01:00
|
|
|
|
2012-09-17 07:42:29 +02:00
|
|
|
#endif // TWOBLUECUBES_CATCH_INTERFACES_REPORTER_H_INCLUDED
|