From 423e1d2ebbd75bb92cb486f83f2c5800b4b9689e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ho=C5=99e=C5=88ovsk=C3=BD?= Date: Mon, 4 Apr 2022 21:07:04 +0200 Subject: [PATCH] Integrate the new reporter specs into Catch2 This means that the CLI interface now uses the new key-value oriented reporter spec, the common reporter base creates the colour implementation based on the reporter-specific configuration, and it also stores the custom configuration options for each reporter instance. Closes #339 as it allows per-reporter forcing of ansi colour codes. --- docs/command-line.md | 28 ++++--- docs/reporters.md | 27 ++++++- src/catch2/catch_config.cpp | 21 ++--- src/catch2/catch_config.hpp | 16 +--- src/catch2/catch_session.cpp | 25 ++++-- .../interfaces/catch_interfaces_reporter.cpp | 18 ++++- .../interfaces/catch_interfaces_reporter.hpp | 11 ++- src/catch2/internal/catch_commandline.cpp | 61 ++++++--------- .../reporters/catch_reporter_common_base.cpp | 4 +- .../reporters/catch_reporter_common_base.hpp | 6 ++ tests/CMakeLists.txt | 31 ++++++-- tests/ExtraTests/CMakeLists.txt | 24 ++++-- .../X29-CustomArgumentsForReporters.cpp | 59 ++++++++++++++ .../Baselines/compact.sw.approved.txt | 40 +++------- .../Baselines/compact.sw.multi.approved.txt | 40 +++------- .../Baselines/console.std.approved.txt | 2 +- .../Baselines/console.sw.approved.txt | 76 +++++------------- .../Baselines/console.sw.multi.approved.txt | 76 +++++------------- .../SelfTest/Baselines/junit.sw.approved.txt | 3 +- .../Baselines/junit.sw.multi.approved.txt | 3 +- .../Baselines/sonarqube.sw.approved.txt | 1 - .../Baselines/sonarqube.sw.multi.approved.txt | 1 - tests/SelfTest/Baselines/tap.sw.approved.txt | 28 +++---- .../Baselines/tap.sw.multi.approved.txt | 28 +++---- tests/SelfTest/Baselines/xml.sw.approved.txt | 78 +++++-------------- .../Baselines/xml.sw.multi.approved.txt | 78 +++++-------------- .../IntrospectiveTests/CmdLine.tests.cpp | 47 +++++------ .../IntrospectiveTests/Reporters.tests.cpp | 9 ++- tools/scripts/approvalTests.py | 2 +- 29 files changed, 395 insertions(+), 448 deletions(-) create mode 100644 tests/ExtraTests/X29-CustomArgumentsForReporters.cpp diff --git a/docs/command-line.md b/docs/command-line.md index 29e6a365..829c156d 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -123,22 +123,33 @@ Test names containing special characters, such as `,` or `[` can specify them on ## Choosing a reporter to use -
-r, --reporter <reporter[::output-file]>
- -> Support for providing output-file through the `-r`, `--reporter` flag was [introduced](https://github.com/catchorg/Catch2/pull/2183) in Catch2 X.Y.Z +
-r, --reporter <reporter[::key=value]*>
Reporters are how the output from Catch2 (results of assertions, tests, benchmarks and so on) is formatted and written out. The default reporter is called the "Console" reporter and is intended to provide relatively verbose and human-friendly output. +Reporters are also individually configurable. To pass configuration options +to the reporter, you append `::key=value` to the reporter specification +as many times as you want, e.g. `--reporter xml::out=someFile.xml`. + +The keys must either be prefixed by "X", in which case they are not parsed +by Catch2 and are only passed down to the reporter, or one of options +hardcoded into Catch2. Currently there are only 2, +["out"](#sending-output-to-a-file), and ["colour-mode"](#colour-mode). + +_Note that the reporter might still check the X-prefixed options for +validity, and throw an error if they are wrong._ + +> Support for passing arguments to reporters through the `-r`, `--reporter` flag was introduced in Catch2 X.Y.Z + There are multiple built-in reporters, you can see what they do by using the [`--list-reporter`](command-line.md#listing-available-tests-tags-or-reporters) flag. If you need a reporter providing custom format outside of the already provided ones, look at the ["write your own reporter" part of the reporter documentation](reporters.md#writing-your-own-reporter). - This option may be passed multiple times to use multiple (different) reporters at the same time. See the [reporter documentation](reporters.md#multiple-reporters) for details on what the resulting behaviour is. Also note that at most one @@ -148,13 +159,12 @@ the [`-o`, `--out`](#sending-output-to-a-file) option. > Support for using multiple different reporters at the same time was [introduced](https://github.com/catchorg/Catch2/pull/2183) in Catch2 X.Y.Z -As with the `--out` option, using `-` for the output file name sends the -output to stdout. _Note: There is currently no way to escape `::` in the reporter spec, -and thus reporter/file names with `::` in them will not work properly. -As `::` in paths is relatively obscure (unlike `:`), we do not consider -this an issue._ +and thus the reporter names, or configuration keys and values, cannot +contain `::`. As `::` in paths is relatively obscure (unlike ':'), we do +not consider this an issue._ + ## Breaking into the debugger diff --git a/docs/reporters.md b/docs/reporters.md index ab6f6453..5f86b02e 100644 --- a/docs/reporters.md +++ b/docs/reporters.md @@ -43,9 +43,14 @@ them write into different destinations. The two main uses of this are Specifying multiple reporter looks like this: ``` ---reporter console::- --reporter JUnit::result-junit.xml +--reporter JUnit::out=result-junit.xml --reporter console::out=-::colour-mode=ansi ``` +This tells Catch2 to use two reporters, `JUnit` reporter that writes +its machine-readable XML output to file `result-junit.xml`, and the +`console` reporter that writes its user-friendly output to stdout and +uses ANSI colour codes for colouring the output. + Using multiple reporters (or one reporter and one-or-more [event listeners](event-listener.md#top)) can have surprisingly complex semantics when using customization points provided to reporters by Catch2, namely @@ -162,6 +167,26 @@ Currently there are two customization options: format includes passing assertions even without the `-s` flag. +### Per-reporter configuration + +> Per-reporter configuration was introduced in Catch2 X.Y.Z + +Catch2 supports some configuration to happen per reporter. The configuration +options fall into one of two categories: + +* Catch2-recognized options +* Reporter-specific options + +The former is a small set of universal options that Catch2 handles for +the reporters, e.g. output file or console colour mode. The latter are +options that the reporters have to handle themselves, but the keys and +values can be arbitrary strings, as long as they don't contain `::`. This +allows writing reporters that can be significantly customized at runtime. + +Reporter-specific options always have to be prefixed with "X" (large +letter X). + + ### Other expected functionality of a reporter When writing a custom reporter, there are few more things that you should diff --git a/src/catch2/catch_config.cpp b/src/catch2/catch_config.cpp index 4c8b3319..75a92883 100644 --- a/src/catch2/catch_config.cpp +++ b/src/catch2/catch_config.cpp @@ -33,17 +33,6 @@ namespace Catch { } // unnamed namespace } // namespace Detail - std::ostream& operator<<( std::ostream& os, - ConfigData::ReporterAndFile const& reporter ) { - os << "{ " << reporter.reporterName << ", "; - if ( reporter.outputFileName ) { - os << *reporter.outputFileName; - } else { - os << ""; - } - return os << " }"; - } - Config::Config( ConfigData const& data ): m_data( data ) { // We need to trim filter specs to avoid trouble with superfluous @@ -76,14 +65,14 @@ namespace Catch { #else "console", #endif - {} + {}, {}, {} } ); } bool defaultOutputUsed = false; m_reporterStreams.reserve( m_data.reporterSpecifications.size() ); - for ( auto const& reporterAndFile : m_data.reporterSpecifications ) { - if ( reporterAndFile.outputFileName.none() ) { + for ( auto const& reporterSpec : m_data.reporterSpecifications ) { + if ( reporterSpec.outputFile().none() ) { CATCH_ENFORCE( !defaultOutputUsed, "Internal error: cannot use default output for " "multiple reporters" ); @@ -93,7 +82,7 @@ namespace Catch { openStream( data.defaultOutputFilename ) ); } else { m_reporterStreams.push_back( - openStream( *reporterAndFile.outputFileName ) ); + openStream( *reporterSpec.outputFile() ) ); } } } @@ -108,7 +97,7 @@ namespace Catch { std::vector const& Config::getTestsOrTags() const { return m_data.testsOrTags; } std::vector const& Config::getSectionsToRun() const { return m_data.sectionsToRun; } - std::vector const& Config::getReportersAndOutputFiles() const { + std::vector const& Config::getReporterSpecs() const { return m_data.reporterSpecifications; } diff --git a/src/catch2/catch_config.hpp b/src/catch2/catch_config.hpp index 91ce4e64..324d634c 100644 --- a/src/catch2/catch_config.hpp +++ b/src/catch2/catch_config.hpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -24,17 +25,6 @@ namespace Catch { class IStream; struct ConfigData { - struct ReporterAndFile { - std::string reporterName; - - // If none, the output goes to the default output. - Optional outputFileName; - - friend bool operator==(ReporterAndFile const& lhs, ReporterAndFile const& rhs) { - return lhs.reporterName == rhs.reporterName && lhs.outputFileName == rhs.outputFileName; - } - friend std::ostream& operator<<(std::ostream &os, ReporterAndFile const& reporter); - }; bool listTests = false; bool listTags = false; @@ -72,7 +62,7 @@ namespace Catch { std::string defaultOutputFilename; std::string name; std::string processName; - std::vector reporterSpecifications; + std::vector reporterSpecifications; std::vector testsOrTags; std::vector sectionsToRun; @@ -90,7 +80,7 @@ namespace Catch { bool listTags() const; bool listReporters() const; - std::vector const& getReportersAndOutputFiles() const; + std::vector const& getReporterSpecs() const; IStream const* getReporterOutputStream(std::size_t reporterIdx) const; std::vector const& getTestsOrTags() const override; diff --git a/src/catch2/catch_session.cpp b/src/catch2/catch_session.cpp index 6af52685..74731cd0 100644 --- a/src/catch2/catch_session.cpp +++ b/src/catch2/catch_session.cpp @@ -41,11 +41,18 @@ namespace Catch { return reporter; } - IStreamingReporterPtr makeReporter(Config const* config) { + IStreamingReporterPtr prepareReporters(Config const* config) { if (Catch::getRegistryHub().getReporterRegistry().getListeners().empty() - && config->getReportersAndOutputFiles().size() == 1) { + && config->getReporterSpecs().size() == 1) { + auto const& spec = config->getReporterSpecs()[0]; auto stream = config->getReporterOutputStream(0); - return createReporter(config->getReportersAndOutputFiles()[0].reporterName, ReporterConfig(config, stream)); + return createReporter( + config->getReporterSpecs()[0].name(), + ReporterConfig( + config, + stream, + spec.colourMode().valueOr( config->colourMode() ), + spec.customOptions() ) ); } auto multi = Detail::make_unique(config); @@ -56,9 +63,15 @@ namespace Catch { } std::size_t reporterIdx = 0; - for (auto const& reporterAndFile : config->getReportersAndOutputFiles()) { + for (auto const& reporterSpec : config->getReporterSpecs()) { auto stream = config->getReporterOutputStream(reporterIdx); - multi->addReporter(createReporter(reporterAndFile.reporterName, ReporterConfig(config, stream))); + multi->addReporter( createReporter( + reporterSpec.name(), + ReporterConfig( config, + stream, + reporterSpec.colourMode().valueOr( + config->colourMode() ), + reporterSpec.customOptions() ) ) ); reporterIdx++; } @@ -304,7 +317,7 @@ namespace Catch { getCurrentMutableContext().setConfig(m_config.get()); // Create reporter(s) so we can route listings through them - auto reporter = makeReporter(m_config.get()); + auto reporter = prepareReporters(m_config.get()); auto const& invalidSpecs = m_config->testSpec().getInvalidSpecs(); if ( !invalidSpecs.empty() ) { diff --git a/src/catch2/interfaces/catch_interfaces_reporter.cpp b/src/catch2/interfaces/catch_interfaces_reporter.cpp index 5f5d274a..585d4365 100644 --- a/src/catch2/interfaces/catch_interfaces_reporter.cpp +++ b/src/catch2/interfaces/catch_interfaces_reporter.cpp @@ -14,17 +14,31 @@ #include #include #include +#include #include #include namespace Catch { - ReporterConfig::ReporterConfig( IConfig const* _fullConfig, IStream const* _stream ) - : m_stream( _stream ), m_fullConfig( _fullConfig ) {} + ReporterConfig::ReporterConfig( + IConfig const* _fullConfig, + IStream const* _stream, + ColourMode colourMode, + std::map customOptions ): + m_stream( _stream ), + m_fullConfig( _fullConfig ), + m_colourMode( colourMode ), + m_customOptions( CATCH_MOVE( customOptions ) ) {} IStream const* ReporterConfig::stream() const { return m_stream; } IConfig const * ReporterConfig::fullConfig() const { return m_fullConfig; } + ColourMode ReporterConfig::colourMode() const { return m_colourMode; } + + std::map const& + ReporterConfig::customOptions() const { + return m_customOptions; + } AssertionStats::AssertionStats( AssertionResult const& _assertionResult, std::vector const& _infoMessages, diff --git a/src/catch2/interfaces/catch_interfaces_reporter.hpp b/src/catch2/interfaces/catch_interfaces_reporter.hpp index 53d5ae60..9c367fe9 100644 --- a/src/catch2/interfaces/catch_interfaces_reporter.hpp +++ b/src/catch2/interfaces/catch_interfaces_reporter.hpp @@ -19,6 +19,7 @@ #include +#include #include #include #include @@ -31,16 +32,24 @@ namespace Catch { class TestCaseHandle; struct IConfig; class IStream; + enum class ColourMode : std::uint8_t; struct ReporterConfig { - ReporterConfig( IConfig const* _fullConfig, IStream const* _stream ); + ReporterConfig( IConfig const* _fullConfig, + IStream const* _stream, + ColourMode colourMode, + std::map customOptions ); IStream const* stream() const; IConfig const* fullConfig() const; + ColourMode colourMode() const; + std::map const& customOptions() const; private: IStream const* m_stream; IConfig const* m_fullConfig; + ColourMode m_colourMode; + std::map m_customOptions; }; struct TestRunInfo { diff --git a/src/catch2/internal/catch_commandline.cpp b/src/catch2/internal/catch_commandline.cpp index 5095f1db..0033a92f 100644 --- a/src/catch2/internal/catch_commandline.cpp +++ b/src/catch2/internal/catch_commandline.cpp @@ -141,57 +141,42 @@ namespace Catch { return ParserResult::runtimeError( "Unrecognised verbosity, '" + verbosity + '\'' ); return ParserResult::ok( ParseResultType::Matched ); }; - auto const setReporter = [&]( std::string const& reporterSpec ) { - if ( reporterSpec.empty() ) { + auto const setReporter = [&]( std::string const& userReporterSpec ) { + if ( userReporterSpec.empty() ) { return ParserResult::runtimeError( "Received empty reporter spec." ); } - // Exactly one of the reporters may be specified without an output - // file, in which case it defaults to the output specified by "-o" - // (or standard output). - static constexpr auto separator = "::"; - static constexpr size_t separatorSize = 2; - auto fileNameSeparatorPos = reporterSpec.find( separator ); - const bool containsFileName = fileNameSeparatorPos != reporterSpec.npos; - if ( containsFileName ) { - auto nextSeparatorPos = reporterSpec.find( - separator, fileNameSeparatorPos + separatorSize ); - if ( nextSeparatorPos != reporterSpec.npos ) { - return ParserResult::runtimeError( - "Too many separators in reporter spec '" + reporterSpec + '\'' ); - } + Optional parsed = + parseReporterSpec( userReporterSpec ); + if ( !parsed ) { + return ParserResult::runtimeError( + "Could not parse reporter spec '" + userReporterSpec + + "'" ); } - std::string reporterName; - Optional outputFileName; - reporterName = reporterSpec.substr( 0, fileNameSeparatorPos ); - if ( reporterName.empty() ) { - return ParserResult::runtimeError( "Reporter name cannot be empty." ); - } - - if ( containsFileName ) { - outputFileName = reporterSpec.substr( - fileNameSeparatorPos + separatorSize, reporterSpec.size() ); - } + auto const& reporterSpec = *parsed; IReporterRegistry::FactoryMap const& factories = getRegistryHub().getReporterRegistry().getFactories(); - auto result = factories.find( reporterName ); + auto result = factories.find( reporterSpec.name() ); - if( result == factories.end() ) - return ParserResult::runtimeError( "Unrecognized reporter, '" + reporterName + "'. Check available with --list-reporters" ); - if( containsFileName && outputFileName->empty() ) - return ParserResult::runtimeError( "Reporter '" + reporterName + "' has empty filename specified as its output. Supply a filename or remove the colons to use the default output." ); + if ( result == factories.end() ) { + return ParserResult::runtimeError( + "Unrecognized reporter, '" + reporterSpec.name() + + "'. Check available with --list-reporters" ); + } - config.reporterSpecifications.push_back({ std::move(reporterName), std::move(outputFileName) }); - // It would be enough to check this only once at the very end, but there is - // not a place where we could call this check, so do it every time it could fail. - // For valid inputs, this is still called at most once. - if (!containsFileName) { + const bool hadOutputFile = reporterSpec.outputFile().some(); + config.reporterSpecifications.push_back( CATCH_MOVE( *parsed ) ); + // It would be enough to check this only once at the very end, but + // there is not a place where we could call this check, so do it + // every time it could fail. For valid inputs, this is still called + // at most once. + if (!hadOutputFile) { int n_reporters_without_file = 0; for (auto const& spec : config.reporterSpecifications) { - if (spec.outputFileName.none()) { + if (spec.outputFile().none()) { n_reporters_without_file++; } } diff --git a/src/catch2/reporters/catch_reporter_common_base.cpp b/src/catch2/reporters/catch_reporter_common_base.cpp index 85092db8..aa2ab70a 100644 --- a/src/catch2/reporters/catch_reporter_common_base.cpp +++ b/src/catch2/reporters/catch_reporter_common_base.cpp @@ -18,7 +18,9 @@ namespace Catch { IEventListener( config.fullConfig() ), m_wrapped_stream( config.stream() ), m_stream( m_wrapped_stream->stream() ), - m_colour( makeColourImpl( m_config->colourMode(), m_wrapped_stream ) ) {} + m_colour( makeColourImpl( config.colourMode(), m_wrapped_stream ) ), + m_customOptions( config.customOptions() ) + {} ReporterBase::~ReporterBase() = default; diff --git a/src/catch2/reporters/catch_reporter_common_base.hpp b/src/catch2/reporters/catch_reporter_common_base.hpp index ba7ab116..e110763b 100644 --- a/src/catch2/reporters/catch_reporter_common_base.hpp +++ b/src/catch2/reporters/catch_reporter_common_base.hpp @@ -10,6 +10,9 @@ #include +#include +#include + namespace Catch { class ColourImpl; @@ -30,7 +33,10 @@ namespace Catch { //! Cached output stream from `m_wrapped_stream` to reduce //! number of indirect calls needed to write output. std::ostream& m_stream; + //! Colour implementation this reporter was configured for Detail::unique_ptr m_colour; + //! The custom reporter options user passed down to the reporter + std::map m_customOptions; public: ReporterBase( ReporterConfig const& config ); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a4d01dc9..46294c3e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -435,8 +435,8 @@ if (NOT MEMORYCHECK_COMMAND) add_test(NAME "MultiReporter::CapturingReportersDontPropagateStdOut" COMMAND $ "Sends stuff to stdout and stderr" - --reporter xml::${_NullFile} - --reporter junit::${_NullFile} + --reporter xml::out=${_NullFile} + --reporter junit::out=${_NullFile} ) set_tests_properties("MultiReporter::CapturingReportersDontPropagateStdOut" PROPERTIES @@ -447,8 +447,8 @@ endif() add_test(NAME "MultiReporter::NonCapturingReportersPropagateStdout" COMMAND $ "Sends stuff to stdout and stderr" - --reporter xml::${_NullFile} - --reporter console::${_NullFile} + --reporter xml::out=${_NullFile} + --reporter console::out=${_NullFile} ) set_tests_properties("MultiReporter::NonCapturingReportersPropagateStdout" PROPERTIES @@ -468,13 +468,34 @@ set_tests_properties("Outputs::DashAsOutLocationSendsOutputToStdout" add_test(NAME "Reporters::DashAsLocationInReporterSpecSendsOutputToStdout" COMMAND $ "Factorials are computed" - --reporter console::- + --reporter console::out=- ) set_tests_properties("Reporters::DashAsLocationInReporterSpecSendsOutputToStdout" PROPERTIES PASS_REGULAR_EXPRESSION "All tests passed \\(5 assertions in 1 test case\\)" ) +add_test(NAME "Reporters::ReporterSpecificColourOverridesDefaultColour" + COMMAND + $ "Factorials are computed" + --reporter console::colour-mode=ansi + --colour-mode none +) +set_tests_properties("Reporters::ReporterSpecificColourOverridesDefaultColour" + PROPERTIES + PASS_REGULAR_EXPRESSION "\\[1\;32mAll tests passed" +) + +add_test(NAME "Reporters::UnrecognizedOptionInSpecCausesError" + COMMAND + $ "Factorials are computed" + --reporter console::bad-option=ansi +) +set_tests_properties("Reporters::UnrecognizedOptionInSpecCausesError" + PROPERTIES + WILL_FAIL ON +) + add_test(NAME "Colours::ColourModeCanBeExplicitlySetToAnsi" COMMAND $ "Factorials are computed" diff --git a/tests/ExtraTests/CMakeLists.txt b/tests/ExtraTests/CMakeLists.txt index 8e6b3094..1a400454 100644 --- a/tests/ExtraTests/CMakeLists.txt +++ b/tests/ExtraTests/CMakeLists.txt @@ -272,8 +272,8 @@ if (NOT MEMORYCHECK_COMMAND) add_test( NAME MultiReporter::NoncapturingListenerDoesntCauseStdoutPassThrough COMMAND ListenerStdoutCaptureInMultireporter - --reporter xml::${_NullFile} - --reporter junit::${_NullFile} + --reporter xml::out=${_NullFile} + --reporter junit::out=${_NullFile} ) set_tests_properties( @@ -290,8 +290,8 @@ target_link_libraries(ListenerCanAskForCapturedStdout PRIVATE Catch2::Catch2With add_test( NAME MultiReporter::CapturingListenerCausesStdoutCapture COMMAND ListenerCanAskForCapturedStdout - --reporter compact::${_NullFile} - --reporter console::${_NullFile} + --reporter compact::out=${_NullFile} + --reporter console::out=${_NullFile} ) set_tests_properties( MultiReporter::CapturingListenerCausesStdoutCapture @@ -317,7 +317,7 @@ add_test( NAME MultiReporter::PreferencesForPassingAssertionsIsRespected COMMAND ReporterPreferencesForPassingAssertionsIsRespected --reporter test-reporter - --reporter console::${_NullFile} + --reporter console::out=${_NullFile} ) set_tests_properties( MultiReporter::PreferencesForPassingAssertionsIsRespected @@ -339,6 +339,20 @@ set_tests_properties( FAIL_REGULAR_EXPRESSION "X28 - ERROR" ) +add_executable(CustomArgumentsForReporters ${TESTS_DIR}/X29-CustomArgumentsForReporters.cpp) +target_link_libraries(CustomArgumentsForReporters PRIVATE Catch2::Catch2WithMain) +add_test( + NAME CustomArgumentsForReporters + COMMAND CustomArgumentsForReporters + --reporter "test-reporter::Xa b=c 1::Xz:e = 1234" +) +set_tests_properties( + CustomArgumentsForReporters + PROPERTIES + PASS_REGULAR_EXPRESSION "Xa b=c 1::Xz:e = 1234" +) + + add_executable(DuplicatedTestCases-SameNameAndTags ${TESTS_DIR}/X31-DuplicatedTestCases.cpp) target_link_libraries(DuplicatedTestCases-SameNameAndTags PRIVATE Catch2::Catch2WithMain) add_test( diff --git a/tests/ExtraTests/X29-CustomArgumentsForReporters.cpp b/tests/ExtraTests/X29-CustomArgumentsForReporters.cpp new file mode 100644 index 00000000..9721775d --- /dev/null +++ b/tests/ExtraTests/X29-CustomArgumentsForReporters.cpp @@ -0,0 +1,59 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 + +/**\file + * Test that custom options are properly passed down to the reporter. + * + * We print out the arguments sorted by key, to have a stable expected + * output. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include + +class TestReporter : public Catch::StreamingReporterBase { +public: + TestReporter( Catch::ReporterConfig const& _config ): + StreamingReporterBase( _config ) { + std::cout << "X29 - TestReporter constructed\n"; + } + + static std::string getDescription() { + return "X29 test reporter"; + } + + void testRunStarting( Catch::TestRunInfo const& ) override { + std::vector> options; + for ( auto const& kv : m_customOptions ) { + options.push_back( kv ); + } + std::sort( options.begin(), options.end() ); + bool first = true; + for ( auto const& kv : options ) { + if ( !first ) { std::cout << "::"; } + std::cout << kv.first << "=" << kv.second; + first = false; + } + std::cout << '\n'; + } + + ~TestReporter() override; +}; + +TestReporter::~TestReporter() = default; + +CATCH_REGISTER_REPORTER( "test-reporter", TestReporter ) + +TEST_CASE( "Just a test case to run things" ) {} diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt index e2ac5551..cabb1733 100644 --- a/tests/SelfTest/Baselines/compact.sw.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.approved.txt @@ -1316,10 +1316,8 @@ CmdLine.tests.cpp:: passed: config.abortAfter == -1 for: -1 == -1 CmdLine.tests.cpp:: passed: config.noThrow == false for: false == false CmdLine.tests.cpp:: passed: config.reporterSpecifications.empty() for: true CmdLine.tests.cpp:: passed: !(cfg.hasTestFilters()) for: !false -CmdLine.tests.cpp:: passed: cfg.getReportersAndOutputFiles().size() == 1 for: 1 == 1 -CmdLine.tests.cpp:: passed: cfg.getReportersAndOutputFiles()[0] == Catch::ConfigData::ReporterAndFile{ expectedReporter, {} } for: { console, } -== -{ console, } +CmdLine.tests.cpp:: passed: cfg.getReporterSpecs().size() == 1 for: 1 == 1 +CmdLine.tests.cpp:: passed: cfg.getReporterSpecs()[0] == Catch::ReporterSpec{ expectedReporter, {}, {}, {} } for: {?} == {?} CmdLine.tests.cpp:: passed: result for: {?} CmdLine.tests.cpp:: passed: cfg.hasTestFilters() for: true CmdLine.tests.cpp:: passed: cfg.testSpec().matches(*fakeTestCase("notIncluded")) == false for: false == false @@ -1333,37 +1331,21 @@ CmdLine.tests.cpp:: passed: cfg.hasTestFilters() for: true CmdLine.tests.cpp:: passed: cfg.testSpec().matches(*fakeTestCase("test1")) == false for: false == false CmdLine.tests.cpp:: passed: cfg.testSpec().matches(*fakeTestCase("alwaysIncluded")) for: true CmdLine.tests.cpp:: passed: result for: {?} with 1 message: 'result.errorMessage() := ""' -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_ReporterAndFile{ {"console", {}} } for: { { console, } } -== -{ { console, } } with 1 message: 'result.errorMessage() := ""' +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "console", {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' CmdLine.tests.cpp:: passed: result for: {?} with 1 message: 'result.errorMessage() := ""' -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_ReporterAndFile{ {"xml", {}} } for: { { xml, } } -== -{ { xml, } } with 1 message: 'result.errorMessage() := ""' +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "xml", {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' CmdLine.tests.cpp:: passed: result for: {?} with 1 message: 'result.errorMessage() := ""' -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_ReporterAndFile{ {"junit", {}} } for: { { junit, } } -== -{ { junit, } } with 1 message: 'result.errorMessage() := ""' +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "junit", {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' CmdLine.tests.cpp:: passed: !result for: true CmdLine.tests.cpp:: passed: result.errorMessage(), ContainsSubstring("Unrecognized reporter") for: "Unrecognized reporter, 'unsupported'. Check available with --list-reporters" contains: "Unrecognized reporter" CmdLine.tests.cpp:: passed: result for: {?} with 1 message: 'result.errorMessage() := ""' -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_ReporterAndFile{ {"console", "out.txt"s} } for: { { console, out.txt } } -== -{ { console, out.txt } } with 1 message: 'result.errorMessage() := ""' +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "console", "out.txt"s, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' CmdLine.tests.cpp:: passed: result for: {?} with 1 message: 'result.errorMessage() := ""' -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_ReporterAndFile{ {"console", "C:\\Temp\\out.txt"s} } for: { { console, C:\Temp\out.txt } } -== -{ { console, C:\Temp\out.txt } } with 1 message: 'result.errorMessage() := ""' -CmdLine.tests.cpp:: passed: !result for: true -CmdLine.tests.cpp:: passed: result.errorMessage(), ContainsSubstring("empty filename") for: "Reporter 'console' has empty filename specified as its output. Supply a filename or remove the colons to use the default output." contains: "empty filename" -CmdLine.tests.cpp:: passed: cli.parse({ "test", "-r", "xml::output.xml", "-r", "junit::output-junit.xml" }) for: {?} -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_ReporterAndFile{ {"xml", "output.xml"s}, {"junit", "output-junit.xml"s} } for: { { xml, output.xml }, { junit, output-junit.xml } } -== -{ { xml, output.xml }, { junit, output-junit.xml } } -CmdLine.tests.cpp:: passed: cli.parse({ "test", "-r", "xml::output.xml", "-r", "console" }) for: {?} -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_ReporterAndFile{ {"xml", "output.xml"s}, {"console", {}} } for: { { xml, output.xml }, { console, } } -== -{ { xml, output.xml }, { console, } } +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' +CmdLine.tests.cpp:: passed: cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "junit::out=output-junit.xml" }) for: {?} +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "junit", "output-junit.xml"s, {}, {} } } for: { {?}, {?} } == { {?}, {?} } +CmdLine.tests.cpp:: passed: cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "console" }) for: {?} +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "console", {}, {}, {} } } for: { {?}, {?} } == { {?}, {?} } CmdLine.tests.cpp:: passed: !result for: true CmdLine.tests.cpp:: passed: result.errorMessage(), ContainsSubstring("Only one reporter may have unspecified output file.") for: "Only one reporter may have unspecified output file." contains: "Only one reporter may have unspecified output file." CmdLine.tests.cpp:: passed: cli.parse({"test", "-b"}) for: {?} diff --git a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt index b46512ed..2b8e2731 100644 --- a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt @@ -1314,10 +1314,8 @@ CmdLine.tests.cpp:: passed: config.abortAfter == -1 for: -1 == -1 CmdLine.tests.cpp:: passed: config.noThrow == false for: false == false CmdLine.tests.cpp:: passed: config.reporterSpecifications.empty() for: true CmdLine.tests.cpp:: passed: !(cfg.hasTestFilters()) for: !false -CmdLine.tests.cpp:: passed: cfg.getReportersAndOutputFiles().size() == 1 for: 1 == 1 -CmdLine.tests.cpp:: passed: cfg.getReportersAndOutputFiles()[0] == Catch::ConfigData::ReporterAndFile{ expectedReporter, {} } for: { console, } -== -{ console, } +CmdLine.tests.cpp:: passed: cfg.getReporterSpecs().size() == 1 for: 1 == 1 +CmdLine.tests.cpp:: passed: cfg.getReporterSpecs()[0] == Catch::ReporterSpec{ expectedReporter, {}, {}, {} } for: {?} == {?} CmdLine.tests.cpp:: passed: result for: {?} CmdLine.tests.cpp:: passed: cfg.hasTestFilters() for: true CmdLine.tests.cpp:: passed: cfg.testSpec().matches(*fakeTestCase("notIncluded")) == false for: false == false @@ -1331,37 +1329,21 @@ CmdLine.tests.cpp:: passed: cfg.hasTestFilters() for: true CmdLine.tests.cpp:: passed: cfg.testSpec().matches(*fakeTestCase("test1")) == false for: false == false CmdLine.tests.cpp:: passed: cfg.testSpec().matches(*fakeTestCase("alwaysIncluded")) for: true CmdLine.tests.cpp:: passed: result for: {?} with 1 message: 'result.errorMessage() := ""' -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_ReporterAndFile{ {"console", {}} } for: { { console, } } -== -{ { console, } } with 1 message: 'result.errorMessage() := ""' +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "console", {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' CmdLine.tests.cpp:: passed: result for: {?} with 1 message: 'result.errorMessage() := ""' -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_ReporterAndFile{ {"xml", {}} } for: { { xml, } } -== -{ { xml, } } with 1 message: 'result.errorMessage() := ""' +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "xml", {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' CmdLine.tests.cpp:: passed: result for: {?} with 1 message: 'result.errorMessage() := ""' -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_ReporterAndFile{ {"junit", {}} } for: { { junit, } } -== -{ { junit, } } with 1 message: 'result.errorMessage() := ""' +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "junit", {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' CmdLine.tests.cpp:: passed: !result for: true CmdLine.tests.cpp:: passed: result.errorMessage(), ContainsSubstring("Unrecognized reporter") for: "Unrecognized reporter, 'unsupported'. Check available with --list-reporters" contains: "Unrecognized reporter" CmdLine.tests.cpp:: passed: result for: {?} with 1 message: 'result.errorMessage() := ""' -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_ReporterAndFile{ {"console", "out.txt"s} } for: { { console, out.txt } } -== -{ { console, out.txt } } with 1 message: 'result.errorMessage() := ""' +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "console", "out.txt"s, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' CmdLine.tests.cpp:: passed: result for: {?} with 1 message: 'result.errorMessage() := ""' -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_ReporterAndFile{ {"console", "C:\\Temp\\out.txt"s} } for: { { console, C:\Temp\out.txt } } -== -{ { console, C:\Temp\out.txt } } with 1 message: 'result.errorMessage() := ""' -CmdLine.tests.cpp:: passed: !result for: true -CmdLine.tests.cpp:: passed: result.errorMessage(), ContainsSubstring("empty filename") for: "Reporter 'console' has empty filename specified as its output. Supply a filename or remove the colons to use the default output." contains: "empty filename" -CmdLine.tests.cpp:: passed: cli.parse({ "test", "-r", "xml::output.xml", "-r", "junit::output-junit.xml" }) for: {?} -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_ReporterAndFile{ {"xml", "output.xml"s}, {"junit", "output-junit.xml"s} } for: { { xml, output.xml }, { junit, output-junit.xml } } -== -{ { xml, output.xml }, { junit, output-junit.xml } } -CmdLine.tests.cpp:: passed: cli.parse({ "test", "-r", "xml::output.xml", "-r", "console" }) for: {?} -CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_ReporterAndFile{ {"xml", "output.xml"s}, {"console", {}} } for: { { xml, output.xml }, { console, } } -== -{ { xml, output.xml }, { console, } } +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' +CmdLine.tests.cpp:: passed: cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "junit::out=output-junit.xml" }) for: {?} +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "junit", "output-junit.xml"s, {}, {} } } for: { {?}, {?} } == { {?}, {?} } +CmdLine.tests.cpp:: passed: cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "console" }) for: {?} +CmdLine.tests.cpp:: passed: config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "console", {}, {}, {} } } for: { {?}, {?} } == { {?}, {?} } CmdLine.tests.cpp:: passed: !result for: true CmdLine.tests.cpp:: passed: result.errorMessage(), ContainsSubstring("Only one reporter may have unspecified output file.") for: "Only one reporter may have unspecified output file." contains: "Only one reporter may have unspecified output file." CmdLine.tests.cpp:: passed: cli.parse({"test", "-b"}) for: {?} diff --git a/tests/SelfTest/Baselines/console.std.approved.txt b/tests/SelfTest/Baselines/console.std.approved.txt index a6847cdf..568a71e2 100644 --- a/tests/SelfTest/Baselines/console.std.approved.txt +++ b/tests/SelfTest/Baselines/console.std.approved.txt @@ -1396,5 +1396,5 @@ due to unexpected exception with message: =============================================================================== test cases: 390 | 314 passed | 69 failed | 7 failed as expected -assertions: 2225 | 2070 passed | 128 failed | 27 failed as expected +assertions: 2223 | 2068 passed | 128 failed | 27 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt index 3091da60..4ae9ebcc 100644 --- a/tests/SelfTest/Baselines/console.sw.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.approved.txt @@ -9433,16 +9433,14 @@ with expansion: !false CmdLine.tests.cpp:: PASSED: - CHECK( cfg.getReportersAndOutputFiles().size() == 1 ) + CHECK( cfg.getReporterSpecs().size() == 1 ) with expansion: 1 == 1 CmdLine.tests.cpp:: PASSED: - CHECK( cfg.getReportersAndOutputFiles()[0] == Catch::ConfigData::ReporterAndFile{ expectedReporter, {} } ) + CHECK( cfg.getReporterSpecs()[0] == Catch::ReporterSpec{ expectedReporter, {}, {}, {} } ) with expansion: - { console, } - == - { console, } + {?} == {?} ------------------------------------------------------------------------------- Process can be configured on command line @@ -9544,11 +9542,9 @@ with message: result.errorMessage() := "" CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_ReporterAndFile{ {"console", {}} } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "console", {}, {}, {} } } ) with expansion: - { { console, } } - == - { { console, } } + { {?} } == { {?} } with message: result.errorMessage() := "" @@ -9568,11 +9564,9 @@ with message: result.errorMessage() := "" CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_ReporterAndFile{ {"xml", {}} } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "xml", {}, {}, {} } } ) with expansion: - { { xml, } } - == - { { xml, } } + { {?} } == { {?} } with message: result.errorMessage() := "" @@ -9592,11 +9586,9 @@ with message: result.errorMessage() := "" CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_ReporterAndFile{ {"junit", {}} } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "junit", {}, {}, {} } } ) with expansion: - { { junit, } } - == - { { junit, } } + { {?} } == { {?} } with message: result.errorMessage() := "" @@ -9635,11 +9627,9 @@ with message: result.errorMessage() := "" CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_ReporterAndFile{ {"console", "out.txt"s} } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "console", "out.txt"s, {}, {} } } ) with expansion: - { { console, out.txt } } - == - { { console, out.txt } } + { {?} } == { {?} } with message: result.errorMessage() := "" @@ -9659,34 +9649,12 @@ with message: result.errorMessage() := "" CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_ReporterAndFile{ {"console", "C:\\Temp\\out.txt"s} } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {} } } ) with expansion: - { { console, C:\Temp\out.txt } } - == - { { console, C:\Temp\out.txt } } + { {?} } == { {?} } with message: result.errorMessage() := "" -------------------------------------------------------------------------------- -Process can be configured on command line - reporter - Output file cannot be empty -------------------------------------------------------------------------------- -CmdLine.tests.cpp: -............................................................................... - -CmdLine.tests.cpp:: PASSED: - CHECK( !result ) -with expansion: - true - -CmdLine.tests.cpp:: PASSED: - REQUIRE_THAT( result.errorMessage(), ContainsSubstring("empty filename") ) -with expansion: - "Reporter 'console' has empty filename specified as its output. Supply a - filename or remove the colons to use the default output." contains: "empty - filename" - ------------------------------------------------------------------------------- Process can be configured on command line reporter @@ -9697,16 +9665,14 @@ CmdLine.tests.cpp: ............................................................................... CmdLine.tests.cpp:: PASSED: - CHECK( cli.parse({ "test", "-r", "xml::output.xml", "-r", "junit::output-junit.xml" }) ) + CHECK( cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "junit::out=output-junit.xml" }) ) with expansion: {?} CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_ReporterAndFile{ {"xml", "output.xml"s}, {"junit", "output-junit.xml"s} } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "junit", "output-junit.xml"s, {}, {} } } ) with expansion: - { { xml, output.xml }, { junit, output-junit.xml } } - == - { { xml, output.xml }, { junit, output-junit.xml } } + { {?}, {?} } == { {?}, {?} } ------------------------------------------------------------------------------- Process can be configured on command line @@ -9718,16 +9684,14 @@ CmdLine.tests.cpp: ............................................................................... CmdLine.tests.cpp:: PASSED: - CHECK( cli.parse({ "test", "-r", "xml::output.xml", "-r", "console" }) ) + CHECK( cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "console" }) ) with expansion: {?} CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_ReporterAndFile{ {"xml", "output.xml"s}, {"console", {}} } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "console", {}, {}, {} } } ) with expansion: - { { xml, output.xml }, { console, } } - == - { { xml, output.xml }, { console, } } + { {?}, {?} } == { {?}, {?} } ------------------------------------------------------------------------------- Process can be configured on command line @@ -17953,5 +17917,5 @@ Misc.tests.cpp:: PASSED: =============================================================================== test cases: 390 | 300 passed | 83 failed | 7 failed as expected -assertions: 2240 | 2070 passed | 143 failed | 27 failed as expected +assertions: 2238 | 2068 passed | 143 failed | 27 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.multi.approved.txt b/tests/SelfTest/Baselines/console.sw.multi.approved.txt index 3847cfc9..356457f3 100644 --- a/tests/SelfTest/Baselines/console.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.multi.approved.txt @@ -9431,16 +9431,14 @@ with expansion: !false CmdLine.tests.cpp:: PASSED: - CHECK( cfg.getReportersAndOutputFiles().size() == 1 ) + CHECK( cfg.getReporterSpecs().size() == 1 ) with expansion: 1 == 1 CmdLine.tests.cpp:: PASSED: - CHECK( cfg.getReportersAndOutputFiles()[0] == Catch::ConfigData::ReporterAndFile{ expectedReporter, {} } ) + CHECK( cfg.getReporterSpecs()[0] == Catch::ReporterSpec{ expectedReporter, {}, {}, {} } ) with expansion: - { console, } - == - { console, } + {?} == {?} ------------------------------------------------------------------------------- Process can be configured on command line @@ -9542,11 +9540,9 @@ with message: result.errorMessage() := "" CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_ReporterAndFile{ {"console", {}} } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "console", {}, {}, {} } } ) with expansion: - { { console, } } - == - { { console, } } + { {?} } == { {?} } with message: result.errorMessage() := "" @@ -9566,11 +9562,9 @@ with message: result.errorMessage() := "" CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_ReporterAndFile{ {"xml", {}} } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "xml", {}, {}, {} } } ) with expansion: - { { xml, } } - == - { { xml, } } + { {?} } == { {?} } with message: result.errorMessage() := "" @@ -9590,11 +9584,9 @@ with message: result.errorMessage() := "" CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_ReporterAndFile{ {"junit", {}} } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "junit", {}, {}, {} } } ) with expansion: - { { junit, } } - == - { { junit, } } + { {?} } == { {?} } with message: result.errorMessage() := "" @@ -9633,11 +9625,9 @@ with message: result.errorMessage() := "" CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_ReporterAndFile{ {"console", "out.txt"s} } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "console", "out.txt"s, {}, {} } } ) with expansion: - { { console, out.txt } } - == - { { console, out.txt } } + { {?} } == { {?} } with message: result.errorMessage() := "" @@ -9657,34 +9647,12 @@ with message: result.errorMessage() := "" CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_ReporterAndFile{ {"console", "C:\\Temp\\out.txt"s} } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {} } } ) with expansion: - { { console, C:\Temp\out.txt } } - == - { { console, C:\Temp\out.txt } } + { {?} } == { {?} } with message: result.errorMessage() := "" -------------------------------------------------------------------------------- -Process can be configured on command line - reporter - Output file cannot be empty -------------------------------------------------------------------------------- -CmdLine.tests.cpp: -............................................................................... - -CmdLine.tests.cpp:: PASSED: - CHECK( !result ) -with expansion: - true - -CmdLine.tests.cpp:: PASSED: - REQUIRE_THAT( result.errorMessage(), ContainsSubstring("empty filename") ) -with expansion: - "Reporter 'console' has empty filename specified as its output. Supply a - filename or remove the colons to use the default output." contains: "empty - filename" - ------------------------------------------------------------------------------- Process can be configured on command line reporter @@ -9695,16 +9663,14 @@ CmdLine.tests.cpp: ............................................................................... CmdLine.tests.cpp:: PASSED: - CHECK( cli.parse({ "test", "-r", "xml::output.xml", "-r", "junit::output-junit.xml" }) ) + CHECK( cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "junit::out=output-junit.xml" }) ) with expansion: {?} CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_ReporterAndFile{ {"xml", "output.xml"s}, {"junit", "output-junit.xml"s} } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "junit", "output-junit.xml"s, {}, {} } } ) with expansion: - { { xml, output.xml }, { junit, output-junit.xml } } - == - { { xml, output.xml }, { junit, output-junit.xml } } + { {?}, {?} } == { {?}, {?} } ------------------------------------------------------------------------------- Process can be configured on command line @@ -9716,16 +9682,14 @@ CmdLine.tests.cpp: ............................................................................... CmdLine.tests.cpp:: PASSED: - CHECK( cli.parse({ "test", "-r", "xml::output.xml", "-r", "console" }) ) + CHECK( cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "console" }) ) with expansion: {?} CmdLine.tests.cpp:: PASSED: - REQUIRE( config.reporterSpecifications == vec_ReporterAndFile{ {"xml", "output.xml"s}, {"console", {}} } ) + REQUIRE( config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "console", {}, {}, {} } } ) with expansion: - { { xml, output.xml }, { console, } } - == - { { xml, output.xml }, { console, } } + { {?}, {?} } == { {?}, {?} } ------------------------------------------------------------------------------- Process can be configured on command line @@ -17945,5 +17909,5 @@ Misc.tests.cpp:: PASSED: =============================================================================== test cases: 390 | 300 passed | 83 failed | 7 failed as expected -assertions: 2240 | 2070 passed | 143 failed | 27 failed as expected +assertions: 2238 | 2068 passed | 143 failed | 27 failed as expected diff --git a/tests/SelfTest/Baselines/junit.sw.approved.txt b/tests/SelfTest/Baselines/junit.sw.approved.txt index 519fdfbc..ce486c27 100644 --- a/tests/SelfTest/Baselines/junit.sw.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ - + @@ -1109,7 +1109,6 @@ Message.tests.cpp: - diff --git a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt index 567b7ab7..11cb526b 100644 --- a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt @@ -1,6 +1,6 @@ - + @@ -1108,7 +1108,6 @@ Message.tests.cpp: - diff --git a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt index 6d41d95b..83b5d3cb 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt @@ -66,7 +66,6 @@ - diff --git a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt index 7f2f076b..2723afe8 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt @@ -65,7 +65,6 @@ - diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt index 1cb2209b..3bbb1b25 100644 --- a/tests/SelfTest/Baselines/tap.sw.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.approved.txt @@ -2479,9 +2479,9 @@ ok {test-number} - config.reporterSpecifications.empty() for: true # Process can be configured on command line ok {test-number} - !(cfg.hasTestFilters()) for: !false # Process can be configured on command line -ok {test-number} - cfg.getReportersAndOutputFiles().size() == 1 for: 1 == 1 +ok {test-number} - cfg.getReporterSpecs().size() == 1 for: 1 == 1 # Process can be configured on command line -ok {test-number} - cfg.getReportersAndOutputFiles()[0] == Catch::ConfigData::ReporterAndFile{ expectedReporter, {} } for: { console, } == { console, } +ok {test-number} - cfg.getReporterSpecs()[0] == Catch::ReporterSpec{ expectedReporter, {}, {}, {} } for: {?} == {?} # Process can be configured on command line ok {test-number} - result for: {?} # Process can be configured on command line @@ -2509,15 +2509,15 @@ ok {test-number} - cfg.testSpec().matches(*fakeTestCase("alwaysIncluded")) for: # Process can be configured on command line ok {test-number} - result for: {?} with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_ReporterAndFile{ {"console", {}} } for: { { console, } } == { { console, } } with 1 message: 'result.errorMessage() := ""' +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "console", {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line ok {test-number} - result for: {?} with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_ReporterAndFile{ {"xml", {}} } for: { { xml, } } == { { xml, } } with 1 message: 'result.errorMessage() := ""' +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "xml", {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line ok {test-number} - result for: {?} with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_ReporterAndFile{ {"junit", {}} } for: { { junit, } } == { { junit, } } with 1 message: 'result.errorMessage() := ""' +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "junit", {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line ok {test-number} - !result for: true # Process can be configured on command line @@ -2525,23 +2525,19 @@ ok {test-number} - result.errorMessage(), ContainsSubstring("Unrecognized report # Process can be configured on command line ok {test-number} - result for: {?} with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_ReporterAndFile{ {"console", "out.txt"s} } for: { { console, out.txt } } == { { console, out.txt } } with 1 message: 'result.errorMessage() := ""' +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "console", "out.txt"s, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line ok {test-number} - result for: {?} with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_ReporterAndFile{ {"console", "C:\\Temp\\out.txt"s} } for: { { console, C:\Temp\out.txt } } == { { console, C:\Temp\out.txt } } with 1 message: 'result.errorMessage() := ""' +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line -ok {test-number} - !result for: true +ok {test-number} - cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "junit::out=output-junit.xml" }) for: {?} # Process can be configured on command line -ok {test-number} - result.errorMessage(), ContainsSubstring("empty filename") for: "Reporter 'console' has empty filename specified as its output. Supply a filename or remove the colons to use the default output." contains: "empty filename" +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "junit", "output-junit.xml"s, {}, {} } } for: { {?}, {?} } == { {?}, {?} } # Process can be configured on command line -ok {test-number} - cli.parse({ "test", "-r", "xml::output.xml", "-r", "junit::output-junit.xml" }) for: {?} +ok {test-number} - cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "console" }) for: {?} # Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_ReporterAndFile{ {"xml", "output.xml"s}, {"junit", "output-junit.xml"s} } for: { { xml, output.xml }, { junit, output-junit.xml } } == { { xml, output.xml }, { junit, output-junit.xml } } -# Process can be configured on command line -ok {test-number} - cli.parse({ "test", "-r", "xml::output.xml", "-r", "console" }) for: {?} -# Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_ReporterAndFile{ {"xml", "output.xml"s}, {"console", {}} } for: { { xml, output.xml }, { console, } } == { { xml, output.xml }, { console, } } +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "console", {}, {}, {} } } for: { {?}, {?} } == { {?}, {?} } # Process can be configured on command line ok {test-number} - !result for: true # Process can be configured on command line @@ -4482,5 +4478,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2240 +1..2238 diff --git a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt index f3e52353..70327011 100644 --- a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt @@ -2477,9 +2477,9 @@ ok {test-number} - config.reporterSpecifications.empty() for: true # Process can be configured on command line ok {test-number} - !(cfg.hasTestFilters()) for: !false # Process can be configured on command line -ok {test-number} - cfg.getReportersAndOutputFiles().size() == 1 for: 1 == 1 +ok {test-number} - cfg.getReporterSpecs().size() == 1 for: 1 == 1 # Process can be configured on command line -ok {test-number} - cfg.getReportersAndOutputFiles()[0] == Catch::ConfigData::ReporterAndFile{ expectedReporter, {} } for: { console, } == { console, } +ok {test-number} - cfg.getReporterSpecs()[0] == Catch::ReporterSpec{ expectedReporter, {}, {}, {} } for: {?} == {?} # Process can be configured on command line ok {test-number} - result for: {?} # Process can be configured on command line @@ -2507,15 +2507,15 @@ ok {test-number} - cfg.testSpec().matches(*fakeTestCase("alwaysIncluded")) for: # Process can be configured on command line ok {test-number} - result for: {?} with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_ReporterAndFile{ {"console", {}} } for: { { console, } } == { { console, } } with 1 message: 'result.errorMessage() := ""' +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "console", {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line ok {test-number} - result for: {?} with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_ReporterAndFile{ {"xml", {}} } for: { { xml, } } == { { xml, } } with 1 message: 'result.errorMessage() := ""' +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "xml", {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line ok {test-number} - result for: {?} with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_ReporterAndFile{ {"junit", {}} } for: { { junit, } } == { { junit, } } with 1 message: 'result.errorMessage() := ""' +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "junit", {}, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line ok {test-number} - !result for: true # Process can be configured on command line @@ -2523,23 +2523,19 @@ ok {test-number} - result.errorMessage(), ContainsSubstring("Unrecognized report # Process can be configured on command line ok {test-number} - result for: {?} with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_ReporterAndFile{ {"console", "out.txt"s} } for: { { console, out.txt } } == { { console, out.txt } } with 1 message: 'result.errorMessage() := ""' +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "console", "out.txt"s, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line ok {test-number} - result for: {?} with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_ReporterAndFile{ {"console", "C:\\Temp\\out.txt"s} } for: { { console, C:\Temp\out.txt } } == { { console, C:\Temp\out.txt } } with 1 message: 'result.errorMessage() := ""' +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {} } } for: { {?} } == { {?} } with 1 message: 'result.errorMessage() := ""' # Process can be configured on command line -ok {test-number} - !result for: true +ok {test-number} - cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "junit::out=output-junit.xml" }) for: {?} # Process can be configured on command line -ok {test-number} - result.errorMessage(), ContainsSubstring("empty filename") for: "Reporter 'console' has empty filename specified as its output. Supply a filename or remove the colons to use the default output." contains: "empty filename" +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "junit", "output-junit.xml"s, {}, {} } } for: { {?}, {?} } == { {?}, {?} } # Process can be configured on command line -ok {test-number} - cli.parse({ "test", "-r", "xml::output.xml", "-r", "junit::output-junit.xml" }) for: {?} +ok {test-number} - cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "console" }) for: {?} # Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_ReporterAndFile{ {"xml", "output.xml"s}, {"junit", "output-junit.xml"s} } for: { { xml, output.xml }, { junit, output-junit.xml } } == { { xml, output.xml }, { junit, output-junit.xml } } -# Process can be configured on command line -ok {test-number} - cli.parse({ "test", "-r", "xml::output.xml", "-r", "console" }) for: {?} -# Process can be configured on command line -ok {test-number} - config.reporterSpecifications == vec_ReporterAndFile{ {"xml", "output.xml"s}, {"console", {}} } for: { { xml, output.xml }, { console, } } == { { xml, output.xml }, { console, } } +ok {test-number} - config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "console", {}, {}, {} } } for: { {?}, {?} } == { {?}, {?} } # Process can be configured on command line ok {test-number} - !result for: true # Process can be configured on command line @@ -4474,5 +4470,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2240 +1..2238 diff --git a/tests/SelfTest/Baselines/xml.sw.approved.txt b/tests/SelfTest/Baselines/xml.sw.approved.txt index 434f8dbe..a491eb75 100644 --- a/tests/SelfTest/Baselines/xml.sw.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.approved.txt @@ -11468,7 +11468,7 @@ C - cfg.getReportersAndOutputFiles().size() == 1 + cfg.getReporterSpecs().size() == 1 1 == 1 @@ -11476,12 +11476,10 @@ C - cfg.getReportersAndOutputFiles()[0] == Catch::ConfigData::ReporterAndFile{ expectedReporter, {} } + cfg.getReporterSpecs()[0] == Catch::ReporterSpec{ expectedReporter, {}, {}, {} } - { console, <default-output> } -== -{ console, <default-output> } + {?} == {?} @@ -11618,12 +11616,10 @@ C - config.reporterSpecifications == vec_ReporterAndFile{ {"console", {}} } + config.reporterSpecifications == vec_Specs{ { "console", {}, {}, {} } } - { { console, <default-output> } } -== -{ { console, <default-output> } } + { {?} } == { {?} } @@ -11648,12 +11644,10 @@ C - config.reporterSpecifications == vec_ReporterAndFile{ {"xml", {}} } + config.reporterSpecifications == vec_Specs{ { "xml", {}, {}, {} } } - { { xml, <default-output> } } -== -{ { xml, <default-output> } } + { {?} } == { {?} } @@ -11678,12 +11672,10 @@ C - config.reporterSpecifications == vec_ReporterAndFile{ {"junit", {}} } + config.reporterSpecifications == vec_Specs{ { "junit", {}, {}, {} } } - { { junit, <default-output> } } -== -{ { junit, <default-output> } } + { {?} } == { {?} } @@ -11730,12 +11722,10 @@ C - config.reporterSpecifications == vec_ReporterAndFile{ {"console", "out.txt"s} } + config.reporterSpecifications == vec_Specs{ { "console", "out.txt"s, {}, {} } } - { { console, out.txt } } -== -{ { console, out.txt } } + { {?} } == { {?} } @@ -11760,34 +11750,10 @@ C - config.reporterSpecifications == vec_ReporterAndFile{ {"console", "C:\\Temp\\out.txt"s} } + config.reporterSpecifications == vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {} } } - { { console, C:\Temp\out.txt } } -== -{ { console, C:\Temp\out.txt } } - - - - - - -
-
- - - !result - - - true - - - - - result.errorMessage(), ContainsSubstring("empty filename") - - - "Reporter 'console' has empty filename specified as its output. Supply a filename or remove the colons to use the default output." contains: "empty filename" + { {?} } == { {?} } @@ -11799,7 +11765,7 @@ C
- cli.parse({ "test", "-r", "xml::output.xml", "-r", "junit::output-junit.xml" }) + cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "junit::out=output-junit.xml" }) {?} @@ -11807,12 +11773,10 @@ C - config.reporterSpecifications == vec_ReporterAndFile{ {"xml", "output.xml"s}, {"junit", "output-junit.xml"s} } + config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "junit", "output-junit.xml"s, {}, {} } } - { { xml, output.xml }, { junit, output-junit.xml } } -== -{ { xml, output.xml }, { junit, output-junit.xml } } + { {?}, {?} } == { {?}, {?} } @@ -11826,7 +11790,7 @@ C
- cli.parse({ "test", "-r", "xml::output.xml", "-r", "console" }) + cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "console" }) {?} @@ -11834,12 +11798,10 @@ C - config.reporterSpecifications == vec_ReporterAndFile{ {"xml", "output.xml"s}, {"console", {}} } + config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "console", {}, {}, {} } } - { { xml, output.xml }, { console, <default-output> } } -== -{ { xml, output.xml }, { console, <default-output> } } + { {?}, {?} } == { {?}, {?} } @@ -21072,6 +21034,6 @@ loose text artifact
- + diff --git a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt index 781d986c..66006e48 100644 --- a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt @@ -11468,7 +11468,7 @@ C - cfg.getReportersAndOutputFiles().size() == 1 + cfg.getReporterSpecs().size() == 1 1 == 1 @@ -11476,12 +11476,10 @@ C - cfg.getReportersAndOutputFiles()[0] == Catch::ConfigData::ReporterAndFile{ expectedReporter, {} } + cfg.getReporterSpecs()[0] == Catch::ReporterSpec{ expectedReporter, {}, {}, {} } - { console, <default-output> } -== -{ console, <default-output> } + {?} == {?} @@ -11618,12 +11616,10 @@ C - config.reporterSpecifications == vec_ReporterAndFile{ {"console", {}} } + config.reporterSpecifications == vec_Specs{ { "console", {}, {}, {} } } - { { console, <default-output> } } -== -{ { console, <default-output> } } + { {?} } == { {?} } @@ -11648,12 +11644,10 @@ C - config.reporterSpecifications == vec_ReporterAndFile{ {"xml", {}} } + config.reporterSpecifications == vec_Specs{ { "xml", {}, {}, {} } } - { { xml, <default-output> } } -== -{ { xml, <default-output> } } + { {?} } == { {?} } @@ -11678,12 +11672,10 @@ C - config.reporterSpecifications == vec_ReporterAndFile{ {"junit", {}} } + config.reporterSpecifications == vec_Specs{ { "junit", {}, {}, {} } } - { { junit, <default-output> } } -== -{ { junit, <default-output> } } + { {?} } == { {?} } @@ -11730,12 +11722,10 @@ C - config.reporterSpecifications == vec_ReporterAndFile{ {"console", "out.txt"s} } + config.reporterSpecifications == vec_Specs{ { "console", "out.txt"s, {}, {} } } - { { console, out.txt } } -== -{ { console, out.txt } } + { {?} } == { {?} } @@ -11760,34 +11750,10 @@ C - config.reporterSpecifications == vec_ReporterAndFile{ {"console", "C:\\Temp\\out.txt"s} } + config.reporterSpecifications == vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {} } } - { { console, C:\Temp\out.txt } } -== -{ { console, C:\Temp\out.txt } } - - - -
- -
-
-
- - - !result - - - true - - - - - result.errorMessage(), ContainsSubstring("empty filename") - - - "Reporter 'console' has empty filename specified as its output. Supply a filename or remove the colons to use the default output." contains: "empty filename" + { {?} } == { {?} } @@ -11799,7 +11765,7 @@ C
- cli.parse({ "test", "-r", "xml::output.xml", "-r", "junit::output-junit.xml" }) + cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "junit::out=output-junit.xml" }) {?} @@ -11807,12 +11773,10 @@ C - config.reporterSpecifications == vec_ReporterAndFile{ {"xml", "output.xml"s}, {"junit", "output-junit.xml"s} } + config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "junit", "output-junit.xml"s, {}, {} } } - { { xml, output.xml }, { junit, output-junit.xml } } -== -{ { xml, output.xml }, { junit, output-junit.xml } } + { {?}, {?} } == { {?}, {?} } @@ -11826,7 +11790,7 @@ C
- cli.parse({ "test", "-r", "xml::output.xml", "-r", "console" }) + cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "console" }) {?} @@ -11834,12 +11798,10 @@ C - config.reporterSpecifications == vec_ReporterAndFile{ {"xml", "output.xml"s}, {"console", {}} } + config.reporterSpecifications == vec_Specs{ { "xml", "output.xml"s, {}, {} }, { "console", {}, {}, {} } } - { { xml, output.xml }, { console, <default-output> } } -== -{ { xml, output.xml }, { console, <default-output> } } + { {?}, {?} } == { {?}, {?} } @@ -21071,6 +21033,6 @@ There is no extra whitespace here
- + diff --git a/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp b/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp index 25d5fbd6..7599adbe 100644 --- a/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp +++ b/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp @@ -361,9 +361,9 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]" #endif ; - CHECK( cfg.getReportersAndOutputFiles().size() == 1 ); - CHECK( cfg.getReportersAndOutputFiles()[0] == - Catch::ConfigData::ReporterAndFile{ expectedReporter, {} } ); + CHECK( cfg.getReporterSpecs().size() == 1 ); + CHECK( cfg.getReporterSpecs()[0] == + Catch::ReporterSpec{ expectedReporter, {}, {}, {} } ); } SECTION("test lists") { @@ -399,28 +399,31 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]" } SECTION("reporter") { - using vec_ReporterAndFile = std::vector; + using vec_Specs = std::vector; using namespace std::string_literals; SECTION("-r/console") { auto result = cli.parse({"test", "-r", "console"}); CAPTURE(result.errorMessage()); CHECK(result); - REQUIRE(config.reporterSpecifications == vec_ReporterAndFile{ {"console", {}} }); + REQUIRE( config.reporterSpecifications == + vec_Specs{ { "console", {}, {}, {} } } ); } SECTION("-r/xml") { auto result = cli.parse({"test", "-r", "xml"}); CAPTURE(result.errorMessage()); CHECK(result); - REQUIRE(config.reporterSpecifications == vec_ReporterAndFile{ {"xml", {}} }); + REQUIRE( config.reporterSpecifications == + vec_Specs{ { "xml", {}, {}, {} } } ); } SECTION("--reporter/junit") { auto result = cli.parse({"test", "--reporter", "junit"}); CAPTURE(result.errorMessage()); CHECK(result); - REQUIRE(config.reporterSpecifications == vec_ReporterAndFile{ {"junit", {}} }); + REQUIRE( config.reporterSpecifications == + vec_Specs{ { "junit", {}, {}, {} } } ); } SECTION("must match one of the available ones") { auto result = cli.parse({"test", "--reporter", "unsupported"}); @@ -429,34 +432,34 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]" REQUIRE_THAT(result.errorMessage(), ContainsSubstring("Unrecognized reporter")); } SECTION("With output file") { - auto result = cli.parse({ "test", "-r", "console::out.txt" }); + auto result = cli.parse({ "test", "-r", "console::out=out.txt" }); CAPTURE(result.errorMessage()); CHECK(result); - REQUIRE(config.reporterSpecifications == vec_ReporterAndFile{ {"console", "out.txt"s} }); + REQUIRE( config.reporterSpecifications == + vec_Specs{ { "console", "out.txt"s, {}, {} } } ); } SECTION("With Windows-like absolute path as output file") { - auto result = cli.parse({ "test", "-r", "console::C:\\Temp\\out.txt" }); + auto result = cli.parse({ "test", "-r", "console::out=C:\\Temp\\out.txt" }); CAPTURE(result.errorMessage()); CHECK(result); - REQUIRE(config.reporterSpecifications == vec_ReporterAndFile{ {"console", "C:\\Temp\\out.txt"s} }); - } - SECTION("Output file cannot be empty") { - auto result = cli.parse({"test", "--reporter", "console::"}); - CHECK(!result); - - REQUIRE_THAT(result.errorMessage(), ContainsSubstring("empty filename")); + REQUIRE( config.reporterSpecifications == + vec_Specs{ { "console", "C:\\Temp\\out.txt"s, {}, {} } } ); } SECTION("Multiple reporters") { SECTION("All with output files") { - CHECK(cli.parse({ "test", "-r", "xml::output.xml", "-r", "junit::output-junit.xml" })); - REQUIRE(config.reporterSpecifications == vec_ReporterAndFile{ {"xml", "output.xml"s}, {"junit", "output-junit.xml"s} }); + CHECK(cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "junit::out=output-junit.xml" })); + REQUIRE( config.reporterSpecifications == + vec_Specs{ { "xml", "output.xml"s, {}, {} }, + { "junit", "output-junit.xml"s, {}, {} } } ); } SECTION("Mixed output files and default output") { - CHECK(cli.parse({ "test", "-r", "xml::output.xml", "-r", "console" })); - REQUIRE(config.reporterSpecifications == vec_ReporterAndFile{ {"xml", "output.xml"s}, {"console", {}} }); + CHECK(cli.parse({ "test", "-r", "xml::out=output.xml", "-r", "console" })); + REQUIRE( config.reporterSpecifications == + vec_Specs{ { "xml", "output.xml"s, {}, {} }, + { "console", {}, {}, {} } } ); } SECTION("cannot have multiple reporters with default output") { - auto result = cli.parse({ "test", "-r", "console", "-r", "xml::output.xml", "-r", "junit" }); + auto result = cli.parse({ "test", "-r", "console", "-r", "xml::out=output.xml", "-r", "junit" }); CHECK(!result); REQUIRE_THAT(result.errorMessage(), ContainsSubstring("Only one reporter may have unspecified output file.")); } diff --git a/tests/SelfTest/IntrospectiveTests/Reporters.tests.cpp b/tests/SelfTest/IntrospectiveTests/Reporters.tests.cpp index 92367915..890afe98 100644 --- a/tests/SelfTest/IntrospectiveTests/Reporters.tests.cpp +++ b/tests/SelfTest/IntrospectiveTests/Reporters.tests.cpp @@ -89,7 +89,8 @@ TEST_CASE( "Reporter's write listings to provided stream", "[reporters]" ) { Catch::ConfigData config_data; Catch::Config config( config_data ); - Catch::ReporterConfig rep_config( &config, &sstream ); + Catch::ReporterConfig rep_config( + &config, &sstream, Catch::ColourMode::None, {} ); auto reporter = factory.second->create( rep_config ); DYNAMIC_SECTION( factory.first << " reporter lists tags" ) { @@ -176,7 +177,8 @@ TEST_CASE("Multireporter calls reporters and listeners in correct order", Catch::ConfigData config_data; Catch::Config config( config_data ); StringIStream sstream; - Catch::ReporterConfig rep_config( &config, &sstream ); + Catch::ReporterConfig rep_config( + &config, &sstream, Catch::ColourMode::None, {} ); // We add reporters before listeners, to check that internally they // get sorted properly, and listeners are called first anyway. @@ -229,7 +231,8 @@ TEST_CASE("Multireporter updates ReporterPreferences properly", Catch::ConfigData config_data; Catch::Config config( config_data ); StringIStream sstream; - Catch::ReporterConfig rep_config( &config, &sstream ); + Catch::ReporterConfig rep_config( + &config, &sstream, Catch::ColourMode::None, {} ); Catch::MultiReporter multiReporter( &config ); // Post init defaults diff --git a/tools/scripts/approvalTests.py b/tools/scripts/approvalTests.py index 36bbdd2f..0febe1cc 100755 --- a/tools/scripts/approvalTests.py +++ b/tools/scripts/approvalTests.py @@ -220,7 +220,7 @@ common_args = ["~[!nonportable]~[!benchmark]~[approvals] *", "-s", "-w", "NoAsse filenames = ['{}.sw.multi'.format(reporter) for reporter in reporters] reporter_args = [] for reporter, filename in zip(reporters, filenames): - reporter_args += ['-r', '{}::{}'.format(reporter, get_rawResultsPath(filename))] + reporter_args += ['-r', '{}::out={}'.format(reporter, get_rawResultsPath(filename))] run_test("default.sw.multi", common_args + reporter_args) check_outputs("default.sw.multi")