mirror of
https://github.com/catchorg/Catch2.git
synced 2025-08-03 22:05:39 +02:00
Add support for multiple parallel reporters
This requires a bunch of different changes across the reporter subsystem. * We need to handle multiple reporters and their differing preferences in `ListeningReporter`, e.g. what to do when we mix reporters that capture and don't capture stdout. * We need to change how the reporter is given output and how we parse reporter's output destination from CLI. * Approval tests need to handle multireporter option
This commit is contained in:
Martin Jeřábek
committed by
Martin Hořeňovský
Martin Hořeňovský
parent
6b55f5d780
commit
ccd67b293d
@@ -9158,9 +9158,11 @@ with expansion:
|
||||
false == false
|
||||
|
||||
CmdLine.tests.cpp:<line number>: PASSED:
|
||||
CHECK( config.reporterName == "console" )
|
||||
CHECK( config.reporterSpecifications == std::vector<Catch::ConfigData::ReporterAndFile>{ {"console", {}} } )
|
||||
with expansion:
|
||||
"console" == "console"
|
||||
{ { console, <default-output> } }
|
||||
==
|
||||
{ { console, <default-output> } }
|
||||
|
||||
CmdLine.tests.cpp:<line number>: PASSED:
|
||||
CHECK_FALSE( cfg.hasTestFilters() )
|
||||
@@ -9260,14 +9262,20 @@ CmdLine.tests.cpp:<line number>
|
||||
...............................................................................
|
||||
|
||||
CmdLine.tests.cpp:<line number>: PASSED:
|
||||
CHECK( cli.parse({"test", "-r", "console"}) )
|
||||
CHECK( result )
|
||||
with expansion:
|
||||
{?}
|
||||
with message:
|
||||
result.errorMessage() := ""
|
||||
|
||||
CmdLine.tests.cpp:<line number>: PASSED:
|
||||
REQUIRE( config.reporterName == "console" )
|
||||
REQUIRE( config.reporterSpecifications == vec_ReporterAndFile{ {"console", {}} } )
|
||||
with expansion:
|
||||
"console" == "console"
|
||||
{ { console, <default-output> } }
|
||||
==
|
||||
{ { console, <default-output> } }
|
||||
with message:
|
||||
result.errorMessage() := ""
|
||||
|
||||
-------------------------------------------------------------------------------
|
||||
Process can be configured on command line
|
||||
@@ -9278,14 +9286,20 @@ CmdLine.tests.cpp:<line number>
|
||||
...............................................................................
|
||||
|
||||
CmdLine.tests.cpp:<line number>: PASSED:
|
||||
CHECK( cli.parse({"test", "-r", "xml"}) )
|
||||
CHECK( result )
|
||||
with expansion:
|
||||
{?}
|
||||
with message:
|
||||
result.errorMessage() := ""
|
||||
|
||||
CmdLine.tests.cpp:<line number>: PASSED:
|
||||
REQUIRE( config.reporterName == "xml" )
|
||||
REQUIRE( config.reporterSpecifications == vec_ReporterAndFile{ {"xml", {}} } )
|
||||
with expansion:
|
||||
"xml" == "xml"
|
||||
{ { xml, <default-output> } }
|
||||
==
|
||||
{ { xml, <default-output> } }
|
||||
with message:
|
||||
result.errorMessage() := ""
|
||||
|
||||
-------------------------------------------------------------------------------
|
||||
Process can be configured on command line
|
||||
@@ -9296,27 +9310,20 @@ CmdLine.tests.cpp:<line number>
|
||||
...............................................................................
|
||||
|
||||
CmdLine.tests.cpp:<line number>: PASSED:
|
||||
CHECK( cli.parse({"test", "--reporter", "junit"}) )
|
||||
CHECK( result )
|
||||
with expansion:
|
||||
{?}
|
||||
with message:
|
||||
result.errorMessage() := ""
|
||||
|
||||
CmdLine.tests.cpp:<line number>: PASSED:
|
||||
REQUIRE( config.reporterName == "junit" )
|
||||
REQUIRE( config.reporterSpecifications == vec_ReporterAndFile{ {"junit", {}} } )
|
||||
with expansion:
|
||||
"junit" == "junit"
|
||||
|
||||
-------------------------------------------------------------------------------
|
||||
Process can be configured on command line
|
||||
reporter
|
||||
Only one reporter is accepted
|
||||
-------------------------------------------------------------------------------
|
||||
CmdLine.tests.cpp:<line number>
|
||||
...............................................................................
|
||||
|
||||
CmdLine.tests.cpp:<line number>: PASSED:
|
||||
REQUIRE_FALSE( cli.parse({ "test", "-r", "xml", "-r", "junit" }) )
|
||||
with expansion:
|
||||
!{?}
|
||||
{ { junit, <default-output> } }
|
||||
==
|
||||
{ { junit, <default-output> } }
|
||||
with message:
|
||||
result.errorMessage() := ""
|
||||
|
||||
-------------------------------------------------------------------------------
|
||||
Process can be configured on command line
|
||||
@@ -9337,6 +9344,136 @@ with expansion:
|
||||
"Unrecognized reporter, 'unsupported'. Check available with --list-reporters"
|
||||
contains: "Unrecognized reporter"
|
||||
|
||||
-------------------------------------------------------------------------------
|
||||
Process can be configured on command line
|
||||
reporter
|
||||
With output file
|
||||
-------------------------------------------------------------------------------
|
||||
CmdLine.tests.cpp:<line number>
|
||||
...............................................................................
|
||||
|
||||
CmdLine.tests.cpp:<line number>: PASSED:
|
||||
CHECK( result )
|
||||
with expansion:
|
||||
{?}
|
||||
with message:
|
||||
result.errorMessage() := ""
|
||||
|
||||
CmdLine.tests.cpp:<line number>: PASSED:
|
||||
REQUIRE( config.reporterSpecifications == vec_ReporterAndFile{ {"console", "out.txt"s} } )
|
||||
with expansion:
|
||||
{ { console, out.txt } }
|
||||
==
|
||||
{ { console, out.txt } }
|
||||
with message:
|
||||
result.errorMessage() := ""
|
||||
|
||||
-------------------------------------------------------------------------------
|
||||
Process can be configured on command line
|
||||
reporter
|
||||
With Windows-like absolute path as output file
|
||||
-------------------------------------------------------------------------------
|
||||
CmdLine.tests.cpp:<line number>
|
||||
...............................................................................
|
||||
|
||||
CmdLine.tests.cpp:<line number>: PASSED:
|
||||
CHECK( result )
|
||||
with expansion:
|
||||
{?}
|
||||
with message:
|
||||
result.errorMessage() := ""
|
||||
|
||||
CmdLine.tests.cpp:<line number>: PASSED:
|
||||
REQUIRE( config.reporterSpecifications == vec_ReporterAndFile{ {"console", "C:\\Temp\\out.txt"s} } )
|
||||
with expansion:
|
||||
{ { console, C:\Temp\out.txt } }
|
||||
==
|
||||
{ { console, C:\Temp\out.txt } }
|
||||
with message:
|
||||
result.errorMessage() := ""
|
||||
|
||||
-------------------------------------------------------------------------------
|
||||
Process can be configured on command line
|
||||
reporter
|
||||
Output file cannot be empty
|
||||
-------------------------------------------------------------------------------
|
||||
CmdLine.tests.cpp:<line number>
|
||||
...............................................................................
|
||||
|
||||
CmdLine.tests.cpp:<line number>: PASSED:
|
||||
CHECK( !result )
|
||||
with expansion:
|
||||
true
|
||||
|
||||
CmdLine.tests.cpp:<line number>: PASSED:
|
||||
REQUIRE_THAT( result.errorMessage(), ContainsSubstring("empty filename") )
|
||||
with expansion:
|
||||
"Reporter 'console' has empty filename specified as its output. Supply a
|
||||
filename or remove the colons to use the default output." contains: "empty
|
||||
filename"
|
||||
|
||||
-------------------------------------------------------------------------------
|
||||
Process can be configured on command line
|
||||
reporter
|
||||
Multiple reporters
|
||||
All with output files
|
||||
-------------------------------------------------------------------------------
|
||||
CmdLine.tests.cpp:<line number>
|
||||
...............................................................................
|
||||
|
||||
CmdLine.tests.cpp:<line number>: PASSED:
|
||||
CHECK( cli.parse({ "test", "-r", "xml::output.xml", "-r", "junit::output-junit.xml" }) )
|
||||
with expansion:
|
||||
{?}
|
||||
|
||||
CmdLine.tests.cpp:<line number>: PASSED:
|
||||
REQUIRE( config.reporterSpecifications == vec_ReporterAndFile{ {"xml", "output.xml"s}, {"junit", "output-junit.xml"s} } )
|
||||
with expansion:
|
||||
{ { xml, output.xml }, { junit, output-junit.xml } }
|
||||
==
|
||||
{ { xml, output.xml }, { junit, output-junit.xml } }
|
||||
|
||||
-------------------------------------------------------------------------------
|
||||
Process can be configured on command line
|
||||
reporter
|
||||
Multiple reporters
|
||||
Mixed output files and default output
|
||||
-------------------------------------------------------------------------------
|
||||
CmdLine.tests.cpp:<line number>
|
||||
...............................................................................
|
||||
|
||||
CmdLine.tests.cpp:<line number>: PASSED:
|
||||
CHECK( cli.parse({ "test", "-r", "xml::output.xml", "-r", "console" }) )
|
||||
with expansion:
|
||||
{?}
|
||||
|
||||
CmdLine.tests.cpp:<line number>: PASSED:
|
||||
REQUIRE( config.reporterSpecifications == vec_ReporterAndFile{ {"xml", "output.xml"s}, {"console", {}} } )
|
||||
with expansion:
|
||||
{ { xml, output.xml }, { console, <default-output> } }
|
||||
==
|
||||
{ { xml, output.xml }, { console, <default-output> } }
|
||||
|
||||
-------------------------------------------------------------------------------
|
||||
Process can be configured on command line
|
||||
reporter
|
||||
Multiple reporters
|
||||
cannot have multiple reporters with default output
|
||||
-------------------------------------------------------------------------------
|
||||
CmdLine.tests.cpp:<line number>
|
||||
...............................................................................
|
||||
|
||||
CmdLine.tests.cpp:<line number>: PASSED:
|
||||
CHECK( !result )
|
||||
with expansion:
|
||||
true
|
||||
|
||||
CmdLine.tests.cpp:<line number>: PASSED:
|
||||
REQUIRE_THAT( result.errorMessage(), ContainsSubstring("Only one reporter may have unspecified output file.") )
|
||||
with expansion:
|
||||
"Only one reporter may have unspecified output file." contains: "Only one
|
||||
reporter may have unspecified output file."
|
||||
|
||||
-------------------------------------------------------------------------------
|
||||
Process can be configured on command line
|
||||
debugger
|
||||
@@ -9574,7 +9711,7 @@ with expansion:
|
||||
{?}
|
||||
|
||||
CmdLine.tests.cpp:<line number>: PASSED:
|
||||
REQUIRE( config.outputFilename == "filename.ext" )
|
||||
REQUIRE( config.defaultOutputFilename == "filename.ext" )
|
||||
with expansion:
|
||||
"filename.ext" == "filename.ext"
|
||||
|
||||
@@ -9592,7 +9729,7 @@ with expansion:
|
||||
{?}
|
||||
|
||||
CmdLine.tests.cpp:<line number>: PASSED:
|
||||
REQUIRE( config.outputFilename == "filename.ext" )
|
||||
REQUIRE( config.defaultOutputFilename == "filename.ext" )
|
||||
with expansion:
|
||||
"filename.ext" == "filename.ext"
|
||||
|
||||
@@ -17517,5 +17654,5 @@ Misc.tests.cpp:<line number>: PASSED:
|
||||
|
||||
===============================================================================
|
||||
test cases: 380 | 290 passed | 83 failed | 7 failed as expected
|
||||
assertions: 2198 | 2028 passed | 143 failed | 27 failed as expected
|
||||
assertions: 2209 | 2039 passed | 143 failed | 27 failed as expected
|
||||
|
||||
|
||||
Reference in New Issue
Block a user