mirror of
https://github.com/catchorg/Catch2.git
synced 2025-08-03 13:55:39 +02:00
Add support for multiple parallel reporters
This requires a bunch of different changes across the reporter subsystem. * We need to handle multiple reporters and their differing preferences in `ListeningReporter`, e.g. what to do when we mix reporters that capture and don't capture stdout. * We need to change how the reporter is given output and how we parse reporter's output destination from CLI. * Approval tests need to handle multireporter option
This commit is contained in:
Martin Jeřábek
committed by
Martin Hořeňovský
Martin Hořeňovský
parent
6b55f5d780
commit
ccd67b293d
@@ -11158,10 +11158,12 @@ Nor would this
|
||||
</Expression>
|
||||
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Original>
|
||||
config.reporterName == "console"
|
||||
config.reporterSpecifications == std::vector<Catch::ConfigData::ReporterAndFile>{ {"console", {}} }
|
||||
</Original>
|
||||
<Expanded>
|
||||
"console" == "console"
|
||||
{ { console, <default-output> } }
|
||||
==
|
||||
{ { console, <default-output> } }
|
||||
</Expanded>
|
||||
</Expression>
|
||||
<Expression success="true" type="CHECK_FALSE" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
@@ -11290,20 +11292,28 @@ Nor would this
|
||||
</Section>
|
||||
<Section name="reporter" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Section name="-r/console" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Info>
|
||||
result.errorMessage() := ""
|
||||
</Info>
|
||||
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Original>
|
||||
cli.parse({"test", "-r", "console"})
|
||||
result
|
||||
</Original>
|
||||
<Expanded>
|
||||
{?}
|
||||
</Expanded>
|
||||
</Expression>
|
||||
<Info>
|
||||
result.errorMessage() := ""
|
||||
</Info>
|
||||
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Original>
|
||||
config.reporterName == "console"
|
||||
config.reporterSpecifications == vec_ReporterAndFile{ {"console", {}} }
|
||||
</Original>
|
||||
<Expanded>
|
||||
"console" == "console"
|
||||
{ { console, <default-output> } }
|
||||
==
|
||||
{ { console, <default-output> } }
|
||||
</Expanded>
|
||||
</Expression>
|
||||
<OverallResults successes="2" failures="0" expectedFailures="0"/>
|
||||
@@ -11312,20 +11322,28 @@ Nor would this
|
||||
</Section>
|
||||
<Section name="reporter" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Section name="-r/xml" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Info>
|
||||
result.errorMessage() := ""
|
||||
</Info>
|
||||
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Original>
|
||||
cli.parse({"test", "-r", "xml"})
|
||||
result
|
||||
</Original>
|
||||
<Expanded>
|
||||
{?}
|
||||
</Expanded>
|
||||
</Expression>
|
||||
<Info>
|
||||
result.errorMessage() := ""
|
||||
</Info>
|
||||
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Original>
|
||||
config.reporterName == "xml"
|
||||
config.reporterSpecifications == vec_ReporterAndFile{ {"xml", {}} }
|
||||
</Original>
|
||||
<Expanded>
|
||||
"xml" == "xml"
|
||||
{ { xml, <default-output> } }
|
||||
==
|
||||
{ { xml, <default-output> } }
|
||||
</Expanded>
|
||||
</Expression>
|
||||
<OverallResults successes="2" failures="0" expectedFailures="0"/>
|
||||
@@ -11334,40 +11352,34 @@ Nor would this
|
||||
</Section>
|
||||
<Section name="reporter" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Section name="--reporter/junit" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Info>
|
||||
result.errorMessage() := ""
|
||||
</Info>
|
||||
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Original>
|
||||
cli.parse({"test", "--reporter", "junit"})
|
||||
result
|
||||
</Original>
|
||||
<Expanded>
|
||||
{?}
|
||||
</Expanded>
|
||||
</Expression>
|
||||
<Info>
|
||||
result.errorMessage() := ""
|
||||
</Info>
|
||||
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Original>
|
||||
config.reporterName == "junit"
|
||||
config.reporterSpecifications == vec_ReporterAndFile{ {"junit", {}} }
|
||||
</Original>
|
||||
<Expanded>
|
||||
"junit" == "junit"
|
||||
{ { junit, <default-output> } }
|
||||
==
|
||||
{ { junit, <default-output> } }
|
||||
</Expanded>
|
||||
</Expression>
|
||||
<OverallResults successes="2" failures="0" expectedFailures="0"/>
|
||||
</Section>
|
||||
<OverallResults successes="2" failures="0" expectedFailures="0"/>
|
||||
</Section>
|
||||
<Section name="reporter" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Section name="Only one reporter is accepted" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Expression success="true" type="REQUIRE_FALSE" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Original>
|
||||
!(cli.parse({ "test", "-r", "xml", "-r", "junit" }))
|
||||
</Original>
|
||||
<Expanded>
|
||||
!{?}
|
||||
</Expanded>
|
||||
</Expression>
|
||||
<OverallResults successes="1" failures="0" expectedFailures="0"/>
|
||||
</Section>
|
||||
<OverallResults successes="1" failures="0" expectedFailures="0"/>
|
||||
</Section>
|
||||
<Section name="reporter" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Section name="must match one of the available ones" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
@@ -11390,6 +11402,167 @@ Nor would this
|
||||
</Section>
|
||||
<OverallResults successes="2" failures="0" expectedFailures="0"/>
|
||||
</Section>
|
||||
<Section name="reporter" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Section name="With output file" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Info>
|
||||
result.errorMessage() := ""
|
||||
</Info>
|
||||
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Original>
|
||||
result
|
||||
</Original>
|
||||
<Expanded>
|
||||
{?}
|
||||
</Expanded>
|
||||
</Expression>
|
||||
<Info>
|
||||
result.errorMessage() := ""
|
||||
</Info>
|
||||
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Original>
|
||||
config.reporterSpecifications == vec_ReporterAndFile{ {"console", "out.txt"s} }
|
||||
</Original>
|
||||
<Expanded>
|
||||
{ { console, out.txt } }
|
||||
==
|
||||
{ { console, out.txt } }
|
||||
</Expanded>
|
||||
</Expression>
|
||||
<OverallResults successes="2" failures="0" expectedFailures="0"/>
|
||||
</Section>
|
||||
<OverallResults successes="2" failures="0" expectedFailures="0"/>
|
||||
</Section>
|
||||
<Section name="reporter" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Section name="With Windows-like absolute path as output file" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Info>
|
||||
result.errorMessage() := ""
|
||||
</Info>
|
||||
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Original>
|
||||
result
|
||||
</Original>
|
||||
<Expanded>
|
||||
{?}
|
||||
</Expanded>
|
||||
</Expression>
|
||||
<Info>
|
||||
result.errorMessage() := ""
|
||||
</Info>
|
||||
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Original>
|
||||
config.reporterSpecifications == vec_ReporterAndFile{ {"console", "C:\\Temp\\out.txt"s} }
|
||||
</Original>
|
||||
<Expanded>
|
||||
{ { console, C:\Temp\out.txt } }
|
||||
==
|
||||
{ { console, C:\Temp\out.txt } }
|
||||
</Expanded>
|
||||
</Expression>
|
||||
<OverallResults successes="2" failures="0" expectedFailures="0"/>
|
||||
</Section>
|
||||
<OverallResults successes="2" failures="0" expectedFailures="0"/>
|
||||
</Section>
|
||||
<Section name="reporter" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Section name="Output file cannot be empty" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Original>
|
||||
!result
|
||||
</Original>
|
||||
<Expanded>
|
||||
true
|
||||
</Expanded>
|
||||
</Expression>
|
||||
<Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Original>
|
||||
result.errorMessage(), ContainsSubstring("empty filename")
|
||||
</Original>
|
||||
<Expanded>
|
||||
"Reporter 'console' has empty filename specified as its output. Supply a filename or remove the colons to use the default output." contains: "empty filename"
|
||||
</Expanded>
|
||||
</Expression>
|
||||
<OverallResults successes="2" failures="0" expectedFailures="0"/>
|
||||
</Section>
|
||||
<OverallResults successes="2" failures="0" expectedFailures="0"/>
|
||||
</Section>
|
||||
<Section name="reporter" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Section name="Multiple reporters" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Section name="All with output files" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Original>
|
||||
cli.parse({ "test", "-r", "xml::output.xml", "-r", "junit::output-junit.xml" })
|
||||
</Original>
|
||||
<Expanded>
|
||||
{?}
|
||||
</Expanded>
|
||||
</Expression>
|
||||
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Original>
|
||||
config.reporterSpecifications == vec_ReporterAndFile{ {"xml", "output.xml"s}, {"junit", "output-junit.xml"s} }
|
||||
</Original>
|
||||
<Expanded>
|
||||
{ { xml, output.xml }, { junit, output-junit.xml } }
|
||||
==
|
||||
{ { xml, output.xml }, { junit, output-junit.xml } }
|
||||
</Expanded>
|
||||
</Expression>
|
||||
<OverallResults successes="2" failures="0" expectedFailures="0"/>
|
||||
</Section>
|
||||
<OverallResults successes="2" failures="0" expectedFailures="0"/>
|
||||
</Section>
|
||||
<OverallResults successes="2" failures="0" expectedFailures="0"/>
|
||||
</Section>
|
||||
<Section name="reporter" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Section name="Multiple reporters" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Section name="Mixed output files and default output" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Original>
|
||||
cli.parse({ "test", "-r", "xml::output.xml", "-r", "console" })
|
||||
</Original>
|
||||
<Expanded>
|
||||
{?}
|
||||
</Expanded>
|
||||
</Expression>
|
||||
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Original>
|
||||
config.reporterSpecifications == vec_ReporterAndFile{ {"xml", "output.xml"s}, {"console", {}} }
|
||||
</Original>
|
||||
<Expanded>
|
||||
{ { xml, output.xml }, { console, <default-output> } }
|
||||
==
|
||||
{ { xml, output.xml }, { console, <default-output> } }
|
||||
</Expanded>
|
||||
</Expression>
|
||||
<OverallResults successes="2" failures="0" expectedFailures="0"/>
|
||||
</Section>
|
||||
<OverallResults successes="2" failures="0" expectedFailures="0"/>
|
||||
</Section>
|
||||
<OverallResults successes="2" failures="0" expectedFailures="0"/>
|
||||
</Section>
|
||||
<Section name="reporter" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Section name="Multiple reporters" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Section name="cannot have multiple reporters with default output" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Original>
|
||||
!result
|
||||
</Original>
|
||||
<Expanded>
|
||||
true
|
||||
</Expanded>
|
||||
</Expression>
|
||||
<Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Original>
|
||||
result.errorMessage(), ContainsSubstring("Only one reporter may have unspecified output file.")
|
||||
</Original>
|
||||
<Expanded>
|
||||
"Only one reporter may have unspecified output file." contains: "Only one reporter may have unspecified output file."
|
||||
</Expanded>
|
||||
</Expression>
|
||||
<OverallResults successes="2" failures="0" expectedFailures="0"/>
|
||||
</Section>
|
||||
<OverallResults successes="2" failures="0" expectedFailures="0"/>
|
||||
</Section>
|
||||
<OverallResults successes="2" failures="0" expectedFailures="0"/>
|
||||
</Section>
|
||||
<Section name="debugger" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Section name="-b" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
@@ -11681,7 +11854,7 @@ Nor would this
|
||||
</Expression>
|
||||
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Original>
|
||||
config.outputFilename == "filename.ext"
|
||||
config.defaultOutputFilename == "filename.ext"
|
||||
</Original>
|
||||
<Expanded>
|
||||
"filename.ext" == "filename.ext"
|
||||
@@ -11703,7 +11876,7 @@ Nor would this
|
||||
</Expression>
|
||||
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
|
||||
<Original>
|
||||
config.outputFilename == "filename.ext"
|
||||
config.defaultOutputFilename == "filename.ext"
|
||||
</Original>
|
||||
<Expanded>
|
||||
"filename.ext" == "filename.ext"
|
||||
@@ -20561,6 +20734,6 @@ loose text artifact
|
||||
</Section>
|
||||
<OverallResult success="true"/>
|
||||
</TestCase>
|
||||
<OverallResults successes="2028" failures="143" expectedFailures="27"/>
|
||||
<OverallResults successes="2039" failures="143" expectedFailures="27"/>
|
||||
<OverallResultsCases successes="290" failures="83" expectedFailures="7"/>
|
||||
</Catch2TestRun>
|
||||
|
||||
Reference in New Issue
Block a user