Add more unit tests

* Call order of listeners/reporters in multireporter
* Adding listeners/reporters properly updates reporter preferences
This commit is contained in:
Martin Hořeňovský 2021-12-19 15:16:56 +01:00
parent ccd67b293d
commit 8baf9c05a3
19 changed files with 836 additions and 19 deletions

View File

@ -30,11 +30,8 @@ namespace Catch {
public:
ListeningReporter( IConfig const* config ):
IStreamingReporter( config ) {
// We will assume that listeners will always want all assertions
m_preferences.shouldReportAllAssertions = true;
}
IStreamingReporter( config )
{}
void addListener( IStreamingReporterPtr&& listener );
void addReporter( IStreamingReporterPtr&& reporter );

View File

@ -170,6 +170,8 @@ Nor would this
:test-result: FAIL Matchers can be negated (Not) with the ! operator - failing
:test-result: XFAIL Mayfail test case with nested sections
:test-result: FAIL Mismatching exception messages failing the test
:test-result: PASS Multireporter calls reporters and listeners in correct order
:test-result: PASS Multireporter updates ReporterPreferences properly
:test-result: PASS Nested generators and captured variables
:test-result: FAIL Nice descriptive name
:test-result: FAIL Non-std exceptions can be translated

View File

@ -168,6 +168,8 @@
:test-result: FAIL Matchers can be negated (Not) with the ! operator - failing
:test-result: XFAIL Mayfail test case with nested sections
:test-result: FAIL Mismatching exception messages failing the test
:test-result: PASS Multireporter calls reporters and listeners in correct order
:test-result: PASS Multireporter updates ReporterPreferences properly
:test-result: PASS Nested generators and captured variables
:test-result: FAIL Nice descriptive name
:test-result: FAIL Non-std exceptions can be translated

View File

@ -941,6 +941,25 @@ Condition.tests.cpp:<line number>: failed: explicitly
Condition.tests.cpp:<line number>: failed: explicitly
Exception.tests.cpp:<line number>: passed: thisThrows(), "expected exception" for: "expected exception" equals: "expected exception"
Exception.tests.cpp:<line number>: failed: thisThrows(), "should fail" for: "expected exception" equals: "should fail"
Reporters.tests.cpp:<line number>: passed: records == expected for: { "Hello", "world", "Goodbye", "world" }
==
{ "Hello", "world", "Goodbye", "world" }
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == false for: false == false
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == false for: false == false
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true
Generators.tests.cpp:<line number>: passed: values > -6 for: 3 > -6
Generators.tests.cpp:<line number>: passed: values > -6 for: 4 > -6
Generators.tests.cpp:<line number>: passed: values > -6 for: 5 > -6

View File

@ -939,6 +939,25 @@ Condition.tests.cpp:<line number>: failed: explicitly
Condition.tests.cpp:<line number>: failed: explicitly
Exception.tests.cpp:<line number>: passed: thisThrows(), "expected exception" for: "expected exception" equals: "expected exception"
Exception.tests.cpp:<line number>: failed: thisThrows(), "should fail" for: "expected exception" equals: "should fail"
Reporters.tests.cpp:<line number>: passed: records == expected for: { "Hello", "world", "Goodbye", "world" }
==
{ "Hello", "world", "Goodbye", "world" }
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == false for: false == false
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == false for: false == false
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
Reporters.tests.cpp:<line number>: passed: multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true
Generators.tests.cpp:<line number>: passed: values > -6 for: 3 > -6
Generators.tests.cpp:<line number>: passed: values > -6 for: 4 > -6
Generators.tests.cpp:<line number>: passed: values > -6 for: 5 > -6

View File

@ -1395,6 +1395,6 @@ due to unexpected exception with message:
Why would you throw a std::string?
===============================================================================
test cases: 380 | 304 passed | 69 failed | 7 failed as expected
assertions: 2194 | 2039 passed | 128 failed | 27 failed as expected
test cases: 382 | 306 passed | 69 failed | 7 failed as expected
assertions: 2211 | 2056 passed | 128 failed | 27 failed as expected

View File

@ -7199,6 +7199,125 @@ Exception.tests.cpp:<line number>: FAILED:
with expansion:
"expected exception" equals: "should fail"
-------------------------------------------------------------------------------
Multireporter calls reporters and listeners in correct order
-------------------------------------------------------------------------------
Reporters.tests.cpp:<line number>
...............................................................................
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( records == expected )
with expansion:
{ "Hello", "world", "Goodbye", "world" }
==
{ "Hello", "world", "Goodbye", "world" }
-------------------------------------------------------------------------------
Multireporter updates ReporterPreferences properly
-------------------------------------------------------------------------------
Reporters.tests.cpp:<line number>
...............................................................................
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == false )
with expansion:
false == false
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false )
with expansion:
false == false
-------------------------------------------------------------------------------
Multireporter updates ReporterPreferences properly
Adding listeners
-------------------------------------------------------------------------------
Reporters.tests.cpp:<line number>
...............................................................................
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true )
with expansion:
true == true
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false )
with expansion:
false == false
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true )
with expansion:
true == true
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true )
with expansion:
true == true
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true )
with expansion:
true == true
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true )
with expansion:
true == true
-------------------------------------------------------------------------------
Multireporter updates ReporterPreferences properly
-------------------------------------------------------------------------------
Reporters.tests.cpp:<line number>
...............................................................................
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == false )
with expansion:
false == false
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false )
with expansion:
false == false
-------------------------------------------------------------------------------
Multireporter updates ReporterPreferences properly
Adding reporters
-------------------------------------------------------------------------------
Reporters.tests.cpp:<line number>
...............................................................................
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true )
with expansion:
true == true
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false )
with expansion:
false == false
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true )
with expansion:
true == true
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true )
with expansion:
true == true
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true )
with expansion:
true == true
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true )
with expansion:
true == true
-------------------------------------------------------------------------------
Nested generators and captured variables
-------------------------------------------------------------------------------
@ -17653,6 +17772,6 @@ Misc.tests.cpp:<line number>
Misc.tests.cpp:<line number>: PASSED:
===============================================================================
test cases: 380 | 290 passed | 83 failed | 7 failed as expected
assertions: 2209 | 2039 passed | 143 failed | 27 failed as expected
test cases: 382 | 292 passed | 83 failed | 7 failed as expected
assertions: 2226 | 2056 passed | 143 failed | 27 failed as expected

View File

@ -7197,6 +7197,125 @@ Exception.tests.cpp:<line number>: FAILED:
with expansion:
"expected exception" equals: "should fail"
-------------------------------------------------------------------------------
Multireporter calls reporters and listeners in correct order
-------------------------------------------------------------------------------
Reporters.tests.cpp:<line number>
...............................................................................
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( records == expected )
with expansion:
{ "Hello", "world", "Goodbye", "world" }
==
{ "Hello", "world", "Goodbye", "world" }
-------------------------------------------------------------------------------
Multireporter updates ReporterPreferences properly
-------------------------------------------------------------------------------
Reporters.tests.cpp:<line number>
...............................................................................
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == false )
with expansion:
false == false
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false )
with expansion:
false == false
-------------------------------------------------------------------------------
Multireporter updates ReporterPreferences properly
Adding listeners
-------------------------------------------------------------------------------
Reporters.tests.cpp:<line number>
...............................................................................
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true )
with expansion:
true == true
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false )
with expansion:
false == false
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true )
with expansion:
true == true
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true )
with expansion:
true == true
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true )
with expansion:
true == true
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true )
with expansion:
true == true
-------------------------------------------------------------------------------
Multireporter updates ReporterPreferences properly
-------------------------------------------------------------------------------
Reporters.tests.cpp:<line number>
...............................................................................
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == false )
with expansion:
false == false
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false )
with expansion:
false == false
-------------------------------------------------------------------------------
Multireporter updates ReporterPreferences properly
Adding reporters
-------------------------------------------------------------------------------
Reporters.tests.cpp:<line number>
...............................................................................
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true )
with expansion:
true == true
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false )
with expansion:
false == false
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true )
with expansion:
true == true
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true )
with expansion:
true == true
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true )
with expansion:
true == true
Reporters.tests.cpp:<line number>: PASSED:
REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true )
with expansion:
true == true
-------------------------------------------------------------------------------
Nested generators and captured variables
-------------------------------------------------------------------------------
@ -17645,6 +17764,6 @@ Misc.tests.cpp:<line number>
Misc.tests.cpp:<line number>: PASSED:
===============================================================================
test cases: 380 | 290 passed | 83 failed | 7 failed as expected
assertions: 2209 | 2039 passed | 143 failed | 27 failed as expected
test cases: 382 | 292 passed | 83 failed | 7 failed as expected
assertions: 2226 | 2056 passed | 143 failed | 27 failed as expected

View File

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuitesloose text artifact
>
<testsuite name="<exe-name>" errors="17" failures="126" tests="2209" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<testsuite name="<exe-name>" errors="17" failures="126" tests="2226" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<properties>
<property name="random-seed" value="1"/>
<property name="filters" value="~[!nonportable]~[!benchmark]~[approvals] *"/>
@ -870,6 +870,10 @@ with expansion:
Exception.tests.cpp:<line number>
</failure>
</testcase>
<testcase classname="<exe-name>.global" name="Multireporter calls reporters and listeners in correct order" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="Multireporter updates ReporterPreferences properly" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="Multireporter updates ReporterPreferences properly/Adding listeners" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="Multireporter updates ReporterPreferences properly/Adding reporters" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="Nested generators and captured variables" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="Nice descriptive name" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="Non-std exceptions can be translated" time="{duration}" status="run">

View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuites>
<testsuite name="<exe-name>" errors="17" failures="126" tests="2209" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<testsuite name="<exe-name>" errors="17" failures="126" tests="2226" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<properties>
<property name="random-seed" value="1"/>
<property name="filters" value="~[!nonportable]~[!benchmark]~[approvals] *"/>
@ -869,6 +869,10 @@ with expansion:
Exception.tests.cpp:<line number>
</failure>
</testcase>
<testcase classname="<exe-name>.global" name="Multireporter calls reporters and listeners in correct order" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="Multireporter updates ReporterPreferences properly" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="Multireporter updates ReporterPreferences properly/Adding listeners" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="Multireporter updates ReporterPreferences properly/Adding reporters" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="Nested generators and captured variables" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="Nice descriptive name" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="Non-std exceptions can be translated" time="{duration}" status="run">

View File

@ -181,6 +181,10 @@
<testCase name="Random seed generation reports unknown methods" duration="{duration}"/>
</file>
<file path="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp">
<testCase name="Multireporter calls reporters and listeners in correct order" duration="{duration}"/>
<testCase name="Multireporter updates ReporterPreferences properly" duration="{duration}"/>
<testCase name="Multireporter updates ReporterPreferences properly/Adding listeners" duration="{duration}"/>
<testCase name="Multireporter updates ReporterPreferences properly/Adding reporters" duration="{duration}"/>
<testCase name="Reporter's write listings to provided stream" duration="{duration}"/>
<testCase name="Reporter's write listings to provided stream/Automake reporter lists tags" duration="{duration}"/>
<testCase name="Reporter's write listings to provided stream/Automake reporter lists reporters" duration="{duration}"/>

View File

@ -180,6 +180,10 @@
<testCase name="Random seed generation reports unknown methods" duration="{duration}"/>
</file>
<file path="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp">
<testCase name="Multireporter calls reporters and listeners in correct order" duration="{duration}"/>
<testCase name="Multireporter updates ReporterPreferences properly" duration="{duration}"/>
<testCase name="Multireporter updates ReporterPreferences properly/Adding listeners" duration="{duration}"/>
<testCase name="Multireporter updates ReporterPreferences properly/Adding reporters" duration="{duration}"/>
<testCase name="Reporter's write listings to provided stream" duration="{duration}"/>
<testCase name="Reporter's write listings to provided stream/Automake reporter lists tags" duration="{duration}"/>
<testCase name="Reporter's write listings to provided stream/Automake reporter lists reporters" duration="{duration}"/>

View File

@ -1864,6 +1864,40 @@ not ok {test-number} - explicitly
ok {test-number} - thisThrows(), "expected exception" for: "expected exception" equals: "expected exception"
# Mismatching exception messages failing the test
not ok {test-number} - thisThrows(), "should fail" for: "expected exception" equals: "should fail"
# Multireporter calls reporters and listeners in correct order
ok {test-number} - records == expected for: { "Hello", "world", "Goodbye", "world" } == { "Hello", "world", "Goodbye", "world" }
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == false for: false == false
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == false for: false == false
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true
# Nested generators and captured variables
ok {test-number} - values > -6 for: 3 > -6
# Nested generators and captured variables
@ -4420,5 +4454,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0
ok {test-number} -
# xmlentitycheck
ok {test-number} -
1..2209
1..2226

View File

@ -1862,6 +1862,40 @@ not ok {test-number} - explicitly
ok {test-number} - thisThrows(), "expected exception" for: "expected exception" equals: "expected exception"
# Mismatching exception messages failing the test
not ok {test-number} - thisThrows(), "should fail" for: "expected exception" equals: "should fail"
# Multireporter calls reporters and listeners in correct order
ok {test-number} - records == expected for: { "Hello", "world", "Goodbye", "world" } == { "Hello", "world", "Goodbye", "world" }
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == false for: false == false
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == false for: false == false
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true
# Multireporter updates ReporterPreferences properly
ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true
# Nested generators and captured variables
ok {test-number} - values > -6 for: 3 > -6
# Nested generators and captured variables
@ -4412,5 +4446,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0
ok {test-number} -
# xmlentitycheck
ok {test-number} -
1..2209
1..2226

View File

@ -423,6 +423,10 @@ Condition.tests.cpp:<line number>|nexplicit failure- failure ignore as test mark
##teamcity[testStarted name='Mismatching exception messages failing the test']
Exception.tests.cpp:<line number>|nexpression failed|n REQUIRE_THROWS_WITH( thisThrows(), "should fail" )|nwith expansion:|n "expected exception" equals: "should fail"|n']
##teamcity[testFinished name='Mismatching exception messages failing the test' duration="{duration}"]
##teamcity[testStarted name='Multireporter calls reporters and listeners in correct order']
##teamcity[testFinished name='Multireporter calls reporters and listeners in correct order' duration="{duration}"]
##teamcity[testStarted name='Multireporter updates ReporterPreferences properly']
##teamcity[testFinished name='Multireporter updates ReporterPreferences properly' duration="{duration}"]
##teamcity[testStarted name='Nested generators and captured variables']
##teamcity[testFinished name='Nested generators and captured variables' duration="{duration}"]
##teamcity[testStarted name='Nice descriptive name']

View File

@ -423,6 +423,10 @@ Condition.tests.cpp:<line number>|nexplicit failure- failure ignore as test mark
##teamcity[testStarted name='Mismatching exception messages failing the test']
Exception.tests.cpp:<line number>|nexpression failed|n REQUIRE_THROWS_WITH( thisThrows(), "should fail" )|nwith expansion:|n "expected exception" equals: "should fail"|n']
##teamcity[testFinished name='Mismatching exception messages failing the test' duration="{duration}"]
##teamcity[testStarted name='Multireporter calls reporters and listeners in correct order']
##teamcity[testFinished name='Multireporter calls reporters and listeners in correct order' duration="{duration}"]
##teamcity[testStarted name='Multireporter updates ReporterPreferences properly']
##teamcity[testFinished name='Multireporter updates ReporterPreferences properly' duration="{duration}"]
##teamcity[testStarted name='Nested generators and captured variables']
##teamcity[testFinished name='Nested generators and captured variables' duration="{duration}"]
##teamcity[testStarted name='Nice descriptive name']

View File

@ -8719,6 +8719,156 @@ Nor would this
</Expression>
<OverallResult success="false"/>
</TestCase>
<TestCase name="Multireporter calls reporters and listeners in correct order" tags="[multi-reporter][reporters]" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
records == expected
</Original>
<Expanded>
{ "Hello", "world", "Goodbye", "world" }
==
{ "Hello", "world", "Goodbye", "world" }
</Expanded>
</Expression>
<OverallResult success="true"/>
</TestCase>
<TestCase name="Multireporter updates ReporterPreferences properly" tags="[multi-reporter][reporters]" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldRedirectStdOut == false
</Original>
<Expanded>
false == false
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldReportAllAssertions == false
</Original>
<Expanded>
false == false
</Expanded>
</Expression>
<Section name="Adding listeners" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldRedirectStdOut == true
</Original>
<Expanded>
true == true
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldReportAllAssertions == false
</Original>
<Expanded>
false == false
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldRedirectStdOut == true
</Original>
<Expanded>
true == true
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldReportAllAssertions == true
</Original>
<Expanded>
true == true
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldRedirectStdOut == true
</Original>
<Expanded>
true == true
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldReportAllAssertions == true
</Original>
<Expanded>
true == true
</Expanded>
</Expression>
<OverallResults successes="6" failures="0" expectedFailures="0"/>
</Section>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldRedirectStdOut == false
</Original>
<Expanded>
false == false
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldReportAllAssertions == false
</Original>
<Expanded>
false == false
</Expanded>
</Expression>
<Section name="Adding reporters" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldRedirectStdOut == true
</Original>
<Expanded>
true == true
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldReportAllAssertions == false
</Original>
<Expanded>
false == false
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldRedirectStdOut == true
</Original>
<Expanded>
true == true
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldReportAllAssertions == true
</Original>
<Expanded>
true == true
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldRedirectStdOut == true
</Original>
<Expanded>
true == true
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldReportAllAssertions == true
</Original>
<Expanded>
true == true
</Expanded>
</Expression>
<OverallResults successes="6" failures="0" expectedFailures="0"/>
</Section>
<OverallResult success="true"/>
</TestCase>
<TestCase name="Nested generators and captured variables" tags="[generators]" filename="tests/<exe-name>/UsageTests/Generators.tests.cpp" >
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/UsageTests/Generators.tests.cpp" >
<Original>
@ -20734,6 +20884,6 @@ loose text artifact
</Section>
<OverallResult success="true"/>
</TestCase>
<OverallResults successes="2039" failures="143" expectedFailures="27"/>
<OverallResultsCases successes="290" failures="83" expectedFailures="7"/>
<OverallResults successes="2056" failures="143" expectedFailures="27"/>
<OverallResultsCases successes="292" failures="83" expectedFailures="7"/>
</Catch2TestRun>

View File

@ -8719,6 +8719,156 @@ Nor would this
</Expression>
<OverallResult success="false"/>
</TestCase>
<TestCase name="Multireporter calls reporters and listeners in correct order" tags="[multi-reporter][reporters]" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
records == expected
</Original>
<Expanded>
{ "Hello", "world", "Goodbye", "world" }
==
{ "Hello", "world", "Goodbye", "world" }
</Expanded>
</Expression>
<OverallResult success="true"/>
</TestCase>
<TestCase name="Multireporter updates ReporterPreferences properly" tags="[multi-reporter][reporters]" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldRedirectStdOut == false
</Original>
<Expanded>
false == false
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldReportAllAssertions == false
</Original>
<Expanded>
false == false
</Expanded>
</Expression>
<Section name="Adding listeners" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldRedirectStdOut == true
</Original>
<Expanded>
true == true
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldReportAllAssertions == false
</Original>
<Expanded>
false == false
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldRedirectStdOut == true
</Original>
<Expanded>
true == true
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldReportAllAssertions == true
</Original>
<Expanded>
true == true
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldRedirectStdOut == true
</Original>
<Expanded>
true == true
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldReportAllAssertions == true
</Original>
<Expanded>
true == true
</Expanded>
</Expression>
<OverallResults successes="6" failures="0" expectedFailures="0"/>
</Section>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldRedirectStdOut == false
</Original>
<Expanded>
false == false
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldReportAllAssertions == false
</Original>
<Expanded>
false == false
</Expanded>
</Expression>
<Section name="Adding reporters" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldRedirectStdOut == true
</Original>
<Expanded>
true == true
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldReportAllAssertions == false
</Original>
<Expanded>
false == false
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldRedirectStdOut == true
</Original>
<Expanded>
true == true
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldReportAllAssertions == true
</Original>
<Expanded>
true == true
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldRedirectStdOut == true
</Original>
<Expanded>
true == true
</Expanded>
</Expression>
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" >
<Original>
multiReporter.getPreferences().shouldReportAllAssertions == true
</Original>
<Expanded>
true == true
</Expanded>
</Expression>
<OverallResults successes="6" failures="0" expectedFailures="0"/>
</Section>
<OverallResult success="true"/>
</TestCase>
<TestCase name="Nested generators and captured variables" tags="[generators]" filename="tests/<exe-name>/UsageTests/Generators.tests.cpp" >
<Expression success="true" type="REQUIRE" filename="tests/<exe-name>/UsageTests/Generators.tests.cpp" >
<Original>
@ -20733,6 +20883,6 @@ There is no extra whitespace here
</Section>
<OverallResult success="true"/>
</TestCase>
<OverallResults successes="2039" failures="143" expectedFailures="27"/>
<OverallResultsCases successes="290" failures="83" expectedFailures="7"/>
<OverallResults successes="2056" failures="143" expectedFailures="27"/>
<OverallResultsCases successes="292" failures="83" expectedFailures="7"/>
</Catch2TestRun>

View File

@ -16,6 +16,9 @@
#include <catch2/internal/catch_list.hpp>
#include <catch2/matchers/catch_matchers_string.hpp>
#include <catch2/reporters/catch_reporter_helpers.hpp>
#include <catch2/reporters/catch_reporter_event_listener.hpp>
#include <catch2/reporters/catch_reporter_streaming_base.hpp>
#include <catch2/reporters/catch_reporter_listening.hpp>
#include <sstream>
@ -112,3 +115,148 @@ TEST_CASE( "Reporter's write listings to provided stream", "[reporters]" ) {
TEST_CASE("Reproducer for #2309 - a very long description past 80 chars (default console width) with a late colon : blablabla", "[console-reporter]") {
SUCCEED();
}
namespace {
// A listener that writes provided string into destination,
// to record order of testRunStarting invocation.
class MockListener : public Catch::EventListenerBase {
std::string m_witness;
std::vector<std::string>& m_recorder;
public:
MockListener( std::string witness,
std::vector<std::string>& recorder,
Catch::ReporterConfig const& config ):
EventListenerBase( config ),
m_witness( witness ),
m_recorder( recorder )
{}
void testRunStarting( Catch::TestRunInfo const& ) override {
m_recorder.push_back( m_witness );
}
};
// A reporter that writes provided string into destination,
// to record order of testRunStarting invocation.
class MockReporter : public Catch::StreamingReporterBase {
std::string m_witness;
std::vector<std::string>& m_recorder;
public:
MockReporter( std::string witness,
std::vector<std::string>& recorder,
Catch::ReporterConfig const& config ):
StreamingReporterBase( config ),
m_witness( witness ),
m_recorder( recorder )
{}
void testRunStarting( Catch::TestRunInfo const& ) override {
m_recorder.push_back( m_witness );
}
};
} // namespace
TEST_CASE("Multireporter calls reporters and listeners in correct order",
"[reporters][multi-reporter]") {
Catch::ConfigData config_data;
Catch::Config config( config_data );
std::stringstream sstream;
Catch::ReporterConfig rep_config( &config, sstream );
// We add reporters before listeners, to check that internally they
// get sorted properly, and listeners are called first anyway.
Catch::ListeningReporter multiReporter( &config );
std::vector<std::string> records;
multiReporter.addReporter( Catch::Detail::make_unique<MockReporter>(
"Goodbye", records, rep_config ) );
multiReporter.addListener( Catch::Detail::make_unique<MockListener>(
"Hello", records, rep_config ) );
multiReporter.addListener( Catch::Detail::make_unique<MockListener>(
"world", records, rep_config ) );
multiReporter.addReporter( Catch::Detail::make_unique<MockReporter>(
"world", records, rep_config ) );
multiReporter.testRunStarting( { "" } );
std::vector<std::string> expected( { "Hello", "world", "Goodbye", "world" } );
REQUIRE( records == expected );
}
namespace {
// A listener that sets it preferences to test that multireporter,
// properly sets up its own preferences
class PreferenceListener : public Catch::EventListenerBase {
public:
PreferenceListener( bool redirectStdout,
bool reportAllAssertions,
Catch::ReporterConfig const& config ):
EventListenerBase( config ) {
m_preferences.shouldRedirectStdOut = redirectStdout;
m_preferences.shouldReportAllAssertions = reportAllAssertions;
}
};
// A reporter that sets it preferences to test that multireporter,
// properly sets up its own preferences
class PreferenceReporter : public Catch::StreamingReporterBase {
public:
PreferenceReporter( bool redirectStdout,
bool reportAllAssertions,
Catch::ReporterConfig const& config ):
StreamingReporterBase( config ) {
m_preferences.shouldRedirectStdOut = redirectStdout;
m_preferences.shouldReportAllAssertions = reportAllAssertions;
}
};
} // namespace
TEST_CASE("Multireporter updates ReporterPreferences properly",
"[reporters][multi-reporter]") {
Catch::ConfigData config_data;
Catch::Config config( config_data );
std::stringstream sstream;
Catch::ReporterConfig rep_config( &config, sstream );
Catch::ListeningReporter multiReporter( &config );
// Post init defaults
REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == false );
REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false );
SECTION( "Adding listeners" ) {
multiReporter.addListener(
Catch::Detail::make_unique<PreferenceListener>(
true, false, rep_config ) );
REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true );
REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false );
multiReporter.addListener(
Catch::Detail::make_unique<PreferenceListener>(
false, true, rep_config ) );
REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true );
REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true);
multiReporter.addListener(
Catch::Detail::make_unique<PreferenceListener>(
false, false, rep_config ) );
REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true );
REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true );
}
SECTION( "Adding reporters" ) {
multiReporter.addReporter(
Catch::Detail::make_unique<PreferenceReporter>(
true, false, rep_config ) );
REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true );
REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false );
multiReporter.addReporter(
Catch::Detail::make_unique<PreferenceReporter>(
false, true, rep_config ) );
REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true );
REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true );
multiReporter.addReporter(
Catch::Detail::make_unique<PreferenceReporter>(
false, false, rep_config ) );
REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true );
REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true );
}
}