From 8baf9c05a3ab8ba2de400678539d0256ec60b4b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ho=C5=99e=C5=88ovsk=C3=BD?= Date: Sun, 19 Dec 2021 15:16:56 +0100 Subject: [PATCH] Add more unit tests * Call order of listeners/reporters in multireporter * Adding listeners/reporters properly updates reporter preferences --- .../reporters/catch_reporter_listening.hpp | 7 +- .../Baselines/automake.sw.approved.txt | 2 + .../Baselines/automake.sw.multi.approved.txt | 2 + .../Baselines/compact.sw.approved.txt | 19 +++ .../Baselines/compact.sw.multi.approved.txt | 19 +++ .../Baselines/console.std.approved.txt | 4 +- .../Baselines/console.sw.approved.txt | 123 +++++++++++++- .../Baselines/console.sw.multi.approved.txt | 123 +++++++++++++- .../SelfTest/Baselines/junit.sw.approved.txt | 6 +- .../Baselines/junit.sw.multi.approved.txt | 6 +- .../Baselines/sonarqube.sw.approved.txt | 4 + .../Baselines/sonarqube.sw.multi.approved.txt | 4 + tests/SelfTest/Baselines/tap.sw.approved.txt | 36 +++- .../Baselines/tap.sw.multi.approved.txt | 36 +++- .../Baselines/teamcity.sw.approved.txt | 4 + .../Baselines/teamcity.sw.multi.approved.txt | 4 + tests/SelfTest/Baselines/xml.sw.approved.txt | 154 +++++++++++++++++- .../Baselines/xml.sw.multi.approved.txt | 154 +++++++++++++++++- .../IntrospectiveTests/Reporters.tests.cpp | 148 +++++++++++++++++ 19 files changed, 836 insertions(+), 19 deletions(-) diff --git a/src/catch2/reporters/catch_reporter_listening.hpp b/src/catch2/reporters/catch_reporter_listening.hpp index f5b3a2ec..133ce8e1 100644 --- a/src/catch2/reporters/catch_reporter_listening.hpp +++ b/src/catch2/reporters/catch_reporter_listening.hpp @@ -30,11 +30,8 @@ namespace Catch { public: ListeningReporter( IConfig const* config ): - IStreamingReporter( config ) { - // We will assume that listeners will always want all assertions - m_preferences.shouldReportAllAssertions = true; - } - + IStreamingReporter( config ) + {} void addListener( IStreamingReporterPtr&& listener ); void addReporter( IStreamingReporterPtr&& reporter ); diff --git a/tests/SelfTest/Baselines/automake.sw.approved.txt b/tests/SelfTest/Baselines/automake.sw.approved.txt index 4465d1e1..34a28749 100644 --- a/tests/SelfTest/Baselines/automake.sw.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.approved.txt @@ -170,6 +170,8 @@ Nor would this :test-result: FAIL Matchers can be negated (Not) with the ! operator - failing :test-result: XFAIL Mayfail test case with nested sections :test-result: FAIL Mismatching exception messages failing the test +:test-result: PASS Multireporter calls reporters and listeners in correct order +:test-result: PASS Multireporter updates ReporterPreferences properly :test-result: PASS Nested generators and captured variables :test-result: FAIL Nice descriptive name :test-result: FAIL Non-std exceptions can be translated diff --git a/tests/SelfTest/Baselines/automake.sw.multi.approved.txt b/tests/SelfTest/Baselines/automake.sw.multi.approved.txt index 328fcf50..e45cec60 100644 --- a/tests/SelfTest/Baselines/automake.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.multi.approved.txt @@ -168,6 +168,8 @@ :test-result: FAIL Matchers can be negated (Not) with the ! operator - failing :test-result: XFAIL Mayfail test case with nested sections :test-result: FAIL Mismatching exception messages failing the test +:test-result: PASS Multireporter calls reporters and listeners in correct order +:test-result: PASS Multireporter updates ReporterPreferences properly :test-result: PASS Nested generators and captured variables :test-result: FAIL Nice descriptive name :test-result: FAIL Non-std exceptions can be translated diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt index 517b4d93..0babaf7d 100644 --- a/tests/SelfTest/Baselines/compact.sw.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.approved.txt @@ -941,6 +941,25 @@ Condition.tests.cpp:: failed: explicitly Condition.tests.cpp:: failed: explicitly Exception.tests.cpp:: passed: thisThrows(), "expected exception" for: "expected exception" equals: "expected exception" Exception.tests.cpp:: failed: thisThrows(), "should fail" for: "expected exception" equals: "should fail" +Reporters.tests.cpp:: passed: records == expected for: { "Hello", "world", "Goodbye", "world" } +== +{ "Hello", "world", "Goodbye", "world" } +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldRedirectStdOut == false for: false == false +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldRedirectStdOut == false for: false == false +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true Generators.tests.cpp:: passed: values > -6 for: 3 > -6 Generators.tests.cpp:: passed: values > -6 for: 4 > -6 Generators.tests.cpp:: passed: values > -6 for: 5 > -6 diff --git a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt index 7196e831..1966443b 100644 --- a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt @@ -939,6 +939,25 @@ Condition.tests.cpp:: failed: explicitly Condition.tests.cpp:: failed: explicitly Exception.tests.cpp:: passed: thisThrows(), "expected exception" for: "expected exception" equals: "expected exception" Exception.tests.cpp:: failed: thisThrows(), "should fail" for: "expected exception" equals: "should fail" +Reporters.tests.cpp:: passed: records == expected for: { "Hello", "world", "Goodbye", "world" } +== +{ "Hello", "world", "Goodbye", "world" } +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldRedirectStdOut == false for: false == false +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldRedirectStdOut == false for: false == false +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true +Reporters.tests.cpp:: passed: multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true Generators.tests.cpp:: passed: values > -6 for: 3 > -6 Generators.tests.cpp:: passed: values > -6 for: 4 > -6 Generators.tests.cpp:: passed: values > -6 for: 5 > -6 diff --git a/tests/SelfTest/Baselines/console.std.approved.txt b/tests/SelfTest/Baselines/console.std.approved.txt index dfc8d3bd..ac979989 100644 --- a/tests/SelfTest/Baselines/console.std.approved.txt +++ b/tests/SelfTest/Baselines/console.std.approved.txt @@ -1395,6 +1395,6 @@ due to unexpected exception with message: Why would you throw a std::string? =============================================================================== -test cases: 380 | 304 passed | 69 failed | 7 failed as expected -assertions: 2194 | 2039 passed | 128 failed | 27 failed as expected +test cases: 382 | 306 passed | 69 failed | 7 failed as expected +assertions: 2211 | 2056 passed | 128 failed | 27 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt index 4b4a2ed0..5d733c77 100644 --- a/tests/SelfTest/Baselines/console.sw.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.approved.txt @@ -7199,6 +7199,125 @@ Exception.tests.cpp:: FAILED: with expansion: "expected exception" equals: "should fail" +------------------------------------------------------------------------------- +Multireporter calls reporters and listeners in correct order +------------------------------------------------------------------------------- +Reporters.tests.cpp: +............................................................................... + +Reporters.tests.cpp:: PASSED: + REQUIRE( records == expected ) +with expansion: + { "Hello", "world", "Goodbye", "world" } + == + { "Hello", "world", "Goodbye", "world" } + +------------------------------------------------------------------------------- +Multireporter updates ReporterPreferences properly +------------------------------------------------------------------------------- +Reporters.tests.cpp: +............................................................................... + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == false ) +with expansion: + false == false + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false ) +with expansion: + false == false + +------------------------------------------------------------------------------- +Multireporter updates ReporterPreferences properly + Adding listeners +------------------------------------------------------------------------------- +Reporters.tests.cpp: +............................................................................... + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true ) +with expansion: + true == true + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false ) +with expansion: + false == false + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true ) +with expansion: + true == true + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true ) +with expansion: + true == true + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true ) +with expansion: + true == true + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true ) +with expansion: + true == true + +------------------------------------------------------------------------------- +Multireporter updates ReporterPreferences properly +------------------------------------------------------------------------------- +Reporters.tests.cpp: +............................................................................... + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == false ) +with expansion: + false == false + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false ) +with expansion: + false == false + +------------------------------------------------------------------------------- +Multireporter updates ReporterPreferences properly + Adding reporters +------------------------------------------------------------------------------- +Reporters.tests.cpp: +............................................................................... + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true ) +with expansion: + true == true + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false ) +with expansion: + false == false + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true ) +with expansion: + true == true + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true ) +with expansion: + true == true + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true ) +with expansion: + true == true + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true ) +with expansion: + true == true + ------------------------------------------------------------------------------- Nested generators and captured variables ------------------------------------------------------------------------------- @@ -17653,6 +17772,6 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 380 | 290 passed | 83 failed | 7 failed as expected -assertions: 2209 | 2039 passed | 143 failed | 27 failed as expected +test cases: 382 | 292 passed | 83 failed | 7 failed as expected +assertions: 2226 | 2056 passed | 143 failed | 27 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.multi.approved.txt b/tests/SelfTest/Baselines/console.sw.multi.approved.txt index 8c8d166f..ad44370c 100644 --- a/tests/SelfTest/Baselines/console.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.multi.approved.txt @@ -7197,6 +7197,125 @@ Exception.tests.cpp:: FAILED: with expansion: "expected exception" equals: "should fail" +------------------------------------------------------------------------------- +Multireporter calls reporters and listeners in correct order +------------------------------------------------------------------------------- +Reporters.tests.cpp: +............................................................................... + +Reporters.tests.cpp:: PASSED: + REQUIRE( records == expected ) +with expansion: + { "Hello", "world", "Goodbye", "world" } + == + { "Hello", "world", "Goodbye", "world" } + +------------------------------------------------------------------------------- +Multireporter updates ReporterPreferences properly +------------------------------------------------------------------------------- +Reporters.tests.cpp: +............................................................................... + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == false ) +with expansion: + false == false + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false ) +with expansion: + false == false + +------------------------------------------------------------------------------- +Multireporter updates ReporterPreferences properly + Adding listeners +------------------------------------------------------------------------------- +Reporters.tests.cpp: +............................................................................... + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true ) +with expansion: + true == true + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false ) +with expansion: + false == false + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true ) +with expansion: + true == true + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true ) +with expansion: + true == true + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true ) +with expansion: + true == true + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true ) +with expansion: + true == true + +------------------------------------------------------------------------------- +Multireporter updates ReporterPreferences properly +------------------------------------------------------------------------------- +Reporters.tests.cpp: +............................................................................... + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == false ) +with expansion: + false == false + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false ) +with expansion: + false == false + +------------------------------------------------------------------------------- +Multireporter updates ReporterPreferences properly + Adding reporters +------------------------------------------------------------------------------- +Reporters.tests.cpp: +............................................................................... + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true ) +with expansion: + true == true + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false ) +with expansion: + false == false + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true ) +with expansion: + true == true + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true ) +with expansion: + true == true + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true ) +with expansion: + true == true + +Reporters.tests.cpp:: PASSED: + REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true ) +with expansion: + true == true + ------------------------------------------------------------------------------- Nested generators and captured variables ------------------------------------------------------------------------------- @@ -17645,6 +17764,6 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 380 | 290 passed | 83 failed | 7 failed as expected -assertions: 2209 | 2039 passed | 143 failed | 27 failed as expected +test cases: 382 | 292 passed | 83 failed | 7 failed as expected +assertions: 2226 | 2056 passed | 143 failed | 27 failed as expected diff --git a/tests/SelfTest/Baselines/junit.sw.approved.txt b/tests/SelfTest/Baselines/junit.sw.approved.txt index cc3038b5..e3398a6a 100644 --- a/tests/SelfTest/Baselines/junit.sw.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ - + @@ -870,6 +870,10 @@ with expansion: Exception.tests.cpp: + + + + diff --git a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt index 22361857..b1ae7efd 100644 --- a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt @@ -1,6 +1,6 @@ - + @@ -869,6 +869,10 @@ with expansion: Exception.tests.cpp: + + + + diff --git a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt index 8ab8d0c0..081ae969 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt @@ -181,6 +181,10 @@ + + + + diff --git a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt index ebbdce72..cf9c15ce 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt @@ -180,6 +180,10 @@ + + + + diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt index 45fa096a..e49f387e 100644 --- a/tests/SelfTest/Baselines/tap.sw.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.approved.txt @@ -1864,6 +1864,40 @@ not ok {test-number} - explicitly ok {test-number} - thisThrows(), "expected exception" for: "expected exception" equals: "expected exception" # Mismatching exception messages failing the test not ok {test-number} - thisThrows(), "should fail" for: "expected exception" equals: "should fail" +# Multireporter calls reporters and listeners in correct order +ok {test-number} - records == expected for: { "Hello", "world", "Goodbye", "world" } == { "Hello", "world", "Goodbye", "world" } +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == false for: false == false +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == false for: false == false +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true # Nested generators and captured variables ok {test-number} - values > -6 for: 3 > -6 # Nested generators and captured variables @@ -4420,5 +4454,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2209 +1..2226 diff --git a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt index 235687a2..f8599833 100644 --- a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt @@ -1862,6 +1862,40 @@ not ok {test-number} - explicitly ok {test-number} - thisThrows(), "expected exception" for: "expected exception" equals: "expected exception" # Mismatching exception messages failing the test not ok {test-number} - thisThrows(), "should fail" for: "expected exception" equals: "should fail" +# Multireporter calls reporters and listeners in correct order +ok {test-number} - records == expected for: { "Hello", "world", "Goodbye", "world" } == { "Hello", "world", "Goodbye", "world" } +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == false for: false == false +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == false for: false == false +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == false for: false == false +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldRedirectStdOut == true for: true == true +# Multireporter updates ReporterPreferences properly +ok {test-number} - multiReporter.getPreferences().shouldReportAllAssertions == true for: true == true # Nested generators and captured variables ok {test-number} - values > -6 for: 3 > -6 # Nested generators and captured variables @@ -4412,5 +4446,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2209 +1..2226 diff --git a/tests/SelfTest/Baselines/teamcity.sw.approved.txt b/tests/SelfTest/Baselines/teamcity.sw.approved.txt index d413fcac..d73f02ae 100644 --- a/tests/SelfTest/Baselines/teamcity.sw.approved.txt +++ b/tests/SelfTest/Baselines/teamcity.sw.approved.txt @@ -423,6 +423,10 @@ Condition.tests.cpp:|nexplicit failure- failure ignore as test mark ##teamcity[testStarted name='Mismatching exception messages failing the test'] Exception.tests.cpp:|nexpression failed|n REQUIRE_THROWS_WITH( thisThrows(), "should fail" )|nwith expansion:|n "expected exception" equals: "should fail"|n'] ##teamcity[testFinished name='Mismatching exception messages failing the test' duration="{duration}"] +##teamcity[testStarted name='Multireporter calls reporters and listeners in correct order'] +##teamcity[testFinished name='Multireporter calls reporters and listeners in correct order' duration="{duration}"] +##teamcity[testStarted name='Multireporter updates ReporterPreferences properly'] +##teamcity[testFinished name='Multireporter updates ReporterPreferences properly' duration="{duration}"] ##teamcity[testStarted name='Nested generators and captured variables'] ##teamcity[testFinished name='Nested generators and captured variables' duration="{duration}"] ##teamcity[testStarted name='Nice descriptive name'] diff --git a/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt b/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt index c57fd14e..3560775f 100644 --- a/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt @@ -423,6 +423,10 @@ Condition.tests.cpp:|nexplicit failure- failure ignore as test mark ##teamcity[testStarted name='Mismatching exception messages failing the test'] Exception.tests.cpp:|nexpression failed|n REQUIRE_THROWS_WITH( thisThrows(), "should fail" )|nwith expansion:|n "expected exception" equals: "should fail"|n'] ##teamcity[testFinished name='Mismatching exception messages failing the test' duration="{duration}"] +##teamcity[testStarted name='Multireporter calls reporters and listeners in correct order'] +##teamcity[testFinished name='Multireporter calls reporters and listeners in correct order' duration="{duration}"] +##teamcity[testStarted name='Multireporter updates ReporterPreferences properly'] +##teamcity[testFinished name='Multireporter updates ReporterPreferences properly' duration="{duration}"] ##teamcity[testStarted name='Nested generators and captured variables'] ##teamcity[testFinished name='Nested generators and captured variables' duration="{duration}"] ##teamcity[testStarted name='Nice descriptive name'] diff --git a/tests/SelfTest/Baselines/xml.sw.approved.txt b/tests/SelfTest/Baselines/xml.sw.approved.txt index c10acdea..b7d5bb15 100644 --- a/tests/SelfTest/Baselines/xml.sw.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.approved.txt @@ -8719,6 +8719,156 @@ Nor would this + + + + records == expected + + + { "Hello", "world", "Goodbye", "world" } +== +{ "Hello", "world", "Goodbye", "world" } + + + + + + + + multiReporter.getPreferences().shouldRedirectStdOut == false + + + false == false + + + + + multiReporter.getPreferences().shouldReportAllAssertions == false + + + false == false + + +
+ + + multiReporter.getPreferences().shouldRedirectStdOut == true + + + true == true + + + + + multiReporter.getPreferences().shouldReportAllAssertions == false + + + false == false + + + + + multiReporter.getPreferences().shouldRedirectStdOut == true + + + true == true + + + + + multiReporter.getPreferences().shouldReportAllAssertions == true + + + true == true + + + + + multiReporter.getPreferences().shouldRedirectStdOut == true + + + true == true + + + + + multiReporter.getPreferences().shouldReportAllAssertions == true + + + true == true + + + +
+ + + multiReporter.getPreferences().shouldRedirectStdOut == false + + + false == false + + + + + multiReporter.getPreferences().shouldReportAllAssertions == false + + + false == false + + +
+ + + multiReporter.getPreferences().shouldRedirectStdOut == true + + + true == true + + + + + multiReporter.getPreferences().shouldReportAllAssertions == false + + + false == false + + + + + multiReporter.getPreferences().shouldRedirectStdOut == true + + + true == true + + + + + multiReporter.getPreferences().shouldReportAllAssertions == true + + + true == true + + + + + multiReporter.getPreferences().shouldRedirectStdOut == true + + + true == true + + + + + multiReporter.getPreferences().shouldReportAllAssertions == true + + + true == true + + + +
+ +
@@ -20734,6 +20884,6 @@ loose text artifact - - + + diff --git a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt index 0491ec50..9a70d37e 100644 --- a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt @@ -8719,6 +8719,156 @@ Nor would this
+ + + + records == expected + + + { "Hello", "world", "Goodbye", "world" } +== +{ "Hello", "world", "Goodbye", "world" } + + + + + + + + multiReporter.getPreferences().shouldRedirectStdOut == false + + + false == false + + + + + multiReporter.getPreferences().shouldReportAllAssertions == false + + + false == false + + +
+ + + multiReporter.getPreferences().shouldRedirectStdOut == true + + + true == true + + + + + multiReporter.getPreferences().shouldReportAllAssertions == false + + + false == false + + + + + multiReporter.getPreferences().shouldRedirectStdOut == true + + + true == true + + + + + multiReporter.getPreferences().shouldReportAllAssertions == true + + + true == true + + + + + multiReporter.getPreferences().shouldRedirectStdOut == true + + + true == true + + + + + multiReporter.getPreferences().shouldReportAllAssertions == true + + + true == true + + + +
+ + + multiReporter.getPreferences().shouldRedirectStdOut == false + + + false == false + + + + + multiReporter.getPreferences().shouldReportAllAssertions == false + + + false == false + + +
+ + + multiReporter.getPreferences().shouldRedirectStdOut == true + + + true == true + + + + + multiReporter.getPreferences().shouldReportAllAssertions == false + + + false == false + + + + + multiReporter.getPreferences().shouldRedirectStdOut == true + + + true == true + + + + + multiReporter.getPreferences().shouldReportAllAssertions == true + + + true == true + + + + + multiReporter.getPreferences().shouldRedirectStdOut == true + + + true == true + + + + + multiReporter.getPreferences().shouldReportAllAssertions == true + + + true == true + + + +
+ +
@@ -20733,6 +20883,6 @@ There is no extra whitespace here - - + + diff --git a/tests/SelfTest/IntrospectiveTests/Reporters.tests.cpp b/tests/SelfTest/IntrospectiveTests/Reporters.tests.cpp index 5c95657d..16fd17dd 100644 --- a/tests/SelfTest/IntrospectiveTests/Reporters.tests.cpp +++ b/tests/SelfTest/IntrospectiveTests/Reporters.tests.cpp @@ -16,6 +16,9 @@ #include #include #include +#include +#include +#include #include @@ -112,3 +115,148 @@ TEST_CASE( "Reporter's write listings to provided stream", "[reporters]" ) { TEST_CASE("Reproducer for #2309 - a very long description past 80 chars (default console width) with a late colon : blablabla", "[console-reporter]") { SUCCEED(); } + +namespace { + // A listener that writes provided string into destination, + // to record order of testRunStarting invocation. + class MockListener : public Catch::EventListenerBase { + std::string m_witness; + std::vector& m_recorder; + public: + MockListener( std::string witness, + std::vector& recorder, + Catch::ReporterConfig const& config ): + EventListenerBase( config ), + m_witness( witness ), + m_recorder( recorder ) + {} + + void testRunStarting( Catch::TestRunInfo const& ) override { + m_recorder.push_back( m_witness ); + } + }; + // A reporter that writes provided string into destination, + // to record order of testRunStarting invocation. + class MockReporter : public Catch::StreamingReporterBase { + std::string m_witness; + std::vector& m_recorder; + public: + MockReporter( std::string witness, + std::vector& recorder, + Catch::ReporterConfig const& config ): + StreamingReporterBase( config ), + m_witness( witness ), + m_recorder( recorder ) + {} + + void testRunStarting( Catch::TestRunInfo const& ) override { + m_recorder.push_back( m_witness ); + } + }; +} // namespace + +TEST_CASE("Multireporter calls reporters and listeners in correct order", + "[reporters][multi-reporter]") { + + Catch::ConfigData config_data; + Catch::Config config( config_data ); + std::stringstream sstream; + Catch::ReporterConfig rep_config( &config, sstream ); + + // We add reporters before listeners, to check that internally they + // get sorted properly, and listeners are called first anyway. + Catch::ListeningReporter multiReporter( &config ); + std::vector records; + multiReporter.addReporter( Catch::Detail::make_unique( + "Goodbye", records, rep_config ) ); + multiReporter.addListener( Catch::Detail::make_unique( + "Hello", records, rep_config ) ); + multiReporter.addListener( Catch::Detail::make_unique( + "world", records, rep_config ) ); + multiReporter.addReporter( Catch::Detail::make_unique( + "world", records, rep_config ) ); + multiReporter.testRunStarting( { "" } ); + + std::vector expected( { "Hello", "world", "Goodbye", "world" } ); + REQUIRE( records == expected ); +} + +namespace { + // A listener that sets it preferences to test that multireporter, + // properly sets up its own preferences + class PreferenceListener : public Catch::EventListenerBase { + public: + PreferenceListener( bool redirectStdout, + bool reportAllAssertions, + Catch::ReporterConfig const& config ): + EventListenerBase( config ) { + m_preferences.shouldRedirectStdOut = redirectStdout; + m_preferences.shouldReportAllAssertions = reportAllAssertions; + } + }; + // A reporter that sets it preferences to test that multireporter, + // properly sets up its own preferences + class PreferenceReporter : public Catch::StreamingReporterBase { + public: + PreferenceReporter( bool redirectStdout, + bool reportAllAssertions, + Catch::ReporterConfig const& config ): + StreamingReporterBase( config ) { + m_preferences.shouldRedirectStdOut = redirectStdout; + m_preferences.shouldReportAllAssertions = reportAllAssertions; + } + }; +} // namespace + +TEST_CASE("Multireporter updates ReporterPreferences properly", + "[reporters][multi-reporter]") { + + Catch::ConfigData config_data; + Catch::Config config( config_data ); + std::stringstream sstream; + Catch::ReporterConfig rep_config( &config, sstream ); + Catch::ListeningReporter multiReporter( &config ); + + // Post init defaults + REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == false ); + REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false ); + + SECTION( "Adding listeners" ) { + multiReporter.addListener( + Catch::Detail::make_unique( + true, false, rep_config ) ); + REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true ); + REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false ); + + multiReporter.addListener( + Catch::Detail::make_unique( + false, true, rep_config ) ); + REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true ); + REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true); + + multiReporter.addListener( + Catch::Detail::make_unique( + false, false, rep_config ) ); + REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true ); + REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true ); + } + SECTION( "Adding reporters" ) { + multiReporter.addReporter( + Catch::Detail::make_unique( + true, false, rep_config ) ); + REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true ); + REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == false ); + + multiReporter.addReporter( + Catch::Detail::make_unique( + false, true, rep_config ) ); + REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true ); + REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true ); + + multiReporter.addReporter( + Catch::Detail::make_unique( + false, false, rep_config ) ); + REQUIRE( multiReporter.getPreferences().shouldRedirectStdOut == true ); + REQUIRE( multiReporter.getPreferences().shouldReportAllAssertions == true ); + } +}