From eb911aa995d806a5989cc026df7e8b69c470c4c8 Mon Sep 17 00:00:00 2001 From: Jozef Grajciar Date: Mon, 10 May 2021 21:42:47 +0200 Subject: [PATCH] Suppress failure of CHECKED_IF and CHECKED_ELSE (#2187) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves #1390 Co-authored-by: Martin Hořeňovský --- docs/deprecations.md | 8 --- docs/other-macros.md | 2 + src/catch2/catch_test_macros.hpp | 8 +-- src/catch2/internal/catch_run_context.cpp | 6 +- .../Baselines/automake.sw.approved.txt | 3 + .../Baselines/compact.sw.approved.txt | 16 ++++- .../Baselines/console.std.approved.txt | 30 +++++---- .../Baselines/console.sw.approved.txt | 52 ++++++++++++-- .../SelfTest/Baselines/junit.sw.approved.txt | 29 ++++---- .../Baselines/sonarqube.sw.approved.txt | 27 ++++---- tests/SelfTest/Baselines/tap.sw.approved.txt | 26 ++++++- .../Baselines/teamcity.sw.approved.txt | 10 ++- tests/SelfTest/Baselines/xml.sw.approved.txt | 67 +++++++++++++++++-- tests/SelfTest/UsageTests/Misc.tests.cpp | 33 +++++++++ 14 files changed, 246 insertions(+), 71 deletions(-) diff --git a/docs/deprecations.md b/docs/deprecations.md index 0649e447..e56b197a 100644 --- a/docs/deprecations.md +++ b/docs/deprecations.md @@ -19,14 +19,6 @@ command line interface instead of parsing C++ code with regular expressions. ## Planned changes -### `CHECKED_IF` and `CHECKED_ELSE` - -To make the `CHECKED_IF` and `CHECKED_ELSE` macros more useful, they will -be marked as "OK to fail" (`Catch::ResultDisposition::SuppressFail` flag -will be added), which means that their failure will not fail the test, -making the `else` actually useful. - - ### Console Colour API The API for Catch2's console colour will be changed to take an extra diff --git a/docs/other-macros.md b/docs/other-macros.md index 50593fae..f23ee6c2 100644 --- a/docs/other-macros.md +++ b/docs/other-macros.md @@ -15,6 +15,8 @@ stringification machinery to the _expr_ and records the result. As with evaluates to `true`. `CHECKED_ELSE( expr )` work similarly, but the block is entered only if the _expr_ evaluated to `false`. +> `CHECKED_X` macros were changed to not count as failure in Catch2 X.Y.Z. + Example: ```cpp int a = ...; diff --git a/src/catch2/catch_test_macros.hpp b/src/catch2/catch_test_macros.hpp index ae509150..cd33a65c 100644 --- a/src/catch2/catch_test_macros.hpp +++ b/src/catch2/catch_test_macros.hpp @@ -32,8 +32,8 @@ #define CATCH_CHECK( ... ) INTERNAL_CATCH_TEST( "CATCH_CHECK", Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) #define CATCH_CHECK_FALSE( ... ) INTERNAL_CATCH_TEST( "CATCH_CHECK_FALSE", Catch::ResultDisposition::ContinueOnFailure | Catch::ResultDisposition::FalseTest, __VA_ARGS__ ) - #define CATCH_CHECKED_IF( ... ) INTERNAL_CATCH_IF( "CATCH_CHECKED_IF", Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) - #define CATCH_CHECKED_ELSE( ... ) INTERNAL_CATCH_ELSE( "CATCH_CHECKED_ELSE", Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) + #define CATCH_CHECKED_IF( ... ) INTERNAL_CATCH_IF( "CATCH_CHECKED_IF", Catch::ResultDisposition::ContinueOnFailure | Catch::ResultDisposition::SuppressFail, __VA_ARGS__ ) + #define CATCH_CHECKED_ELSE( ... ) INTERNAL_CATCH_ELSE( "CATCH_CHECKED_ELSE", Catch::ResultDisposition::ContinueOnFailure | Catch::ResultDisposition::SuppressFail, __VA_ARGS__ ) #define CATCH_CHECK_NOFAIL( ... ) INTERNAL_CATCH_TEST( "CATCH_CHECK_NOFAIL", Catch::ResultDisposition::ContinueOnFailure | Catch::ResultDisposition::SuppressFail, __VA_ARGS__ ) #define CATCH_CHECK_THROWS( ... ) INTERNAL_CATCH_THROWS( "CATCH_CHECK_THROWS", Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) @@ -123,8 +123,8 @@ #define CHECK( ... ) INTERNAL_CATCH_TEST( "CHECK", Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) #define CHECK_FALSE( ... ) INTERNAL_CATCH_TEST( "CHECK_FALSE", Catch::ResultDisposition::ContinueOnFailure | Catch::ResultDisposition::FalseTest, __VA_ARGS__ ) - #define CHECKED_IF( ... ) INTERNAL_CATCH_IF( "CHECKED_IF", Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) - #define CHECKED_ELSE( ... ) INTERNAL_CATCH_ELSE( "CHECKED_ELSE", Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) + #define CHECKED_IF( ... ) INTERNAL_CATCH_IF( "CHECKED_IF", Catch::ResultDisposition::ContinueOnFailure | Catch::ResultDisposition::SuppressFail, __VA_ARGS__ ) + #define CHECKED_ELSE( ... ) INTERNAL_CATCH_ELSE( "CHECKED_ELSE", Catch::ResultDisposition::ContinueOnFailure | Catch::ResultDisposition::SuppressFail, __VA_ARGS__ ) #define CHECK_NOFAIL( ... ) INTERNAL_CATCH_TEST( "CHECK_NOFAIL", Catch::ResultDisposition::ContinueOnFailure | Catch::ResultDisposition::SuppressFail, __VA_ARGS__ ) #define CHECK_THROWS( ... ) INTERNAL_CATCH_THROWS( "CHECK_THROWS", Catch::ResultDisposition::ContinueOnFailure, __VA_ARGS__ ) diff --git a/src/catch2/internal/catch_run_context.cpp b/src/catch2/internal/catch_run_context.cpp index 5df38651..d3c22303 100644 --- a/src/catch2/internal/catch_run_context.cpp +++ b/src/catch2/internal/catch_run_context.cpp @@ -230,9 +230,11 @@ namespace Catch { if (result.getResultType() == ResultWas::Ok) { m_totals.assertions.passed++; m_lastAssertionPassed = true; - } else if (!result.isOk()) { + } else if (!result.succeeded()) { m_lastAssertionPassed = false; - if( m_activeTestCase->getTestCaseInfo().okToFail() ) + if (result.isOk()) { + } + else if( m_activeTestCase->getTestCaseInfo().okToFail() ) m_totals.assertions.failedButOk++; else m_totals.assertions.failed++; diff --git a/tests/SelfTest/Baselines/automake.sw.approved.txt b/tests/SelfTest/Baselines/automake.sw.approved.txt index 983d9764..c4ad5b1e 100644 --- a/tests/SelfTest/Baselines/automake.sw.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.approved.txt @@ -231,6 +231,9 @@ Message from section two :test-result: PASS Test case with one argument :test-result: PASS Test enum bit values :test-result: PASS Test with special, characters "in name +:test-result: PASS Testing checked-if +:test-result: XFAIL Testing checked-if 2 +:test-result: XFAIL Testing checked-if 3 :test-result: FAIL The NO_FAIL macro reports a failure but does not fail the test :test-result: PASS The default listing implementation write to provided stream :test-result: FAIL This test 'should' fail but doesn't diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt index e757222d..92488f5f 100644 --- a/tests/SelfTest/Baselines/compact.sw.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.approved.txt @@ -1714,6 +1714,16 @@ Misc.tests.cpp:: passed: v.capacity() >= V for: 15 >= 15 VariadicMacros.tests.cpp:: passed: with 1 message: 'no assertions' Tricky.tests.cpp:: passed: 0x == bit30and31 for: 3221225472 (0x) == 3221225472 CmdLine.tests.cpp:: passed: +Misc.tests.cpp:: passed: true +Misc.tests.cpp:: passed: +Misc.tests.cpp:: failed - but was ok: false +Misc.tests.cpp:: passed: true +Misc.tests.cpp:: failed - but was ok: false +Misc.tests.cpp:: passed: +Misc.tests.cpp:: passed: true +Misc.tests.cpp:: failed: explicitly +Misc.tests.cpp:: failed - but was ok: false +Misc.tests.cpp:: failed: explicitly Message.tests.cpp:: failed - but was ok: 1 == 2 Reporters.tests.cpp:: passed: listingString, Contains("[fakeTag]"s) for: "All available tags: 1 [fakeTag] @@ -1996,11 +2006,11 @@ InternalBenchmark.tests.cpp:: passed: called == 1 for: 1 == 1 Tricky.tests.cpp:: passed: obj.prop != 0 for: 0x != 0 Misc.tests.cpp:: passed: flag for: true Misc.tests.cpp:: passed: testCheckedElse( true ) for: true -Misc.tests.cpp:: failed: flag for: false +Misc.tests.cpp:: failed - but was ok: flag for: false Misc.tests.cpp:: failed: testCheckedElse( false ) for: false Misc.tests.cpp:: passed: flag for: true Misc.tests.cpp:: passed: testCheckedIf( true ) for: true -Misc.tests.cpp:: failed: flag for: false +Misc.tests.cpp:: failed - but was ok: flag for: false Misc.tests.cpp:: failed: testCheckedIf( false ) for: false InternalBenchmark.tests.cpp:: passed: o.samples_seen == static_cast(x.size()) for: 6 == 6 InternalBenchmark.tests.cpp:: passed: o.low_severe == los for: 0 == 0 @@ -2342,5 +2352,5 @@ InternalBenchmark.tests.cpp:: passed: med == 18. for: 18.0 == 18.0 InternalBenchmark.tests.cpp:: passed: q3 == 23. for: 23.0 == 23.0 Misc.tests.cpp:: passed: Misc.tests.cpp:: passed: -Failed 86 test cases, failed 148 assertions. +Failed 86 test cases, failed 146 assertions. diff --git a/tests/SelfTest/Baselines/console.std.approved.txt b/tests/SelfTest/Baselines/console.std.approved.txt index f91f4b80..d9e8f2b8 100644 --- a/tests/SelfTest/Baselines/console.std.approved.txt +++ b/tests/SelfTest/Baselines/console.std.approved.txt @@ -922,6 +922,22 @@ with expansion: } " +------------------------------------------------------------------------------- +Testing checked-if 2 +------------------------------------------------------------------------------- +Misc.tests.cpp: +............................................................................... + +Misc.tests.cpp:: FAILED: + +------------------------------------------------------------------------------- +Testing checked-if 3 +------------------------------------------------------------------------------- +Misc.tests.cpp: +............................................................................... + +Misc.tests.cpp:: FAILED: + ------------------------------------------------------------------------------- Thrown string literals are translated ------------------------------------------------------------------------------- @@ -1135,11 +1151,6 @@ checkedElse, failing Misc.tests.cpp: ............................................................................... -Misc.tests.cpp:: FAILED: - CHECKED_ELSE( flag ) -with expansion: - false - Misc.tests.cpp:: FAILED: REQUIRE( testCheckedElse( false ) ) with expansion: @@ -1151,11 +1162,6 @@ checkedIf, failing Misc.tests.cpp: ............................................................................... -Misc.tests.cpp:: FAILED: - CHECKED_IF( flag ) -with expansion: - false - Misc.tests.cpp:: FAILED: REQUIRE( testCheckedIf( false ) ) with expansion: @@ -1380,6 +1386,6 @@ due to unexpected exception with message: Why would you throw a std::string? =============================================================================== -test cases: 356 | 282 passed | 70 failed | 4 failed as expected -assertions: 2077 | 1925 passed | 131 failed | 21 failed as expected +test cases: 359 | 283 passed | 70 failed | 6 failed as expected +assertions: 2082 | 1930 passed | 129 failed | 23 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt index 5f77647a..7e53a539 100644 --- a/tests/SelfTest/Baselines/console.sw.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.approved.txt @@ -12298,6 +12298,50 @@ CmdLine.tests.cpp: CmdLine.tests.cpp:: PASSED: +------------------------------------------------------------------------------- +Testing checked-if +------------------------------------------------------------------------------- +Misc.tests.cpp: +............................................................................... + +Misc.tests.cpp:: PASSED: + CHECKED_IF( true ) + +Misc.tests.cpp:: PASSED: + +Misc.tests.cpp:: FAILED - but was ok: + CHECKED_IF( false ) + +Misc.tests.cpp:: PASSED: + CHECKED_ELSE( true ) + +Misc.tests.cpp:: FAILED - but was ok: + CHECKED_ELSE( false ) + +Misc.tests.cpp:: PASSED: + +------------------------------------------------------------------------------- +Testing checked-if 2 +------------------------------------------------------------------------------- +Misc.tests.cpp: +............................................................................... + +Misc.tests.cpp:: PASSED: + CHECKED_IF( true ) + +Misc.tests.cpp:: FAILED: + +------------------------------------------------------------------------------- +Testing checked-if 3 +------------------------------------------------------------------------------- +Misc.tests.cpp: +............................................................................... + +Misc.tests.cpp:: FAILED - but was ok: + CHECKED_ELSE( false ) + +Misc.tests.cpp:: FAILED: + ------------------------------------------------------------------------------- The NO_FAIL macro reports a failure but does not fail the test ------------------------------------------------------------------------------- @@ -14176,7 +14220,7 @@ checkedElse, failing Misc.tests.cpp: ............................................................................... -Misc.tests.cpp:: FAILED: +Misc.tests.cpp:: FAILED - but was ok: CHECKED_ELSE( flag ) with expansion: false @@ -14208,7 +14252,7 @@ checkedIf, failing Misc.tests.cpp: ............................................................................... -Misc.tests.cpp:: FAILED: +Misc.tests.cpp:: FAILED - but was ok: CHECKED_IF( flag ) with expansion: false @@ -16722,6 +16766,6 @@ Misc.tests.cpp: Misc.tests.cpp:: PASSED: =============================================================================== -test cases: 356 | 266 passed | 86 failed | 4 failed as expected -assertions: 2094 | 1925 passed | 148 failed | 21 failed as expected +test cases: 359 | 267 passed | 86 failed | 6 failed as expected +assertions: 2099 | 1930 passed | 146 failed | 23 failed as expected diff --git a/tests/SelfTest/Baselines/junit.sw.approved.txt b/tests/SelfTest/Baselines/junit.sw.approved.txt index 12a1456f..d90b1645 100644 --- a/tests/SelfTest/Baselines/junit.sw.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ - + @@ -1276,6 +1276,19 @@ Misc.tests.cpp: + + + +FAILED: +Misc.tests.cpp: + + + + +FAILED: +Misc.tests.cpp: + + @@ -1505,13 +1518,6 @@ Exception.tests.cpp: - -FAILED: - CHECKED_ELSE( flag ) -with expansion: - false -Misc.tests.cpp: - FAILED: REQUIRE( testCheckedElse( false ) ) @@ -1522,13 +1528,6 @@ Misc.tests.cpp: - -FAILED: - CHECKED_IF( flag ) -with expansion: - false -Misc.tests.cpp: - FAILED: REQUIRE( testCheckedIf( false ) ) diff --git a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt index 1f0cc5da..b900a55c 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt @@ -1583,17 +1583,23 @@ Misc.tests.cpp: + + + +FAILED: +Misc.tests.cpp: + + + + +FAILED: +Misc.tests.cpp: + + - -FAILED: - CHECKED_ELSE( flag ) -with expansion: - false -Misc.tests.cpp: - FAILED: REQUIRE( testCheckedElse( false ) ) @@ -1604,13 +1610,6 @@ Misc.tests.cpp: - -FAILED: - CHECKED_IF( flag ) -with expansion: - false -Misc.tests.cpp: - FAILED: REQUIRE( testCheckedIf( false ) ) diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt index 8bb648f5..a6e63f6d 100644 --- a/tests/SelfTest/Baselines/tap.sw.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.approved.txt @@ -3073,6 +3073,26 @@ ok {test-number} - with 1 message: 'no assertions' ok {test-number} - 0x == bit30and31 for: 3221225472 (0x) == 3221225472 # Test with special, characters "in name ok {test-number} - +# Testing checked-if +ok {test-number} - true +# Testing checked-if +ok {test-number} - +# Testing checked-if +ok {test-number} - false # TODO +# Testing checked-if +ok {test-number} - true +# Testing checked-if +ok {test-number} - false # TODO +# Testing checked-if +ok {test-number} - +# Testing checked-if 2 +ok {test-number} - true +# Testing checked-if 2 +not ok {test-number} - explicitly +# Testing checked-if 3 +ok {test-number} - false # TODO +# Testing checked-if 3 +not ok {test-number} - explicitly # The NO_FAIL macro reports a failure but does not fail the test ok {test-number} - 1 == 2 # TODO # The default listing implementation write to provided stream @@ -3570,7 +3590,7 @@ ok {test-number} - flag for: true # checkedElse ok {test-number} - testCheckedElse( true ) for: true # checkedElse, failing -not ok {test-number} - flag for: false +ok {test-number} - flag for: false # TODO # checkedElse, failing not ok {test-number} - testCheckedElse( false ) for: false # checkedIf @@ -3578,7 +3598,7 @@ ok {test-number} - flag for: true # checkedIf ok {test-number} - testCheckedIf( true ) for: true # checkedIf, failing -not ok {test-number} - flag for: false +ok {test-number} - flag for: false # TODO # checkedIf, failing not ok {test-number} - testCheckedIf( false ) for: false # classify_outliers @@ -4180,5 +4200,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2094 +1..2099 diff --git a/tests/SelfTest/Baselines/teamcity.sw.approved.txt b/tests/SelfTest/Baselines/teamcity.sw.approved.txt index e64c984f..2c1a8c5c 100644 --- a/tests/SelfTest/Baselines/teamcity.sw.approved.txt +++ b/tests/SelfTest/Baselines/teamcity.sw.approved.txt @@ -563,6 +563,14 @@ Misc.tests.cpp:|nexpression failed|n CHECK( s1 == s2 )|nwith expan ##teamcity[testFinished name='Test enum bit values' duration="{duration}"] ##teamcity[testStarted name='Test with special, characters "in name'] ##teamcity[testFinished name='Test with special, characters "in name' duration="{duration}"] +##teamcity[testStarted name='Testing checked-if'] +##teamcity[testFinished name='Testing checked-if' duration="{duration}"] +##teamcity[testStarted name='Testing checked-if 2'] +Misc.tests.cpp:|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n'] +##teamcity[testFinished name='Testing checked-if 2' duration="{duration}"] +##teamcity[testStarted name='Testing checked-if 3'] +Misc.tests.cpp:|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n'] +##teamcity[testFinished name='Testing checked-if 3' duration="{duration}"] ##teamcity[testStarted name='The NO_FAIL macro reports a failure but does not fail the test'] ##teamcity[testFinished name='The NO_FAIL macro reports a failure but does not fail the test' duration="{duration}"] ##teamcity[testStarted name='The default listing implementation write to provided stream'] @@ -661,13 +669,11 @@ Exception.tests.cpp:|nunexpected exception with message:|n "unexpe ##teamcity[testStarted name='checkedElse'] ##teamcity[testFinished name='checkedElse' duration="{duration}"] ##teamcity[testStarted name='checkedElse, failing'] -Misc.tests.cpp:|nexpression failed|n CHECKED_ELSE( flag )|nwith expansion:|n false|n'] Misc.tests.cpp:|nexpression failed|n REQUIRE( testCheckedElse( false ) )|nwith expansion:|n false|n'] ##teamcity[testFinished name='checkedElse, failing' duration="{duration}"] ##teamcity[testStarted name='checkedIf'] ##teamcity[testFinished name='checkedIf' duration="{duration}"] ##teamcity[testStarted name='checkedIf, failing'] -Misc.tests.cpp:|nexpression failed|n CHECKED_IF( flag )|nwith expansion:|n false|n'] Misc.tests.cpp:|nexpression failed|n REQUIRE( testCheckedIf( false ) )|nwith expansion:|n false|n'] ##teamcity[testFinished name='checkedIf, failing' duration="{duration}"] ##teamcity[testStarted name='classify_outliers'] diff --git a/tests/SelfTest/Baselines/xml.sw.approved.txt b/tests/SelfTest/Baselines/xml.sw.approved.txt index a624d5bd..4aecb91e 100644 --- a/tests/SelfTest/Baselines/xml.sw.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.approved.txt @@ -14425,6 +14425,65 @@ Message from section two + + + + true + + + true + + + + + false + + + false + + + + + true + + + true + + + + + false + + + false + + + + + + + + true + + + true + + + + + + + + + false + + + false + + + + + @@ -19667,9 +19726,9 @@ loose text artifact - - + + - - + + diff --git a/tests/SelfTest/UsageTests/Misc.tests.cpp b/tests/SelfTest/UsageTests/Misc.tests.cpp index 277723a8..5dfef6b1 100644 --- a/tests/SelfTest/UsageTests/Misc.tests.cpp +++ b/tests/SelfTest/UsageTests/Misc.tests.cpp @@ -182,6 +182,39 @@ TEST_CASE( "checkedElse, failing", "[failing][.]" ) { REQUIRE( testCheckedElse( false ) ); } +TEST_CASE("Testing checked-if", "[checked-if]") { + CHECKED_IF(true) { + SUCCEED(); + } + CHECKED_IF(false) { + FAIL(); + } + CHECKED_ELSE(true) { + FAIL(); + } + CHECKED_ELSE(false) { + SUCCEED(); + } +} + +TEST_CASE("Testing checked-if 2", "[checked-if][!shouldfail]") { + CHECKED_IF(true) { + FAIL(); + } + // If the checked if is not entered, this passes and the test + // fails, because of the [!shouldfail] tag. + SUCCEED(); +} + +TEST_CASE("Testing checked-if 3", "[checked-if][!shouldfail]") { + CHECKED_ELSE(false) { + FAIL(); + } + // If the checked false is not entered, this passes and the test + // fails, because of the [!shouldfail] tag. + SUCCEED(); +} + TEST_CASE( "xmlentitycheck" ) { SECTION( "embedded xml: it should be possible to embed xml characters, such as <, \" or &, or even whole documents within an attribute" ) { SUCCEED(); // We need this here to stop it failing due to no tests