Add new SKIP macro for skipping tests at runtime (#2360)

* Add new SKIP macro for skipping tests at runtime

This adds a new `SKIP` macro for dynamically skipping tests at runtime.
The "skipped" status of a test case is treated as a first-class citizen,
like "succeeded" or "failed", and is reported with a new color on the
console.

* Don't show "skipped assertions" in console/compact reporters

Also extend skip tests to cover a few more use cases.

* Return exit code 4 if all test cases are skipped

* Use LightGrey for the skip colour

This isn't great, but is better than the deep blue that was borderline
invisible on dark backgrounds. The fix is to redo the colouring
a bit, including introducing light-blue that is actually visible.

* Add support for explicit skips in all reporters

* --allow-running-no-tests also allows all tests to be skipped

* Add docs for SKIP macro, deprecate IEventListener::skipTest

Co-authored-by: Martin Hořeňovský <martin.horenovsky@gmail.com>
This commit is contained in:
Philip Salzmann
2023-01-12 15:01:47 +01:00
committed by GitHub
parent 52066dbc2a
commit d548be26e3
47 changed files with 3722 additions and 2171 deletions

View File

@@ -116,6 +116,7 @@ set(TEST_SOURCES
${SELF_TEST_DIR}/UsageTests/Generators.tests.cpp
${SELF_TEST_DIR}/UsageTests/Message.tests.cpp
${SELF_TEST_DIR}/UsageTests/Misc.tests.cpp
${SELF_TEST_DIR}/UsageTests/Skip.tests.cpp
${SELF_TEST_DIR}/UsageTests/ToStringByte.tests.cpp
${SELF_TEST_DIR}/UsageTests/ToStringChrono.tests.cpp
${SELF_TEST_DIR}/UsageTests/ToStringGeneral.tests.cpp
@@ -272,6 +273,10 @@ add_test(NAME TestSpecs::OverrideFailureWithNoMatchedTests
COMMAND $<TARGET_FILE:SelfTest> "___nonexistent_test___" --allow-running-no-tests
)
add_test(NAME TestSpecs::OverrideAllSkipFailure
COMMAND $<TARGET_FILE:SelfTest> "tests can be skipped dynamically at runtime" --allow-running-no-tests
)
add_test(NAME TestSpecs::NonMatchingTestSpecIsRoundTrippable
COMMAND $<TARGET_FILE:SelfTest> Tracker, "this test does not exist" "[nor does this tag]"
)

View File

@@ -488,15 +488,32 @@ set_tests_properties(TestSpecs::EmptySpecWithNoTestsFails
PROPERTIES
WILL_FAIL ON
)
add_test(
NAME TestSpecs::OverrideFailureWithEmptySpec
COMMAND $<TARGET_FILE:NoTests> --allow-running-no-tests
)
add_test(
NAME List::Listeners::WorksWithoutRegisteredListeners
COMMAND $<TARGET_FILE:NoTests> --list-listeners
)
add_executable(AllSkipped ${TESTS_DIR}/X93-AllSkipped.cpp)
target_link_libraries(AllSkipped PRIVATE Catch2::Catch2WithMain)
add_test(
NAME TestSpecs::SkippingAllTestsFails
COMMAND $<TARGET_FILE:AllSkipped>
)
set_tests_properties(TestSpecs::SkippingAllTestsFails
PROPERTIES
WILL_FAIL ON
)
set( EXTRA_TEST_BINARIES
AllSkipped
PrefixedMacros
DisabledMacros
DisabledExceptions-DefaultHandler

View File

@@ -0,0 +1,16 @@
// Copyright Catch2 Authors
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE.txt or copy at
// https://www.boost.org/LICENSE_1_0.txt)
// SPDX-License-Identifier: BSL-1.0
#include <catch2/catch_test_macros.hpp>
TEST_CASE( "this test case is being skipped" ) { SKIP(); }
TEST_CASE( "all sections in this test case are being skipped" ) {
SECTION( "A" ) { SKIP(); }
SECTION( "B" ) { SKIP(); }
}

View File

@@ -297,6 +297,7 @@ Message from section two
:test-result: PASS X/level/1/b
:test-result: PASS XmlEncode
:test-result: PASS XmlWriter writes boolean attributes as true/false
:test-result: SKIP a succeeding test can still be skipped
:test-result: PASS analyse no analysis
:test-result: PASS array<int, N> -> toString
:test-result: PASS benchmark function call
@@ -309,10 +310,14 @@ Message from section two
:test-result: PASS comparisons between const int variables
:test-result: PASS comparisons between int variables
:test-result: PASS convertToBits
:test-result: SKIP dynamic skipping works with generators
:test-result: PASS empty tags are not allowed
:test-result: PASS erfc_inv
:test-result: PASS estimate_clock_resolution
:test-result: PASS even more nested SECTION tests
:test-result: XFAIL failed assertions before SKIP cause test case to fail
:test-result: XFAIL failing for some generator values causes entire test case to fail
:test-result: XFAIL failing in some unskipped sections causes entire test case to fail
:test-result: FAIL first tag
loose text artifact
:test-result: FAIL has printf
@@ -331,6 +336,10 @@ loose text artifact
:test-result: FAIL mix info, unscoped info and warning
:test-result: FAIL more nested SECTION tests
:test-result: PASS nested SECTION tests
a!
b1!
!
:test-result: FAIL nested sections can be skipped dynamically at runtime
:test-result: PASS non streamable - with conv. op
:test-result: PASS non-copyable objects
:test-result: PASS normal_cdf
@@ -352,9 +361,11 @@ loose text artifact
:test-result: PASS run_for_at_least, chronometer
:test-result: PASS run_for_at_least, int
:test-result: FAIL second tag
:test-result: SKIP sections can be skipped dynamically at runtime
:test-result: FAIL send a single char to INFO
:test-result: FAIL sends information to INFO
:test-result: PASS shortened hide tags are split apart
:test-result: SKIP skipped tests can optionally provide a reason
:test-result: PASS splitString
:test-result: FAIL stacks unscoped info in loops
:test-result: PASS startsWith
@@ -376,6 +387,7 @@ loose text artifact
:test-result: PASS strlen3
:test-result: PASS tables
:test-result: PASS tags with dots in later positions are not parsed as hidden
:test-result: SKIP tests can be skipped dynamically at runtime
:test-result: FAIL thrown std::strings are translated
:test-result: PASS toString on const wchar_t const pointer returns the string contents
:test-result: PASS toString on const wchar_t pointer returns the string contents

View File

@@ -290,6 +290,7 @@
:test-result: PASS X/level/1/b
:test-result: PASS XmlEncode
:test-result: PASS XmlWriter writes boolean attributes as true/false
:test-result: SKIP a succeeding test can still be skipped
:test-result: PASS analyse no analysis
:test-result: PASS array<int, N> -> toString
:test-result: PASS benchmark function call
@@ -302,10 +303,14 @@
:test-result: PASS comparisons between const int variables
:test-result: PASS comparisons between int variables
:test-result: PASS convertToBits
:test-result: SKIP dynamic skipping works with generators
:test-result: PASS empty tags are not allowed
:test-result: PASS erfc_inv
:test-result: PASS estimate_clock_resolution
:test-result: PASS even more nested SECTION tests
:test-result: XFAIL failed assertions before SKIP cause test case to fail
:test-result: XFAIL failing for some generator values causes entire test case to fail
:test-result: XFAIL failing in some unskipped sections causes entire test case to fail
:test-result: FAIL first tag
:test-result: FAIL has printf
:test-result: PASS is_unary_function
@@ -323,6 +328,7 @@
:test-result: FAIL mix info, unscoped info and warning
:test-result: FAIL more nested SECTION tests
:test-result: PASS nested SECTION tests
:test-result: FAIL nested sections can be skipped dynamically at runtime
:test-result: PASS non streamable - with conv. op
:test-result: PASS non-copyable objects
:test-result: PASS normal_cdf
@@ -344,9 +350,11 @@
:test-result: PASS run_for_at_least, chronometer
:test-result: PASS run_for_at_least, int
:test-result: FAIL second tag
:test-result: SKIP sections can be skipped dynamically at runtime
:test-result: FAIL send a single char to INFO
:test-result: FAIL sends information to INFO
:test-result: PASS shortened hide tags are split apart
:test-result: SKIP skipped tests can optionally provide a reason
:test-result: PASS splitString
:test-result: FAIL stacks unscoped info in loops
:test-result: PASS startsWith
@@ -368,6 +376,7 @@
:test-result: PASS strlen3
:test-result: PASS tables
:test-result: PASS tags with dots in later positions are not parsed as hidden
:test-result: SKIP tests can be skipped dynamically at runtime
:test-result: FAIL thrown std::strings are translated
:test-result: PASS toString on const wchar_t const pointer returns the string contents
:test-result: PASS toString on const wchar_t pointer returns the string contents

View File

@@ -2060,6 +2060,8 @@ Xml.tests.cpp:<line number>: passed: encode( "[\x7F]" ) == "[\\x7F]" for: "[\x7F
Xml.tests.cpp:<line number>: passed: stream.str(), ContainsSubstring(R"(attr1="true")") && ContainsSubstring(R"(attr2="false")") for: "<?xml version="1.0" encoding="UTF-8"?>
<Element1 attr1="true" attr2="false"/>
" ( contains: "attr1="true"" and contains: "attr2="false"" )
Skip.tests.cpp:<line number>: passed:
Skip.tests.cpp:<line number>: skipped:
InternalBenchmark.tests.cpp:<line number>: passed: analysis.mean.point.count() == 23 for: 23.0 == 23
InternalBenchmark.tests.cpp:<line number>: passed: analysis.mean.lower_bound.count() == 23 for: 23.0 == 23
InternalBenchmark.tests.cpp:<line number>: passed: analysis.mean.upper_bound.count() == 23 for: 23.0 == 23
@@ -2149,6 +2151,9 @@ FloatingPoint.tests.cpp:<line number>: passed: convertToBits( -0. ) == ( 1ULL <<
9223372036854775808 (0x<hex digits>)
FloatingPoint.tests.cpp:<line number>: passed: convertToBits( std::numeric_limits<float>::denorm_min() ) == 1 for: 1 == 1
FloatingPoint.tests.cpp:<line number>: passed: convertToBits( std::numeric_limits<double>::denorm_min() ) == 1 for: 1 == 1
Skip.tests.cpp:<line number>: skipped: 'skipping because answer = 41'
Skip.tests.cpp:<line number>: passed:
Skip.tests.cpp:<line number>: skipped: 'skipping because answer = 43'
Tag.tests.cpp:<line number>: passed: Catch::TestCaseInfo("", { "test with an empty tag", "[]" }, dummySourceLineInfo)
InternalBenchmark.tests.cpp:<line number>: passed: erfc_inv(1.103560) == Approx(-0.09203687623843015) for: -0.0920368762 == Approx( -0.0920368762 )
InternalBenchmark.tests.cpp:<line number>: passed: erfc_inv(1.067400) == Approx(-0.05980291115763361) for: -0.0598029112 == Approx( -0.0598029112 )
@@ -2158,6 +2163,14 @@ InternalBenchmark.tests.cpp:<line number>: passed: res.outliers.total() == 0 for
Misc.tests.cpp:<line number>: passed:
Misc.tests.cpp:<line number>: passed:
Misc.tests.cpp:<line number>: passed:
Skip.tests.cpp:<line number>: failed: 3 == 4
Skip.tests.cpp:<line number>: skipped:
Skip.tests.cpp:<line number>: failed: explicitly
Skip.tests.cpp:<line number>: skipped:
Skip.tests.cpp:<line number>: failed: explicitly
Skip.tests.cpp:<line number>: skipped:
Skip.tests.cpp:<line number>: skipped:
Skip.tests.cpp:<line number>: failed: explicitly
loose text artifact
Clara.tests.cpp:<line number>: passed: with 1 message: 'Catch::Clara::Detail::is_unary_function<decltype(unary1)>::value'
Clara.tests.cpp:<line number>: passed: with 1 message: 'Catch::Clara::Detail::is_unary_function<decltype(unary2)>::value'
@@ -2215,6 +2228,10 @@ Misc.tests.cpp:<line number>: passed: a < b for: 1 < 2
Misc.tests.cpp:<line number>: passed: a != b for: 1 != 2
Misc.tests.cpp:<line number>: passed: b != a for: 2 != 1
Misc.tests.cpp:<line number>: passed: a != b for: 1 != 2
a!
b1!
Skip.tests.cpp:<line number>: skipped:
!
Tricky.tests.cpp:<line number>: passed: s == "7" for: "7" == "7"
Tricky.tests.cpp:<line number>: passed: ti == typeid(int) for: {?} == {?}
InternalBenchmark.tests.cpp:<line number>: passed: normal_cdf(0.000000) == Approx(0.50000000000000000) for: 0.5 == Approx( 0.5 )
@@ -2299,9 +2316,13 @@ InternalBenchmark.tests.cpp:<line number>: passed: x >= old_x for: 128 >= 64
InternalBenchmark.tests.cpp:<line number>: passed: Timing.elapsed >= time for: 128 ns >= 100 ns
InternalBenchmark.tests.cpp:<line number>: passed: Timing.result == Timing.iterations + 17 for: 145 == 145
InternalBenchmark.tests.cpp:<line number>: passed: Timing.iterations >= time.count() for: 128 >= 100
Skip.tests.cpp:<line number>: passed:
Skip.tests.cpp:<line number>: skipped:
Skip.tests.cpp:<line number>: passed:
Misc.tests.cpp:<line number>: failed: false with 1 message: '3'
Message.tests.cpp:<line number>: failed: false with 2 messages: 'hi' and 'i := 7'
Tag.tests.cpp:<line number>: passed: testcase.tags, VectorContains( Tag( "magic-tag" ) ) && VectorContains( Tag( "."_catch_sr ) ) for: { {?}, {?} } ( Contains: {?} and Contains: {?} )
Skip.tests.cpp:<line number>: skipped: 'skipping because answer = 43'
StringManip.tests.cpp:<line number>: passed: splitStringRef("", ','), Equals(std::vector<StringRef>()) for: { } Equals: { }
StringManip.tests.cpp:<line number>: passed: splitStringRef("abc", ','), Equals(std::vector<StringRef>{"abc"}) for: { abc } Equals: { abc }
StringManip.tests.cpp:<line number>: passed: splitStringRef("abc,def", ','), Equals(std::vector<StringRef>{"abc", "def"}) for: { abc, def } Equals: { abc, def }
@@ -2367,6 +2388,7 @@ Generators.tests.cpp:<line number>: passed: strlen(std::get<0>(data)) == static_
Generators.tests.cpp:<line number>: passed: strlen(std::get<0>(data)) == static_cast<size_t>(std::get<1>(data)) for: 6 == 6
Tag.tests.cpp:<line number>: passed: testcase.tags.size() == 1 for: 1 == 1
Tag.tests.cpp:<line number>: passed: testcase.tags[0].original == "magic.tag"_catch_sr for: magic.tag == magic.tag
Skip.tests.cpp:<line number>: skipped:
Exception.tests.cpp:<line number>: failed: unexpected exception with message: 'Why would you throw a std::string?'
Misc.tests.cpp:<line number>: passed: result == "\"wide load\"" for: ""wide load"" == ""wide load""
Misc.tests.cpp:<line number>: passed: result == "\"wide load\"" for: ""wide load"" == ""wide load""
@@ -2463,7 +2485,7 @@ InternalBenchmark.tests.cpp:<line number>: passed: med == 18. for: 18.0 == 18.0
InternalBenchmark.tests.cpp:<line number>: passed: q3 == 23. for: 23.0 == 23.0
Misc.tests.cpp:<line number>: passed:
Misc.tests.cpp:<line number>: passed:
test cases: 395 | 305 passed | 83 failed | 7 failed as expected
assertions: 2163 | 1993 passed | 143 failed | 27 failed as expected
test cases: 404 | 305 passed | 84 failed | 5 skipped | 10 failed as expected
assertions: 2173 | 1997 passed | 145 failed | 31 failed as expected

View File

@@ -2053,6 +2053,8 @@ Xml.tests.cpp:<line number>: passed: encode( "[\x7F]" ) == "[\\x7F]" for: "[\x7F
Xml.tests.cpp:<line number>: passed: stream.str(), ContainsSubstring(R"(attr1="true")") && ContainsSubstring(R"(attr2="false")") for: "<?xml version="1.0" encoding="UTF-8"?>
<Element1 attr1="true" attr2="false"/>
" ( contains: "attr1="true"" and contains: "attr2="false"" )
Skip.tests.cpp:<line number>: passed:
Skip.tests.cpp:<line number>: skipped:
InternalBenchmark.tests.cpp:<line number>: passed: analysis.mean.point.count() == 23 for: 23.0 == 23
InternalBenchmark.tests.cpp:<line number>: passed: analysis.mean.lower_bound.count() == 23 for: 23.0 == 23
InternalBenchmark.tests.cpp:<line number>: passed: analysis.mean.upper_bound.count() == 23 for: 23.0 == 23
@@ -2142,6 +2144,9 @@ FloatingPoint.tests.cpp:<line number>: passed: convertToBits( -0. ) == ( 1ULL <<
9223372036854775808 (0x<hex digits>)
FloatingPoint.tests.cpp:<line number>: passed: convertToBits( std::numeric_limits<float>::denorm_min() ) == 1 for: 1 == 1
FloatingPoint.tests.cpp:<line number>: passed: convertToBits( std::numeric_limits<double>::denorm_min() ) == 1 for: 1 == 1
Skip.tests.cpp:<line number>: skipped: 'skipping because answer = 41'
Skip.tests.cpp:<line number>: passed:
Skip.tests.cpp:<line number>: skipped: 'skipping because answer = 43'
Tag.tests.cpp:<line number>: passed: Catch::TestCaseInfo("", { "test with an empty tag", "[]" }, dummySourceLineInfo)
InternalBenchmark.tests.cpp:<line number>: passed: erfc_inv(1.103560) == Approx(-0.09203687623843015) for: -0.0920368762 == Approx( -0.0920368762 )
InternalBenchmark.tests.cpp:<line number>: passed: erfc_inv(1.067400) == Approx(-0.05980291115763361) for: -0.0598029112 == Approx( -0.0598029112 )
@@ -2151,6 +2156,14 @@ InternalBenchmark.tests.cpp:<line number>: passed: res.outliers.total() == 0 for
Misc.tests.cpp:<line number>: passed:
Misc.tests.cpp:<line number>: passed:
Misc.tests.cpp:<line number>: passed:
Skip.tests.cpp:<line number>: failed: 3 == 4
Skip.tests.cpp:<line number>: skipped:
Skip.tests.cpp:<line number>: failed: explicitly
Skip.tests.cpp:<line number>: skipped:
Skip.tests.cpp:<line number>: failed: explicitly
Skip.tests.cpp:<line number>: skipped:
Skip.tests.cpp:<line number>: skipped:
Skip.tests.cpp:<line number>: failed: explicitly
Clara.tests.cpp:<line number>: passed: with 1 message: 'Catch::Clara::Detail::is_unary_function<decltype(unary1)>::value'
Clara.tests.cpp:<line number>: passed: with 1 message: 'Catch::Clara::Detail::is_unary_function<decltype(unary2)>::value'
Clara.tests.cpp:<line number>: passed: with 1 message: 'Catch::Clara::Detail::is_unary_function<decltype(unary3)>::value'
@@ -2207,6 +2220,7 @@ Misc.tests.cpp:<line number>: passed: a < b for: 1 < 2
Misc.tests.cpp:<line number>: passed: a != b for: 1 != 2
Misc.tests.cpp:<line number>: passed: b != a for: 2 != 1
Misc.tests.cpp:<line number>: passed: a != b for: 1 != 2
Skip.tests.cpp:<line number>: skipped:
Tricky.tests.cpp:<line number>: passed: s == "7" for: "7" == "7"
Tricky.tests.cpp:<line number>: passed: ti == typeid(int) for: {?} == {?}
InternalBenchmark.tests.cpp:<line number>: passed: normal_cdf(0.000000) == Approx(0.50000000000000000) for: 0.5 == Approx( 0.5 )
@@ -2291,9 +2305,13 @@ InternalBenchmark.tests.cpp:<line number>: passed: x >= old_x for: 128 >= 64
InternalBenchmark.tests.cpp:<line number>: passed: Timing.elapsed >= time for: 128 ns >= 100 ns
InternalBenchmark.tests.cpp:<line number>: passed: Timing.result == Timing.iterations + 17 for: 145 == 145
InternalBenchmark.tests.cpp:<line number>: passed: Timing.iterations >= time.count() for: 128 >= 100
Skip.tests.cpp:<line number>: passed:
Skip.tests.cpp:<line number>: skipped:
Skip.tests.cpp:<line number>: passed:
Misc.tests.cpp:<line number>: failed: false with 1 message: '3'
Message.tests.cpp:<line number>: failed: false with 2 messages: 'hi' and 'i := 7'
Tag.tests.cpp:<line number>: passed: testcase.tags, VectorContains( Tag( "magic-tag" ) ) && VectorContains( Tag( "."_catch_sr ) ) for: { {?}, {?} } ( Contains: {?} and Contains: {?} )
Skip.tests.cpp:<line number>: skipped: 'skipping because answer = 43'
StringManip.tests.cpp:<line number>: passed: splitStringRef("", ','), Equals(std::vector<StringRef>()) for: { } Equals: { }
StringManip.tests.cpp:<line number>: passed: splitStringRef("abc", ','), Equals(std::vector<StringRef>{"abc"}) for: { abc } Equals: { abc }
StringManip.tests.cpp:<line number>: passed: splitStringRef("abc,def", ','), Equals(std::vector<StringRef>{"abc", "def"}) for: { abc, def } Equals: { abc, def }
@@ -2359,6 +2377,7 @@ Generators.tests.cpp:<line number>: passed: strlen(std::get<0>(data)) == static_
Generators.tests.cpp:<line number>: passed: strlen(std::get<0>(data)) == static_cast<size_t>(std::get<1>(data)) for: 6 == 6
Tag.tests.cpp:<line number>: passed: testcase.tags.size() == 1 for: 1 == 1
Tag.tests.cpp:<line number>: passed: testcase.tags[0].original == "magic.tag"_catch_sr for: magic.tag == magic.tag
Skip.tests.cpp:<line number>: skipped:
Exception.tests.cpp:<line number>: failed: unexpected exception with message: 'Why would you throw a std::string?'
Misc.tests.cpp:<line number>: passed: result == "\"wide load\"" for: ""wide load"" == ""wide load""
Misc.tests.cpp:<line number>: passed: result == "\"wide load\"" for: ""wide load"" == ""wide load""
@@ -2455,7 +2474,7 @@ InternalBenchmark.tests.cpp:<line number>: passed: med == 18. for: 18.0 == 18.0
InternalBenchmark.tests.cpp:<line number>: passed: q3 == 23. for: 23.0 == 23.0
Misc.tests.cpp:<line number>: passed:
Misc.tests.cpp:<line number>: passed:
test cases: 395 | 305 passed | 83 failed | 7 failed as expected
assertions: 2163 | 1993 passed | 143 failed | 27 failed as expected
test cases: 404 | 305 passed | 84 failed | 5 skipped | 10 failed as expected
assertions: 2173 | 1997 passed | 145 failed | 31 failed as expected

View File

@@ -1164,6 +1164,14 @@ Exception.tests.cpp:<line number>: FAILED:
due to unexpected exception with message:
unexpected exception
-------------------------------------------------------------------------------
a succeeding test can still be skipped
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
-------------------------------------------------------------------------------
checkedElse, failing
-------------------------------------------------------------------------------
@@ -1186,6 +1194,87 @@ Misc.tests.cpp:<line number>: FAILED:
with expansion:
false
-------------------------------------------------------------------------------
dynamic skipping works with generators
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
explicitly with message:
skipping because answer = 41
-------------------------------------------------------------------------------
dynamic skipping works with generators
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
explicitly with message:
skipping because answer = 43
-------------------------------------------------------------------------------
failed assertions before SKIP cause test case to fail
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: FAILED:
CHECK( 3 == 4 )
Skip.tests.cpp:<line number>: SKIPPED:
-------------------------------------------------------------------------------
failing for some generator values causes entire test case to fail
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: FAILED:
-------------------------------------------------------------------------------
failing for some generator values causes entire test case to fail
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
-------------------------------------------------------------------------------
failing for some generator values causes entire test case to fail
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: FAILED:
-------------------------------------------------------------------------------
failing for some generator values causes entire test case to fail
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
-------------------------------------------------------------------------------
failing in some unskipped sections causes entire test case to fail
skipped
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
-------------------------------------------------------------------------------
failing in some unskipped sections causes entire test case to fail
not skipped
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: FAILED:
loose text artifact
-------------------------------------------------------------------------------
just failure
@@ -1304,6 +1393,19 @@ Misc.tests.cpp:<line number>: FAILED:
with expansion:
1 == 2
a!
b1!
-------------------------------------------------------------------------------
nested sections can be skipped dynamically at runtime
B
B2
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
!
-------------------------------------------------------------------------------
not prints unscoped info from previous failures
-------------------------------------------------------------------------------
@@ -1338,6 +1440,15 @@ Message.tests.cpp:<line number>: FAILED:
with message:
this SHOULD be seen only ONCE
-------------------------------------------------------------------------------
sections can be skipped dynamically at runtime
skipped
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
-------------------------------------------------------------------------------
send a single char to INFO
-------------------------------------------------------------------------------
@@ -1361,6 +1472,16 @@ with messages:
hi
i := 7
-------------------------------------------------------------------------------
skipped tests can optionally provide a reason
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
explicitly with message:
skipping because answer = 43
-------------------------------------------------------------------------------
stacks unscoped info in loops
-------------------------------------------------------------------------------
@@ -1383,6 +1504,14 @@ with messages:
5
6
-------------------------------------------------------------------------------
tests can be skipped dynamically at runtime
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
-------------------------------------------------------------------------------
thrown std::strings are translated
-------------------------------------------------------------------------------
@@ -1394,6 +1523,6 @@ due to unexpected exception with message:
Why would you throw a std::string?
===============================================================================
test cases: 395 | 319 passed | 69 failed | 7 failed as expected
assertions: 2148 | 1993 passed | 128 failed | 27 failed as expected
test cases: 404 | 319 passed | 69 failed | 6 skipped | 10 failed as expected
assertions: 2156 | 1997 passed | 128 failed | 31 failed as expected

View File

@@ -14659,6 +14659,16 @@ with expansion:
<Element1 attr1="true" attr2="false"/>
" ( contains: "attr1="true"" and contains: "attr2="false"" )
-------------------------------------------------------------------------------
a succeeding test can still be skipped
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: PASSED:
Skip.tests.cpp:<line number>: SKIPPED:
-------------------------------------------------------------------------------
analyse no analysis
-------------------------------------------------------------------------------
@@ -15204,6 +15214,34 @@ FloatingPoint.tests.cpp:<line number>: PASSED:
with expansion:
1 == 1
-------------------------------------------------------------------------------
dynamic skipping works with generators
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
explicitly with message:
skipping because answer = 41
-------------------------------------------------------------------------------
dynamic skipping works with generators
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: PASSED:
-------------------------------------------------------------------------------
dynamic skipping works with generators
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
explicitly with message:
skipping because answer = 43
-------------------------------------------------------------------------------
empty tags are not allowed
-------------------------------------------------------------------------------
@@ -15279,6 +15317,67 @@ Misc.tests.cpp:<line number>
Misc.tests.cpp:<line number>: PASSED:
-------------------------------------------------------------------------------
failed assertions before SKIP cause test case to fail
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: FAILED:
CHECK( 3 == 4 )
Skip.tests.cpp:<line number>: SKIPPED:
-------------------------------------------------------------------------------
failing for some generator values causes entire test case to fail
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: FAILED:
-------------------------------------------------------------------------------
failing for some generator values causes entire test case to fail
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
-------------------------------------------------------------------------------
failing for some generator values causes entire test case to fail
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: FAILED:
-------------------------------------------------------------------------------
failing for some generator values causes entire test case to fail
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
-------------------------------------------------------------------------------
failing in some unskipped sections causes entire test case to fail
skipped
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
-------------------------------------------------------------------------------
failing in some unskipped sections causes entire test case to fail
not skipped
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: FAILED:
-------------------------------------------------------------------------------
first tag
-------------------------------------------------------------------------------
@@ -15775,6 +15874,40 @@ Misc.tests.cpp:<line number>: PASSED:
with expansion:
1 != 2
a-------------------------------------------------------------------------------
nested sections can be skipped dynamically at runtime
A
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
No assertions in section 'A'
!
b1-------------------------------------------------------------------------------
nested sections can be skipped dynamically at runtime
B
B1
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
No assertions in section 'B1'
!
-------------------------------------------------------------------------------
nested sections can be skipped dynamically at runtime
B
B2
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
!
-------------------------------------------------------------------------------
non streamable - with conv. op
-------------------------------------------------------------------------------
@@ -16376,6 +16509,33 @@ Misc.tests.cpp:<line number>
No assertions in test case 'second tag'
-------------------------------------------------------------------------------
sections can be skipped dynamically at runtime
not skipped
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: PASSED:
-------------------------------------------------------------------------------
sections can be skipped dynamically at runtime
skipped
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
-------------------------------------------------------------------------------
sections can be skipped dynamically at runtime
also not skipped
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: PASSED:
-------------------------------------------------------------------------------
send a single char to INFO
-------------------------------------------------------------------------------
@@ -16410,6 +16570,16 @@ Tag.tests.cpp:<line number>: PASSED:
with expansion:
{ {?}, {?} } ( Contains: {?} and Contains: {?} )
-------------------------------------------------------------------------------
skipped tests can optionally provide a reason
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
explicitly with message:
skipping because answer = 43
-------------------------------------------------------------------------------
splitString
-------------------------------------------------------------------------------
@@ -16837,6 +17007,14 @@ Tag.tests.cpp:<line number>: PASSED:
with expansion:
magic.tag == magic.tag
-------------------------------------------------------------------------------
tests can be skipped dynamically at runtime
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
-------------------------------------------------------------------------------
thrown std::strings are translated
-------------------------------------------------------------------------------
@@ -17544,6 +17722,6 @@ Misc.tests.cpp:<line number>
Misc.tests.cpp:<line number>: PASSED:
===============================================================================
test cases: 395 | 305 passed | 83 failed | 7 failed as expected
assertions: 2163 | 1993 passed | 143 failed | 27 failed as expected
test cases: 404 | 305 passed | 84 failed | 5 skipped | 10 failed as expected
assertions: 2173 | 1997 passed | 145 failed | 31 failed as expected

View File

@@ -14652,6 +14652,16 @@ with expansion:
<Element1 attr1="true" attr2="false"/>
" ( contains: "attr1="true"" and contains: "attr2="false"" )
-------------------------------------------------------------------------------
a succeeding test can still be skipped
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: PASSED:
Skip.tests.cpp:<line number>: SKIPPED:
-------------------------------------------------------------------------------
analyse no analysis
-------------------------------------------------------------------------------
@@ -15197,6 +15207,34 @@ FloatingPoint.tests.cpp:<line number>: PASSED:
with expansion:
1 == 1
-------------------------------------------------------------------------------
dynamic skipping works with generators
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
explicitly with message:
skipping because answer = 41
-------------------------------------------------------------------------------
dynamic skipping works with generators
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: PASSED:
-------------------------------------------------------------------------------
dynamic skipping works with generators
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
explicitly with message:
skipping because answer = 43
-------------------------------------------------------------------------------
empty tags are not allowed
-------------------------------------------------------------------------------
@@ -15272,6 +15310,67 @@ Misc.tests.cpp:<line number>
Misc.tests.cpp:<line number>: PASSED:
-------------------------------------------------------------------------------
failed assertions before SKIP cause test case to fail
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: FAILED:
CHECK( 3 == 4 )
Skip.tests.cpp:<line number>: SKIPPED:
-------------------------------------------------------------------------------
failing for some generator values causes entire test case to fail
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: FAILED:
-------------------------------------------------------------------------------
failing for some generator values causes entire test case to fail
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
-------------------------------------------------------------------------------
failing for some generator values causes entire test case to fail
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: FAILED:
-------------------------------------------------------------------------------
failing for some generator values causes entire test case to fail
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
-------------------------------------------------------------------------------
failing in some unskipped sections causes entire test case to fail
skipped
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
-------------------------------------------------------------------------------
failing in some unskipped sections causes entire test case to fail
not skipped
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: FAILED:
-------------------------------------------------------------------------------
first tag
-------------------------------------------------------------------------------
@@ -15767,6 +15866,37 @@ Misc.tests.cpp:<line number>: PASSED:
with expansion:
1 != 2
-------------------------------------------------------------------------------
nested sections can be skipped dynamically at runtime
A
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
No assertions in section 'A'
-------------------------------------------------------------------------------
nested sections can be skipped dynamically at runtime
B
B1
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
No assertions in section 'B1'
-------------------------------------------------------------------------------
nested sections can be skipped dynamically at runtime
B
B2
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
-------------------------------------------------------------------------------
non streamable - with conv. op
-------------------------------------------------------------------------------
@@ -16368,6 +16498,33 @@ Misc.tests.cpp:<line number>
No assertions in test case 'second tag'
-------------------------------------------------------------------------------
sections can be skipped dynamically at runtime
not skipped
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: PASSED:
-------------------------------------------------------------------------------
sections can be skipped dynamically at runtime
skipped
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
-------------------------------------------------------------------------------
sections can be skipped dynamically at runtime
also not skipped
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: PASSED:
-------------------------------------------------------------------------------
send a single char to INFO
-------------------------------------------------------------------------------
@@ -16402,6 +16559,16 @@ Tag.tests.cpp:<line number>: PASSED:
with expansion:
{ {?}, {?} } ( Contains: {?} and Contains: {?} )
-------------------------------------------------------------------------------
skipped tests can optionally provide a reason
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
explicitly with message:
skipping because answer = 43
-------------------------------------------------------------------------------
splitString
-------------------------------------------------------------------------------
@@ -16829,6 +16996,14 @@ Tag.tests.cpp:<line number>: PASSED:
with expansion:
magic.tag == magic.tag
-------------------------------------------------------------------------------
tests can be skipped dynamically at runtime
-------------------------------------------------------------------------------
Skip.tests.cpp:<line number>
...............................................................................
Skip.tests.cpp:<line number>: SKIPPED:
-------------------------------------------------------------------------------
thrown std::strings are translated
-------------------------------------------------------------------------------
@@ -17536,6 +17711,6 @@ Misc.tests.cpp:<line number>
Misc.tests.cpp:<line number>: PASSED:
===============================================================================
test cases: 395 | 305 passed | 83 failed | 7 failed as expected
assertions: 2163 | 1993 passed | 143 failed | 27 failed as expected
test cases: 404 | 305 passed | 84 failed | 5 skipped | 10 failed as expected
assertions: 2173 | 1997 passed | 145 failed | 31 failed as expected

View File

@@ -6,3 +6,6 @@ A string sent to stderr via clog
Message from section one
Message from section two
loose text artifact
a!
b1!
!

View File

@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuitesloose text artifact
>
<testsuite name="<exe-name>" errors="17" failures="126" tests="2163" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<testsuite name="<exe-name>" errors="17" failures="128" skipped="11" tests="2184" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<properties>
<property name="random-seed" value="1"/>
<property name="filters" value="&quot;*&quot; ~[!nonportable] ~[!benchmark] ~[approvals]"/>
@@ -1591,6 +1591,12 @@ at Exception.tests.cpp:<line number>
<testcase classname="<exe-name>.global" name="XmlEncode/string with control char (1)" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="XmlEncode/string with control char (x7F)" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="XmlWriter writes boolean attributes as true/false" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="a succeeding test can still be skipped" time="{duration}" status="run">
<skipped type="SKIP">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testcase>
<testcase classname="<exe-name>.global" name="analyse no analysis" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="array&lt;int, N> -> toString" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="benchmark function call/without chronometer" time="{duration}" status="run"/>
@@ -1625,12 +1631,67 @@ at Misc.tests.cpp:<line number>
<testcase classname="<exe-name>.global" name="comparisons between const int variables" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="comparisons between int variables" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="convertToBits" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="dynamic skipping works with generators" time="{duration}" status="run">
<skipped type="SKIP">
SKIPPED
skipping because answer = 41
at Skip.tests.cpp:<line number>
</skipped>
<skipped type="SKIP">
SKIPPED
skipping because answer = 43
at Skip.tests.cpp:<line number>
</skipped>
</testcase>
<testcase classname="<exe-name>.global" name="empty tags are not allowed" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="erfc_inv" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="estimate_clock_resolution" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="even more nested SECTION tests/c/d (leaf)" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="even more nested SECTION tests/c/e (leaf)" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="even more nested SECTION tests/f (leaf)" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="failed assertions before SKIP cause test case to fail" time="{duration}" status="run">
<skipped message="TEST_CASE tagged with !mayfail"/>
<failure message="3 == 4" type="CHECK">
FAILED:
CHECK( 3 == 4 )
at Skip.tests.cpp:<line number>
</failure>
<skipped type="SKIP">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testcase>
<testcase classname="<exe-name>.global" name="failing for some generator values causes entire test case to fail" time="{duration}" status="run">
<failure type="FAIL">
FAILED:
at Skip.tests.cpp:<line number>
</failure>
<skipped type="SKIP">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
<failure type="FAIL">
FAILED:
at Skip.tests.cpp:<line number>
</failure>
<skipped type="SKIP">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testcase>
<testcase classname="<exe-name>.global" name="failing in some unskipped sections causes entire test case to fail/skipped" time="{duration}" status="run">
<skipped type="SKIP">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testcase>
<testcase classname="<exe-name>.global" name="failing in some unskipped sections causes entire test case to fail/not skipped" time="{duration}" status="run">
<skipped message="TEST_CASE tagged with !mayfail"/>
<failure type="FAIL">
FAILED:
at Skip.tests.cpp:<line number>
</failure>
</testcase>
<testcase classname="<exe-name>.global" name="is_unary_function" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="just failure" time="{duration}" status="run">
<failure type="FAIL">
@@ -1743,6 +1804,19 @@ at Misc.tests.cpp:<line number>
<testcase classname="<exe-name>.global" name="more nested SECTION tests/doesn't equal/less than" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="nested SECTION tests/doesn't equal" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="nested SECTION tests/doesn't equal/not equal" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="nested sections can be skipped dynamically at runtime/B2/B" time="{duration}" status="run">
<skipped type="SKIP">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testcase>
<testcase classname="<exe-name>.global" name="nested sections can be skipped dynamically at runtime/B" time="{duration}" status="run">
<system-out>
a!
b1!
!
</system-out>
</testcase>
<testcase classname="<exe-name>.global" name="non streamable - with conv. op" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="non-copyable objects" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="normal_cdf" time="{duration}" status="run"/>
@@ -1794,6 +1868,14 @@ at Message.tests.cpp:<line number>
<testcase classname="<exe-name>.global" name="resolution" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="run_for_at_least, chronometer" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="run_for_at_least, int" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="sections can be skipped dynamically at runtime/not skipped" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="sections can be skipped dynamically at runtime/skipped" time="{duration}" status="run">
<skipped type="SKIP">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testcase>
<testcase classname="<exe-name>.global" name="sections can be skipped dynamically at runtime/also not skipped" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="send a single char to INFO" time="{duration}" status="run">
<failure message="false" type="REQUIRE">
FAILED:
@@ -1812,6 +1894,13 @@ at Message.tests.cpp:<line number>
</failure>
</testcase>
<testcase classname="<exe-name>.global" name="shortened hide tags are split apart" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="skipped tests can optionally provide a reason" time="{duration}" status="run">
<skipped type="SKIP">
SKIPPED
skipping because answer = 43
at Skip.tests.cpp:<line number>
</skipped>
</testcase>
<testcase classname="<exe-name>.global" name="splitString" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="stacks unscoped info in loops" time="{duration}" status="run">
<failure message="false" type="CHECK">
@@ -1856,6 +1945,12 @@ at Message.tests.cpp:<line number>
<testcase classname="<exe-name>.global" name="strlen3" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="tables" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="tags with dots in later positions are not parsed as hidden" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="tests can be skipped dynamically at runtime" time="{duration}" status="run">
<skipped type="SKIP">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testcase>
<testcase classname="<exe-name>.global" name="thrown std::strings are translated" time="{duration}" status="run">
<error type="TEST_CASE">
FAILED:
@@ -1905,6 +2000,9 @@ This would not be caught previously
A string sent directly to stdout
Message from section one
Message from section two
a!
b1!
!
</system-out>
<system-err>
Nor would this

View File

@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuites>
<testsuite name="<exe-name>" errors="17" failures="126" tests="2163" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<testsuite name="<exe-name>" errors="17" failures="128" skipped="11" tests="2184" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<properties>
<property name="random-seed" value="1"/>
<property name="filters" value="&quot;*&quot; ~[!nonportable] ~[!benchmark] ~[approvals]"/>
@@ -1590,6 +1590,12 @@ at Exception.tests.cpp:<line number>
<testcase classname="<exe-name>.global" name="XmlEncode/string with control char (1)" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="XmlEncode/string with control char (x7F)" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="XmlWriter writes boolean attributes as true/false" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="a succeeding test can still be skipped" time="{duration}" status="run">
<skipped type="SKIP">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testcase>
<testcase classname="<exe-name>.global" name="analyse no analysis" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="array&lt;int, N> -> toString" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="benchmark function call/without chronometer" time="{duration}" status="run"/>
@@ -1624,12 +1630,67 @@ at Misc.tests.cpp:<line number>
<testcase classname="<exe-name>.global" name="comparisons between const int variables" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="comparisons between int variables" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="convertToBits" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="dynamic skipping works with generators" time="{duration}" status="run">
<skipped type="SKIP">
SKIPPED
skipping because answer = 41
at Skip.tests.cpp:<line number>
</skipped>
<skipped type="SKIP">
SKIPPED
skipping because answer = 43
at Skip.tests.cpp:<line number>
</skipped>
</testcase>
<testcase classname="<exe-name>.global" name="empty tags are not allowed" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="erfc_inv" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="estimate_clock_resolution" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="even more nested SECTION tests/c/d (leaf)" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="even more nested SECTION tests/c/e (leaf)" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="even more nested SECTION tests/f (leaf)" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="failed assertions before SKIP cause test case to fail" time="{duration}" status="run">
<skipped message="TEST_CASE tagged with !mayfail"/>
<failure message="3 == 4" type="CHECK">
FAILED:
CHECK( 3 == 4 )
at Skip.tests.cpp:<line number>
</failure>
<skipped type="SKIP">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testcase>
<testcase classname="<exe-name>.global" name="failing for some generator values causes entire test case to fail" time="{duration}" status="run">
<failure type="FAIL">
FAILED:
at Skip.tests.cpp:<line number>
</failure>
<skipped type="SKIP">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
<failure type="FAIL">
FAILED:
at Skip.tests.cpp:<line number>
</failure>
<skipped type="SKIP">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testcase>
<testcase classname="<exe-name>.global" name="failing in some unskipped sections causes entire test case to fail/skipped" time="{duration}" status="run">
<skipped type="SKIP">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testcase>
<testcase classname="<exe-name>.global" name="failing in some unskipped sections causes entire test case to fail/not skipped" time="{duration}" status="run">
<skipped message="TEST_CASE tagged with !mayfail"/>
<failure type="FAIL">
FAILED:
at Skip.tests.cpp:<line number>
</failure>
</testcase>
<testcase classname="<exe-name>.global" name="is_unary_function" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="just failure" time="{duration}" status="run">
<failure type="FAIL">
@@ -1742,6 +1803,19 @@ at Misc.tests.cpp:<line number>
<testcase classname="<exe-name>.global" name="more nested SECTION tests/doesn't equal/less than" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="nested SECTION tests/doesn't equal" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="nested SECTION tests/doesn't equal/not equal" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="nested sections can be skipped dynamically at runtime/B2/B" time="{duration}" status="run">
<skipped type="SKIP">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testcase>
<testcase classname="<exe-name>.global" name="nested sections can be skipped dynamically at runtime/B" time="{duration}" status="run">
<system-out>
a!
b1!
!
</system-out>
</testcase>
<testcase classname="<exe-name>.global" name="non streamable - with conv. op" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="non-copyable objects" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="normal_cdf" time="{duration}" status="run"/>
@@ -1793,6 +1867,14 @@ at Message.tests.cpp:<line number>
<testcase classname="<exe-name>.global" name="resolution" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="run_for_at_least, chronometer" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="run_for_at_least, int" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="sections can be skipped dynamically at runtime/not skipped" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="sections can be skipped dynamically at runtime/skipped" time="{duration}" status="run">
<skipped type="SKIP">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testcase>
<testcase classname="<exe-name>.global" name="sections can be skipped dynamically at runtime/also not skipped" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="send a single char to INFO" time="{duration}" status="run">
<failure message="false" type="REQUIRE">
FAILED:
@@ -1811,6 +1893,13 @@ at Message.tests.cpp:<line number>
</failure>
</testcase>
<testcase classname="<exe-name>.global" name="shortened hide tags are split apart" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="skipped tests can optionally provide a reason" time="{duration}" status="run">
<skipped type="SKIP">
SKIPPED
skipping because answer = 43
at Skip.tests.cpp:<line number>
</skipped>
</testcase>
<testcase classname="<exe-name>.global" name="splitString" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="stacks unscoped info in loops" time="{duration}" status="run">
<failure message="false" type="CHECK">
@@ -1855,6 +1944,12 @@ at Message.tests.cpp:<line number>
<testcase classname="<exe-name>.global" name="strlen3" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="tables" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="tags with dots in later positions are not parsed as hidden" time="{duration}" status="run"/>
<testcase classname="<exe-name>.global" name="tests can be skipped dynamically at runtime" time="{duration}" status="run">
<skipped type="SKIP">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testcase>
<testcase classname="<exe-name>.global" name="thrown std::strings are translated" time="{duration}" status="run">
<error type="TEST_CASE">
FAILED:
@@ -1904,6 +1999,9 @@ This would not be caught previously
A string sent directly to stdout
Message from section one
Message from section two
a!
b1!
!
</system-out>
<system-err>
Nor would this

View File

@@ -1832,6 +1832,95 @@ at Misc.tests.cpp:<line number>
<testCase name="xmlentitycheck/embedded xml: &lt;test>it should be possible to embed xml characters, such as &lt;, &quot; or &amp;, or even whole &lt;xml>documents&lt;/xml> within an attribute&lt;/test>" duration="{duration}"/>
<testCase name="xmlentitycheck/encoded chars: these should all be encoded: &amp;&amp;&amp;&quot;&quot;&quot;&lt;&lt;&lt;&amp;&quot;&lt;&lt;&amp;&quot;" duration="{duration}"/>
</file>
<file path="tests/<exe-name>/UsageTests/Skip.tests.cpp">
<testCase name="a succeeding test can still be skipped" duration="{duration}">
<skipped message="SKIP()">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testCase>
<testCase name="dynamic skipping works with generators" duration="{duration}">
<skipped message="SKIP()">
SKIPPED
skipping because answer = 41
at Skip.tests.cpp:<line number>
</skipped>
<skipped message="SKIP()">
SKIPPED
skipping because answer = 43
at Skip.tests.cpp:<line number>
</skipped>
</testCase>
<testCase name="failed assertions before SKIP cause test case to fail" duration="{duration}">
<skipped message="CHECK(3 == 4)">
FAILED:
CHECK( 3 == 4 )
at Skip.tests.cpp:<line number>
</skipped>
<skipped message="SKIP()">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testCase>
<testCase name="failing for some generator values causes entire test case to fail" duration="{duration}">
<skipped message="FAIL()">
FAILED:
at Skip.tests.cpp:<line number>
</skipped>
<skipped message="SKIP()">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
<skipped message="FAIL()">
FAILED:
at Skip.tests.cpp:<line number>
</skipped>
<skipped message="SKIP()">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testCase>
<testCase name="failing in some unskipped sections causes entire test case to fail/skipped" duration="{duration}">
<skipped message="SKIP()">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testCase>
<testCase name="failing in some unskipped sections causes entire test case to fail/not skipped" duration="{duration}">
<skipped message="FAIL()">
FAILED:
at Skip.tests.cpp:<line number>
</skipped>
</testCase>
<testCase name="nested sections can be skipped dynamically at runtime/B2/B" duration="{duration}">
<skipped message="SKIP()">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testCase>
<testCase name="nested sections can be skipped dynamically at runtime/B" duration="{duration}"/>
<testCase name="sections can be skipped dynamically at runtime/not skipped" duration="{duration}"/>
<testCase name="sections can be skipped dynamically at runtime/skipped" duration="{duration}">
<skipped message="SKIP()">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testCase>
<testCase name="sections can be skipped dynamically at runtime/also not skipped" duration="{duration}"/>
<testCase name="skipped tests can optionally provide a reason" duration="{duration}">
<skipped message="SKIP()">
SKIPPED
skipping because answer = 43
at Skip.tests.cpp:<line number>
</skipped>
</testCase>
<testCase name="tests can be skipped dynamically at runtime" duration="{duration}">
<skipped message="SKIP()">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testCase>
</file>
<file path="tests/<exe-name>/UsageTests/ToStringChrono.tests.cpp">
<testCase name="Stringifying std::chrono::duration helpers" duration="{duration}"/>
<testCase name="Stringifying std::chrono::duration with weird ratios" duration="{duration}"/>

View File

@@ -1831,6 +1831,95 @@ at Misc.tests.cpp:<line number>
<testCase name="xmlentitycheck/embedded xml: &lt;test>it should be possible to embed xml characters, such as &lt;, &quot; or &amp;, or even whole &lt;xml>documents&lt;/xml> within an attribute&lt;/test>" duration="{duration}"/>
<testCase name="xmlentitycheck/encoded chars: these should all be encoded: &amp;&amp;&amp;&quot;&quot;&quot;&lt;&lt;&lt;&amp;&quot;&lt;&lt;&amp;&quot;" duration="{duration}"/>
</file>
<file path="tests/<exe-name>/UsageTests/Skip.tests.cpp">
<testCase name="a succeeding test can still be skipped" duration="{duration}">
<skipped message="SKIP()">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testCase>
<testCase name="dynamic skipping works with generators" duration="{duration}">
<skipped message="SKIP()">
SKIPPED
skipping because answer = 41
at Skip.tests.cpp:<line number>
</skipped>
<skipped message="SKIP()">
SKIPPED
skipping because answer = 43
at Skip.tests.cpp:<line number>
</skipped>
</testCase>
<testCase name="failed assertions before SKIP cause test case to fail" duration="{duration}">
<skipped message="CHECK(3 == 4)">
FAILED:
CHECK( 3 == 4 )
at Skip.tests.cpp:<line number>
</skipped>
<skipped message="SKIP()">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testCase>
<testCase name="failing for some generator values causes entire test case to fail" duration="{duration}">
<skipped message="FAIL()">
FAILED:
at Skip.tests.cpp:<line number>
</skipped>
<skipped message="SKIP()">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
<skipped message="FAIL()">
FAILED:
at Skip.tests.cpp:<line number>
</skipped>
<skipped message="SKIP()">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testCase>
<testCase name="failing in some unskipped sections causes entire test case to fail/skipped" duration="{duration}">
<skipped message="SKIP()">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testCase>
<testCase name="failing in some unskipped sections causes entire test case to fail/not skipped" duration="{duration}">
<skipped message="FAIL()">
FAILED:
at Skip.tests.cpp:<line number>
</skipped>
</testCase>
<testCase name="nested sections can be skipped dynamically at runtime/B2/B" duration="{duration}">
<skipped message="SKIP()">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testCase>
<testCase name="nested sections can be skipped dynamically at runtime/B" duration="{duration}"/>
<testCase name="sections can be skipped dynamically at runtime/not skipped" duration="{duration}"/>
<testCase name="sections can be skipped dynamically at runtime/skipped" duration="{duration}">
<skipped message="SKIP()">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testCase>
<testCase name="sections can be skipped dynamically at runtime/also not skipped" duration="{duration}"/>
<testCase name="skipped tests can optionally provide a reason" duration="{duration}">
<skipped message="SKIP()">
SKIPPED
skipping because answer = 43
at Skip.tests.cpp:<line number>
</skipped>
</testCase>
<testCase name="tests can be skipped dynamically at runtime" duration="{duration}">
<skipped message="SKIP()">
SKIPPED
at Skip.tests.cpp:<line number>
</skipped>
</testCase>
</file>
<file path="tests/<exe-name>/UsageTests/ToStringChrono.tests.cpp">
<testCase name="Stringifying std::chrono::duration helpers" duration="{duration}"/>
<testCase name="Stringifying std::chrono::duration with weird ratios" duration="{duration}"/>

View File

@@ -3609,6 +3609,10 @@ ok {test-number} - encode( "[\x01]" ) == "[\\x01]" for: "[\x01]" == "[\x01]"
ok {test-number} - encode( "[\x7F]" ) == "[\\x7F]" for: "[\x7F]" == "[\x7F]"
# XmlWriter writes boolean attributes as true/false
ok {test-number} - stream.str(), ContainsSubstring(R"(attr1="true")") && ContainsSubstring(R"(attr2="false")") for: "<?xml version="1.0" encoding="UTF-8"?> <Element1 attr1="true" attr2="false"/> " ( contains: "attr1="true"" and contains: "attr2="false"" )
# a succeeding test can still be skipped
ok {test-number} -
# a succeeding test can still be skipped
ok {test-number} - # SKIP
# analyse no analysis
ok {test-number} - analysis.mean.point.count() == 23 for: 23.0 == 23
# analyse no analysis
@@ -3779,6 +3783,12 @@ ok {test-number} - convertToBits( -0. ) == ( 1ULL << 63 ) for: 92233720368547758
ok {test-number} - convertToBits( std::numeric_limits<float>::denorm_min() ) == 1 for: 1 == 1
# convertToBits
ok {test-number} - convertToBits( std::numeric_limits<double>::denorm_min() ) == 1 for: 1 == 1
# dynamic skipping works with generators
ok {test-number} - # SKIP 'skipping because answer = 41'
# dynamic skipping works with generators
ok {test-number} -
# dynamic skipping works with generators
ok {test-number} - # SKIP 'skipping because answer = 43'
# empty tags are not allowed
ok {test-number} - Catch::TestCaseInfo("", { "test with an empty tag", "[]" }, dummySourceLineInfo)
# erfc_inv
@@ -3797,6 +3807,22 @@ ok {test-number} -
ok {test-number} -
# even more nested SECTION tests
ok {test-number} -
# failed assertions before SKIP cause test case to fail
not ok {test-number} - 3 == 4
# failed assertions before SKIP cause test case to fail
ok {test-number} - # SKIP
# failing for some generator values causes entire test case to fail
not ok {test-number} - explicitly
# failing for some generator values causes entire test case to fail
ok {test-number} - # SKIP
# failing for some generator values causes entire test case to fail
not ok {test-number} - explicitly
# failing for some generator values causes entire test case to fail
ok {test-number} - # SKIP
# failing in some unskipped sections causes entire test case to fail
ok {test-number} - # SKIP
# failing in some unskipped sections causes entire test case to fail
not ok {test-number} - explicitly
loose text artifact
# is_unary_function
ok {test-number} - with 1 message: 'Catch::Clara::Detail::is_unary_function<decltype(unary1)>::value'
@@ -3906,6 +3932,11 @@ ok {test-number} - a != b for: 1 != 2
ok {test-number} - b != a for: 2 != 1
# nested SECTION tests
ok {test-number} - a != b for: 1 != 2
a!
b1!
# nested sections can be skipped dynamically at runtime
ok {test-number} - # SKIP
!
# non streamable - with conv. op
ok {test-number} - s == "7" for: "7" == "7"
# non-copyable objects
@@ -4070,12 +4101,20 @@ ok {test-number} - Timing.elapsed >= time for: 128 ns >= 100 ns
ok {test-number} - Timing.result == Timing.iterations + 17 for: 145 == 145
# run_for_at_least, int
ok {test-number} - Timing.iterations >= time.count() for: 128 >= 100
# sections can be skipped dynamically at runtime
ok {test-number} -
# sections can be skipped dynamically at runtime
ok {test-number} - # SKIP
# sections can be skipped dynamically at runtime
ok {test-number} -
# send a single char to INFO
not ok {test-number} - false with 1 message: '3'
# sends information to INFO
not ok {test-number} - false with 2 messages: 'hi' and 'i := 7'
# shortened hide tags are split apart
ok {test-number} - testcase.tags, VectorContains( Tag( "magic-tag" ) ) && VectorContains( Tag( "."_catch_sr ) ) for: { {?}, {?} } ( Contains: {?} and Contains: {?} )
# skipped tests can optionally provide a reason
ok {test-number} - # SKIP 'skipping because answer = 43'
# splitString
ok {test-number} - splitStringRef("", ','), Equals(std::vector<StringRef>()) for: { } Equals: { }
# splitString
@@ -4158,6 +4197,8 @@ ok {test-number} - strlen(std::get<0>(data)) == static_cast<size_t>(std::get<1>(
ok {test-number} - testcase.tags.size() == 1 for: 1 == 1
# tags with dots in later positions are not parsed as hidden
ok {test-number} - testcase.tags[0].original == "magic.tag"_catch_sr for: magic.tag == magic.tag
# tests can be skipped dynamically at runtime
ok {test-number} - # SKIP
# thrown std::strings are translated
not ok {test-number} - unexpected exception with message: 'Why would you throw a std::string?'
# toString on const wchar_t const pointer returns the string contents
@@ -4330,5 +4371,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0
ok {test-number} -
# xmlentitycheck
ok {test-number} -
1..2163
1..2184

View File

@@ -3602,6 +3602,10 @@ ok {test-number} - encode( "[\x01]" ) == "[\\x01]" for: "[\x01]" == "[\x01]"
ok {test-number} - encode( "[\x7F]" ) == "[\\x7F]" for: "[\x7F]" == "[\x7F]"
# XmlWriter writes boolean attributes as true/false
ok {test-number} - stream.str(), ContainsSubstring(R"(attr1="true")") && ContainsSubstring(R"(attr2="false")") for: "<?xml version="1.0" encoding="UTF-8"?> <Element1 attr1="true" attr2="false"/> " ( contains: "attr1="true"" and contains: "attr2="false"" )
# a succeeding test can still be skipped
ok {test-number} -
# a succeeding test can still be skipped
ok {test-number} - # SKIP
# analyse no analysis
ok {test-number} - analysis.mean.point.count() == 23 for: 23.0 == 23
# analyse no analysis
@@ -3772,6 +3776,12 @@ ok {test-number} - convertToBits( -0. ) == ( 1ULL << 63 ) for: 92233720368547758
ok {test-number} - convertToBits( std::numeric_limits<float>::denorm_min() ) == 1 for: 1 == 1
# convertToBits
ok {test-number} - convertToBits( std::numeric_limits<double>::denorm_min() ) == 1 for: 1 == 1
# dynamic skipping works with generators
ok {test-number} - # SKIP 'skipping because answer = 41'
# dynamic skipping works with generators
ok {test-number} -
# dynamic skipping works with generators
ok {test-number} - # SKIP 'skipping because answer = 43'
# empty tags are not allowed
ok {test-number} - Catch::TestCaseInfo("", { "test with an empty tag", "[]" }, dummySourceLineInfo)
# erfc_inv
@@ -3790,6 +3800,22 @@ ok {test-number} -
ok {test-number} -
# even more nested SECTION tests
ok {test-number} -
# failed assertions before SKIP cause test case to fail
not ok {test-number} - 3 == 4
# failed assertions before SKIP cause test case to fail
ok {test-number} - # SKIP
# failing for some generator values causes entire test case to fail
not ok {test-number} - explicitly
# failing for some generator values causes entire test case to fail
ok {test-number} - # SKIP
# failing for some generator values causes entire test case to fail
not ok {test-number} - explicitly
# failing for some generator values causes entire test case to fail
ok {test-number} - # SKIP
# failing in some unskipped sections causes entire test case to fail
ok {test-number} - # SKIP
# failing in some unskipped sections causes entire test case to fail
not ok {test-number} - explicitly
# is_unary_function
ok {test-number} - with 1 message: 'Catch::Clara::Detail::is_unary_function<decltype(unary1)>::value'
# is_unary_function
@@ -3898,6 +3924,8 @@ ok {test-number} - a != b for: 1 != 2
ok {test-number} - b != a for: 2 != 1
# nested SECTION tests
ok {test-number} - a != b for: 1 != 2
# nested sections can be skipped dynamically at runtime
ok {test-number} - # SKIP
# non streamable - with conv. op
ok {test-number} - s == "7" for: "7" == "7"
# non-copyable objects
@@ -4062,12 +4090,20 @@ ok {test-number} - Timing.elapsed >= time for: 128 ns >= 100 ns
ok {test-number} - Timing.result == Timing.iterations + 17 for: 145 == 145
# run_for_at_least, int
ok {test-number} - Timing.iterations >= time.count() for: 128 >= 100
# sections can be skipped dynamically at runtime
ok {test-number} -
# sections can be skipped dynamically at runtime
ok {test-number} - # SKIP
# sections can be skipped dynamically at runtime
ok {test-number} -
# send a single char to INFO
not ok {test-number} - false with 1 message: '3'
# sends information to INFO
not ok {test-number} - false with 2 messages: 'hi' and 'i := 7'
# shortened hide tags are split apart
ok {test-number} - testcase.tags, VectorContains( Tag( "magic-tag" ) ) && VectorContains( Tag( "."_catch_sr ) ) for: { {?}, {?} } ( Contains: {?} and Contains: {?} )
# skipped tests can optionally provide a reason
ok {test-number} - # SKIP 'skipping because answer = 43'
# splitString
ok {test-number} - splitStringRef("", ','), Equals(std::vector<StringRef>()) for: { } Equals: { }
# splitString
@@ -4150,6 +4186,8 @@ ok {test-number} - strlen(std::get<0>(data)) == static_cast<size_t>(std::get<1>(
ok {test-number} - testcase.tags.size() == 1 for: 1 == 1
# tags with dots in later positions are not parsed as hidden
ok {test-number} - testcase.tags[0].original == "magic.tag"_catch_sr for: magic.tag == magic.tag
# tests can be skipped dynamically at runtime
ok {test-number} - # SKIP
# thrown std::strings are translated
not ok {test-number} - unexpected exception with message: 'Why would you throw a std::string?'
# toString on const wchar_t const pointer returns the string contents
@@ -4322,5 +4360,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0
ok {test-number} -
# xmlentitycheck
ok {test-number} -
1..2163
1..2184

View File

@@ -722,6 +722,9 @@
##teamcity[testFinished name='XmlEncode' duration="{duration}"]
##teamcity[testStarted name='XmlWriter writes boolean attributes as true/false']
##teamcity[testFinished name='XmlWriter writes boolean attributes as true/false' duration="{duration}"]
##teamcity[testStarted name='a succeeding test can still be skipped']
##teamcity[testIgnored name='a succeeding test can still be skipped' message='Skip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit skip']
##teamcity[testFinished name='a succeeding test can still be skipped' duration="{duration}"]
##teamcity[testStarted name='analyse no analysis']
##teamcity[testFinished name='analyse no analysis' duration="{duration}"]
##teamcity[testStarted name='array<int, N> -> toString']
@@ -748,6 +751,10 @@
##teamcity[testFinished name='comparisons between int variables' duration="{duration}"]
##teamcity[testStarted name='convertToBits']
##teamcity[testFinished name='convertToBits' duration="{duration}"]
##teamcity[testStarted name='dynamic skipping works with generators']
##teamcity[testIgnored name='dynamic skipping works with generators' message='Skip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit skip with message:|n "skipping because answer = 41"']
##teamcity[testIgnored name='dynamic skipping works with generators' message='Skip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit skip with message:|n "skipping because answer = 43"']
##teamcity[testFinished name='dynamic skipping works with generators' duration="{duration}"]
##teamcity[testStarted name='empty tags are not allowed']
##teamcity[testFinished name='empty tags are not allowed' duration="{duration}"]
##teamcity[testStarted name='erfc_inv']
@@ -756,6 +763,20 @@
##teamcity[testFinished name='estimate_clock_resolution' duration="{duration}"]
##teamcity[testStarted name='even more nested SECTION tests']
##teamcity[testFinished name='even more nested SECTION tests' duration="{duration}"]
##teamcity[testStarted name='failed assertions before SKIP cause test case to fail']
##teamcity[testIgnored name='failed assertions before SKIP cause test case to fail' message='Skip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexpression failed|n CHECK( 3 == 4 )|nwith expansion:|n 3 == 4|n- failure ignore as test marked as |'ok to fail|'|n']
##teamcity[testIgnored name='failed assertions before SKIP cause test case to fail' message='Skip.tests.cpp:<line number>|nexplicit skip']
##teamcity[testFinished name='failed assertions before SKIP cause test case to fail' duration="{duration}"]
##teamcity[testStarted name='failing for some generator values causes entire test case to fail']
##teamcity[testIgnored name='failing for some generator values causes entire test case to fail' message='Skip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n']
##teamcity[testIgnored name='failing for some generator values causes entire test case to fail' message='Skip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit skip']
##teamcity[testIgnored name='failing for some generator values causes entire test case to fail' message='Skip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n']
##teamcity[testIgnored name='failing for some generator values causes entire test case to fail' message='Skip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit skip']
##teamcity[testFinished name='failing for some generator values causes entire test case to fail' duration="{duration}"]
##teamcity[testStarted name='failing in some unskipped sections causes entire test case to fail']
##teamcity[testIgnored name='failing in some unskipped sections causes entire test case to fail' message='-------------------------------------------------------------------------------|nskipped|n-------------------------------------------------------------------------------|nSkip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit skip']
##teamcity[testIgnored name='failing in some unskipped sections causes entire test case to fail' message='-------------------------------------------------------------------------------|nnot skipped|n-------------------------------------------------------------------------------|nSkip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n']
##teamcity[testFinished name='failing in some unskipped sections causes entire test case to fail' duration="{duration}"]
##teamcity[testStarted name='first tag']
##teamcity[testFinished name='first tag' duration="{duration}"]
##teamcity[testStarted name='has printf']
@@ -802,6 +823,10 @@ loose text artifact
##teamcity[testFinished name='more nested SECTION tests' duration="{duration}"]
##teamcity[testStarted name='nested SECTION tests']
##teamcity[testFinished name='nested SECTION tests' duration="{duration}"]
##teamcity[testStarted name='nested sections can be skipped dynamically at runtime']
##teamcity[testIgnored name='nested sections can be skipped dynamically at runtime' message='-------------------------------------------------------------------------------|nB|nB2|n-------------------------------------------------------------------------------|nSkip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit skip']
##teamcity[testStdOut name='nested sections can be skipped dynamically at runtime' out='a!|nb1!|n!|n']
##teamcity[testFinished name='nested sections can be skipped dynamically at runtime' duration="{duration}"]
##teamcity[testStarted name='non streamable - with conv. op']
##teamcity[testFinished name='non streamable - with conv. op' duration="{duration}"]
##teamcity[testStarted name='non-copyable objects']
@@ -847,6 +872,9 @@ loose text artifact
##teamcity[testFinished name='run_for_at_least, int' duration="{duration}"]
##teamcity[testStarted name='second tag']
##teamcity[testFinished name='second tag' duration="{duration}"]
##teamcity[testStarted name='sections can be skipped dynamically at runtime']
##teamcity[testIgnored name='sections can be skipped dynamically at runtime' message='-------------------------------------------------------------------------------|nskipped|n-------------------------------------------------------------------------------|nSkip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit skip']
##teamcity[testFinished name='sections can be skipped dynamically at runtime' duration="{duration}"]
##teamcity[testStarted name='send a single char to INFO']
##teamcity[testFailed name='send a single char to INFO' message='Misc.tests.cpp:<line number>|n...............................................................................|n|nMisc.tests.cpp:<line number>|nexpression failed with message:|n "3"|n REQUIRE( false )|nwith expansion:|n false|n']
##teamcity[testFinished name='send a single char to INFO' duration="{duration}"]
@@ -855,6 +883,9 @@ loose text artifact
##teamcity[testFinished name='sends information to INFO' duration="{duration}"]
##teamcity[testStarted name='shortened hide tags are split apart']
##teamcity[testFinished name='shortened hide tags are split apart' duration="{duration}"]
##teamcity[testStarted name='skipped tests can optionally provide a reason']
##teamcity[testIgnored name='skipped tests can optionally provide a reason' message='Skip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit skip with message:|n "skipping because answer = 43"']
##teamcity[testFinished name='skipped tests can optionally provide a reason' duration="{duration}"]
##teamcity[testStarted name='splitString']
##teamcity[testFinished name='splitString' duration="{duration}"]
##teamcity[testStarted name='stacks unscoped info in loops']
@@ -899,6 +930,9 @@ loose text artifact
##teamcity[testFinished name='tables' duration="{duration}"]
##teamcity[testStarted name='tags with dots in later positions are not parsed as hidden']
##teamcity[testFinished name='tags with dots in later positions are not parsed as hidden' duration="{duration}"]
##teamcity[testStarted name='tests can be skipped dynamically at runtime']
##teamcity[testIgnored name='tests can be skipped dynamically at runtime' message='Skip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit skip']
##teamcity[testFinished name='tests can be skipped dynamically at runtime' duration="{duration}"]
##teamcity[testStarted name='thrown std::strings are translated']
##teamcity[testFailed name='thrown std::strings are translated' message='Exception.tests.cpp:<line number>|n...............................................................................|n|nException.tests.cpp:<line number>|nunexpected exception with message:|n "Why would you throw a std::string?"']
##teamcity[testFinished name='thrown std::strings are translated' duration="{duration}"]

View File

@@ -722,6 +722,9 @@
##teamcity[testFinished name='XmlEncode' duration="{duration}"]
##teamcity[testStarted name='XmlWriter writes boolean attributes as true/false']
##teamcity[testFinished name='XmlWriter writes boolean attributes as true/false' duration="{duration}"]
##teamcity[testStarted name='a succeeding test can still be skipped']
##teamcity[testIgnored name='a succeeding test can still be skipped' message='Skip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit skip']
##teamcity[testFinished name='a succeeding test can still be skipped' duration="{duration}"]
##teamcity[testStarted name='analyse no analysis']
##teamcity[testFinished name='analyse no analysis' duration="{duration}"]
##teamcity[testStarted name='array<int, N> -> toString']
@@ -748,6 +751,10 @@
##teamcity[testFinished name='comparisons between int variables' duration="{duration}"]
##teamcity[testStarted name='convertToBits']
##teamcity[testFinished name='convertToBits' duration="{duration}"]
##teamcity[testStarted name='dynamic skipping works with generators']
##teamcity[testIgnored name='dynamic skipping works with generators' message='Skip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit skip with message:|n "skipping because answer = 41"']
##teamcity[testIgnored name='dynamic skipping works with generators' message='Skip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit skip with message:|n "skipping because answer = 43"']
##teamcity[testFinished name='dynamic skipping works with generators' duration="{duration}"]
##teamcity[testStarted name='empty tags are not allowed']
##teamcity[testFinished name='empty tags are not allowed' duration="{duration}"]
##teamcity[testStarted name='erfc_inv']
@@ -756,6 +763,20 @@
##teamcity[testFinished name='estimate_clock_resolution' duration="{duration}"]
##teamcity[testStarted name='even more nested SECTION tests']
##teamcity[testFinished name='even more nested SECTION tests' duration="{duration}"]
##teamcity[testStarted name='failed assertions before SKIP cause test case to fail']
##teamcity[testIgnored name='failed assertions before SKIP cause test case to fail' message='Skip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexpression failed|n CHECK( 3 == 4 )|nwith expansion:|n 3 == 4|n- failure ignore as test marked as |'ok to fail|'|n']
##teamcity[testIgnored name='failed assertions before SKIP cause test case to fail' message='Skip.tests.cpp:<line number>|nexplicit skip']
##teamcity[testFinished name='failed assertions before SKIP cause test case to fail' duration="{duration}"]
##teamcity[testStarted name='failing for some generator values causes entire test case to fail']
##teamcity[testIgnored name='failing for some generator values causes entire test case to fail' message='Skip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n']
##teamcity[testIgnored name='failing for some generator values causes entire test case to fail' message='Skip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit skip']
##teamcity[testIgnored name='failing for some generator values causes entire test case to fail' message='Skip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n']
##teamcity[testIgnored name='failing for some generator values causes entire test case to fail' message='Skip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit skip']
##teamcity[testFinished name='failing for some generator values causes entire test case to fail' duration="{duration}"]
##teamcity[testStarted name='failing in some unskipped sections causes entire test case to fail']
##teamcity[testIgnored name='failing in some unskipped sections causes entire test case to fail' message='-------------------------------------------------------------------------------|nskipped|n-------------------------------------------------------------------------------|nSkip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit skip']
##teamcity[testIgnored name='failing in some unskipped sections causes entire test case to fail' message='-------------------------------------------------------------------------------|nnot skipped|n-------------------------------------------------------------------------------|nSkip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit failure- failure ignore as test marked as |'ok to fail|'|n']
##teamcity[testFinished name='failing in some unskipped sections causes entire test case to fail' duration="{duration}"]
##teamcity[testStarted name='first tag']
##teamcity[testFinished name='first tag' duration="{duration}"]
##teamcity[testStarted name='has printf']
@@ -801,6 +822,10 @@
##teamcity[testFinished name='more nested SECTION tests' duration="{duration}"]
##teamcity[testStarted name='nested SECTION tests']
##teamcity[testFinished name='nested SECTION tests' duration="{duration}"]
##teamcity[testStarted name='nested sections can be skipped dynamically at runtime']
##teamcity[testIgnored name='nested sections can be skipped dynamically at runtime' message='-------------------------------------------------------------------------------|nB|nB2|n-------------------------------------------------------------------------------|nSkip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit skip']
##teamcity[testStdOut name='nested sections can be skipped dynamically at runtime' out='a!|nb1!|n!|n']
##teamcity[testFinished name='nested sections can be skipped dynamically at runtime' duration="{duration}"]
##teamcity[testStarted name='non streamable - with conv. op']
##teamcity[testFinished name='non streamable - with conv. op' duration="{duration}"]
##teamcity[testStarted name='non-copyable objects']
@@ -846,6 +871,9 @@
##teamcity[testFinished name='run_for_at_least, int' duration="{duration}"]
##teamcity[testStarted name='second tag']
##teamcity[testFinished name='second tag' duration="{duration}"]
##teamcity[testStarted name='sections can be skipped dynamically at runtime']
##teamcity[testIgnored name='sections can be skipped dynamically at runtime' message='-------------------------------------------------------------------------------|nskipped|n-------------------------------------------------------------------------------|nSkip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit skip']
##teamcity[testFinished name='sections can be skipped dynamically at runtime' duration="{duration}"]
##teamcity[testStarted name='send a single char to INFO']
##teamcity[testFailed name='send a single char to INFO' message='Misc.tests.cpp:<line number>|n...............................................................................|n|nMisc.tests.cpp:<line number>|nexpression failed with message:|n "3"|n REQUIRE( false )|nwith expansion:|n false|n']
##teamcity[testFinished name='send a single char to INFO' duration="{duration}"]
@@ -854,6 +882,9 @@
##teamcity[testFinished name='sends information to INFO' duration="{duration}"]
##teamcity[testStarted name='shortened hide tags are split apart']
##teamcity[testFinished name='shortened hide tags are split apart' duration="{duration}"]
##teamcity[testStarted name='skipped tests can optionally provide a reason']
##teamcity[testIgnored name='skipped tests can optionally provide a reason' message='Skip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit skip with message:|n "skipping because answer = 43"']
##teamcity[testFinished name='skipped tests can optionally provide a reason' duration="{duration}"]
##teamcity[testStarted name='splitString']
##teamcity[testFinished name='splitString' duration="{duration}"]
##teamcity[testStarted name='stacks unscoped info in loops']
@@ -898,6 +929,9 @@
##teamcity[testFinished name='tables' duration="{duration}"]
##teamcity[testStarted name='tags with dots in later positions are not parsed as hidden']
##teamcity[testFinished name='tags with dots in later positions are not parsed as hidden' duration="{duration}"]
##teamcity[testStarted name='tests can be skipped dynamically at runtime']
##teamcity[testIgnored name='tests can be skipped dynamically at runtime' message='Skip.tests.cpp:<line number>|n...............................................................................|n|nSkip.tests.cpp:<line number>|nexplicit skip']
##teamcity[testFinished name='tests can be skipped dynamically at runtime' duration="{duration}"]
##teamcity[testStarted name='thrown std::strings are translated']
##teamcity[testFailed name='thrown std::strings are translated' message='Exception.tests.cpp:<line number>|n...............................................................................|n|nException.tests.cpp:<line number>|nunexpected exception with message:|n "Why would you throw a std::string?"']
##teamcity[testFinished name='thrown std::strings are translated' duration="{duration}"]

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,73 @@
// Copyright Catch2 Authors
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE.txt or copy at
// https://www.boost.org/LICENSE_1_0.txt)
// SPDX-License-Identifier: BSL-1.0
#include <catch2/catch_test_macros.hpp>
#include <catch2/generators/catch_generators_range.hpp>
#include <iostream>
TEST_CASE( "tests can be skipped dynamically at runtime", "[skipping]" ) {
SKIP();
FAIL( "this is not reached" );
}
TEST_CASE( "skipped tests can optionally provide a reason", "[skipping]" ) {
const int answer = 43;
SKIP( "skipping because answer = " << answer );
FAIL( "this is not reached" );
}
TEST_CASE( "sections can be skipped dynamically at runtime", "[skipping]" ) {
SECTION( "not skipped" ) { SUCCEED(); }
SECTION( "skipped" ) { SKIP(); }
SECTION( "also not skipped" ) { SUCCEED(); }
}
TEST_CASE( "nested sections can be skipped dynamically at runtime",
"[skipping]" ) {
SECTION( "A" ) { std::cout << "a"; }
SECTION( "B" ) {
SECTION( "B1" ) { std::cout << "b1"; }
SECTION( "B2" ) { SKIP(); }
}
std::cout << "!\n";
}
TEST_CASE( "dynamic skipping works with generators", "[skipping]" ) {
const int answer = GENERATE( 41, 42, 43 );
if ( answer != 42 ) { SKIP( "skipping because answer = " << answer ); }
SUCCEED();
}
TEST_CASE( "failed assertions before SKIP cause test case to fail",
"[skipping][!shouldfail]" ) {
CHECK( 3 == 4 );
SKIP();
}
TEST_CASE( "a succeeding test can still be skipped",
"[skipping][!shouldfail]" ) {
SUCCEED();
SKIP();
}
TEST_CASE( "failing in some unskipped sections causes entire test case to fail",
"[skipping][!shouldfail]" ) {
SECTION( "skipped" ) { SKIP(); }
SECTION( "not skipped" ) { FAIL(); }
}
TEST_CASE( "failing for some generator values causes entire test case to fail",
"[skipping][!shouldfail]" ) {
int i = GENERATE( 1, 2, 3, 4 );
if ( i % 2 == 0 ) {
SKIP();
} else {
FAIL();
}
}