Catch exceptions from StringMakers inside Detail::stringify

This stops tests failing falsely if the assertion passed, but the stringification itself failed as the assertion was sent to the reporter. I don't think that stringification should be fallible, but the overhead in compilation ended up being small enough (<0.5% on `SelfTest`) that it might be worth implementing, in case there is more users with weird `StringMaker`s than just MongoDB. Closes #2980
2025-08-02 13:25:41 +02:00 · 2025-08-01 23:23:34 +02:00
parent 17fe5eaa5c
commit c5e0ef4e67
20 changed files with 129 additions and 20 deletions
--- a/tests/SelfTest/Baselines/automake.sw.approved.txt
+++ b/tests/SelfTest/Baselines/automake.sw.approved.txt
@@ -248,6 +248,7 @@ Message from section two
 :test-result: PASS String matchers
 :test-result: PASS StringRef
 :test-result: PASS StringRef at compilation time
+:test-result: PASS Stringifying bla bla bla
 :test-result: PASS Stringifying char arrays with statically known sizes - char
 :test-result: PASS Stringifying char arrays with statically known sizes - signed char
 :test-result: PASS Stringifying char arrays with statically known sizes - unsigned char
--- a/tests/SelfTest/Baselines/automake.sw.multi.approved.txt
+++ b/tests/SelfTest/Baselines/automake.sw.multi.approved.txt
@@ -241,6 +241,7 @@
 :test-result: PASS String matchers
 :test-result: PASS StringRef
 :test-result: PASS StringRef at compilation time
+:test-result: PASS Stringifying bla bla bla
 :test-result: PASS Stringifying char arrays with statically known sizes - char
 :test-result: PASS Stringifying char arrays with statically known sizes - signed char
 :test-result: PASS Stringifying char arrays with statically known sizes - unsigned char
--- a/tests/SelfTest/Baselines/compact.sw.approved.txt
+++ b/tests/SelfTest/Baselines/compact.sw.approved.txt
@@ -1848,6 +1848,9 @@ String.tests.cpp:<line number>: passed: with 1 message: '!(sr1.empty())'
 String.tests.cpp:<line number>: passed: with 1 message: 'sr1.size() == 3'
 String.tests.cpp:<line number>: passed: with 1 message: 'sr2.empty()'
 String.tests.cpp:<line number>: passed: with 1 message: 'sr2.size() == 0'
+ToString.tests.cpp:<line number>: passed: tos == tos for: { stringification failed with an exception: "Invalid" }
+==
+{ stringification failed with an exception: "Invalid" }
 ToString.tests.cpp:<line number>: passed: ::Catch::Detail::stringify( with_null_terminator ) == R"("abc")"s for: ""abc"" == ""abc""
 ToString.tests.cpp:<line number>: passed: ::Catch::Detail::stringify( no_null_terminator ) == R"("abc")"s for: ""abc"" == ""abc""
 ToString.tests.cpp:<line number>: passed: ::Catch::Detail::stringify( with_null_terminator ) == R"("abc")"s for: ""abc"" == ""abc""
@@ -2874,7 +2877,7 @@ InternalBenchmark.tests.cpp:<line number>: passed: med == 18. for: 18.0 == 18.0
 InternalBenchmark.tests.cpp:<line number>: passed: q3 == 23. for: 23.0 == 23.0
 Misc.tests.cpp:<line number>: passed:
 Misc.tests.cpp:<line number>: passed:
-test cases:  433 |  315 passed |  95 failed | 6 skipped | 17 failed as expected
-assertions: 2291 | 2093 passed | 157 failed | 41 failed as expected
+test cases:  434 |  316 passed |  95 failed | 6 skipped | 17 failed as expected
+assertions: 2292 | 2094 passed | 157 failed | 41 failed as expected


--- a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt
+++ b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt
@@ -1841,6 +1841,9 @@ String.tests.cpp:<line number>: passed: with 1 message: '!(sr1.empty())'
 String.tests.cpp:<line number>: passed: with 1 message: 'sr1.size() == 3'
 String.tests.cpp:<line number>: passed: with 1 message: 'sr2.empty()'
 String.tests.cpp:<line number>: passed: with 1 message: 'sr2.size() == 0'
+ToString.tests.cpp:<line number>: passed: tos == tos for: { stringification failed with an exception: "Invalid" }
+==
+{ stringification failed with an exception: "Invalid" }
 ToString.tests.cpp:<line number>: passed: ::Catch::Detail::stringify( with_null_terminator ) == R"("abc")"s for: ""abc"" == ""abc""
 ToString.tests.cpp:<line number>: passed: ::Catch::Detail::stringify( no_null_terminator ) == R"("abc")"s for: ""abc"" == ""abc""
 ToString.tests.cpp:<line number>: passed: ::Catch::Detail::stringify( with_null_terminator ) == R"("abc")"s for: ""abc"" == ""abc""
@@ -2863,7 +2866,7 @@ InternalBenchmark.tests.cpp:<line number>: passed: med == 18. for: 18.0 == 18.0
 InternalBenchmark.tests.cpp:<line number>: passed: q3 == 23. for: 23.0 == 23.0
 Misc.tests.cpp:<line number>: passed:
 Misc.tests.cpp:<line number>: passed:
-test cases:  433 |  315 passed |  95 failed | 6 skipped | 17 failed as expected
-assertions: 2291 | 2093 passed | 157 failed | 41 failed as expected
+test cases:  434 |  316 passed |  95 failed | 6 skipped | 17 failed as expected
+assertions: 2292 | 2094 passed | 157 failed | 41 failed as expected


--- a/tests/SelfTest/Baselines/console.std.approved.txt
+++ b/tests/SelfTest/Baselines/console.std.approved.txt
@@ -1719,6 +1719,6 @@ due to unexpected exception with message:
  Why would you throw a std::string?

 ===============================================================================
-test cases:  433 |  333 passed |  76 failed | 7 skipped | 17 failed as expected
-assertions: 2270 | 2093 passed | 136 failed | 41 failed as expected
+test cases:  434 |  334 passed |  76 failed | 7 skipped | 17 failed as expected
+assertions: 2271 | 2094 passed | 136 failed | 41 failed as expected

--- a/tests/SelfTest/Baselines/console.sw.approved.txt
+++ b/tests/SelfTest/Baselines/console.sw.approved.txt
@@ -11789,6 +11789,19 @@ String.tests.cpp:<line number>: PASSED:
 with message:
  sr2.size() == 0

+-------------------------------------------------------------------------------
+Stringifying bla bla bla
+-------------------------------------------------------------------------------
+ToString.tests.cpp:<line number>
+...............................................................................
+
+ToString.tests.cpp:<line number>: PASSED:
+  CHECK( tos == tos )
+with expansion:
+  { stringification failed with an exception: "Invalid" }
+  ==
+  { stringification failed with an exception: "Invalid" }
+
 -------------------------------------------------------------------------------
 Stringifying char arrays with statically known sizes - char
 -------------------------------------------------------------------------------
@@ -19213,6 +19226,6 @@ Misc.tests.cpp:<line number>
 Misc.tests.cpp:<line number>: PASSED:

 ===============================================================================
-test cases:  433 |  315 passed |  95 failed | 6 skipped | 17 failed as expected
-assertions: 2291 | 2093 passed | 157 failed | 41 failed as expected
+test cases:  434 |  316 passed |  95 failed | 6 skipped | 17 failed as expected
+assertions: 2292 | 2094 passed | 157 failed | 41 failed as expected

--- a/tests/SelfTest/Baselines/console.sw.multi.approved.txt
+++ b/tests/SelfTest/Baselines/console.sw.multi.approved.txt
@@ -11782,6 +11782,19 @@ String.tests.cpp:<line number>: PASSED:
 with message:
  sr2.size() == 0

+-------------------------------------------------------------------------------
+Stringifying bla bla bla
+-------------------------------------------------------------------------------
+ToString.tests.cpp:<line number>
+...............................................................................
+
+ToString.tests.cpp:<line number>: PASSED:
+  CHECK( tos == tos )
+with expansion:
+  { stringification failed with an exception: "Invalid" }
+  ==
+  { stringification failed with an exception: "Invalid" }
+
 -------------------------------------------------------------------------------
 Stringifying char arrays with statically known sizes - char
 -------------------------------------------------------------------------------
@@ -19202,6 +19215,6 @@ Misc.tests.cpp:<line number>
 Misc.tests.cpp:<line number>: PASSED:

 ===============================================================================
-test cases:  433 |  315 passed |  95 failed | 6 skipped | 17 failed as expected
-assertions: 2291 | 2093 passed | 157 failed | 41 failed as expected
+test cases:  434 |  316 passed |  95 failed | 6 skipped | 17 failed as expected
+assertions: 2292 | 2094 passed | 157 failed | 41 failed as expected

--- a/tests/SelfTest/Baselines/junit.sw.approved.txt
+++ b/tests/SelfTest/Baselines/junit.sw.approved.txt
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <testsuitesloose text artifact
 >
-  <testsuite name="<exe-name>" errors="17" failures="140" skipped="12" tests="2303" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+  <testsuite name="<exe-name>" errors="17" failures="140" skipped="12" tests="2304" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
    <properties>
      <property name="random-seed" value="1"/>
      <property name="filters" value="&quot;*&quot; ~[!nonportable] ~[!benchmark] ~[approvals]"/>
@@ -1511,6 +1511,7 @@ at Matchers.tests.cpp:<line number>
    <testcase classname="<exe-name>.global" name="StringRef at compilation time" time="{duration}" status="run"/>
    <testcase classname="<exe-name>.global" name="StringRef at compilation time/Simple constructors" time="{duration}" status="run"/>
    <testcase classname="<exe-name>.global" name="StringRef at compilation time/UDL construction" time="{duration}" status="run"/>
+    <testcase classname="<exe-name>.global" name="Stringifying bla bla bla" time="{duration}" status="run"/>
    <testcase classname="<exe-name>.global" name="Stringifying char arrays with statically known sizes - char" time="{duration}" status="run"/>
    <testcase classname="<exe-name>.global" name="Stringifying char arrays with statically known sizes - signed char" time="{duration}" status="run"/>
    <testcase classname="<exe-name>.global" name="Stringifying char arrays with statically known sizes - unsigned char" time="{duration}" status="run"/>
--- a/tests/SelfTest/Baselines/junit.sw.multi.approved.txt
+++ b/tests/SelfTest/Baselines/junit.sw.multi.approved.txt
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <testsuites>
-  <testsuite name="<exe-name>" errors="17" failures="140" skipped="12" tests="2303" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+  <testsuite name="<exe-name>" errors="17" failures="140" skipped="12" tests="2304" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
    <properties>
      <property name="random-seed" value="1"/>
      <property name="filters" value="&quot;*&quot; ~[!nonportable] ~[!benchmark] ~[approvals]"/>
@@ -1510,6 +1510,7 @@ at Matchers.tests.cpp:<line number>
    <testcase classname="<exe-name>.global" name="StringRef at compilation time" time="{duration}" status="run"/>
    <testcase classname="<exe-name>.global" name="StringRef at compilation time/Simple constructors" time="{duration}" status="run"/>
    <testcase classname="<exe-name>.global" name="StringRef at compilation time/UDL construction" time="{duration}" status="run"/>
+    <testcase classname="<exe-name>.global" name="Stringifying bla bla bla" time="{duration}" status="run"/>
    <testcase classname="<exe-name>.global" name="Stringifying char arrays with statically known sizes - char" time="{duration}" status="run"/>
    <testcase classname="<exe-name>.global" name="Stringifying char arrays with statically known sizes - signed char" time="{duration}" status="run"/>
    <testcase classname="<exe-name>.global" name="Stringifying char arrays with statically known sizes - unsigned char" time="{duration}" status="run"/>
--- a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt
+++ b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt
@@ -388,6 +388,7 @@ at AssertionHandler.tests.cpp:<line number>
  <file path="tests/<exe-name>/IntrospectiveTests/ToString.tests.cpp">
    <testCase name="Directly creating an EnumInfo" duration="{duration}"/>
    <testCase name="Range type with sentinel" duration="{duration}"/>
+    <testCase name="Stringifying bla bla bla" duration="{duration}"/>
    <testCase name="Stringifying char arrays with statically known sizes - char" duration="{duration}"/>
    <testCase name="Stringifying char arrays with statically known sizes - signed char" duration="{duration}"/>
    <testCase name="Stringifying char arrays with statically known sizes - unsigned char" duration="{duration}"/>
--- a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt
+++ b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt
@@ -387,6 +387,7 @@ at AssertionHandler.tests.cpp:<line number>
  <file path="tests/<exe-name>/IntrospectiveTests/ToString.tests.cpp">
    <testCase name="Directly creating an EnumInfo" duration="{duration}"/>
    <testCase name="Range type with sentinel" duration="{duration}"/>
+    <testCase name="Stringifying bla bla bla" duration="{duration}"/>
    <testCase name="Stringifying char arrays with statically known sizes - char" duration="{duration}"/>
    <testCase name="Stringifying char arrays with statically known sizes - signed char" duration="{duration}"/>
    <testCase name="Stringifying char arrays with statically known sizes - unsigned char" duration="{duration}"/>
--- a/tests/SelfTest/Baselines/tap.sw.approved.txt
+++ b/tests/SelfTest/Baselines/tap.sw.approved.txt
@@ -2803,6 +2803,8 @@ ok {test-number} - with 1 message: 'sr1.size() == 3'
 ok {test-number} - with 1 message: 'sr2.empty()'
 # StringRef at compilation time
 ok {test-number} - with 1 message: 'sr2.size() == 0'
+# Stringifying bla bla bla
+ok {test-number} - tos == tos for: { stringification failed with an exception: "Invalid" } == { stringification failed with an exception: "Invalid" }
 # Stringifying char arrays with statically known sizes - char
 ok {test-number} - ::Catch::Detail::stringify( with_null_terminator ) == R"("abc")"s for: ""abc"" == ""abc""
 # Stringifying char arrays with statically known sizes - char
@@ -4603,5 +4605,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0
 ok {test-number} -
 # xmlentitycheck
 ok {test-number} -
-1..2303
+1..2304

--- a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt
+++ b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt
@@ -2796,6 +2796,8 @@ ok {test-number} - with 1 message: 'sr1.size() == 3'
 ok {test-number} - with 1 message: 'sr2.empty()'
 # StringRef at compilation time
 ok {test-number} - with 1 message: 'sr2.size() == 0'
+# Stringifying bla bla bla
+ok {test-number} - tos == tos for: { stringification failed with an exception: "Invalid" } == { stringification failed with an exception: "Invalid" }
 # Stringifying char arrays with statically known sizes - char
 ok {test-number} - ::Catch::Detail::stringify( with_null_terminator ) == R"("abc")"s for: ""abc"" == ""abc""
 # Stringifying char arrays with statically known sizes - char
@@ -4592,5 +4594,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0
 ok {test-number} -
 # xmlentitycheck
 ok {test-number} -
-1..2303
+1..2304

--- a/tests/SelfTest/Baselines/teamcity.sw.approved.txt
+++ b/tests/SelfTest/Baselines/teamcity.sw.approved.txt
@@ -617,6 +617,8 @@
 ##teamcity[testFinished name='StringRef' duration="{duration}"]
 ##teamcity[testStarted name='StringRef at compilation time']
 ##teamcity[testFinished name='StringRef at compilation time' duration="{duration}"]
+##teamcity[testStarted name='Stringifying bla bla bla']
+##teamcity[testFinished name='Stringifying bla bla bla' duration="{duration}"]
 ##teamcity[testStarted name='Stringifying char arrays with statically known sizes - char']
 ##teamcity[testFinished name='Stringifying char arrays with statically known sizes - char' duration="{duration}"]
 ##teamcity[testStarted name='Stringifying char arrays with statically known sizes - signed char']
--- a/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt
+++ b/tests/SelfTest/Baselines/teamcity.sw.multi.approved.txt
@@ -617,6 +617,8 @@
 ##teamcity[testFinished name='StringRef' duration="{duration}"]
 ##teamcity[testStarted name='StringRef at compilation time']
 ##teamcity[testFinished name='StringRef at compilation time' duration="{duration}"]
+##teamcity[testStarted name='Stringifying bla bla bla']
+##teamcity[testFinished name='Stringifying bla bla bla' duration="{duration}"]
 ##teamcity[testStarted name='Stringifying char arrays with statically known sizes - char']
 ##teamcity[testFinished name='Stringifying char arrays with statically known sizes - char' duration="{duration}"]
 ##teamcity[testStarted name='Stringifying char arrays with statically known sizes - signed char']
--- a/tests/SelfTest/Baselines/xml.sw.approved.txt
+++ b/tests/SelfTest/Baselines/xml.sw.approved.txt
@@ -13725,6 +13725,19 @@ Message from section two
    </Section>
    <OverallResult success="true" skips="0"/>
  </TestCase>
+  <TestCase name="Stringifying bla bla bla" filename="tests/<exe-name>/IntrospectiveTests/ToString.tests.cpp" >
+    <Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/ToString.tests.cpp" >
+      <Original>
+        tos == tos
+      </Original>
+      <Expanded>
+        { stringification failed with an exception: "Invalid" }
+==
+{ stringification failed with an exception: "Invalid" }
+      </Expanded>
+    </Expression>
+    <OverallResult success="true" skips="0"/>
+  </TestCase>
  <TestCase name="Stringifying char arrays with statically known sizes - char" tags="[toString]" filename="tests/<exe-name>/IntrospectiveTests/ToString.tests.cpp" >
    <Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/ToString.tests.cpp" >
      <Original>
@@ -22214,6 +22227,6 @@ Approx( -1.95996398454005449 )
    </Section>
    <OverallResult success="true" skips="0"/>
  </TestCase>
-  <OverallResults successes="2093" failures="157" expectedFailures="41" skips="12"/>
-  <OverallResultsCases successes="315" failures="95" expectedFailures="17" skips="6"/>
+  <OverallResults successes="2094" failures="157" expectedFailures="41" skips="12"/>
+  <OverallResultsCases successes="316" failures="95" expectedFailures="17" skips="6"/>
 </Catch2TestRun>
--- a/tests/SelfTest/Baselines/xml.sw.multi.approved.txt
+++ b/tests/SelfTest/Baselines/xml.sw.multi.approved.txt
@@ -13725,6 +13725,19 @@ Message from section two
    </Section>
    <OverallResult success="true" skips="0"/>
  </TestCase>
+  <TestCase name="Stringifying bla bla bla" filename="tests/<exe-name>/IntrospectiveTests/ToString.tests.cpp" >
+    <Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/ToString.tests.cpp" >
+      <Original>
+        tos == tos
+      </Original>
+      <Expanded>
+        { stringification failed with an exception: "Invalid" }
+==
+{ stringification failed with an exception: "Invalid" }
+      </Expanded>
+    </Expression>
+    <OverallResult success="true" skips="0"/>
+  </TestCase>
  <TestCase name="Stringifying char arrays with statically known sizes - char" tags="[toString]" filename="tests/<exe-name>/IntrospectiveTests/ToString.tests.cpp" >
    <Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/ToString.tests.cpp" >
      <Original>
@@ -22213,6 +22226,6 @@ Approx( -1.95996398454005449 )
    </Section>
    <OverallResult success="true" skips="0"/>
  </TestCase>
-  <OverallResults successes="2093" failures="157" expectedFailures="41" skips="12"/>
-  <OverallResultsCases successes="315" failures="95" expectedFailures="17" skips="6"/>
+  <OverallResults successes="2094" failures="157" expectedFailures="41" skips="12"/>
+  <OverallResultsCases successes="316" failures="95" expectedFailures="17" skips="6"/>
 </Catch2TestRun>
--- a/tests/SelfTest/IntrospectiveTests/ToString.tests.cpp
+++ b/tests/SelfTest/IntrospectiveTests/ToString.tests.cpp
@@ -118,3 +118,24 @@ TEST_CASE( "#2944 - Stringifying dates before 1970 should not crash", "[.approva
            Equals( "gmtime from provided timepoint has failed. This "
                    "happens e.g. with pre-1970 dates using Microsoft libc" ) );
 }
+
+namespace {
+    struct ThrowsOnStringification {
+        friend bool operator==( ThrowsOnStringification,
+                                ThrowsOnStringification ) {
+            return true;
+        }
+    };
+}
+
+template <>
+struct Catch::StringMaker<ThrowsOnStringification> {
+    static std::string convert(ThrowsOnStringification) {
+        throw std::runtime_error( "Invalid" );
+    }
+};
+
+TEST_CASE( "Stringifying bla bla bla" ) {
+    ThrowsOnStringification tos;
+    CHECK( tos == tos );
+}