Added benchmark tests

CMakeLists.txt

@@ -55,6 +55,7 @@ endfunction()
 set(TEST_SOURCES
         ${SELF_TEST_DIR}/ApproxTests.cpp
         ${SELF_TEST_DIR}/BDDTests.cpp
+        ${SELF_TEST_DIR}/Benchmark.tests.cpp
         ${SELF_TEST_DIR}/ClassTests.cpp
         ${SELF_TEST_DIR}/CmdLineTests.cpp
         ${SELF_TEST_DIR}/CompilationTests.cpp
@@ -96,9 +97,9 @@ set(SURROGATE_SOURCES
         ${SELF_TEST_DIR}/SurrogateCpps/catch_option.cpp
         ${SELF_TEST_DIR}/SurrogateCpps/catch_stream.cpp
         ${SELF_TEST_DIR}/SurrogateCpps/catch_streambuf.cpp
+        ${SELF_TEST_DIR}/SurrogateCpps/catch_test_case_tracker.cpp
         ${SELF_TEST_DIR}/SurrogateCpps/catch_test_spec.cpp
         ${SELF_TEST_DIR}/SurrogateCpps/catch_xmlwriter.cpp
-        ${SELF_TEST_DIR}/SurrogateCpps/catch_test_case_tracker.cpp
         )
 CheckFileList(SURROGATE_SOURCES ${SELF_TEST_DIR}/SurrogateCpps)
 

projects/SelfTest/Baselines/console.std.approved.txt

@@ -850,6 +850,22 @@ warning:
   Uncomment the code in this test to check that it gives a sensible compiler
   error
 
+-------------------------------------------------------------------------------
+benchmarked
+-------------------------------------------------------------------------------
+String.tests.cpp:<line number>
+...............................................................................
+
++-----------------------------------------------------------------------------+
+| benchmark name                           |  iters | elapsed ns |    average |
++-----------------------------------------------------------------------------+
+
+| Load up a vector                         |      1 |       8459 |    8459 ns |
+| Load up a map                            |      1 |      28774 |   28774 ns |
+| Reserved vector                          |      1 |       2430 |    2430 ns |
+| A fixed size array that should require   |        |            |            |
+| no allocations                           |     10 |       2563 |     256 ns |
++-----------------------------------------------------------------------------+
 -------------------------------------------------------------------------------
 checkedElse, failing
 -------------------------------------------------------------------------------
@@ -1016,6 +1032,6 @@ with expansion:
   "{?}" == "1"
 
 ===============================================================================
-test cases: 182 | 131 passed | 47 failed |  4 failed as expected
+test cases: 183 | 132 passed | 47 failed |  4 failed as expected
-assertions: 896 | 779 passed | 96 failed | 21 failed as expected
+assertions: 900 | 783 passed | 96 failed | 21 failed as expected
 

projects/SelfTest/Baselines/console.sw.approved.txt

@@ -6381,6 +6381,67 @@ PASSED:
 with expansion:
   0 == 0
 
+-------------------------------------------------------------------------------
+benchmarked
+-------------------------------------------------------------------------------
+String.tests.cpp:<line number>
+...............................................................................
+
++-----------------------------------------------------------------------------+
+| benchmark name                           |  iters | elapsed ns |    average |
++-----------------------------------------------------------------------------+
+
+| Load up a vector                         |      1 |       6957 |    6957 ns |
++-----------------------------------------------------------------------------+
+String.tests.cpp:<line number>:
+PASSED:
+  REQUIRE( v.size() == size )
+with expansion:
+  100 == 100
+
+
++-----------------------------------------------------------------------------+
+
+| benchmark name                           |  iters | elapsed ns |    average |
++-----------------------------------------------------------------------------+
+
+| Load up a map                            |      1 |      27868 |   27868 ns |
++-----------------------------------------------------------------------------+
+String.tests.cpp:<line number>:
+PASSED:
+  REQUIRE( m.size() == size )
+with expansion:
+  100 == 100
+
+
++-----------------------------------------------------------------------------+
+
+| benchmark name                           |  iters | elapsed ns |    average |
++-----------------------------------------------------------------------------+
+
+| Reserved vector                          |      1 |       2711 |    2711 ns |
++-----------------------------------------------------------------------------+
+String.tests.cpp:<line number>:
+PASSED:
+  REQUIRE( v.size() == size )
+with expansion:
+  100 == 100
+
+
++-----------------------------------------------------------------------------+
+
+| benchmark name                           |  iters | elapsed ns |    average |
++-----------------------------------------------------------------------------+
+
+| A fixed size array that should require   |        |            |            |
+| no allocations                           |      1 |        530 |     530 ns |
++-----------------------------------------------------------------------------+
+String.tests.cpp:<line number>:
+PASSED:
+  REQUIRE( sum > size )
+with expansion:
+  4950 (0x<hex digits>) > 100
+
 -------------------------------------------------------------------------------
 boolean member
 -------------------------------------------------------------------------------
@@ -7590,6 +7651,6 @@ MiscTests.cpp:<line number>:
 PASSED:
 
 ===============================================================================
-test cases: 182 | 130 passed | 48 failed |  4 failed as expected
+test cases: 183 | 131 passed | 48 failed |  4 failed as expected
-assertions: 898 | 779 passed | 98 failed | 21 failed as expected
+assertions: 902 | 783 passed | 98 failed | 21 failed as expected
 

projects/SelfTest/Baselines/junit.sw.approved.txt

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <testsuitesspanner>
-  <testsuite name="<exe-name>" errors="15" failures="84" tests="899" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+  <testsuite name="<exe-name>" errors="15" failures="84" tests="903" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
     <testcase classname="<exe-name>.global" name="# A test name that starts with a #" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="#748 - captures with unexpected exceptions/outside assertions" time="{duration}">
       <error type="TEST_CASE">
@@ -641,6 +641,7 @@ ExceptionTests.cpp:<line number>
     <testcase classname="<exe-name>.global" name="XmlEncode/string with control char (1)" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="XmlEncode/string with control char (x7F)" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="atomic if" time="{duration}"/>
+    <testcase classname="<exe-name>.global" name="benchmarked" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="boolean member" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="checkedElse" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="checkedElse, failing" time="{duration}">

projects/SelfTest/Baselines/xml.sw.approved.txt

@@ -7201,6 +7201,41 @@ Message from section two
       </Expression>
       <OverallResult success="true"/>
     </TestCase>
+    <TestCase name="benchmarked" tags="[benchmark]" filename="projects/<exe-name>/String.tests.cpp" >
+      <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/String.tests.cpp" >
+        <Original>
+          v.size() == size
+        </Original>
+        <Expanded>
+          100 == 100
+        </Expanded>
+      </Expression>
+      <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/String.tests.cpp" >
+        <Original>
+          m.size() == size
+        </Original>
+        <Expanded>
+          100 == 100
+        </Expanded>
+      </Expression>
+      <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/String.tests.cpp" >
+        <Original>
+          v.size() == size
+        </Original>
+        <Expanded>
+          100 == 100
+        </Expanded>
+      </Expression>
+      <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/String.tests.cpp" >
+        <Original>
+          sum > size
+        </Original>
+        <Expanded>
+          4950 (0x<hex digits>) > 100
+        </Expanded>
+      </Expression>
+      <OverallResult success="true"/>
+    </TestCase>
     <TestCase name="boolean member" tags="[Tricky]" filename="projects/<exe-name>/TrickyTests.cpp" >
       <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/TrickyTests.cpp" >
         <Original>
@@ -8441,7 +8476,7 @@ spanner      <OverallResult success="true"/>
       </Section>
       <OverallResult success="true"/>
     </TestCase>
-    <OverallResults successes="779" failures="99" expectedFailures="21"/>
+    <OverallResults successes="783" failures="99" expectedFailures="21"/>
   </Group>
-  <OverallResults successes="779" failures="98" expectedFailures="21"/>
+  <OverallResults successes="783" failures="98" expectedFailures="21"/>
 </Catch>

projects/SelfTest/Benchmark.tests.cpp

@@ -0,0 +1,43 @@
+#include "catch.hpp"
+
+#include <map>
+
+TEST_CASE( "benchmarked", "[.][benchmark]" ) {
+
+    static const int size = 100;
+
+    std::vector<int> v;
+    std::map<int, int> m;
+
+    BENCHMARK( "Load up a vector" ) {
+        v = std::vector<int>();
+        for(int i =0; i < size; ++i )
+            v.push_back( i );
+    }
+    REQUIRE( v.size() == size );
+
+    BENCHMARK( "Load up a map" ) {
+        m = std::map<int, int>();
+        for(int i =0; i < size; ++i )
+            m.insert( { i, i+1 } );
+    }
+    REQUIRE( m.size() == size );
+
+    BENCHMARK( "Reserved vector" ) {
+        v = std::vector<int>();
+        v.reserve(size);
+        for(int i =0; i < size; ++i )
+            v.push_back( i );
+    }
+    REQUIRE( v.size() == size );
+
+    int array[size];
+    BENCHMARK( "A fixed size array that should require no allocations" ) {
+        for(int i =0; i < size; ++i )
+            array[i] = i;
+    }
+    int sum = 0;
+    for(int i =0; i < size; ++i )
+        sum += array[i];
+    REQUIRE( sum > size );
+}