Sort test ordering during Approval testing

projects/SelfTest/Baselines/console.std.approved.txt

@@ -3,6 +3,46 @@
 CatchSelfTest is a <version> host application.
 Run with -? for options
 
+-------------------------------------------------------------------------------
+'Not' checks that should fail
+-------------------------------------------------------------------------------
+ConditionTests.cpp:<line number>
+...............................................................................
+
+ConditionTests.cpp:<line number>: FAILED:
+  CHECK( false != false )
+
+ConditionTests.cpp:<line number>: FAILED:
+  CHECK( true != true )
+
+ConditionTests.cpp:<line number>: FAILED:
+  CHECK( !true )
+with expansion:
+  false
+
+ConditionTests.cpp:<line number>: FAILED:
+  CHECK_FALSE( true )
+
+ConditionTests.cpp:<line number>: FAILED:
+  CHECK( !trueValue )
+with expansion:
+  false
+
+ConditionTests.cpp:<line number>: FAILED:
+  CHECK_FALSE( trueValue )
+with expansion:
+  !true
+
+ConditionTests.cpp:<line number>: FAILED:
+  CHECK( !(1 == 1) )
+with expansion:
+  false
+
+ConditionTests.cpp:<line number>: FAILED:
+  CHECK_FALSE( 1 == 1 )
+with expansion:
+  !(1 == 1)
+
 -------------------------------------------------------------------------------
 A METHOD_AS_TEST_CASE based test run that fails
 -------------------------------------------------------------------------------
@@ -25,6 +65,97 @@ ClassTests.cpp:<line number>: FAILED:
 with expansion:
   1 == 2
 
+-------------------------------------------------------------------------------
+A couple of nested sections followed by a failure
+-------------------------------------------------------------------------------
+MiscTests.cpp:<line number>
+...............................................................................
+
+MiscTests.cpp:<line number>: FAILED:
+explicitly with message:
+  to infinity and beyond
+
+-------------------------------------------------------------------------------
+A failing expression with a non streamable type is still captured
+-------------------------------------------------------------------------------
+TrickyTests.cpp:<line number>
+...............................................................................
+
+TrickyTests.cpp:<line number>: FAILED:
+  CHECK( &o1 == &o2 )
+with expansion:
+  0x<hex digits> == 0x<hex digits>
+
+TrickyTests.cpp:<line number>: FAILED:
+  CHECK( o1 == o2 )
+with expansion:
+  {?} == {?}
+
+-------------------------------------------------------------------------------
+An unchecked exception reports the line of the last assertion
+-------------------------------------------------------------------------------
+ExceptionTests.cpp:<line number>
+...............................................................................
+
+ExceptionTests.cpp:<line number>: FAILED:
+  {Unknown expression after the reported line}
+due to unexpected exception with message:
+  unexpected exception
+
+-------------------------------------------------------------------------------
+Contains string matcher
+-------------------------------------------------------------------------------
+MiscTests.cpp:<line number>
+...............................................................................
+
+MiscTests.cpp:<line number>: FAILED:
+  CHECK_THAT( testStringForMatching(), Contains( "not there" ) )
+with expansion:
+  "this string contains 'abc' as a substring" contains: "not there"
+
+-------------------------------------------------------------------------------
+Custom exceptions can be translated when testing for nothrow
+-------------------------------------------------------------------------------
+ExceptionTests.cpp:<line number>
+...............................................................................
+
+ExceptionTests.cpp:<line number>: FAILED:
+  REQUIRE_NOTHROW( throwCustom() )
+due to unexpected exception with message:
+  custom exception - not std
+
+-------------------------------------------------------------------------------
+Custom exceptions can be translated when testing for throwing as something else
+-------------------------------------------------------------------------------
+ExceptionTests.cpp:<line number>
+...............................................................................
+
+ExceptionTests.cpp:<line number>: FAILED:
+  REQUIRE_THROWS_AS( throwCustom() )
+due to unexpected exception with message:
+  custom exception - not std
+
+-------------------------------------------------------------------------------
+Custom std-exceptions can be custom translated
+-------------------------------------------------------------------------------
+ExceptionTests.cpp:<line number>
+...............................................................................
+
+ExceptionTests.cpp:<line number>: FAILED:
+due to unexpected exception with message:
+  custom std exception
+
+-------------------------------------------------------------------------------
+EndsWith string matcher
+-------------------------------------------------------------------------------
+MiscTests.cpp:<line number>
+...............................................................................
+
+MiscTests.cpp:<line number>: FAILED:
+  CHECK_THAT( testStringForMatching(), EndsWith( "this" ) )
+with expansion:
+  "this string contains 'abc' as a substring" ends with: "this"
+
 -------------------------------------------------------------------------------
 Equality checks that should fail
 -------------------------------------------------------------------------------
@@ -96,6 +227,100 @@ ConditionTests.cpp:<line number>: FAILED:
 with expansion:
   1.3 == Approx( 1.301 )
 
+-------------------------------------------------------------------------------
+Equals string matcher
+-------------------------------------------------------------------------------
+MiscTests.cpp:<line number>
+...............................................................................
+
+MiscTests.cpp:<line number>: FAILED:
+  CHECK_THAT( testStringForMatching(), Equals( "something else" ) )
+with expansion:
+  "this string contains 'abc' as a substring" equals: "something else"
+
+-------------------------------------------------------------------------------
+Expected exceptions that don't throw or unexpected exceptions fail the test
+-------------------------------------------------------------------------------
+ExceptionTests.cpp:<line number>
+...............................................................................
+
+ExceptionTests.cpp:<line number>: FAILED:
+  CHECK_THROWS_AS( thisThrows() )
+due to unexpected exception with message:
+  expected exception
+
+ExceptionTests.cpp:<line number>: FAILED:
+  CHECK_THROWS_AS( thisDoesntThrow() )
+because no exception was thrown where one was expected:
+
+ExceptionTests.cpp:<line number>: FAILED:
+  CHECK_NOTHROW( thisThrows() )
+due to unexpected exception with message:
+  expected exception
+
+-------------------------------------------------------------------------------
+FAIL aborts the test
+-------------------------------------------------------------------------------
+MessageTests.cpp:<line number>
+...............................................................................
+
+MessageTests.cpp:<line number>: FAILED:
+explicitly with message:
+  This is a failure
+
+-------------------------------------------------------------------------------
+FAIL does not require an argument
+-------------------------------------------------------------------------------
+MessageTests.cpp:<line number>
+...............................................................................
+
+MessageTests.cpp:<line number>: FAILED:
+
+-------------------------------------------------------------------------------
+INFO and WARN do not abort tests
+-------------------------------------------------------------------------------
+MessageTests.cpp:<line number>
+...............................................................................
+
+MessageTests.cpp:<line number>:
+warning:
+  this is a warning
+
+-------------------------------------------------------------------------------
+INFO gets logged on failure
+-------------------------------------------------------------------------------
+MessageTests.cpp:<line number>
+...............................................................................
+
+MessageTests.cpp:<line number>: FAILED:
+  REQUIRE( a == 1 )
+with expansion:
+  2 == 1
+with messages:
+  this message should be logged
+  so should this
+
+-------------------------------------------------------------------------------
+INFO gets logged on failure, even if captured before successful assertions
+-------------------------------------------------------------------------------
+MessageTests.cpp:<line number>
+...............................................................................
+
+MessageTests.cpp:<line number>: FAILED:
+  CHECK( a == 1 )
+with expansion:
+  2 == 1
+with messages:
+  this message may be logged later
+  this message should be logged
+
+MessageTests.cpp:<line number>: FAILED:
+  CHECK( a == 0 )
+with expansion:
+  2 == 0
+with message:
+  and this, but later
+
 -------------------------------------------------------------------------------
 Inequality checks that should fail
 -------------------------------------------------------------------------------
@@ -127,6 +352,60 @@ ConditionTests.cpp:<line number>: FAILED:
 with expansion:
   5 != 5
 
+-------------------------------------------------------------------------------
+Matchers can be composed with both && and || - failing
+-------------------------------------------------------------------------------
+MiscTests.cpp:<line number>
+...............................................................................
+
+MiscTests.cpp:<line number>: FAILED:
+  CHECK_THAT( testStringForMatching(), ( Contains( "string" ) || Contains( "different" ) ) && Contains( "random" ) )
+with expansion:
+  "this string contains 'abc' as a substring" ( ( contains: "string" or
+  contains: "different" ) and contains: "random" )
+
+-------------------------------------------------------------------------------
+Matchers can be negated (Not) with the ! operator - failing
+-------------------------------------------------------------------------------
+MiscTests.cpp:<line number>
+...............................................................................
+
+MiscTests.cpp:<line number>: FAILED:
+  CHECK_THAT( testStringForMatching(), !Contains( "substring" ) )
+with expansion:
+  "this string contains 'abc' as a substring" not contains: "substring"
+
+-------------------------------------------------------------------------------
+Mismatching exception messages failing the test
+-------------------------------------------------------------------------------
+ExceptionTests.cpp:<line number>
+...............................................................................
+
+ExceptionTests.cpp:<line number>: FAILED:
+  REQUIRE_THROWS_WITH( thisThrows(), "should fail" )
+with expansion:
+  expected exception
+
+-------------------------------------------------------------------------------
+Nice descriptive name
+-------------------------------------------------------------------------------
+MiscTests.cpp:<line number>
+...............................................................................
+
+MiscTests.cpp:<line number>:
+warning:
+  This one ran
+
+-------------------------------------------------------------------------------
+Non-std exceptions can be translated
+-------------------------------------------------------------------------------
+ExceptionTests.cpp:<line number>
+...............................................................................
+
+ExceptionTests.cpp:<line number>: FAILED:
+due to unexpected exception with message:
+  custom exception
+
 -------------------------------------------------------------------------------
 Ordering comparison checks that should fail
 -------------------------------------------------------------------------------
@@ -229,64 +508,99 @@ with expansion:
   "hello" <= "a"
 
 -------------------------------------------------------------------------------
-'Not' checks that should fail
+Output from all sections is reported
+  one
 -------------------------------------------------------------------------------
-ConditionTests.cpp:<line number>
+MessageTests.cpp:<line number>
 ...............................................................................
 
-ConditionTests.cpp:<line number>: FAILED:
+MessageTests.cpp:<line number>: FAILED:
-  CHECK( false != false )
+explicitly with message:
-
+  Message from section one
-ConditionTests.cpp:<line number>: FAILED:
-  CHECK( true != true )
-
-ConditionTests.cpp:<line number>: FAILED:
-  CHECK( !true )
-with expansion:
-  false
-
-ConditionTests.cpp:<line number>: FAILED:
-  CHECK_FALSE( true )
-
-ConditionTests.cpp:<line number>: FAILED:
-  CHECK( !trueValue )
-with expansion:
-  false
-
-ConditionTests.cpp:<line number>: FAILED:
-  CHECK_FALSE( trueValue )
-with expansion:
-  !true
-
-ConditionTests.cpp:<line number>: FAILED:
-  CHECK( !(1 == 1) )
-with expansion:
-  false
-
-ConditionTests.cpp:<line number>: FAILED:
-  CHECK_FALSE( 1 == 1 )
-with expansion:
-  !(1 == 1)
 
 -------------------------------------------------------------------------------
-Expected exceptions that don't throw or unexpected exceptions fail the test
+Output from all sections is reported
+  two
+-------------------------------------------------------------------------------
+MessageTests.cpp:<line number>
+...............................................................................
+
+MessageTests.cpp:<line number>: FAILED:
+explicitly with message:
+  Message from section two
+
+-------------------------------------------------------------------------------
+Pointers can be converted to strings
+-------------------------------------------------------------------------------
+MessageTests.cpp:<line number>
+...............................................................................
+
+MessageTests.cpp:<line number>:
+warning:
+  actual address of p: 0x<hex digits>
+
+MessageTests.cpp:<line number>:
+warning:
+  toString(p): 0x<hex digits>
+
+-------------------------------------------------------------------------------
+SCOPED_INFO is reset for each loop
+-------------------------------------------------------------------------------
+MessageTests.cpp:<line number>
+...............................................................................
+
+MessageTests.cpp:<line number>: FAILED:
+  REQUIRE( i < 10 )
+with expansion:
+  10 < 10
+with messages:
+  current counter 10
+  i := 10
+
+A string sent directly to stdout
+A string sent directly to stderr
+Message from section one
+Message from section two
+-------------------------------------------------------------------------------
+StartsWith string matcher
+-------------------------------------------------------------------------------
+MiscTests.cpp:<line number>
+...............................................................................
+
+MiscTests.cpp:<line number>: FAILED:
+  CHECK_THAT( testStringForMatching(), StartsWith( "string" ) )
+with expansion:
+  "this string contains 'abc' as a substring" starts with: "string"
+
+hello
+hello
+-------------------------------------------------------------------------------
+Tabs and newlines show in output
+-------------------------------------------------------------------------------
+MiscTests.cpp:<line number>
+...............................................................................
+
+MiscTests.cpp:<line number>: FAILED:
+  CHECK( s1 == s2 )
+with expansion:
+  "if ($b == 10) {
+  		$a= 20;
+  }"
+  ==
+  "if ($b == 10) {
+  $a = 20;
+  }
+  "
+
+-------------------------------------------------------------------------------
+Unexpected exceptions can be translated
 -------------------------------------------------------------------------------
 ExceptionTests.cpp:<line number>
 ...............................................................................
 
 ExceptionTests.cpp:<line number>: FAILED:
-  CHECK_THROWS_AS( thisThrows() )
 due to unexpected exception with message:
-  expected exception
+  3.14
-
-ExceptionTests.cpp:<line number>: FAILED:
-  CHECK_THROWS_AS( thisDoesntThrow() )
-because no exception was thrown where one was expected:
-
-ExceptionTests.cpp:<line number>: FAILED:
-  CHECK_NOTHROW( thisThrows() )
-due to unexpected exception with message:
-  expected exception
 
 -------------------------------------------------------------------------------
 When unchecked exceptions are thrown directly they are always failures
@@ -299,29 +613,8 @@ due to unexpected exception with message:
   unexpected exception
 
 -------------------------------------------------------------------------------
-An unchecked exception reports the line of the last assertion
+When unchecked exceptions are thrown during a CHECK the test should abort and
--------------------------------------------------------------------------------
+fail
-ExceptionTests.cpp:<line number>
-...............................................................................
-
-ExceptionTests.cpp:<line number>: FAILED:
-  {Unknown expression after the reported line}
-due to unexpected exception with message:
-  unexpected exception
-
--------------------------------------------------------------------------------
-When unchecked exceptions are thrown from sections they are always failures
-  section name
--------------------------------------------------------------------------------
-ExceptionTests.cpp:<line number>
-...............................................................................
-
-ExceptionTests.cpp:<line number>: FAILED:
-due to unexpected exception with message:
-  unexpected exception
-
--------------------------------------------------------------------------------
-When unchecked exceptions are thrown from functions they are always failures
 -------------------------------------------------------------------------------
 ExceptionTests.cpp:<line number>
 ...............................................................................
@@ -344,8 +637,7 @@ due to unexpected exception with message:
   expected exception
 
 -------------------------------------------------------------------------------
-When unchecked exceptions are thrown during a CHECK the test should abort and
+When unchecked exceptions are thrown from functions they are always failures
-fail
 -------------------------------------------------------------------------------
 ExceptionTests.cpp:<line number>
 ...............................................................................
@@ -356,168 +648,69 @@ due to unexpected exception with message:
   expected exception
 
 -------------------------------------------------------------------------------
-Non-std exceptions can be translated
+When unchecked exceptions are thrown from sections they are always failures
+  section name
 -------------------------------------------------------------------------------
 ExceptionTests.cpp:<line number>
 ...............................................................................
 
 ExceptionTests.cpp:<line number>: FAILED:
 due to unexpected exception with message:
-  custom exception
+  unexpected exception
 
 -------------------------------------------------------------------------------
-Custom std-exceptions can be custom translated
+Where the LHS is not a simple value
 -------------------------------------------------------------------------------
-ExceptionTests.cpp:<line number>
+TrickyTests.cpp:<line number>
 ...............................................................................
 
-ExceptionTests.cpp:<line number>: FAILED:
+TrickyTests.cpp:<line number>:
-due to unexpected exception with message:
-  custom std exception
-
--------------------------------------------------------------------------------
-Custom exceptions can be translated when testing for nothrow
--------------------------------------------------------------------------------
-ExceptionTests.cpp:<line number>
-...............................................................................
-
-ExceptionTests.cpp:<line number>: FAILED:
-  REQUIRE_NOTHROW( throwCustom() )
-due to unexpected exception with message:
-  custom exception - not std
-
--------------------------------------------------------------------------------
-Custom exceptions can be translated when testing for throwing as something else
--------------------------------------------------------------------------------
-ExceptionTests.cpp:<line number>
-...............................................................................
-
-ExceptionTests.cpp:<line number>: FAILED:
-  REQUIRE_THROWS_AS( throwCustom() )
-due to unexpected exception with message:
-  custom exception - not std
-
--------------------------------------------------------------------------------
-Unexpected exceptions can be translated
--------------------------------------------------------------------------------
-ExceptionTests.cpp:<line number>
-...............................................................................
-
-ExceptionTests.cpp:<line number>: FAILED:
-due to unexpected exception with message:
-  3.14
-
--------------------------------------------------------------------------------
-Mismatching exception messages failing the test
--------------------------------------------------------------------------------
-ExceptionTests.cpp:<line number>
-...............................................................................
-
-ExceptionTests.cpp:<line number>: FAILED:
-  REQUIRE_THROWS_WITH( thisThrows(), "should fail" )
-with expansion:
-  expected exception
-
--------------------------------------------------------------------------------
-INFO and WARN do not abort tests
--------------------------------------------------------------------------------
-MessageTests.cpp:<line number>
-...............................................................................
-
-MessageTests.cpp:<line number>:
 warning:
-  this is a warning
+  Uncomment the code in this test to check that it gives a sensible compiler
+  error
 
 -------------------------------------------------------------------------------
-INFO gets logged on failure
+Where there is more to the expression after the RHS
 -------------------------------------------------------------------------------
-MessageTests.cpp:<line number>
+TrickyTests.cpp:<line number>
 ...............................................................................
 
-MessageTests.cpp:<line number>: FAILED:
+TrickyTests.cpp:<line number>:
-  REQUIRE( a == 1 )
+warning:
+  Uncomment the code in this test to check that it gives a sensible compiler
+  error
+
+-------------------------------------------------------------------------------
+checkedElse, failing
+-------------------------------------------------------------------------------
+MiscTests.cpp:<line number>
+...............................................................................
+
+MiscTests.cpp:<line number>: FAILED:
+  CHECKED_ELSE( flag )
 with expansion:
-  2 == 1
+  false
-with messages:
-  this message should be logged
-  so should this
 
--------------------------------------------------------------------------------
+MiscTests.cpp:<line number>: FAILED:
-INFO gets logged on failure, even if captured before successful assertions
+  REQUIRE( testCheckedElse( false ) )
--------------------------------------------------------------------------------
-MessageTests.cpp:<line number>
-...............................................................................
-
-MessageTests.cpp:<line number>: FAILED:
-  CHECK( a == 1 )
 with expansion:
-  2 == 1
+  false
-with messages:
-  this message may be logged later
-  this message should be logged
 
-MessageTests.cpp:<line number>: FAILED:
+-------------------------------------------------------------------------------
-  CHECK( a == 0 )
+checkedIf, failing
+-------------------------------------------------------------------------------
+MiscTests.cpp:<line number>
+...............................................................................
+
+MiscTests.cpp:<line number>: FAILED:
+  CHECKED_IF( flag )
 with expansion:
-  2 == 0
+  false
-with message:
-  and this, but later
 
--------------------------------------------------------------------------------
+MiscTests.cpp:<line number>: FAILED:
-FAIL aborts the test
+  REQUIRE( testCheckedIf( false ) )
--------------------------------------------------------------------------------
-MessageTests.cpp:<line number>
-...............................................................................
-
-MessageTests.cpp:<line number>: FAILED:
-explicitly with message:
-  This is a failure
-
--------------------------------------------------------------------------------
-FAIL does not require an argument
--------------------------------------------------------------------------------
-MessageTests.cpp:<line number>
-...............................................................................
-
-MessageTests.cpp:<line number>: FAILED:
-
--------------------------------------------------------------------------------
-Output from all sections is reported
-  one
--------------------------------------------------------------------------------
-MessageTests.cpp:<line number>
-...............................................................................
-
-MessageTests.cpp:<line number>: FAILED:
-explicitly with message:
-  Message from section one
-
--------------------------------------------------------------------------------
-Output from all sections is reported
-  two
--------------------------------------------------------------------------------
-MessageTests.cpp:<line number>
-...............................................................................
-
-MessageTests.cpp:<line number>: FAILED:
-explicitly with message:
-  Message from section two
-
-Message from section one
-Message from section two
--------------------------------------------------------------------------------
-SCOPED_INFO is reset for each loop
--------------------------------------------------------------------------------
-MessageTests.cpp:<line number>
-...............................................................................
-
-MessageTests.cpp:<line number>: FAILED:
-  REQUIRE( i < 10 )
 with expansion:
-  10 < 10
+  false
-with messages:
-  current counter 10
-  i := 10
 
 -------------------------------------------------------------------------------
 just failure
@@ -529,45 +722,6 @@ MessageTests.cpp:<line number>: FAILED:
 explicitly with message:
   Previous info should not be seen
 
--------------------------------------------------------------------------------
-sends information to INFO
--------------------------------------------------------------------------------
-MessageTests.cpp:<line number>
-...............................................................................
-
-MessageTests.cpp:<line number>: FAILED:
-  REQUIRE( false )
-with messages:
-  hi
-  i := 7
-
--------------------------------------------------------------------------------
-Pointers can be converted to strings
--------------------------------------------------------------------------------
-MessageTests.cpp:<line number>
-...............................................................................
-
-MessageTests.cpp:<line number>:
-warning:
-  actual address of p: 0x<hex digits>
-
-MessageTests.cpp:<line number>:
-warning:
-  toString(p): 0x<hex digits>
-
--------------------------------------------------------------------------------
-more nested SECTION tests
-  s1
-  s2
--------------------------------------------------------------------------------
-MiscTests.cpp:<line number>
-...............................................................................
-
-MiscTests.cpp:<line number>: FAILED:
-  REQUIRE( a == b )
-with expansion:
-  1 == 2
-
 -------------------------------------------------------------------------------
 looped SECTION tests
   s1
@@ -628,39 +782,18 @@ with expansion:
 with message:
   Testing if fib[7] (21) is even
 
-A string sent directly to stdout
-A string sent directly to stderr
 -------------------------------------------------------------------------------
-checkedIf, failing
+more nested SECTION tests
+  s1
+  s2
 -------------------------------------------------------------------------------
 MiscTests.cpp:<line number>
 ...............................................................................
 
 MiscTests.cpp:<line number>: FAILED:
-  CHECKED_IF( flag )
+  REQUIRE( a == b )
 with expansion:
-  false
+  1 == 2
-
-MiscTests.cpp:<line number>: FAILED:
-  REQUIRE( testCheckedIf( false ) )
-with expansion:
-  false
-
--------------------------------------------------------------------------------
-checkedElse, failing
--------------------------------------------------------------------------------
-MiscTests.cpp:<line number>
-...............................................................................
-
-MiscTests.cpp:<line number>: FAILED:
-  CHECKED_ELSE( flag )
-with expansion:
-  false
-
-MiscTests.cpp:<line number>: FAILED:
-  REQUIRE( testCheckedElse( false ) )
-with expansion:
-  false
 
 -------------------------------------------------------------------------------
 send a single char to INFO
@@ -674,149 +807,16 @@ with message:
   3
 
 -------------------------------------------------------------------------------
-Contains string matcher
+sends information to INFO
 -------------------------------------------------------------------------------
-MiscTests.cpp:<line number>
+MessageTests.cpp:<line number>
 ...............................................................................
 
-MiscTests.cpp:<line number>: FAILED:
+MessageTests.cpp:<line number>: FAILED:
-  CHECK_THAT( testStringForMatching(), Contains( "not there" ) )
+  REQUIRE( false )
-with expansion:
+with messages:
-  "this string contains 'abc' as a substring" contains: "not there"
+  hi
-
+  i := 7
--------------------------------------------------------------------------------
-StartsWith string matcher
--------------------------------------------------------------------------------
-MiscTests.cpp:<line number>
-...............................................................................
-
-MiscTests.cpp:<line number>: FAILED:
-  CHECK_THAT( testStringForMatching(), StartsWith( "string" ) )
-with expansion:
-  "this string contains 'abc' as a substring" starts with: "string"
-
--------------------------------------------------------------------------------
-EndsWith string matcher
--------------------------------------------------------------------------------
-MiscTests.cpp:<line number>
-...............................................................................
-
-MiscTests.cpp:<line number>: FAILED:
-  CHECK_THAT( testStringForMatching(), EndsWith( "this" ) )
-with expansion:
-  "this string contains 'abc' as a substring" ends with: "this"
-
--------------------------------------------------------------------------------
-Equals string matcher
--------------------------------------------------------------------------------
-MiscTests.cpp:<line number>
-...............................................................................
-
-MiscTests.cpp:<line number>: FAILED:
-  CHECK_THAT( testStringForMatching(), Equals( "something else" ) )
-with expansion:
-  "this string contains 'abc' as a substring" equals: "something else"
-
--------------------------------------------------------------------------------
-Matchers can be composed with both && and || - failing
--------------------------------------------------------------------------------
-MiscTests.cpp:<line number>
-...............................................................................
-
-MiscTests.cpp:<line number>: FAILED:
-  CHECK_THAT( testStringForMatching(), ( Contains( "string" ) || Contains( "different" ) ) && Contains( "random" ) )
-with expansion:
-  "this string contains 'abc' as a substring" ( ( contains: "string" or
-  contains: "different" ) and contains: "random" )
-
--------------------------------------------------------------------------------
-Matchers can be negated (Not) with the ! operator - failing
--------------------------------------------------------------------------------
-MiscTests.cpp:<line number>
-...............................................................................
-
-MiscTests.cpp:<line number>: FAILED:
-  CHECK_THAT( testStringForMatching(), !Contains( "substring" ) )
-with expansion:
-  "this string contains 'abc' as a substring" not contains: "substring"
-
--------------------------------------------------------------------------------
-Nice descriptive name
--------------------------------------------------------------------------------
-MiscTests.cpp:<line number>
-...............................................................................
-
-MiscTests.cpp:<line number>:
-warning:
-  This one ran
-
--------------------------------------------------------------------------------
-A couple of nested sections followed by a failure
--------------------------------------------------------------------------------
-MiscTests.cpp:<line number>
-...............................................................................
-
-MiscTests.cpp:<line number>: FAILED:
-explicitly with message:
-  to infinity and beyond
-
--------------------------------------------------------------------------------
-Tabs and newlines show in output
--------------------------------------------------------------------------------
-MiscTests.cpp:<line number>
-...............................................................................
-
-MiscTests.cpp:<line number>: FAILED:
-  CHECK( s1 == s2 )
-with expansion:
-  "if ($b == 10) {
-  		$a= 20;
-  }"
-  ==
-  "if ($b == 10) {
-  $a = 20;
-  }
-  "
-
-hello
-hello
--------------------------------------------------------------------------------
-Where there is more to the expression after the RHS
--------------------------------------------------------------------------------
-TrickyTests.cpp:<line number>
-...............................................................................
-
-TrickyTests.cpp:<line number>:
-warning:
-  Uncomment the code in this test to check that it gives a sensible compiler
-  error
-
--------------------------------------------------------------------------------
-Where the LHS is not a simple value
--------------------------------------------------------------------------------
-TrickyTests.cpp:<line number>
-...............................................................................
-
-TrickyTests.cpp:<line number>:
-warning:
-  Uncomment the code in this test to check that it gives a sensible compiler
-  error
-
--------------------------------------------------------------------------------
-A failing expression with a non streamable type is still captured
--------------------------------------------------------------------------------
-TrickyTests.cpp:<line number>
-...............................................................................
-
-TrickyTests.cpp:<line number>: FAILED:
-  CHECK( &o1 == &o2 )
-with expansion:
-  0x<hex digits> == 0x<hex digits>
-
-TrickyTests.cpp:<line number>: FAILED:
-  CHECK( o1 == o2 )
-with expansion:
-  {?} == {?}
 
 -------------------------------------------------------------------------------
 string literals of different sizes can be compared

projects/SelfTest/Baselines/console.swa4.approved.txt

@@ -4,612 +4,37 @@ CatchSelfTest is a <version> host application.
 Run with -? for options
 
 -------------------------------------------------------------------------------
-toString(enum)
+# A test name that starts with a #
 -------------------------------------------------------------------------------
-EnumToString.cpp:<line number>
+MiscTests.cpp:<line number>
 ...............................................................................
 
-EnumToString.cpp:<line number>:
+MiscTests.cpp:<line number>:
 PASSED:
-  CHECK( Catch::toString(e0) == "0" )
+with message:
-with expansion:
+  yay
-  "0" == "0"
-
-EnumToString.cpp:<line number>:
-PASSED:
-  CHECK( Catch::toString(e1) == "1" )
-with expansion:
-  "1" == "1"
 
 -------------------------------------------------------------------------------
-toString(enum w/operator<<)
+'Not' checks that should fail
--------------------------------------------------------------------------------
-EnumToString.cpp:<line number>
-...............................................................................
-
-EnumToString.cpp:<line number>:
-PASSED:
-  CHECK( Catch::toString(e0) == "E2{0}" )
-with expansion:
-  "E2{0}" == "E2{0}"
-
-EnumToString.cpp:<line number>:
-PASSED:
-  CHECK( Catch::toString(e1) == "E2{1}" )
-with expansion:
-  "E2{1}" == "E2{1}"
-
--------------------------------------------------------------------------------
-toString(enum class)
--------------------------------------------------------------------------------
-EnumToString.cpp:<line number>
-...............................................................................
-
-EnumToString.cpp:<line number>:
-PASSED:
-  CHECK( Catch::toString(e0) == "0" )
-with expansion:
-  "0" == "0"
-
-EnumToString.cpp:<line number>:
-PASSED:
-  CHECK( Catch::toString(e1) == "1" )
-with expansion:
-  "1" == "1"
-
--------------------------------------------------------------------------------
-toString(enum class w/operator<<)
--------------------------------------------------------------------------------
-EnumToString.cpp:<line number>
-...............................................................................
-
-EnumToString.cpp:<line number>:
-PASSED:
-  CHECK( Catch::toString(e0) == "E2/V0" )
-with expansion:
-  "E2/V0" == "E2/V0"
-
-EnumToString.cpp:<line number>:
-PASSED:
-  CHECK( Catch::toString(e1) == "E2/V1" )
-with expansion:
-  "E2/V1" == "E2/V1"
-
-EnumToString.cpp:<line number>:
-PASSED:
-  CHECK( Catch::toString(e3) == "Unknown enum value 10" )
-with expansion:
-  "Unknown enum value 10"
-  ==
-  "Unknown enum value 10"
-
--------------------------------------------------------------------------------
-Some simple comparisons between doubles
--------------------------------------------------------------------------------
-ApproxTests.cpp:<line number>
-...............................................................................
-
-ApproxTests.cpp:<line number>:
-PASSED:
-  REQUIRE( d == Approx( 1.23 ) )
-with expansion:
-  1.23 == Approx( 1.23 )
-
-ApproxTests.cpp:<line number>:
-PASSED:
-  REQUIRE( d != Approx( 1.22 ) )
-with expansion:
-  1.23 != Approx( 1.22 )
-
-ApproxTests.cpp:<line number>:
-PASSED:
-  REQUIRE( d != Approx( 1.24 ) )
-with expansion:
-  1.23 != Approx( 1.24 )
-
-ApproxTests.cpp:<line number>:
-PASSED:
-  REQUIRE( Approx( d ) == 1.23 )
-with expansion:
-  Approx( 1.23 ) == 1.23
-
-ApproxTests.cpp:<line number>:
-PASSED:
-  REQUIRE( Approx( d ) != 1.22 )
-with expansion:
-  Approx( 1.23 ) != 1.22
-
-ApproxTests.cpp:<line number>:
-PASSED:
-  REQUIRE( Approx( d ) != 1.24 )
-with expansion:
-  Approx( 1.23 ) != 1.24
-
--------------------------------------------------------------------------------
-Approximate comparisons with different epsilons
--------------------------------------------------------------------------------
-ApproxTests.cpp:<line number>
-...............................................................................
-
-ApproxTests.cpp:<line number>:
-PASSED:
-  REQUIRE( d != Approx( 1.231 ) )
-with expansion:
-  1.23 != Approx( 1.231 )
-
-ApproxTests.cpp:<line number>:
-PASSED:
-  REQUIRE( d == Approx( 1.231 ).epsilon( 0.1 ) )
-with expansion:
-  1.23 == Approx( 1.231 )
-
--------------------------------------------------------------------------------
-Approximate comparisons with floats
--------------------------------------------------------------------------------
-ApproxTests.cpp:<line number>
-...............................................................................
-
-ApproxTests.cpp:<line number>:
-PASSED:
-  REQUIRE( 1.23f == Approx( 1.23f ) )
-with expansion:
-  1.23f == Approx( 1.2300000191 )
-
-ApproxTests.cpp:<line number>:
-PASSED:
-  REQUIRE( 0.0f == Approx( 0.0f ) )
-with expansion:
-  0.0f == Approx( 0.0 )
-
--------------------------------------------------------------------------------
-Approximate comparisons with ints
--------------------------------------------------------------------------------
-ApproxTests.cpp:<line number>
-...............................................................................
-
-ApproxTests.cpp:<line number>:
-PASSED:
-  REQUIRE( 1 == Approx( 1 ) )
-with expansion:
-  1 == Approx( 1.0 )
-
-ApproxTests.cpp:<line number>:
-PASSED:
-  REQUIRE( 0 == Approx( 0 ) )
-with expansion:
-  0 == Approx( 0.0 )
-
--------------------------------------------------------------------------------
-Approximate comparisons with mixed numeric types
--------------------------------------------------------------------------------
-ApproxTests.cpp:<line number>
-...............................................................................
-
-ApproxTests.cpp:<line number>:
-PASSED:
-  REQUIRE( 1.0f == Approx( 1 ) )
-with expansion:
-  1.0f == Approx( 1.0 )
-
-ApproxTests.cpp:<line number>:
-PASSED:
-  REQUIRE( 0 == Approx( dZero) )
-with expansion:
-  0 == Approx( 0.0 )
-
-ApproxTests.cpp:<line number>:
-PASSED:
-  REQUIRE( 0 == Approx( dSmall ).epsilon( 0.001 ) )
-with expansion:
-  0 == Approx( 0.00001 )
-
-ApproxTests.cpp:<line number>:
-PASSED:
-  REQUIRE( 1.234f == Approx( dMedium ) )
-with expansion:
-  1.234f == Approx( 1.234 )
-
-ApproxTests.cpp:<line number>:
-PASSED:
-  REQUIRE( dMedium == Approx( 1.234f ) )
-with expansion:
-  1.234 == Approx( 1.2339999676 )
-
--------------------------------------------------------------------------------
-Use a custom approx
--------------------------------------------------------------------------------
-ApproxTests.cpp:<line number>
-...............................................................................
-
-ApproxTests.cpp:<line number>:
-PASSED:
-  REQUIRE( d == approx( 1.23 ) )
-with expansion:
-  1.23 == Approx( 1.23 )
-
-ApproxTests.cpp:<line number>:
-PASSED:
-  REQUIRE( d == approx( 1.22 ) )
-with expansion:
-  1.23 == Approx( 1.22 )
-
-ApproxTests.cpp:<line number>:
-PASSED:
-  REQUIRE( d == approx( 1.24 ) )
-with expansion:
-  1.23 == Approx( 1.24 )
-
-ApproxTests.cpp:<line number>:
-PASSED:
-  REQUIRE( d != approx( 1.25 ) )
-with expansion:
-  1.23 != Approx( 1.25 )
-
-ApproxTests.cpp:<line number>:
-PASSED:
-  REQUIRE( approx( d ) == 1.23 )
-with expansion:
-  Approx( 1.23 ) == 1.23
-
-ApproxTests.cpp:<line number>:
-PASSED:
-  REQUIRE( approx( d ) == 1.22 )
-with expansion:
-  Approx( 1.23 ) == 1.22
-
-ApproxTests.cpp:<line number>:
-PASSED:
-  REQUIRE( approx( d ) == 1.24 )
-with expansion:
-  Approx( 1.23 ) == 1.24
-
-ApproxTests.cpp:<line number>:
-PASSED:
-  REQUIRE( approx( d ) != 1.25 )
-with expansion:
-  Approx( 1.23 ) != 1.25
-
--------------------------------------------------------------------------------
-Approximate PI
--------------------------------------------------------------------------------
-ApproxTests.cpp:<line number>
-...............................................................................
-
-ApproxTests.cpp:<line number>:
-PASSED:
-  REQUIRE( divide( 22, 7 ) == Approx( 3.141 ).epsilon( 0.001 ) )
-with expansion:
-  3.1428571429 == Approx( 3.141 )
-
-ApproxTests.cpp:<line number>:
-PASSED:
-  REQUIRE( divide( 22, 7 ) != Approx( 3.141 ).epsilon( 0.0001 ) )
-with expansion:
-  3.1428571429 != Approx( 3.141 )
-
--------------------------------------------------------------------------------
-A METHOD_AS_TEST_CASE based test run that succeeds
--------------------------------------------------------------------------------
-ClassTests.cpp:<line number>
-...............................................................................
-
-ClassTests.cpp:<line number>:
-PASSED:
-  REQUIRE( s == "hello" )
-with expansion:
-  "hello" == "hello"
-
--------------------------------------------------------------------------------
-A METHOD_AS_TEST_CASE based test run that fails
--------------------------------------------------------------------------------
-ClassTests.cpp:<line number>
-...............................................................................
-
-ClassTests.cpp:<line number>: FAILED:
-  REQUIRE( s == "world" )
-with expansion:
-  "hello" == "world"
-
--------------------------------------------------------------------------------
-A TEST_CASE_METHOD based test run that succeeds
--------------------------------------------------------------------------------
-ClassTests.cpp:<line number>
-...............................................................................
-
-ClassTests.cpp:<line number>:
-PASSED:
-  REQUIRE( m_a == 1 )
-with expansion:
-  1 == 1
-
--------------------------------------------------------------------------------
-A TEST_CASE_METHOD based test run that fails
--------------------------------------------------------------------------------
-ClassTests.cpp:<line number>
-...............................................................................
-
-ClassTests.cpp:<line number>: FAILED:
-  REQUIRE( m_a == 2 )
-with expansion:
-  1 == 2
-
--------------------------------------------------------------------------------
-Equality checks that should succeed
--------------------------------------------------------------------------------
-ConditionTests.cpp:<line number>
-...............................................................................
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.int_seven == 7 )
-with expansion:
-  7 == 7
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.float_nine_point_one == Approx( 9.1f ) )
-with expansion:
-  9.1f == Approx( 9.1000003815 )
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.double_pi == Approx( 3.1415926535 ) )
-with expansion:
-  3.1415926535 == Approx( 3.1415926535 )
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.str_hello == "hello" )
-with expansion:
-  "hello" == "hello"
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( "hello" == data.str_hello )
-with expansion:
-  "hello" == "hello"
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.str_hello.size() == 5 )
-with expansion:
-  5 == 5
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( x == Approx( 1.3 ) )
-with expansion:
-  1.3 == Approx( 1.3 )
-
--------------------------------------------------------------------------------
-Equality checks that should fail
 -------------------------------------------------------------------------------
 ConditionTests.cpp:<line number>
 ...............................................................................
 
 ConditionTests.cpp:<line number>: FAILED:
-  CHECK( data.int_seven == 6 )
+  CHECK( false != false )
-with expansion:
-  7 == 6
 
 ConditionTests.cpp:<line number>: FAILED:
-  CHECK( data.int_seven == 8 )
+  CHECK( true != true )
-with expansion:
-  7 == 8
-
--------------------------------------------------------------------------------
-Inequality checks that should succeed
--------------------------------------------------------------------------------
-ConditionTests.cpp:<line number>
-...............................................................................
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.int_seven != 6 )
-with expansion:
-  7 != 6
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.int_seven != 8 )
-with expansion:
-  7 != 8
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.float_nine_point_one != Approx( 9.11f ) )
-with expansion:
-  9.1f != Approx( 9.1099996567 )
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.float_nine_point_one != Approx( 9.0f ) )
-with expansion:
-  9.1f != Approx( 9.0 )
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.float_nine_point_one != Approx( 1 ) )
-with expansion:
-  9.1f != Approx( 1.0 )
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.float_nine_point_one != Approx( 0 ) )
-with expansion:
-  9.1f != Approx( 0.0 )
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.double_pi != Approx( 3.1415 ) )
-with expansion:
-  3.1415926535 != Approx( 3.1415 )
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.str_hello != "goodbye" )
-with expansion:
-  "hello" != "goodbye"
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.str_hello != "hell" )
-with expansion:
-  "hello" != "hell"
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.str_hello != "hello1" )
-with expansion:
-  "hello" != "hello1"
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.str_hello.size() != 6 )
-with expansion:
-  5 != 6
-
--------------------------------------------------------------------------------
-Inequality checks that should fail
--------------------------------------------------------------------------------
-ConditionTests.cpp:<line number>
-...............................................................................
 
 ConditionTests.cpp:<line number>: FAILED:
-  CHECK( data.int_seven != 7 )
+  CHECK( !true )
 with expansion:
-  7 != 7
+  false
 
 ConditionTests.cpp:<line number>: FAILED:
-  CHECK( data.float_nine_point_one != Approx( 9.1f ) )
+  CHECK_FALSE( true )
-with expansion:
-  9.1f != Approx( 9.1000003815 )
-
--------------------------------------------------------------------------------
-Ordering comparison checks that should succeed
--------------------------------------------------------------------------------
-ConditionTests.cpp:<line number>
-...............................................................................
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.int_seven < 8 )
-with expansion:
-  7 < 8
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.int_seven > 6 )
-with expansion:
-  7 > 6
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.int_seven > 0 )
-with expansion:
-  7 > 0
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.int_seven > -1 )
-with expansion:
-  7 > -1
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.int_seven >= 7 )
-with expansion:
-  7 >= 7
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.int_seven >= 6 )
-with expansion:
-  7 >= 6
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.int_seven <= 7 )
-with expansion:
-  7 <= 7
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.int_seven <= 8 )
-with expansion:
-  7 <= 8
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.float_nine_point_one > 9 )
-with expansion:
-  9.1f > 9
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.float_nine_point_one < 10 )
-with expansion:
-  9.1f < 10
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.float_nine_point_one < 9.2 )
-with expansion:
-  9.1f < 9.2
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.str_hello <= "hello" )
-with expansion:
-  "hello" <= "hello"
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.str_hello >= "hello" )
-with expansion:
-  "hello" >= "hello"
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.str_hello < "hellp" )
-with expansion:
-  "hello" < "hellp"
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.str_hello < "zebra" )
-with expansion:
-  "hello" < "zebra"
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.str_hello > "hellm" )
-with expansion:
-  "hello" > "hellm"
-
-ConditionTests.cpp:<line number>:
-PASSED:
-  REQUIRE( data.str_hello > "a" )
-with expansion:
-  "hello" > "a"
-
--------------------------------------------------------------------------------
-Ordering comparison checks that should fail
--------------------------------------------------------------------------------
-ConditionTests.cpp:<line number>
-...............................................................................
-
-ConditionTests.cpp:<line number>: FAILED:
-  CHECK( data.int_seven > 7 )
-with expansion:
-  7 > 7
-
-ConditionTests.cpp:<line number>: FAILED:
-  CHECK( data.int_seven < 7 )
-with expansion:
-  7 < 7
 
 ===============================================================================
-test cases: 21 | 16 passed | 3 failed | 2 failed as expected
+test cases: 2 | 1 passed | 1 failed
-assertions: 81 | 73 passed | 4 failed | 4 failed as expected
+assertions: 5 | 1 passed | 4 failed
 

scripts/approvalTests.py

@@ -22,6 +22,7 @@ if len(sys.argv) == 2:
 	cmdPath = sys.argv[1]
 else:
 	cmdPath = os.path.join( catchPath, 'projects/XCode/CatchSelfTest/DerivedData/CatchSelfTest/Build/Products/Debug/CatchSelfTest' )
+#	cmdPath = os.path.join( catchPath, 'projects/CMake/cmake-build-debug/SelfTest' )
 
 overallResult = 0
 
@@ -100,15 +101,15 @@ def approve( baseName, args ):
 			overallResult = 1
 
 # Standard console reporter
-approve( "console.std", ["~_"] )
+approve( "console.std", ["~_", "--order", "lex"] )
 # console reporter, include passes, warn about No Assertions
-approve( "console.sw", ["~_", "-s", "-w", "NoAssertions"] )
+approve( "console.sw", ["~_", "-s", "-w", "NoAssertions", "--order", "lex"] )
 # console reporter, include passes, warn about No Assertions, limit failures to first 4
-approve( "console.swa4", ["~_", "-s", "-w", "NoAssertions", "-x", "4"] )
+approve( "console.swa4", ["~_", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"] )
 # junit reporter, include passes, warn about No Assertions
-approve( "junit.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "junit"] )
+approve( "junit.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"] )
 # xml reporter, include passes, warn about No Assertions
-approve( "xml.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "xml"] )
+approve( "xml.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"] )
 
 if overallResult != 0:
 	print( "run approve.py to approve new baselines" )