From 239fa28e4679b2889b715f98243d2af380beadaf Mon Sep 17 00:00:00 2001 From: Phil Nash Date: Fri, 28 Jun 2013 16:23:14 +0100 Subject: [PATCH] Added run without -s to approvals --- .../SelfTest/Baselines/approvedResults.txt | 681 ++++++++++++++++++ scripts/approvalTests.py | 1 + 2 files changed, 682 insertions(+) diff --git a/projects/SelfTest/Baselines/approvedResults.txt b/projects/SelfTest/Baselines/approvedResults.txt index 8279682b..1de7f385 100644 --- a/projects/SelfTest/Baselines/approvedResults.txt +++ b/projects/SelfTest/Baselines/approvedResults.txt @@ -1,4 +1,685 @@ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +CatchSelfTest is a Catch v1.0 b1 host application. +Run with -? for options + +------------------------------------------------------------------------------- +./failing/TestClass/failingCase +------------------------------------------------------------------------------- +ClassTests.cpp:35 +............................................................................... + +ClassTests.cpp:28: FAILED: + REQUIRE( s == "world" ) +with expansion: + "hello" == "world" + +------------------------------------------------------------------------------- +./failing/Fixture/failingCase +------------------------------------------------------------------------------- +ClassTests.cpp:53 +............................................................................... + +ClassTests.cpp:55: FAILED: + REQUIRE( m_a == 2 ) +with expansion: + 1 == 2 + +------------------------------------------------------------------------------- +./failing/conditions/equality +------------------------------------------------------------------------------- +ConditionTests.cpp:67 +............................................................................... + +ConditionTests.cpp:71: FAILED: + CHECK( data.int_seven == 6 ) +with expansion: + 7 == 6 + +ConditionTests.cpp:72: FAILED: + CHECK( data.int_seven == 8 ) +with expansion: + 7 == 8 + +ConditionTests.cpp:73: FAILED: + CHECK( data.int_seven == 0 ) +with expansion: + 7 == 0 + +ConditionTests.cpp:74: FAILED: + CHECK( data.float_nine_point_one == Approx( 9.11f ) ) +with expansion: + 9.1 == Approx( 9.11 ) + +ConditionTests.cpp:75: FAILED: + CHECK( data.float_nine_point_one == Approx( 9.0f ) ) +with expansion: + 9.1 == Approx( 9 ) + +ConditionTests.cpp:76: FAILED: + CHECK( data.float_nine_point_one == Approx( 1 ) ) +with expansion: + 9.1 == Approx( 1 ) + +ConditionTests.cpp:77: FAILED: + CHECK( data.float_nine_point_one == Approx( 0 ) ) +with expansion: + 9.1 == Approx( 0 ) + +ConditionTests.cpp:78: FAILED: + CHECK( data.double_pi == Approx( 3.1415 ) ) +with expansion: + 3.1415926535 == Approx( 3.1415 ) + +ConditionTests.cpp:79: FAILED: + CHECK( data.str_hello == "goodbye" ) +with expansion: + "hello" == "goodbye" + +ConditionTests.cpp:80: FAILED: + CHECK( data.str_hello == "hell" ) +with expansion: + "hello" == "hell" + +ConditionTests.cpp:81: FAILED: + CHECK( data.str_hello == "hello1" ) +with expansion: + "hello" == "hello1" + +ConditionTests.cpp:82: FAILED: + CHECK( data.str_hello.size() == 6 ) +with expansion: + 5 == 6 + +ConditionTests.cpp:85: FAILED: + CHECK( x == Approx( 1.301 ) ) +with expansion: + 1.3 == Approx( 1.301 ) + +------------------------------------------------------------------------------- +./failing/conditions/inequality +------------------------------------------------------------------------------- +ConditionTests.cpp:107 +............................................................................... + +ConditionTests.cpp:111: FAILED: + CHECK( data.int_seven != 7 ) +with expansion: + 7 != 7 + +ConditionTests.cpp:112: FAILED: + CHECK( data.float_nine_point_one != Approx( 9.1f ) ) +with expansion: + 9.1 != Approx( 9.1 ) + +ConditionTests.cpp:113: FAILED: + CHECK( data.double_pi != Approx( 3.1415926535 ) ) +with expansion: + 3.1415926535 != Approx( 3.14159 ) + +ConditionTests.cpp:114: FAILED: + CHECK( data.str_hello != "hello" ) +with expansion: + "hello" != "hello" + +ConditionTests.cpp:115: FAILED: + CHECK( data.str_hello.size() != 5 ) +with expansion: + 5 != 5 + +------------------------------------------------------------------------------- +./failing/conditions/ordered +------------------------------------------------------------------------------- +ConditionTests.cpp:148 +............................................................................... + +ConditionTests.cpp:152: FAILED: + CHECK( data.int_seven > 7 ) +with expansion: + 7 > 7 + +ConditionTests.cpp:153: FAILED: + CHECK( data.int_seven < 7 ) +with expansion: + 7 < 7 + +ConditionTests.cpp:154: FAILED: + CHECK( data.int_seven > 8 ) +with expansion: + 7 > 8 + +ConditionTests.cpp:155: FAILED: + CHECK( data.int_seven < 6 ) +with expansion: + 7 < 6 + +ConditionTests.cpp:156: FAILED: + CHECK( data.int_seven < 0 ) +with expansion: + 7 < 0 + +ConditionTests.cpp:157: FAILED: + CHECK( data.int_seven < -1 ) +with expansion: + 7 < -1 + +ConditionTests.cpp:159: FAILED: + CHECK( data.int_seven >= 8 ) +with expansion: + 7 >= 8 + +ConditionTests.cpp:160: FAILED: + CHECK( data.int_seven <= 6 ) +with expansion: + 7 <= 6 + +ConditionTests.cpp:162: FAILED: + CHECK( data.float_nine_point_one < 9 ) +with expansion: + 9.1 < 9 + +ConditionTests.cpp:163: FAILED: + CHECK( data.float_nine_point_one > 10 ) +with expansion: + 9.1 > 10 + +ConditionTests.cpp:164: FAILED: + CHECK( data.float_nine_point_one > 9.2 ) +with expansion: + 9.1 > 9.199999999999999 + +ConditionTests.cpp:166: FAILED: + CHECK( data.str_hello > "hello" ) +with expansion: + "hello" > "hello" + +ConditionTests.cpp:167: FAILED: + CHECK( data.str_hello < "hello" ) +with expansion: + "hello" < "hello" + +ConditionTests.cpp:168: FAILED: + CHECK( data.str_hello > "hellp" ) +with expansion: + "hello" > "hellp" + +ConditionTests.cpp:169: FAILED: + CHECK( data.str_hello > "z" ) +with expansion: + "hello" > "z" + +ConditionTests.cpp:170: FAILED: + CHECK( data.str_hello < "hellm" ) +with expansion: + "hello" < "hellm" + +ConditionTests.cpp:171: FAILED: + CHECK( data.str_hello < "a" ) +with expansion: + "hello" < "a" + +ConditionTests.cpp:173: FAILED: + CHECK( data.str_hello >= "z" ) +with expansion: + "hello" >= "z" + +ConditionTests.cpp:174: FAILED: + CHECK( data.str_hello <= "a" ) +with expansion: + "hello" <= "a" + +------------------------------------------------------------------------------- +./failing/conditions/not +------------------------------------------------------------------------------- +ConditionTests.cpp:333 +............................................................................... + +ConditionTests.cpp:337: FAILED: + CHECK( false != false ) + +ConditionTests.cpp:338: FAILED: + CHECK( true != true ) + +ConditionTests.cpp:339: FAILED: + CHECK( !true ) +with expansion: + false + +ConditionTests.cpp:340: FAILED: + CHECK_FALSE( true ) + +ConditionTests.cpp:342: FAILED: + CHECK( !trueValue ) +with expansion: + false + +ConditionTests.cpp:343: FAILED: + CHECK_FALSE( trueValue ) +with expansion: + !true + +ConditionTests.cpp:345: FAILED: + CHECK( !(1 == 1) ) +with expansion: + false + +ConditionTests.cpp:346: FAILED: + CHECK_FALSE( 1 == 1 ) +with expansion: + !(1 == 1) + +------------------------------------------------------------------------------- +./failing/exceptions/explicit +------------------------------------------------------------------------------- +ExceptionTests.cpp:38 +............................................................................... + +ExceptionTests.cpp:40: FAILED: + CHECK_THROWS_AS( thisThrows() ) +due to unexpected exception with message: + expected exception + +ExceptionTests.cpp:41: FAILED: + CHECK_THROWS_AS( thisDoesntThrow() ) +because no exception was thrown where one was expected: + +ExceptionTests.cpp:42: FAILED: + CHECK_NOTHROW( thisThrows() ) +due to unexpected exception with message: + expected exception + +------------------------------------------------------------------------------- +./failing/exceptions/implicit +------------------------------------------------------------------------------- +ExceptionTests.cpp:45 +............................................................................... + +ExceptionTests.cpp:45: FAILED: +due to unexpected exception with message: + unexpected exception + +------------------------------------------------------------------------------- +./failing/exceptions/implicit/2 +------------------------------------------------------------------------------- +ExceptionTests.cpp:51 +............................................................................... + +ExceptionTests.cpp:53: FAILED: + {Unknown expression after the reported line} +due to unexpected exception with message: + unexpected exception + +------------------------------------------------------------------------------- +./failing/exceptions/implicit/3 + section name +------------------------------------------------------------------------------- +ExceptionTests.cpp:59 +............................................................................... + +ExceptionTests.cpp:59: FAILED: +due to unexpected exception with message: + unexpected exception + +------------------------------------------------------------------------------- +./failing/exceptions/implicit/4 +------------------------------------------------------------------------------- +ExceptionTests.cpp:66 +............................................................................... + +ExceptionTests.cpp:68: FAILED: + CHECK( thisThrows() == 0 ) +due to unexpected exception with message: + expected exception + +------------------------------------------------------------------------------- +./failing/exceptions/custom +------------------------------------------------------------------------------- +ExceptionTests.cpp:108 +............................................................................... + +ExceptionTests.cpp:108: FAILED: +due to unexpected exception with message: + custom exception + +------------------------------------------------------------------------------- +./failing/exceptions/custom/nothrow +------------------------------------------------------------------------------- +ExceptionTests.cpp:119 +............................................................................... + +ExceptionTests.cpp:121: FAILED: + REQUIRE_NOTHROW( throwCustom() ) +due to unexpected exception with message: + custom exception - not std + +------------------------------------------------------------------------------- +./failing/exceptions/custom/throw +------------------------------------------------------------------------------- +ExceptionTests.cpp:124 +............................................................................... + +ExceptionTests.cpp:126: FAILED: + REQUIRE_THROWS_AS( throwCustom() ) +due to unexpected exception with message: + custom exception - not std + +------------------------------------------------------------------------------- +./failing/exceptions/custom/double +------------------------------------------------------------------------------- +ExceptionTests.cpp:130 +............................................................................... + +ExceptionTests.cpp:130: FAILED: +due to unexpected exception with message: + 3.14 + +------------------------------------------------------------------------------- +./failing/message/info/1 +------------------------------------------------------------------------------- +MessageTests.cpp:21 +............................................................................... + +MessageTests.cpp:26: FAILED: + REQUIRE( a == 1 ) +with expansion: + 2 == 1 +with messages: + this message should be logged + so should this + +------------------------------------------------------------------------------- +./mixed/message/info/2 +------------------------------------------------------------------------------- +MessageTests.cpp:29 +............................................................................... + +MessageTests.cpp:37: FAILED: + CHECK( a == 1 ) +with expansion: + 2 == 1 +with message: + this message should be logged + +MessageTests.cpp:41: FAILED: + CHECK( a == 0 ) +with expansion: + 2 == 0 +with message: + and this, but later + +------------------------------------------------------------------------------- +./failing/message/fail +------------------------------------------------------------------------------- +MessageTests.cpp:48 +............................................................................... + +MessageTests.cpp:51: FAILED: +explicitly with message: + This is a failure + +------------------------------------------------------------------------------- +./failing/message/sections + one +------------------------------------------------------------------------------- +MessageTests.cpp:56 +............................................................................... + +MessageTests.cpp:58: FAILED: +explicitly with message: + Message from section one + +------------------------------------------------------------------------------- +./failing/message/sections + two +------------------------------------------------------------------------------- +MessageTests.cpp:61 +............................................................................... + +MessageTests.cpp:63: FAILED: +explicitly with message: + Message from section two + +Message from section one +Message from section two +------------------------------------------------------------------------------- +./mixed/message/scoped +------------------------------------------------------------------------------- +MessageTests.cpp:80 +............................................................................... + +MessageTests.cpp:86: FAILED: + REQUIRE( i < 10 ) +with expansion: + 10 < 10 +with messages: + current counter 10 + i := 10 + +------------------------------------------------------------------------------- +just failure +------------------------------------------------------------------------------- +MessageTests.cpp:99 +............................................................................... + +MessageTests.cpp:101: FAILED: +explicitly with message: + Previous info should not be seen + +------------------------------------------------------------------------------- +./mixed/Misc/Sections/nested2 + s1 + s2 +------------------------------------------------------------------------------- +MiscTests.cpp:55 +............................................................................... + +MiscTests.cpp:57: FAILED: + REQUIRE( a == b ) +with expansion: + 1 == 2 + +------------------------------------------------------------------------------- +./mixed/Misc/Sections/loops + s1 +------------------------------------------------------------------------------- +MiscTests.cpp:97 +............................................................................... + +MiscTests.cpp:99: FAILED: + CHECK( b > a ) +with expansion: + 0 > 1 + +------------------------------------------------------------------------------- +./mixed/Misc/loops +------------------------------------------------------------------------------- +MiscTests.cpp:104 +............................................................................... + +MiscTests.cpp:111: FAILED: + CHECK( ( fib[i] % 2 ) == 0 ) +with expansion: + 1 == 0 +with message: + Testing if fib[0] (1) is even + +MiscTests.cpp:111: FAILED: + CHECK( ( fib[i] % 2 ) == 0 ) +with expansion: + 1 == 0 +with message: + Testing if fib[1] (1) is even + +MiscTests.cpp:111: FAILED: + CHECK( ( fib[i] % 2 ) == 0 ) +with expansion: + 1 == 0 +with message: + Testing if fib[3] (3) is even + +MiscTests.cpp:111: FAILED: + CHECK( ( fib[i] % 2 ) == 0 ) +with expansion: + 1 == 0 +with message: + Testing if fib[4] (5) is even + +MiscTests.cpp:111: FAILED: + CHECK( ( fib[i] % 2 ) == 0 ) +with expansion: + 1 == 0 +with message: + Testing if fib[6] (13) is even + +MiscTests.cpp:111: FAILED: + CHECK( ( fib[i] % 2 ) == 0 ) +with expansion: + 1 == 0 +with message: + Testing if fib[7] (21) is even + +Some information +An error +------------------------------------------------------------------------------- +./failing/info +------------------------------------------------------------------------------- +MiscTests.cpp:133 +............................................................................... + +MiscTests.cpp:138: FAILED: + REQUIRE( false ) +with messages: + hi + i := 7 + +------------------------------------------------------------------------------- +./failing/checkedif +------------------------------------------------------------------------------- +MiscTests.cpp:154 +............................................................................... + +MiscTests.cpp:143: FAILED: + CHECKED_IF( flag ) +with expansion: + false + +MiscTests.cpp:156: FAILED: + REQUIRE( testCheckedIf( false ) ) +with expansion: + false + +------------------------------------------------------------------------------- +./failing/checkedelse +------------------------------------------------------------------------------- +MiscTests.cpp:172 +............................................................................... + +MiscTests.cpp:161: FAILED: + CHECKED_ELSE( flag ) +with expansion: + false + +MiscTests.cpp:174: FAILED: + REQUIRE( testCheckedElse( false ) ) +with expansion: + false + +------------------------------------------------------------------------------- +./manual/onechar +------------------------------------------------------------------------------- +MiscTests.cpp:189 +............................................................................... + +MiscTests.cpp:192: FAILED: + REQUIRE( false ) +with message: + 3 + +------------------------------------------------------------------------------- +./failing/matchers/Contains +------------------------------------------------------------------------------- +MiscTests.cpp:219 +............................................................................... + +MiscTests.cpp:221: FAILED: + CHECK_THAT( testStringForMatching() Contains( "not there" ) ) +with expansion: + "this string contains 'abc' as a substring" contains: "not there" + +------------------------------------------------------------------------------- +./failing/matchers/StartsWith +------------------------------------------------------------------------------- +MiscTests.cpp:224 +............................................................................... + +MiscTests.cpp:226: FAILED: + CHECK_THAT( testStringForMatching() StartsWith( "string" ) ) +with expansion: + "this string contains 'abc' as a substring" starts with: "string" + +------------------------------------------------------------------------------- +./failing/matchers/EndsWith +------------------------------------------------------------------------------- +MiscTests.cpp:229 +............................................................................... + +MiscTests.cpp:231: FAILED: + CHECK_THAT( testStringForMatching() EndsWith( "this" ) ) +with expansion: + "this string contains 'abc' as a substring" ends with: "this" + +------------------------------------------------------------------------------- +./failing/matchers/Equals +------------------------------------------------------------------------------- +MiscTests.cpp:234 +............................................................................... + +MiscTests.cpp:236: FAILED: + CHECK_THAT( testStringForMatching() Equals( "something else" ) ) +with expansion: + "this string contains 'abc' as a substring" equals: "something else" + +Message from section one +Message from section two +Some information +An error +Message from section one +Message from section two +Some information +An error +hello +hello +------------------------------------------------------------------------------- +./failing/Tricky/non streamable type +------------------------------------------------------------------------------- +TrickyTests.cpp:88 +............................................................................... + +TrickyTests.cpp:95: FAILED: + CHECK( &o1 == &o2 ) +with expansion: + 0x == 0x + +TrickyTests.cpp:96: FAILED: + CHECK( o1 == o2 ) +with expansion: + {?} == {?} + +------------------------------------------------------------------------------- +./failing/string literals +------------------------------------------------------------------------------- +TrickyTests.cpp:104 +............................................................................... + +TrickyTests.cpp:106: FAILED: + REQUIRE( std::string( "first" ) == "second" ) +with expansion: + "first" == "second" + +=============================================================================== +113 test cases - 34 failed (686 assertions - 87 failed) + + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ CatchSelfTest is a Catch v1.0 b1 host application. Run with -? for options diff --git a/scripts/approvalTests.py b/scripts/approvalTests.py index 14a5354e..d0eba1cf 100644 --- a/scripts/approvalTests.py +++ b/scripts/approvalTests.py @@ -20,6 +20,7 @@ else: cmdPath = os.path.join( catchPath, 'projects/XCode4/CatchSelfTest/DerivedData/CatchSelfTest/Build/Products/Debug/CatchSelfTest' ) f = open( rawResultsPath, 'w' ) +subprocess.call([ cmdPath, "~dummy", "-r", "console" ], stdout=f, stderr=f ) subprocess.call([ cmdPath, "~dummy", "-s", "-w", "NoAssertions", "-r", "console" ], stdout=f, stderr=f ) subprocess.call([ cmdPath, "~dummy", "-s", "-w", "NoAssertions", "-r", "console", "-a", "4" ], stdout=f, stderr=f ) subprocess.call([ cmdPath, "~dummy", "-s", "-w", "NoAssertions", "-r", "junit" ], stdout=f, stderr=f )