From f133277910c59ba5526a081e94d5e4ece406e517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ho=C5=99e=C5=88ovsk=C3=BD?= Date: Fri, 15 May 2020 21:00:19 +0200 Subject: [PATCH] Add status attribute to JUnit's section reporting This brings our output inline with GTest's. We do not handle skipped tests properly, but that should be currently less important than having the attribute exist with proper value for non-skipped tests. Thanks @joda-01. Closes #1899 --- include/reporters/catch_reporter_junit.cpp | 5 + .../SelfTest/Baselines/junit.sw.approved.txt | 1122 ++++++++--------- 2 files changed, 566 insertions(+), 561 deletions(-) diff --git a/include/reporters/catch_reporter_junit.cpp b/include/reporters/catch_reporter_junit.cpp index 7416a523..31f0802f 100644 --- a/include/reporters/catch_reporter_junit.cpp +++ b/include/reporters/catch_reporter_junit.cpp @@ -193,6 +193,11 @@ namespace Catch { xml.writeAttribute( "name", name ); } xml.writeAttribute( "time", ::Catch::Detail::stringify( sectionNode.stats.durationInSeconds ) ); + // This is not ideal, but it should be enough to mimic gtest's + // junit output. + // Ideally the JUnit reporter would also handle `skipTest` + // events and write those out appropriately. + xml.writeAttribute( "status", "run" ); writeAssertions( sectionNode ); diff --git a/projects/SelfTest/Baselines/junit.sw.approved.txt b/projects/SelfTest/Baselines/junit.sw.approved.txt index 603e4d0b..57237c15 100644 --- a/projects/SelfTest/Baselines/junit.sw.approved.txt +++ b/projects/SelfTest/Baselines/junit.sw.approved.txt @@ -6,17 +6,17 @@ - - - - - - - - - - - + + + + + + + + + + + FAILED: 1514 @@ -29,11 +29,11 @@ This would not be caught previously Nor would this - - - - - + + + + + FAILED: expected exception @@ -41,7 +41,7 @@ answer := 42 Exception.tests.cpp: - + FAILED: REQUIRE_NOTHROW( thisThrows() ) @@ -50,10 +50,10 @@ answer := 42 Exception.tests.cpp: - - - - + + + + FAILED: CHECK( f() == 0 ) @@ -62,13 +62,13 @@ with expansion: Misc.tests.cpp: - - - - - - - + + + + + + + FAILED: CHECK( false != false ) @@ -120,14 +120,14 @@ FAILED: Condition.tests.cpp: - - - - - - - - + + + + + + + + FAILED: REQUIRE( s == "world" ) @@ -136,8 +136,8 @@ with expansion: Class.tests.cpp: - - + + FAILED: REQUIRE( Template_Fixture_2<TestType>::m_a.size() == 1 ) @@ -146,7 +146,7 @@ with expansion: Class.tests.cpp: - + FAILED: REQUIRE( Template_Fixture_2<TestType>::m_a.size() == 1 ) @@ -155,7 +155,7 @@ with expansion: Class.tests.cpp: - + FAILED: REQUIRE( Template_Fixture_2<TestType>::m_a.size() == 1 ) @@ -164,7 +164,7 @@ with expansion: Class.tests.cpp: - + FAILED: REQUIRE( Template_Fixture_2<TestType>::m_a.size() == 1 ) @@ -173,11 +173,11 @@ with expansion: Class.tests.cpp: - - - - - + + + + + FAILED: REQUIRE( Template_Fixture_2<TestType>{}.m_a.size() < 2 ) @@ -186,7 +186,7 @@ with expansion: Class.tests.cpp: - + FAILED: REQUIRE( Template_Fixture_2<TestType>{}.m_a.size() < 2 ) @@ -195,7 +195,7 @@ with expansion: Class.tests.cpp: - + FAILED: REQUIRE( Template_Fixture_2<TestType>{}.m_a.size() < 2 ) @@ -204,7 +204,7 @@ with expansion: Class.tests.cpp: - + FAILED: REQUIRE( Template_Fixture_2<TestType>{}.m_a.size() < 2 ) @@ -213,11 +213,11 @@ with expansion: Class.tests.cpp: - - - - - + + + + + FAILED: REQUIRE( Template_Fixture<TestType>::m_a == 2 ) @@ -226,7 +226,7 @@ with expansion: Class.tests.cpp: - + FAILED: REQUIRE( Template_Fixture<TestType>::m_a == 2 ) @@ -235,7 +235,7 @@ with expansion: Class.tests.cpp: - + FAILED: REQUIRE( Template_Fixture<TestType>::m_a == 2 ) @@ -244,10 +244,10 @@ with expansion: Class.tests.cpp: - - - - + + + + FAILED: REQUIRE( Nttp_Fixture<V>::value == 0 ) @@ -256,7 +256,7 @@ with expansion: Class.tests.cpp: - + FAILED: REQUIRE( Nttp_Fixture<V>::value == 0 ) @@ -265,7 +265,7 @@ with expansion: Class.tests.cpp: - + FAILED: REQUIRE( Nttp_Fixture<V>::value == 0 ) @@ -274,10 +274,10 @@ with expansion: Class.tests.cpp: - - - - + + + + FAILED: REQUIRE( m_a == 2 ) @@ -286,25 +286,25 @@ with expansion: Class.tests.cpp: - - - - - - - - - - - + + + + + + + + + + + FAILED: to infinity and beyond Misc.tests.cpp: - - + + FAILED: CHECK( &o1 == &o2 ) @@ -320,9 +320,9 @@ with expansion: Tricky.tests.cpp: - - - + + + FAILED: {Unknown expression after the reported line} @@ -330,38 +330,38 @@ unexpected exception Exception.tests.cpp: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + FAILED: CHECK_THAT( testStringForMatching(), Contains("not there", Catch::CaseSensitive::No) ) @@ -378,10 +378,10 @@ with expansion: Matchers.tests.cpp: - - - - + + + + FAILED: REQUIRE_NOTHROW( throwCustom() ) @@ -389,7 +389,7 @@ custom exception - not std Exception.tests.cpp: - + FAILED: REQUIRE_THROWS_AS( throwCustom(), std::exception ) @@ -397,16 +397,16 @@ custom exception - not std Exception.tests.cpp: - + FAILED: custom std exception Exception.tests.cpp: - - - + + + FAILED: CHECK_THAT( testStringForMatching(), EndsWith("Substring") ) @@ -423,10 +423,10 @@ with expansion: Matchers.tests.cpp: - - - - + + + + FAILED: CHECK( data.int_seven == 6 ) @@ -519,9 +519,9 @@ with expansion: Condition.tests.cpp: - - - + + + FAILED: CHECK_THAT( testStringForMatching(), Equals("this string contains 'ABC' as a substring") ) @@ -539,8 +539,8 @@ with expansion: Matchers.tests.cpp: - - + + FAILED: CHECK_THROWS_MATCHES( doesNotThrow(), SpecialException, ExceptionMatcher{1} ) @@ -552,7 +552,7 @@ FAILED: Matchers.tests.cpp: - + FAILED: CHECK_THROWS_MATCHES( throwsAsInt(1), SpecialException, ExceptionMatcher{1} ) @@ -566,7 +566,7 @@ Unknown exception Matchers.tests.cpp: - + FAILED: CHECK_THROWS_MATCHES( throwsSpecialException(3), SpecialException, ExceptionMatcher{1} ) @@ -582,12 +582,12 @@ with expansion: Matchers.tests.cpp: - - - - - - + + + + + + FAILED: CHECK_THROWS_AS( thisThrows(), std::string ) @@ -606,77 +606,77 @@ expected exception Exception.tests.cpp: - + FAILED: This is a failure Message.tests.cpp: - + FAILED: Message.tests.cpp: - + FAILED: This is a failure Message.tests.cpp: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + FAILED: REQUIRE( a == 1 ) @@ -687,7 +687,7 @@ so should this Message.tests.cpp: - + FAILED: CHECK( a == 1 ) @@ -708,7 +708,7 @@ and this, but later Message.tests.cpp: - + FAILED: REQUIRE( i < 10 ) @@ -719,7 +719,7 @@ i := 10 Message.tests.cpp: - + FAILED: CHECK( data.int_seven != 7 ) @@ -756,14 +756,14 @@ with expansion: Condition.tests.cpp: - - - - - - - - + + + + + + + + FAILED: CHECK_THAT( testStringForMatching(), (Contains("string") || Contains("different")) && Contains("random") ) @@ -773,8 +773,8 @@ with expansion: Matchers.tests.cpp: - - + + FAILED: CHECK_THAT( testStringForMatching(), !Contains("substring") ) @@ -783,7 +783,7 @@ with expansion: Matchers.tests.cpp: - + FAILED: REQUIRE_THROWS_WITH( thisThrows(), "should fail" ) @@ -792,18 +792,18 @@ with expansion: Exception.tests.cpp: - - - + + + FAILED: custom exception Exception.tests.cpp: - - - + + + FAILED: CHECK( data.int_seven > 7 ) @@ -938,98 +938,98 @@ with expansion: Condition.tests.cpp: - - - - + + + + FAILED: Message from section one Message.tests.cpp: - + FAILED: Message from section two Message.tests.cpp: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + FAILED: CHECK( truthy(false) ) @@ -1038,7 +1038,7 @@ with expansion: Decomposition.tests.cpp: - + FAILED: CHECK_THAT( testStringForMatching(), Matches("this STRING contains 'abc' as a substring") ) @@ -1064,19 +1064,19 @@ with expansion: Matchers.tests.cpp: - - - - - - - - - - - - - + + + + + + + + + + + + + A string sent directly to stdout @@ -1085,14 +1085,14 @@ A string sent directly to stderr A string sent to stderr via clog - - + + Message from section one Message from section two - + FAILED: CHECK_THAT( testStringForMatching(), StartsWith("This String") ) @@ -1109,31 +1109,31 @@ with expansion: Matchers.tests.cpp: - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + FAILED: CHECK( s1 == s2 ) @@ -1149,103 +1149,103 @@ with expansion: Misc.tests.cpp: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + FAILED: For some reason someone is throwing a string literal! Exception.tests.cpp: - - - - - - - - - - - - + + + + + + + + + + + + FAILED: 3.14 Exception.tests.cpp: - - - - - - - + + + + + + + FAILED: CHECK_THAT( empty, Approx(t1) ) @@ -1254,7 +1254,7 @@ with expansion: Matchers.tests.cpp: - + FAILED: CHECK_THAT( v1, Approx(v2) ) @@ -1263,12 +1263,12 @@ with expansion: Matchers.tests.cpp: - - - - - - + + + + + + FAILED: CHECK_THAT( v, VectorContains(-1) ) @@ -1284,7 +1284,7 @@ with expansion: Matchers.tests.cpp: - + FAILED: CHECK_THAT( empty, Contains(v) ) @@ -1300,7 +1300,7 @@ with expansion: Matchers.tests.cpp: - + FAILED: CHECK_THAT( v, Equals(v2) ) @@ -1330,7 +1330,7 @@ with expansion: Matchers.tests.cpp: - + FAILED: CHECK_THAT( v, UnorderedEquals(empty) ) @@ -1360,15 +1360,15 @@ with expansion: Matchers.tests.cpp: - - + + FAILED: unexpected exception Exception.tests.cpp: - + FAILED: CHECK( thisThrows() == 0 ) @@ -1376,7 +1376,7 @@ expected exception Exception.tests.cpp: - + FAILED: REQUIRE( thisThrows() == 0 ) @@ -1384,7 +1384,7 @@ expected exception Exception.tests.cpp: - + FAILED: CHECK( thisThrows() == 0 ) @@ -1392,33 +1392,33 @@ expected exception Exception.tests.cpp: - + FAILED: unexpected exception Exception.tests.cpp: - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + FAILED: CHECKED_ELSE( flag ) @@ -1434,8 +1434,8 @@ with expansion: Misc.tests.cpp: - - + + FAILED: CHECKED_IF( flag ) @@ -1451,27 +1451,27 @@ with expansion: Misc.tests.cpp: - - - - - - + + + + + + FAILED: Previous info should not be seen Message.tests.cpp: - + FAILED: previous unscoped info SHOULD not be seen Message.tests.cpp: - - + + FAILED: CHECK( b > a ) @@ -1480,7 +1480,7 @@ with expansion: Misc.tests.cpp: - + FAILED: CHECK( b > a ) @@ -1489,15 +1489,15 @@ with expansion: Misc.tests.cpp: - - - - - - - - - + + + + + + + + + FAILED: CHECK( ( fib[i] % 2 ) == 0 ) @@ -1547,8 +1547,8 @@ Testing if fib[7] (21) is even Misc.tests.cpp: - - + + FAILED: REQUIRE( a == b ) @@ -1557,14 +1557,14 @@ with expansion: Misc.tests.cpp: - - - - - - - - + + + + + + + + FAILED: REQUIRE( false ) @@ -1572,15 +1572,15 @@ this SHOULD be seen Message.tests.cpp: - - - - - - - - - + + + + + + + + + FAILED: REQUIRE( false ) @@ -1589,7 +1589,7 @@ this SHOULD also be seen Message.tests.cpp: - + FAILED: CHECK( false ) @@ -1597,16 +1597,16 @@ this SHOULD be seen only ONCE Message.tests.cpp: - - - - - - - - - - + + + + + + + + + + FAILED: REQUIRE( false ) @@ -1614,7 +1614,7 @@ FAILED: Misc.tests.cpp: - + FAILED: REQUIRE( false ) @@ -1623,9 +1623,9 @@ i := 7 Message.tests.cpp: - - - + + + FAILED: CHECK( false ) @@ -1645,16 +1645,16 @@ Count 4 to 6... Message.tests.cpp: - - - - - - - - - - + + + + + + + + + + FAILED: REQUIRE( std::string( "first" ) == "second" ) @@ -1663,51 +1663,51 @@ with expansion: Tricky.tests.cpp: - - - - - - - - - - - - + + + + + + + + + + + + FAILED: Why would you throw a std::string? Exception.tests.cpp: - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + This would not be caught previously A string sent directly to stdout