From f3fe2dcb110a2fbdf55ff02f2dfeea072e782cdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ho=C5=99e=C5=88ovsk=C3=BD?= Date: Fri, 15 May 2020 21:00:19 +0200 Subject: [PATCH] Add status attribute to JUnit's section reporting This brings our output inline with GTest's. We do not handle skipped tests properly, but that should be currently less important than having the attribute exist with proper value for non-skipped tests. Thanks @joda-01. Closes #1899 --- src/catch2/reporters/catch_reporter_junit.cpp | 5 + .../SelfTest/Baselines/junit.sw.approved.txt | 1232 ++++++++--------- 2 files changed, 621 insertions(+), 616 deletions(-) diff --git a/src/catch2/reporters/catch_reporter_junit.cpp b/src/catch2/reporters/catch_reporter_junit.cpp index c18b03e8..7e1d6829 100644 --- a/src/catch2/reporters/catch_reporter_junit.cpp +++ b/src/catch2/reporters/catch_reporter_junit.cpp @@ -198,6 +198,11 @@ namespace Catch { xml.writeAttribute( "name", name ); } xml.writeAttribute( "time", ::Catch::Detail::stringify( sectionNode.stats.durationInSeconds ) ); + // This is not ideal, but it should be enough to mimic gtest's + // junit output. + // Ideally the JUnit reporter would also handle `skipTest` + // events and write those out appropriately. + xml.writeAttribute( "status", "run" ); writeAssertions( sectionNode ); diff --git a/tests/SelfTest/Baselines/junit.sw.approved.txt b/tests/SelfTest/Baselines/junit.sw.approved.txt index a0bfb944..a29e54aa 100644 --- a/tests/SelfTest/Baselines/junit.sw.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.approved.txt @@ -6,17 +6,17 @@ - - - - - - - - - - - + + + + + + + + + + + FAILED: 1514 @@ -29,11 +29,11 @@ This would not be caught previously Nor would this - - - - - + + + + + FAILED: expected exception @@ -41,7 +41,7 @@ answer := 42 Exception.tests.cpp: - + FAILED: REQUIRE_NOTHROW( thisThrows() ) @@ -50,10 +50,10 @@ answer := 42 Exception.tests.cpp: - - - - + + + + FAILED: CHECK( f() == 0 ) @@ -62,13 +62,13 @@ with expansion: Misc.tests.cpp: - - - - - - - + + + + + + + FAILED: CHECK( false != false ) @@ -120,14 +120,14 @@ FAILED: Condition.tests.cpp: - - - - - - - - + + + + + + + + FAILED: REQUIRE( s == "world" ) @@ -136,8 +136,8 @@ with expansion: Class.tests.cpp: - - + + FAILED: REQUIRE( Template_Fixture_2<TestType>::m_a.size() == 1 ) @@ -146,7 +146,7 @@ with expansion: Class.tests.cpp: - + FAILED: REQUIRE( Template_Fixture_2<TestType>::m_a.size() == 1 ) @@ -155,7 +155,7 @@ with expansion: Class.tests.cpp: - + FAILED: REQUIRE( Template_Fixture_2<TestType>::m_a.size() == 1 ) @@ -164,7 +164,7 @@ with expansion: Class.tests.cpp: - + FAILED: REQUIRE( Template_Fixture_2<TestType>::m_a.size() == 1 ) @@ -173,11 +173,11 @@ with expansion: Class.tests.cpp: - - - - - + + + + + FAILED: REQUIRE( Template_Fixture_2<TestType>{}.m_a.size() < 2 ) @@ -186,7 +186,7 @@ with expansion: Class.tests.cpp: - + FAILED: REQUIRE( Template_Fixture_2<TestType>{}.m_a.size() < 2 ) @@ -195,7 +195,7 @@ with expansion: Class.tests.cpp: - + FAILED: REQUIRE( Template_Fixture_2<TestType>{}.m_a.size() < 2 ) @@ -204,7 +204,7 @@ with expansion: Class.tests.cpp: - + FAILED: REQUIRE( Template_Fixture_2<TestType>{}.m_a.size() < 2 ) @@ -213,11 +213,11 @@ with expansion: Class.tests.cpp: - - - - - + + + + + FAILED: REQUIRE( Template_Fixture<TestType>::m_a == 2 ) @@ -226,7 +226,7 @@ with expansion: Class.tests.cpp: - + FAILED: REQUIRE( Template_Fixture<TestType>::m_a == 2 ) @@ -235,7 +235,7 @@ with expansion: Class.tests.cpp: - + FAILED: REQUIRE( Template_Fixture<TestType>::m_a == 2 ) @@ -244,10 +244,10 @@ with expansion: Class.tests.cpp: - - - - + + + + FAILED: REQUIRE( Nttp_Fixture<V>::value == 0 ) @@ -256,7 +256,7 @@ with expansion: Class.tests.cpp: - + FAILED: REQUIRE( Nttp_Fixture<V>::value == 0 ) @@ -265,7 +265,7 @@ with expansion: Class.tests.cpp: - + FAILED: REQUIRE( Nttp_Fixture<V>::value == 0 ) @@ -274,10 +274,10 @@ with expansion: Class.tests.cpp: - - - - + + + + FAILED: REQUIRE( m_a == 2 ) @@ -286,25 +286,25 @@ with expansion: Class.tests.cpp: - - - - - - - - - - - + + + + + + + + + + + FAILED: to infinity and beyond Misc.tests.cpp: - - + + FAILED: CHECK( &o1 == &o2 ) @@ -320,9 +320,9 @@ with expansion: Tricky.tests.cpp: - - - + + + FAILED: {Unknown expression after the reported line} @@ -330,53 +330,53 @@ unexpected exception Exception.tests.cpp: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + FAILED: CHECK_THAT( testStringForMatching(), Contains("not there", Catch::CaseSensitive::No) ) @@ -393,10 +393,10 @@ with expansion: Matchers.tests.cpp: - - - - + + + + FAILED: REQUIRE_NOTHROW( throwCustom() ) @@ -404,7 +404,7 @@ custom exception - not std Exception.tests.cpp: - + FAILED: REQUIRE_THROWS_AS( throwCustom(), std::exception ) @@ -412,16 +412,16 @@ custom exception - not std Exception.tests.cpp: - + FAILED: custom std exception Exception.tests.cpp: - - - + + + FAILED: CHECK_THAT( testStringForMatching(), EndsWith("Substring") ) @@ -438,10 +438,10 @@ with expansion: Matchers.tests.cpp: - - - - + + + + FAILED: CHECK( data.int_seven == 6 ) @@ -534,9 +534,9 @@ with expansion: Condition.tests.cpp: - - - + + + FAILED: CHECK_THAT( testStringForMatching(), Equals("this string contains 'ABC' as a substring") ) @@ -554,8 +554,8 @@ with expansion: Matchers.tests.cpp: - - + + FAILED: CHECK_THROWS_MATCHES( doesNotThrow(), SpecialException, ExceptionMatcher{1} ) @@ -567,7 +567,7 @@ FAILED: Matchers.tests.cpp: - + FAILED: CHECK_THROWS_MATCHES( throwsAsInt(1), SpecialException, ExceptionMatcher{1} ) @@ -581,7 +581,7 @@ Unknown exception Matchers.tests.cpp: - + FAILED: CHECK_THROWS_MATCHES( throwsSpecialException(3), SpecialException, ExceptionMatcher{1} ) @@ -597,12 +597,12 @@ with expansion: Matchers.tests.cpp: - - - - - - + + + + + + FAILED: CHECK_THROWS_AS( thisThrows(), std::string ) @@ -621,77 +621,77 @@ expected exception Exception.tests.cpp: - + FAILED: This is a failure Message.tests.cpp: - + FAILED: Message.tests.cpp: - + FAILED: This is a failure Message.tests.cpp: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + FAILED: REQUIRE( a == 1 ) @@ -702,7 +702,7 @@ so should this Message.tests.cpp: - + FAILED: CHECK( a == 1 ) @@ -723,7 +723,7 @@ and this, but later Message.tests.cpp: - + FAILED: REQUIRE( i < 10 ) @@ -734,7 +734,7 @@ i := 10 Message.tests.cpp: - + FAILED: CHECK( data.int_seven != 7 ) @@ -771,14 +771,14 @@ with expansion: Condition.tests.cpp: - - - - - - - - + + + + + + + + FAILED: CHECK_THAT( testStringForMatching(), (Contains("string") || Contains("different")) && Contains("random") ) @@ -788,8 +788,8 @@ with expansion: Matchers.tests.cpp: - - + + FAILED: CHECK_THAT( testStringForMatching(), !Contains("substring") ) @@ -798,7 +798,7 @@ with expansion: Matchers.tests.cpp: - + FAILED: REQUIRE_THROWS_WITH( thisThrows(), "should fail" ) @@ -807,18 +807,18 @@ with expansion: Exception.tests.cpp: - - - + + + FAILED: custom exception Exception.tests.cpp: - - - + + + FAILED: CHECK( data.int_seven > 7 ) @@ -953,99 +953,99 @@ with expansion: Condition.tests.cpp: - - - - + + + + FAILED: Message from section one Message.tests.cpp: - + FAILED: Message from section two Message.tests.cpp: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + FAILED: CHECK( truthy(false) ) @@ -1054,7 +1054,7 @@ with expansion: Decomposition.tests.cpp: - + FAILED: CHECK_THAT( testStringForMatching(), Matches("this STRING contains 'abc' as a substring") ) @@ -1080,19 +1080,19 @@ with expansion: Matchers.tests.cpp: - - - - - - - - - - - - - + + + + + + + + + + + + + A string sent directly to stdout @@ -1101,14 +1101,14 @@ A string sent directly to stderr A string sent to stderr via clog - - + + Message from section one Message from section two - + FAILED: CHECK_THAT( testStringForMatching(), StartsWith("This String") ) @@ -1125,34 +1125,34 @@ with expansion: Matchers.tests.cpp: - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + FAILED: CHECK( s1 == s2 ) @@ -1168,108 +1168,108 @@ with expansion: Misc.tests.cpp: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + FAILED: For some reason someone is throwing a string literal! Exception.tests.cpp: - - - - - - - - - - - - + + + + + + + + + + + + FAILED: 3.14 Exception.tests.cpp: - - - - - - - - - - - - + + + + + + + + + + + + FAILED: CHECK_THAT( empty, Approx(t1) ) @@ -1278,7 +1278,7 @@ with expansion: Matchers.tests.cpp: - + FAILED: CHECK_THAT( v1, Approx(v2) ) @@ -1287,12 +1287,12 @@ with expansion: Matchers.tests.cpp: - - - - - - + + + + + + FAILED: CHECK_THAT( v, VectorContains(-1) ) @@ -1308,7 +1308,7 @@ with expansion: Matchers.tests.cpp: - + FAILED: CHECK_THAT( empty, Contains(v) ) @@ -1324,7 +1324,7 @@ with expansion: Matchers.tests.cpp: - + FAILED: CHECK_THAT( v, Equals(v2) ) @@ -1354,7 +1354,7 @@ with expansion: Matchers.tests.cpp: - + FAILED: CHECK_THAT( v, UnorderedEquals(empty) ) @@ -1384,15 +1384,15 @@ with expansion: Matchers.tests.cpp: - - + + FAILED: unexpected exception Exception.tests.cpp: - + FAILED: CHECK( thisThrows() == 0 ) @@ -1400,7 +1400,7 @@ expected exception Exception.tests.cpp: - + FAILED: REQUIRE( thisThrows() == 0 ) @@ -1408,7 +1408,7 @@ expected exception Exception.tests.cpp: - + FAILED: CHECK( thisThrows() == 0 ) @@ -1416,35 +1416,35 @@ expected exception Exception.tests.cpp: - + FAILED: unexpected exception Exception.tests.cpp: - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + FAILED: CHECKED_ELSE( flag ) @@ -1460,8 +1460,8 @@ with expansion: Misc.tests.cpp: - - + + FAILED: CHECKED_IF( flag ) @@ -1477,35 +1477,35 @@ with expansion: Misc.tests.cpp: - - - - - - - - - - - - - - + + + + + + + + + + + + + + FAILED: Previous info should not be seen Message.tests.cpp: - + FAILED: previous unscoped info SHOULD not be seen Message.tests.cpp: - - + + FAILED: CHECK( b > a ) @@ -1514,7 +1514,7 @@ with expansion: Misc.tests.cpp: - + FAILED: CHECK( b > a ) @@ -1523,15 +1523,15 @@ with expansion: Misc.tests.cpp: - - - - - - - - - + + + + + + + + + FAILED: CHECK( ( fib[i] % 2 ) == 0 ) @@ -1581,13 +1581,13 @@ Testing if fib[7] (21) is even Misc.tests.cpp: - - - - - - - + + + + + + + FAILED: REQUIRE( a == b ) @@ -1596,16 +1596,16 @@ with expansion: Misc.tests.cpp: - - - - - - - - - - + + + + + + + + + + FAILED: REQUIRE( false ) @@ -1613,15 +1613,15 @@ this SHOULD be seen Message.tests.cpp: - - - - - - - - - + + + + + + + + + FAILED: REQUIRE( false ) @@ -1630,7 +1630,7 @@ this SHOULD also be seen Message.tests.cpp: - + FAILED: CHECK( false ) @@ -1638,19 +1638,19 @@ this SHOULD be seen only ONCE Message.tests.cpp: - - - - - - - - - - - - - + + + + + + + + + + + + + FAILED: REQUIRE( false ) @@ -1658,7 +1658,7 @@ FAILED: Misc.tests.cpp: - + FAILED: REQUIRE( false ) @@ -1667,9 +1667,9 @@ i := 7 Message.tests.cpp: - - - + + + FAILED: CHECK( false ) @@ -1689,16 +1689,16 @@ Count 4 to 6... Message.tests.cpp: - - - - - - - - - - + + + + + + + + + + FAILED: REQUIRE( std::string( "first" ) == "second" ) @@ -1707,62 +1707,62 @@ with expansion: Tricky.tests.cpp: - - - - - - - - - - - - + + + + + + + + + + + + FAILED: Why would you throw a std::string? Exception.tests.cpp: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This would not be caught previously A string sent directly to stdout