mirror of
https://github.com/catchorg/Catch2.git
synced 2024-11-07 06:39:55 +01:00
Add SonarQube Generic Test Data reporter
It outputs reports in the `Generic Execution Test Data` format, see https://docs.sonarqube.org/latest/analysis/generic-test/, specifically https://docs.sonarqube.org/latest/analysis/generic-test/#header-2 Close #1738 (this is a cherry-pick and fixup of that PR)
This commit is contained in:
parent
9a558171d8
commit
51b29ced1a
@ -14,6 +14,7 @@ coverage:
|
||||
- "**/catch_reporter_tap.hpp"
|
||||
- "**/catch_reporter_automake.hpp"
|
||||
- "**/catch_reporter_teamcity.hpp"
|
||||
- "**/catch_reporter_sonarqube.hpp"
|
||||
- "**/external/clara.hpp"
|
||||
|
||||
|
||||
|
@ -12,7 +12,7 @@ Build Systems may refer to low-level tools, like CMake, or larger systems that r
|
||||
|
||||
## Continuous Integration systems
|
||||
|
||||
Probably the most important aspect to using Catch with a build server is the use of different reporters. Catch comes bundled with three reporters that should cover the majority of build servers out there - although adding more for better integration with some is always a possibility (currently we also offer TeamCity, TAP and Automake reporters).
|
||||
Probably the most important aspect to using Catch with a build server is the use of different reporters. Catch comes bundled with three reporters that should cover the majority of build servers out there - although adding more for better integration with some is always a possibility (currently we also offer TeamCity, TAP, Automake and SonarQube reporters).
|
||||
|
||||
Two of these reporters are built in (XML and JUnit) and the third (TeamCity) is included as a separate header. It's possible that the other two may be split out in the future too - as that would make the core of Catch smaller for those that don't need them.
|
||||
|
||||
@ -65,6 +65,10 @@ The Automake Reporter writes out the [meta tags](https://www.gnu.org/software/au
|
||||
|
||||
Because of the incremental nature of Catch's test suites and ability to run specific tests, our implementation of TAP reporter writes out the number of tests in a suite last.
|
||||
|
||||
### SonarQube Reporter
|
||||
```-r sonarqube```
|
||||
[SonarQube Generic Test Data](https://docs.sonarqube.org/latest/analysis/generic-test/) XML format for tests metrics.
|
||||
|
||||
## Low-level tools
|
||||
|
||||
### Precompiled headers (PCHs)
|
||||
|
@ -42,8 +42,8 @@ Tag version and release title should be same as the new version,
|
||||
description should contain the release notes for the current release.
|
||||
Single header version of `catch.hpp` *needs* to be attached as a binary,
|
||||
as that is where the official download link links to. Preferably
|
||||
it should use linux line endings. All non-bundled reporters (Automake,
|
||||
TAP, TeamCity) should also be attached as binaries, as they might be
|
||||
it should use linux line endings. All non-bundled reporters (Automake, TAP,
|
||||
TeamCity, SonarQube) should also be attached as binaries, as they might be
|
||||
dependent on a specific version of the single-include header.
|
||||
|
||||
Since 2.5.0, the release tag and the "binaries" (headers) should be PGP
|
||||
|
@ -29,6 +29,7 @@ Do this in one source file - the same one you have `CATCH_CONFIG_MAIN` or `CATCH
|
||||
Use this when building as part of a TeamCity build to see results as they happen ([code example](../examples/207-Rpt-TeamCityReporter.cpp)).
|
||||
* `tap` writes in the TAP ([Test Anything Protocol](https://en.wikipedia.org/wiki/Test_Anything_Protocol)) format.
|
||||
* `automake` writes in a format that correspond to [automake .trs](https://www.gnu.org/software/automake/manual/html_node/Log-files-generation-and-test-results-recording.html) files
|
||||
* `sonarqube` writes the [SonarQube Generic Test Data](https://docs.sonarqube.org/latest/analysis/generic-test/) XML format.
|
||||
|
||||
You see what reporters are available from the command line by running with `--list-reporters`.
|
||||
|
||||
|
181
include/reporters/catch_reporter_sonarqube.hpp
Normal file
181
include/reporters/catch_reporter_sonarqube.hpp
Normal file
@ -0,0 +1,181 @@
|
||||
/*
|
||||
* Created by Daniel Garcia on 2018-12-04.
|
||||
* Copyright Social Point SL. All rights reserved.
|
||||
*
|
||||
* Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
* file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
*/
|
||||
#ifndef CATCH_REPORTER_SONARQUBE_HPP_INCLUDED
|
||||
#define CATCH_REPORTER_SONARQUBE_HPP_INCLUDED
|
||||
|
||||
|
||||
// Don't #include any Catch headers here - we can assume they are already
|
||||
// included before this header.
|
||||
// This is not good practice in general but is necessary in this case so this
|
||||
// file can be distributed as a single header that works with the main
|
||||
// Catch single header.
|
||||
|
||||
#include <map>
|
||||
|
||||
namespace Catch {
|
||||
|
||||
struct SonarQubeReporter : CumulativeReporterBase<SonarQubeReporter> {
|
||||
|
||||
SonarQubeReporter(ReporterConfig const& config)
|
||||
: CumulativeReporterBase(config)
|
||||
, xml(config.stream()) {
|
||||
m_reporterPrefs.shouldRedirectStdOut = true;
|
||||
m_reporterPrefs.shouldReportAllAssertions = true;
|
||||
}
|
||||
|
||||
~SonarQubeReporter() override;
|
||||
|
||||
static std::string getDescription() {
|
||||
return "Reports test results in the Generic Test Data SonarQube XML format";
|
||||
}
|
||||
|
||||
static std::set<Verbosity> getSupportedVerbosities() {
|
||||
return { Verbosity::Normal };
|
||||
}
|
||||
|
||||
void noMatchingTestCases(std::string const& /*spec*/) override {}
|
||||
|
||||
void testRunStarting(TestRunInfo const& testRunInfo) override {
|
||||
CumulativeReporterBase::testRunStarting(testRunInfo);
|
||||
xml.startElement("testExecutions");
|
||||
xml.writeAttribute("version", "1");
|
||||
}
|
||||
|
||||
void testGroupEnded(TestGroupStats const& testGroupStats) override {
|
||||
CumulativeReporterBase::testGroupEnded(testGroupStats);
|
||||
writeGroup(*m_testGroups.back());
|
||||
}
|
||||
|
||||
void testRunEndedCumulative() override {
|
||||
xml.endElement();
|
||||
}
|
||||
|
||||
void writeGroup(TestGroupNode const& groupNode) {
|
||||
std::map<std::string, TestGroupNode::ChildNodes> testsPerFile;
|
||||
for(auto const& child : groupNode.children)
|
||||
testsPerFile[child->value.testInfo.lineInfo.file].push_back(child);
|
||||
|
||||
for(auto const& kv : testsPerFile)
|
||||
writeTestFile(kv.first.c_str(), kv.second);
|
||||
}
|
||||
|
||||
void writeTestFile(const char* filename, TestGroupNode::ChildNodes const& testCaseNodes) {
|
||||
XmlWriter::ScopedElement e = xml.scopedElement("file");
|
||||
xml.writeAttribute("path", filename);
|
||||
|
||||
for(auto const& child : testCaseNodes)
|
||||
writeTestCase(*child);
|
||||
}
|
||||
|
||||
void writeTestCase(TestCaseNode const& testCaseNode) {
|
||||
// All test cases have exactly one section - which represents the
|
||||
// test case itself. That section may have 0-n nested sections
|
||||
assert(testCaseNode.children.size() == 1);
|
||||
SectionNode const& rootSection = *testCaseNode.children.front();
|
||||
writeSection("", rootSection, testCaseNode.value.testInfo.okToFail());
|
||||
}
|
||||
|
||||
void writeSection(std::string const& rootName, SectionNode const& sectionNode, bool okToFail) {
|
||||
std::string name = trim(sectionNode.stats.sectionInfo.name);
|
||||
if(!rootName.empty())
|
||||
name = rootName + '/' + name;
|
||||
|
||||
if(!sectionNode.assertions.empty() || !sectionNode.stdOut.empty() || !sectionNode.stdErr.empty()) {
|
||||
XmlWriter::ScopedElement e = xml.scopedElement("testCase");
|
||||
xml.writeAttribute("name", name);
|
||||
xml.writeAttribute("duration", static_cast<long>(sectionNode.stats.durationInSeconds * 1000));
|
||||
|
||||
writeAssertions(sectionNode, okToFail);
|
||||
}
|
||||
|
||||
for(auto const& childNode : sectionNode.childSections)
|
||||
writeSection(name, *childNode, okToFail);
|
||||
}
|
||||
|
||||
void writeAssertions(SectionNode const& sectionNode, bool okToFail) {
|
||||
for(auto const& assertion : sectionNode.assertions)
|
||||
writeAssertion( assertion, okToFail);
|
||||
}
|
||||
|
||||
void writeAssertion(AssertionStats const& stats, bool okToFail) {
|
||||
AssertionResult const& result = stats.assertionResult;
|
||||
if(!result.isOk()) {
|
||||
std::string elementName;
|
||||
if(okToFail) {
|
||||
elementName = "skipped";
|
||||
}
|
||||
else {
|
||||
switch(result.getResultType()) {
|
||||
case ResultWas::ThrewException:
|
||||
case ResultWas::FatalErrorCondition:
|
||||
elementName = "error";
|
||||
break;
|
||||
case ResultWas::ExplicitFailure:
|
||||
elementName = "failure";
|
||||
break;
|
||||
case ResultWas::ExpressionFailed:
|
||||
elementName = "failure";
|
||||
break;
|
||||
case ResultWas::DidntThrowException:
|
||||
elementName = "failure";
|
||||
break;
|
||||
|
||||
// We should never see these here:
|
||||
case ResultWas::Info:
|
||||
case ResultWas::Warning:
|
||||
case ResultWas::Ok:
|
||||
case ResultWas::Unknown:
|
||||
case ResultWas::FailureBit:
|
||||
case ResultWas::Exception:
|
||||
elementName = "internalError";
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
XmlWriter::ScopedElement e = xml.scopedElement(elementName);
|
||||
|
||||
ReusableStringStream messageRss;
|
||||
messageRss << result.getTestMacroName() << "(" << result.getExpression() << ")";
|
||||
xml.writeAttribute("message", messageRss.str());
|
||||
|
||||
ReusableStringStream textRss;
|
||||
if (stats.totals.assertions.total() > 0) {
|
||||
textRss << "FAILED:\n";
|
||||
if (result.hasExpression()) {
|
||||
textRss << "\t" << result.getExpressionInMacro() << "\n";
|
||||
}
|
||||
if (result.hasExpandedExpression()) {
|
||||
textRss << "with expansion:\n\t" << result.getExpandedExpression() << "\n";
|
||||
}
|
||||
}
|
||||
|
||||
if(!result.getMessage().empty())
|
||||
textRss << result.getMessage() << "\n";
|
||||
|
||||
for(auto const& msg : stats.infoMessages)
|
||||
if(msg.type == ResultWas::Info)
|
||||
textRss << msg.message << "\n";
|
||||
|
||||
textRss << "at " << result.getSourceInfo();
|
||||
xml.writeText(textRss.str(), false);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
XmlWriter xml;
|
||||
};
|
||||
|
||||
#ifdef CATCH_IMPL
|
||||
SonarQubeReporter::~SonarQubeReporter() {}
|
||||
#endif
|
||||
|
||||
CATCH_REGISTER_REPORTER( "sonarqube", SonarQubeReporter )
|
||||
|
||||
} // end namespace Catch
|
||||
|
||||
#endif // CATCH_REPORTER_SONARQUBE_HPP_INCLUDED
|
@ -282,6 +282,7 @@ set(REPORTER_HEADERS
|
||||
${HEADER_DIR}/reporters/catch_reporter_tap.hpp
|
||||
${HEADER_DIR}/reporters/catch_reporter_teamcity.hpp
|
||||
${HEADER_DIR}/reporters/catch_reporter_xml.h
|
||||
${HEADER_DIR}/reporters/catch_reporter_sonarqube.hpp
|
||||
)
|
||||
set(REPORTER_SOURCES
|
||||
${HEADER_DIR}/reporters/catch_reporter_bases.cpp
|
||||
|
1730
projects/SelfTest/Baselines/sonarqube.sw.approved.txt
Normal file
1730
projects/SelfTest/Baselines/sonarqube.sw.approved.txt
Normal file
File diff suppressed because it is too large
Load Diff
@ -13,6 +13,7 @@
|
||||
#include "reporters/catch_reporter_teamcity.hpp"
|
||||
#include "reporters/catch_reporter_tap.hpp"
|
||||
#include "reporters/catch_reporter_automake.hpp"
|
||||
#include "reporters/catch_reporter_sonarqube.hpp"
|
||||
|
||||
|
||||
// Some example tag aliases
|
||||
|
@ -29,6 +29,7 @@ filelocParser = re.compile(r'''
|
||||
lineNumberParser = re.compile(r' line="[0-9]*"')
|
||||
hexParser = re.compile(r'\b(0[xX][0-9a-fA-F]+)\b')
|
||||
durationsParser = re.compile(r' time="[0-9]*\.[0-9]*"')
|
||||
sonarqubeDurationParser = re.compile(r' duration="[0-9]+"')
|
||||
timestampsParser = re.compile(r'\d{4}-\d{2}-\d{2}T\d{2}\:\d{2}\:\d{2}Z')
|
||||
versionParser = re.compile(r'Catch v[0-9]+\.[0-9]+\.[0-9]+(-develop\.[0-9]+)?')
|
||||
nullParser = re.compile(r'\b(__null|nullptr)\b')
|
||||
@ -138,6 +139,7 @@ def filterLine(line, isCompact):
|
||||
|
||||
# strip durations and timestamps
|
||||
line = durationsParser.sub(' time="{duration}"', line)
|
||||
line = sonarqubeDurationParser.sub(' duration="{duration}"', line)
|
||||
line = timestampsParser.sub('{iso8601-timestamp}', line)
|
||||
line = specialCaseParser.sub('file:\g<1>', line)
|
||||
line = errnoParser.sub('errno', line)
|
||||
@ -204,6 +206,8 @@ approve("junit.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "No
|
||||
approve("xml.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex", "--rng-seed", "1"])
|
||||
# compact reporter, include passes, warn about No Assertions
|
||||
approve('compact.sw', ['~[!nonportable]~[!benchmark]~[approvals]', '-s', '-w', 'NoAssertions', '-r', 'compact', '--order', 'lex', "--rng-seed", "1"])
|
||||
# sonarqube reporter, include passes, warn about No Assertions
|
||||
approve("sonarqube.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-r", "sonarqube", "--order", "lex", "--rng-seed", "1"])
|
||||
|
||||
if overallResult != 0:
|
||||
print("If these differences are expected, run approve.py to approve new baselines.")
|
||||
|
@ -156,7 +156,7 @@ def performUpdates(version):
|
||||
# We probably should have some kind of convention to select which reporters need to be copied automagically,
|
||||
# but this works for now
|
||||
import shutil
|
||||
for rep in ('automake', 'tap', 'teamcity'):
|
||||
for rep in ('automake', 'tap', 'teamcity', 'sonarqube'):
|
||||
sourceFile = os.path.join(catchPath, 'include/reporters/catch_reporter_{}.hpp'.format(rep))
|
||||
destFile = os.path.join(catchPath, 'single_include', 'catch2', 'catch_reporter_{}.hpp'.format(rep))
|
||||
shutil.copyfile(sourceFile, destFile)
|
||||
|
Loading…
Reference in New Issue
Block a user