Add default reporter for Bazel integration (#2399)

When the added Bazel configuration flag is enabled,
a default JUnit reporter will be added if the XML
envrioment variable is defined.
Fix include paths for generated config header.
Enable Bazel config by default when building with
Bazel.


Co-authored-by: Martin Hořeňovský <martin.horenovsky@gmail.com>

.bazelrc

@@ -0,0 +1,4 @@
+build:gcc9 --cxxopt=-std=c++2a
+build:clang13 --cxxopt=-std=c++17
+build:vs2019 --cxxopt=/std:c++17
+build:vs2022 --cxxopt=/std:c++17

BUILD.bazel

@@ -1,13 +1,14 @@
 # Load the cc_library rule.
 load("@rules_cc//cc:defs.bzl", "cc_library")
-
 load("@bazel_skylib//rules:expand_template.bzl", "expand_template")
 
 expand_template(
-    name = "catch_user_config.hpp",
+    name = "catch_user_config",
     out = "catch2/catch_user_config.hpp",
     substitutions = {
         "#cmakedefine CATCH_CONFIG_ANDROID_LOGWRITE": "",
+        "#cmakedefine CATCH_CONFIG_BAZEL_SUPPORT": "#define CATCH_CONFIG_BAZEL_SUPPORT",
+        "#cmakedefine CATCH_CONFIG_NO_COLOUR_WIN32": "",
         "#cmakedefine CATCH_CONFIG_COLOUR_WIN32": "",
         "#cmakedefine CATCH_CONFIG_COUNTER": "",
         "#cmakedefine CATCH_CONFIG_CPP11_TO_STRING": "",
@@ -55,23 +56,36 @@ expand_template(
     template = "src/catch2/catch_user_config.hpp.in",
 )
 
+# Generated header library, modifies the include prefix to account for
+# generation path so that we can include <catch2/catch_user_config.hpp>
+# correctly.
+cc_library(
+    name = "catch2_generated",
+    hdrs = ["catch2/catch_user_config.hpp"],
+    include_prefix = ".",  # to manipulate -I of dependenices
+    visibility = ["//visibility:public"],
+)
+
 # Static library, without main.
 cc_library(
     name = "catch2",
-    hdrs = glob(["src/catch2/**/*.hpp"]) + ["catch_user_config.hpp"],
-    srcs = glob(["src/catch2/**/*.cpp"],
-                exclude=[ "src/catch2/internal/catch_main.cpp"]),
-    visibility = ["//visibility:public"],
-    linkstatic = True,
+    srcs = glob(
+        ["src/catch2/**/*.cpp"],
+        exclude = ["src/catch2/internal/catch_main.cpp"],
+    ),
+    hdrs = glob(["src/catch2/**/*.hpp"]),
     includes = ["src/"],
+    linkstatic = True,
+    visibility = ["//visibility:public"],
+    deps = [":catch2_generated"],
 )
 
 # Static library, with main.
 cc_library(
     name = "catch2_main",
     srcs = ["src/catch2/internal/catch_main.cpp"],
-    deps = [":catch2"],
-    visibility = ["//visibility:public"],
-    linkstatic = True,
     includes = ["src/"],
+    linkstatic = True,
+    visibility = ["//visibility:public"],
+    deps = [":catch2"],
 )

CMake/CatchConfigOptions.cmake

@@ -26,6 +26,7 @@ endmacro()
 
 set(_OverridableOptions
   "ANDROID_LOGWRITE"
+  "BAZEL_SUPPORT"
   "COLOUR_WIN32"
   "COUNTER"
   "CPP11_TO_STRING"

WORKSPACE

@@ -1,11 +1,14 @@
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
 http_archive(
     name = "bazel_skylib",
+    strip_prefix = "bazel-skylib-2a87d4a62af886fb320883aba102255aba87275e",
     urls = [
-        "https://github.com/Vertexwahn/bazel-skylib/archive/b0cd4bbd4bf4af76c380e1f8fafdbe3964161aff.tar.gz",
+        "https://github.com/bazelbuild/bazel-skylib/archive/2a87d4a62af886fb320883aba102255aba87275e.tar.gz",
     ],
-    strip_prefix = "bazel-skylib-b0cd4bbd4bf4af76c380e1f8fafdbe3964161aff",
-    sha256 = "e57f3ff541c65678f3c2b344c73945531838e86ea0be71c63eea862ab43e792b",
+    sha256 = "d847b08d6702d2779e9eb399b54ff8920fa7521dc45e3e53572d1d8907767de7",
 )
+
 load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
+
 bazel_skylib_workspace()

docs/configuration.md

@@ -8,6 +8,7 @@
 [stdout](#stdout)<br>
 [Fallback stringifier](#fallback-stringifier)<br>
 [Default reporter](#default-reporter)<br>
+[Bazel support](#bazel-support)<br>
 [C++11 toggles](#c11-toggles)<br>
 [C++17 toggles](#c17-toggles)<br>
 [Other toggles](#other-toggles)<br>
@@ -96,6 +97,12 @@ This means that defining `CATCH_CONFIG_DEFAULT_REPORTER` to `"console"`
 is equivalent with the out-of-the-box experience.
 
 
+## Bazel support
+When `CATCH_CONFIG_BAZEL_SUPPORT` is defined, Catch2 will register a `JUnit`
+reporter writing to a path pointed by `XML_OUTPUT_FILE` provided by Bazel.
+
+> `CATCH_CONFIG_BAZEL_SUPPORT` was [introduced](https://github.com/catchorg/Catch2/pull/2399) in Catch2 X.Y.Z.
+
 ## C++11 toggles
 
     CATCH_CONFIG_CPP11_TO_STRING // Use `std::to_string`

src/catch2/catch_config.cpp

@@ -69,6 +69,28 @@ namespace Catch {
             } );
         }
 
+#if defined( CATCH_CONFIG_BAZEL_SUPPORT )
+        // Register a JUnit reporter for Bazel. Bazel sets an environment
+        // variable with the path to XML output. If this file is written to
+        // during test, Bazel will not generate a default XML output.
+        // This allows the XML output file to contain higher level of detail
+        // than what is possible otherwise.
+#    if defined( _MSC_VER )
+        // On Windows getenv throws a warning as there is no input validation,
+        // since the key is hardcoded, this should not be an issue.
+#        pragma warning( push )
+#        pragma warning( disable : 4996 )
+#    endif
+        const auto bazelOutputFilePtr = std::getenv( "XML_OUTPUT_FILE" );
+#    if defined( _MSC_VER )
+#        pragma warning( pop )
+#    endif
+        if ( bazelOutputFilePtr != nullptr ) {
+            m_data.reporterSpecifications.push_back(
+                { "junit", std::string( bazelOutputFilePtr ), {}, {} } );
+        }
+#endif
+
         bool defaultOutputUsed = false;
         m_reporterStreams.reserve( m_data.reporterSpecifications.size() );
         for ( auto const& reporterSpec : m_data.reporterSpecifications ) {

src/catch2/catch_user_config.hpp.in

@@ -165,6 +165,7 @@
 // ------
 
 
+#cmakedefine CATCH_CONFIG_BAZEL_SUPPORT
 #cmakedefine CATCH_CONFIG_DISABLE_EXCEPTIONS
 #cmakedefine CATCH_CONFIG_DISABLE_EXCEPTIONS_CUSTOM_HANDLER
 #cmakedefine CATCH_CONFIG_DISABLE

tests/ExtraTests/CMakeLists.txt

@@ -121,6 +121,14 @@ set_tests_properties(
     FAIL_REGULAR_EXPRESSION "abort;terminate;fatal"
 )
 
+add_executable( BazelReporter ${TESTS_DIR}/X30-BazelReporter.cpp )
+target_compile_definitions( BazelReporter PRIVATE CATCH_CONFIG_BAZEL_SUPPORT )
+target_link_libraries(BazelReporter Catch2_buildall_interface)
+add_test(NAME CATCH_CONFIG_BAZEL_REPORTER-1
+  COMMAND
+  "${PYTHON_EXECUTABLE}" "${CATCH_DIR}/tests/TestScripts/testBazelReporter.py" $<TARGET_FILE:BazelReporter> "${CMAKE_CURRENT_BINARY_DIR}"
+)
+
 
 # The default handler on Windows leads to the just-in-time debugger firing,
 # which makes this test unsuitable for CI and headless runs, as it opens

tests/ExtraTests/X30-BazelReporter.cpp

@@ -0,0 +1,17 @@
+
+//              Copyright Catch2 Authors
+// Distributed under the Boost Software License, Version 1.0.
+//   (See accompanying file LICENSE_1_0.txt or copy at
+//        https://www.boost.org/LICENSE_1_0.txt)
+
+// SPDX-License-Identifier: BSL-1.0
+
+/**\file
+ * Test the Bazel report functionality with a simple set
+ * of dummy test cases.
+ */
+
+#include <catch2/catch_test_macros.hpp>
+
+TEST_CASE( "Passing test case" ) { REQUIRE( 1 == 1 ); }
+TEST_CASE( "Failing test case" ) { REQUIRE( 2 == 1 ); }

tests/TestScripts/testBazelReporter.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+
+#              Copyright Catch2 Authors
+# Distributed under the Boost Software License, Version 1.0.
+#   (See accompanying file LICENSE_1_0.txt or copy at
+#        https://www.boost.org/LICENSE_1_0.txt)
+
+# SPDX-License-Identifier: BSL-1.0
+
+import os
+import re
+import sys
+import xml.etree.ElementTree as ET
+import subprocess
+
+"""
+Test that Catch2 recognizes `XML_OUTPUT_FILE` env variable and creates
+a junit reporter that writes to the provided path.
+
+Requires 2 arguments, path to Catch2 binary configured with
+`CATCH_CONFIG_BAZEL_SUPPORT`, and the output directory for the output file.
+"""
+if len(sys.argv) != 3:
+    print("Wrong number of arguments: {}".format(len(sys.argv)))
+    print("Usage: {} test-bin-path output-dir".format(sys.argv[0]))
+    exit(1)
+
+
+bin_path = os.path.abspath(sys.argv[1])
+output_dir = os.path.abspath(sys.argv[2])
+xml_out_path = os.path.join(output_dir, "bazel-out.xml")
+
+# Ensure no file exists from previous test runs
+if os.path.isfile(xml_out_path):
+    os.remove(xml_out_path)
+
+print('bin path:', bin_path)
+print('xml out path:', xml_out_path)
+
+env = os.environ.copy()
+env["XML_OUTPUT_FILE"] = xml_out_path
+test_passing = True
+
+try:
+    ret = subprocess.run(
+        bin_path,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        check=True,
+        universal_newlines=True,
+        env=env
+    )
+    stdout = ret.stdout
+except subprocess.SubprocessError as ex:
+    if ex.returncode == 1:
+        # The test cases are allowed to fail.
+        test_passing = False
+        stdout = ex.stdout
+    else:
+        print('Could not run "{}"'.format(args))
+        print("Return code: {}".format(ex.returncode))
+        print("stdout: {}".format(ex.stdout))
+        print("stderr: {}".format(ex.stdout))
+        raise
+
+# Check for valid XML output
+try:
+    tree = ET.parse(xml_out_path)
+except ET.ParseError as ex:
+    print("Invalid XML: '{}'".format(ex))
+    raise
+except FileNotFoundError as ex:
+    print("Could not find '{}'".format(xml_out_path))
+    raise
+
+bin_name = os.path.basename(bin_path)
+# Check for matching testsuite
+if not tree.find('.//testsuite[@name="{}"]'.format(bin_name)):
+    print("Could not find '{}' testsuite".format(bin_name))
+    exit(2)
+
+# Check that we haven't disabled the default reporter
+summary_test_cases = re.findall(r'test cases: \d* \| \d* passed \| \d* failed', stdout)
+if len(summary_test_cases) == 0:
+    print("Could not find test summary in {}".format(stdout))
+    exit(2)
+
+total, passed, failed = [int(s) for s in summary_test_cases[0].split() if s.isdigit()]
+
+if failed == 0 and not test_passing:
+    print("Expected at least 1 test failure!")
+    exit(2)
+
+if len(tree.findall('.//testcase')) != total:
+    print("Unexpected number of test cases!")
+    exit(2)
+
+if len(tree.findall('.//failure')) != failed:
+    print("Unexpected number of test failures!")
+    exit(2)
+
+if (passed + failed) != total:
+    print("Something has gone very wrong, ({} + {}) != {}".format(passed, failed, total))
+    exit(2)