Merge pull request #2023 from globberwops/feature/add-reporter-and-output-dir

Add REPORTER and OUTPUT_DIR args to catch_discover_tests

contrib/Catch.cmake

@@ -33,6 +33,10 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``.
                          [TEST_SUFFIX suffix]
                          [PROPERTIES name1 value1...]
                          [TEST_LIST var]
+                         [REPORTER reporter]
+                         [OUTPUT_DIR dir]
+                         [OUTPUT_PREFIX prefix}
+                         [OUTPUT_SUFFIX suffix]
     )
 
   ``catch_discover_tests`` sets up a post-build command on the test executable
@@ -90,6 +94,28 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``.
     executable is being used in multiple calls to ``catch_discover_tests()``.
     Note that this variable is only available in CTest.
 
+  ``REPORTER reporter``
+    Use the specified reporter when running the test case. The reporter will
+    be passed to the Catch executable as ``--reporter reporter``.
+
+  ``OUTPUT_DIR dir``
+    If specified, the parameter is passed along as
+    ``--out dir/<test_name>`` to Catch executable. The actual file name is the
+    same as the test name. This should be used instead of
+    ``EXTRA_ARGS --out foo`` to avoid race conditions writing the result output
+    when using parallel test execution.
+
+  ``OUTPUT_PREFIX prefix``
+    May be used in conjunction with ``OUTPUT_DIR``.
+    If specified, ``prefix`` is added to each output file name, like so
+    ``--out dir/prefix<test_name>``.
+
+  ``OUTPUT_SUFFIX suffix``
+    May be used in conjunction with ``OUTPUT_DIR``.
+    If specified, ``suffix`` is added to each output file name, like so
+    ``--out dir/<test_name>suffix``. This can be used to add a file extension to
+    the output e.g. ".xml".
+
 #]=======================================================================]
 
 #------------------------------------------------------------------------------
@@ -97,7 +123,7 @@ function(catch_discover_tests TARGET)
   cmake_parse_arguments(
     ""
     ""
-    "TEST_PREFIX;TEST_SUFFIX;WORKING_DIRECTORY;TEST_LIST"
+    "TEST_PREFIX;TEST_SUFFIX;WORKING_DIRECTORY;TEST_LIST;REPORTER;OUTPUT_DIR;OUTPUT_PREFIX;OUTPUT_SUFFIX"
     "TEST_SPEC;EXTRA_ARGS;PROPERTIES"
     ${ARGN}
   )
@@ -110,7 +136,7 @@ function(catch_discover_tests TARGET)
   endif()
 
   ## Generate a unique name based on the extra arguments
-  string(SHA1 args_hash "${_TEST_SPEC} ${_EXTRA_ARGS}")
+  string(SHA1 args_hash "${_TEST_SPEC} ${_EXTRA_ARGS} ${_REPORTER} ${_OUTPUT_DIR} ${_OUTPUT_PREFIX} ${_OUTPUT_SUFFIX}")
   string(SUBSTRING ${args_hash} 0 7 args_hash)
 
   # Define rule to generate test list for aforementioned test executable
@@ -134,6 +160,10 @@ function(catch_discover_tests TARGET)
             -D "TEST_PREFIX=${_TEST_PREFIX}"
             -D "TEST_SUFFIX=${_TEST_SUFFIX}"
             -D "TEST_LIST=${_TEST_LIST}"
+            -D "TEST_REPORTER=${_REPORTER}"
+            -D "TEST_OUTPUT_DIR=${_OUTPUT_DIR}"
+            -D "TEST_OUTPUT_PREFIX=${_OUTPUT_PREFIX}"
+            -D "TEST_OUTPUT_SUFFIX=${_OUTPUT_SUFFIX}"
             -D "CTEST_FILE=${ctest_tests_file}"
             -P "${_CATCH_DISCOVER_TESTS_SCRIPT}"
     VERBATIM

contrib/CatchAddTests.cmake

@@ -6,6 +6,10 @@ set(suffix "${TEST_SUFFIX}")
 set(spec ${TEST_SPEC})
 set(extra_args ${TEST_EXTRA_ARGS})
 set(properties ${TEST_PROPERTIES})
+set(reporter ${TEST_REPORTER})
+set(output_dir ${TEST_OUTPUT_DIR})
+set(output_prefix ${TEST_OUTPUT_PREFIX})
+set(output_suffix ${TEST_OUTPUT_SUFFIX})
 set(script)
 set(suite)
 set(tests)
@@ -48,6 +52,43 @@ endif()
 
 string(REPLACE "\n" ";" output "${output}")
 
+# Run test executable to get list of available reporters
+execute_process(
+  COMMAND ${TEST_EXECUTOR} "${TEST_EXECUTABLE}" ${spec} --list-reporters
+  OUTPUT_VARIABLE reporters_output
+  RESULT_VARIABLE reporters_result
+)
+if(${reporters_result} EQUAL 0)
+  message(WARNING
+    "Test executable '${TEST_EXECUTABLE}' contains no reporters!\n"
+  )
+elseif(${reporters_result} LESS 0)
+  message(FATAL_ERROR
+    "Error running test executable '${TEST_EXECUTABLE}':\n"
+    "  Result: ${reporters_result}\n"
+    "  Output: ${reporters_output}\n"
+  )
+endif()
+string(FIND "${reporters_output}" "${reporter}" reporter_is_valid)
+if(reporter AND ${reporter_is_valid} EQUAL -1)
+  message(FATAL_ERROR
+    "\"${reporter}\" is not a valid reporter!\n"
+  )
+endif()
+
+# Prepare reporter
+if(reporter)
+  set(reporter_arg "--reporter ${reporter}")
+endif()
+
+# Prepare output dir
+if(output_dir AND NOT IS_ABSOLUTE ${output_dir})
+  set(output_dir "${TEST_WORKING_DIR}/${output_dir}")
+  if(NOT EXISTS ${output_dir})
+    file(MAKE_DIRECTORY ${output_dir})
+  endif()
+endif()
+
 # Parse output
 foreach(line ${output})
   set(test ${line})
@@ -56,6 +97,12 @@ foreach(line ${output})
   foreach(char , [ ])
     string(REPLACE ${char} "\\${char}" test_name ${test_name})
   endforeach(char)
+  # ...add output dir
+  if(output_dir)
+    string(REGEX REPLACE "[^A-Za-z0-9_]" "_" test_name_clean ${test_name})
+    set(output_dir_arg "--out ${output_dir}/${output_prefix}${test_name_clean}${output_suffix}")
+  endif()
+  
   # ...and add to script
   add_command(add_test
     "${prefix}${test}${suffix}"
@@ -63,6 +110,8 @@ foreach(line ${output})
     "${TEST_EXECUTABLE}"
     "${test_name}"
     ${extra_args}
+    "${reporter_arg}"
+    "${output_dir_arg}"
   )
   add_command(set_tests_properties
     "${prefix}${test}${suffix}"

docs/cmake-integration.md

@@ -97,6 +97,10 @@ catch_discover_tests(target
                      [TEST_SUFFIX suffix]
                      [PROPERTIES name1 value1...]
                      [TEST_LIST var]
+                     [REPORTER reporter]
+                     [OUTPUT_DIR dir]
+                     [OUTPUT_PREFIX prefix]
+                     [OUTPUT_SUFFIX suffix]
 )
 ```
 
@@ -143,6 +147,32 @@ default `<target>_TESTS`.  This can be useful when the same test
 executable is being used in multiple calls to `catch_discover_tests()`.
 Note that this variable is only available in CTest.
 
+* `REPORTER reporter`
+
+Use the specified reporter when running the test case. The reporter will
+be passed to the test runner as `--reporter reporter`.
+
+* `OUTPUT_DIR dir`
+
+If specified, the parameter is passed along as
+`--out dir/<test_name>` to test executable. The actual file name is the
+same as the test name. This should be used instead of
+`EXTRA_ARGS --out foo` to avoid race conditions writing the result output
+when using parallel test execution.
+
+* `OUTPUT_PREFIX prefix`
+
+May be used in conjunction with `OUTPUT_DIR`.
+If specified, `prefix` is added to each output file name, like so
+`--out dir/prefix<test_name>`.
+
+* `OUTPUT_SUFFIX suffix`
+
+May be used in conjunction with `OUTPUT_DIR`.
+If specified, `suffix` is added to each output file name, like so
+`--out dir/<test_name>suffix`. This can be used to add a file extension to
+the output file name e.g. ".xml".
+
 
 ### `ParseAndAddCatchTests.cmake`