From c522e88afa04d66004e8d36e3a729fef90e36aaa Mon Sep 17 00:00:00 2001 From: Martin Stump <11492152+globberwops@users.noreply.github.com> Date: Sat, 12 Sep 2020 11:12:30 +0200 Subject: [PATCH] Add REPORTER and OUTPUT_* args --- docs/cmake-integration.md | 30 +++++++++++++++++++++++ extras/Catch.cmake | 34 ++++++++++++++++++++++++-- extras/CatchAddTests.cmake | 49 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 2 deletions(-) diff --git a/docs/cmake-integration.md b/docs/cmake-integration.md index 1835a043..952efa8f 100644 --- a/docs/cmake-integration.md +++ b/docs/cmake-integration.md @@ -97,6 +97,10 @@ catch_discover_tests(target [TEST_SUFFIX suffix] [PROPERTIES name1 value1...] [TEST_LIST var] + [REPORTER reporter] + [OUTPUT_DIR dir] + [OUTPUT_PREFIX prefix] + [OUTPUT_SUFFIX suffix] ) ``` @@ -143,6 +147,32 @@ default `_TESTS`. This can be useful when the same test executable is being used in multiple calls to `catch_discover_tests()`. Note that this variable is only available in CTest. +* `REPORTER reporter` + +Use the specified reporter when running the test case. The reporter will +be passed to the test runner as `--reporter reporter`. + +* `OUTPUT_DIR dir` + +If specified, the parameter is passed along as +`--out dir/` to test executable. The actual file name is the +same as the test name. This should be used instead of +`EXTRA_ARGS --out foo` to avoid race conditions writing the result output +when using parallel test execution. + +* `OUTPUT_PREFIX prefix` + +May be used in conjunction with `OUTPUT_DIR`. +If specified, `prefix` is added to each output file name, like so +`--out dir/prefix`. + +* `OUTPUT_SUFFIX suffix` + +May be used in conjunction with `OUTPUT_DIR`. +If specified, `suffix` is added to each output file name, like so +`--out dir/suffix`. This can be used to add a file extension to +the output file name e.g. ".xml". + ### `ParseAndAddCatchTests.cmake` diff --git a/extras/Catch.cmake b/extras/Catch.cmake index 486e3233..b3f90a79 100644 --- a/extras/Catch.cmake +++ b/extras/Catch.cmake @@ -33,6 +33,10 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``. [TEST_SUFFIX suffix] [PROPERTIES name1 value1...] [TEST_LIST var] + [REPORTER reporter] + [OUTPUT_DIR dir] + [OUTPUT_PREFIX prefix} + [OUTPUT_SUFFIX suffix] ) ``catch_discover_tests`` sets up a post-build command on the test executable @@ -90,6 +94,28 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``. executable is being used in multiple calls to ``catch_discover_tests()``. Note that this variable is only available in CTest. + ``REPORTER reporter`` + Use the specified reporter when running the test case. The reporter will + be passed to the Catch executable as ``--reporter reporter``. + + ``OUTPUT_DIR dir`` + If specified, the parameter is passed along as + ``--out dir/`` to Catch executable. The actual file name is the + same as the test name. This should be used instead of + ``EXTRA_ARGS --out foo`` to avoid race conditions writing the result output + when using parallel test execution. + + ``OUTPUT_PREFIX prefix`` + May be used in conjunction with ``OUTPUT_DIR``. + If specified, ``prefix`` is added to each output file name, like so + ``--out dir/prefix``. + + ``OUTPUT_SUFFIX suffix`` + May be used in conjunction with ``OUTPUT_DIR``. + If specified, ``suffix`` is added to each output file name, like so + ``--out dir/suffix``. This can be used to add a file extension to + the output e.g. ".xml". + #]=======================================================================] #------------------------------------------------------------------------------ @@ -97,7 +123,7 @@ function(catch_discover_tests TARGET) cmake_parse_arguments( "" "" - "TEST_PREFIX;TEST_SUFFIX;WORKING_DIRECTORY;TEST_LIST" + "TEST_PREFIX;TEST_SUFFIX;WORKING_DIRECTORY;TEST_LIST;REPORTER;OUTPUT_DIR;OUTPUT_PREFIX;OUTPUT_SUFFIX" "TEST_SPEC;EXTRA_ARGS;PROPERTIES" ${ARGN} ) @@ -110,7 +136,7 @@ function(catch_discover_tests TARGET) endif() ## Generate a unique name based on the extra arguments - string(SHA1 args_hash "${_TEST_SPEC} ${_EXTRA_ARGS}") + string(SHA1 args_hash "${_TEST_SPEC} ${_EXTRA_ARGS} ${_REPORTER} ${_OUTPUT_DIR} ${_OUTPUT_PREFIX} ${_OUTPUT_SUFFIX}") string(SUBSTRING ${args_hash} 0 7 args_hash) # Define rule to generate test list for aforementioned test executable @@ -134,6 +160,10 @@ function(catch_discover_tests TARGET) -D "TEST_PREFIX=${_TEST_PREFIX}" -D "TEST_SUFFIX=${_TEST_SUFFIX}" -D "TEST_LIST=${_TEST_LIST}" + -D "TEST_REPORTER=${_REPORTER}" + -D "TEST_OUTPUT_DIR=${_OUTPUT_DIR}" + -D "TEST_OUTPUT_PREFIX=${_OUTPUT_PREFIX}" + -D "TEST_OUTPUT_SUFFIX=${_OUTPUT_SUFFIX}" -D "CTEST_FILE=${ctest_tests_file}" -P "${_CATCH_DISCOVER_TESTS_SCRIPT}" VERBATIM diff --git a/extras/CatchAddTests.cmake b/extras/CatchAddTests.cmake index 01c01386..82c986df 100644 --- a/extras/CatchAddTests.cmake +++ b/extras/CatchAddTests.cmake @@ -6,6 +6,10 @@ set(suffix "${TEST_SUFFIX}") set(spec ${TEST_SPEC}) set(extra_args ${TEST_EXTRA_ARGS}) set(properties ${TEST_PROPERTIES}) +set(reporter ${TEST_REPORTER}) +set(output_dir ${TEST_OUTPUT_DIR}) +set(output_prefix ${TEST_OUTPUT_PREFIX}) +set(output_suffix ${TEST_OUTPUT_SUFFIX}) set(script) set(suite) set(tests) @@ -43,6 +47,43 @@ endif() string(REPLACE "\n" ";" output "${output}") +# Run test executable to get list of available reporters +execute_process( + COMMAND ${TEST_EXECUTOR} "${TEST_EXECUTABLE}" ${spec} --list-reporters + OUTPUT_VARIABLE reporters_output + RESULT_VARIABLE reporters_result +) +if(${reporters_result} EQUAL 0) + message(WARNING + "Test executable '${TEST_EXECUTABLE}' contains no reporters!\n" + ) +elseif(${reporters_result} LESS 0) + message(FATAL_ERROR + "Error running test executable '${TEST_EXECUTABLE}':\n" + " Result: ${reporters_result}\n" + " Output: ${reporters_output}\n" + ) +endif() +string(FIND "${reporters_output}" "${reporter}" reporter_is_valid) +if(reporter AND ${reporter_is_valid} EQUAL -1) + message(FATAL_ERROR + "\"${reporter}\" is not a valid reporter!\n" + ) +endif() + +# Prepare reporter +if(reporter) + set(reporter_arg "--reporter ${reporter}") +endif() + +# Prepare output dir +if(output_dir AND NOT IS_ABSOLUTE ${output_dir}) + set(output_dir "${TEST_WORKING_DIR}/${output_dir}") + if(NOT EXISTS ${output_dir}) + file(MAKE_DIRECTORY ${output_dir}) + endif() +endif() + # Parse output foreach(line ${output}) set(test ${line}) @@ -51,6 +92,12 @@ foreach(line ${output}) foreach(char , [ ]) string(REPLACE ${char} "\\${char}" test_name ${test_name}) endforeach(char) + # ...add output dir + if(output_dir) + string(REGEX REPLACE "[^A-Za-z0-9_]" "_" test_name_clean ${test_name}) + set(output_dir_arg "--out ${output_dir}/${output_prefix}${test_name_clean}${output_suffix}") + endif() + # ...and add to script add_command(add_test "${prefix}${test}${suffix}" @@ -58,6 +105,8 @@ foreach(line ${output}) "${TEST_EXECUTABLE}" "${test_name}" ${extra_args} + "${reporter_arg}" + "${output_dir_arg}" ) add_command(set_tests_properties "${prefix}${test}${suffix}"