Merge pull request #2023 from globberwops/feature/add-reporter-and-output-dir

Add REPORTER and OUTPUT_DIR args to catch_discover_tests
This commit is contained in:
Martin Hořeňovský 2020-09-23 22:47:21 +02:00 committed by GitHub
commit 314a05d809
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 111 additions and 2 deletions

View File

@ -33,6 +33,10 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``.
[TEST_SUFFIX suffix] [TEST_SUFFIX suffix]
[PROPERTIES name1 value1...] [PROPERTIES name1 value1...]
[TEST_LIST var] [TEST_LIST var]
[REPORTER reporter]
[OUTPUT_DIR dir]
[OUTPUT_PREFIX prefix}
[OUTPUT_SUFFIX suffix]
) )
``catch_discover_tests`` sets up a post-build command on the test executable ``catch_discover_tests`` sets up a post-build command on the test executable
@ -90,6 +94,28 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``.
executable is being used in multiple calls to ``catch_discover_tests()``. executable is being used in multiple calls to ``catch_discover_tests()``.
Note that this variable is only available in CTest. Note that this variable is only available in CTest.
``REPORTER reporter``
Use the specified reporter when running the test case. The reporter will
be passed to the Catch executable as ``--reporter reporter``.
``OUTPUT_DIR dir``
If specified, the parameter is passed along as
``--out dir/<test_name>`` to Catch executable. The actual file name is the
same as the test name. This should be used instead of
``EXTRA_ARGS --out foo`` to avoid race conditions writing the result output
when using parallel test execution.
``OUTPUT_PREFIX prefix``
May be used in conjunction with ``OUTPUT_DIR``.
If specified, ``prefix`` is added to each output file name, like so
``--out dir/prefix<test_name>``.
``OUTPUT_SUFFIX suffix``
May be used in conjunction with ``OUTPUT_DIR``.
If specified, ``suffix`` is added to each output file name, like so
``--out dir/<test_name>suffix``. This can be used to add a file extension to
the output e.g. ".xml".
#]=======================================================================] #]=======================================================================]
#------------------------------------------------------------------------------ #------------------------------------------------------------------------------
@ -97,7 +123,7 @@ function(catch_discover_tests TARGET)
cmake_parse_arguments( cmake_parse_arguments(
"" ""
"" ""
"TEST_PREFIX;TEST_SUFFIX;WORKING_DIRECTORY;TEST_LIST" "TEST_PREFIX;TEST_SUFFIX;WORKING_DIRECTORY;TEST_LIST;REPORTER;OUTPUT_DIR;OUTPUT_PREFIX;OUTPUT_SUFFIX"
"TEST_SPEC;EXTRA_ARGS;PROPERTIES" "TEST_SPEC;EXTRA_ARGS;PROPERTIES"
${ARGN} ${ARGN}
) )
@ -110,7 +136,7 @@ function(catch_discover_tests TARGET)
endif() endif()
## Generate a unique name based on the extra arguments ## Generate a unique name based on the extra arguments
string(SHA1 args_hash "${_TEST_SPEC} ${_EXTRA_ARGS}") string(SHA1 args_hash "${_TEST_SPEC} ${_EXTRA_ARGS} ${_REPORTER} ${_OUTPUT_DIR} ${_OUTPUT_PREFIX} ${_OUTPUT_SUFFIX}")
string(SUBSTRING ${args_hash} 0 7 args_hash) string(SUBSTRING ${args_hash} 0 7 args_hash)
# Define rule to generate test list for aforementioned test executable # Define rule to generate test list for aforementioned test executable
@ -134,6 +160,10 @@ function(catch_discover_tests TARGET)
-D "TEST_PREFIX=${_TEST_PREFIX}" -D "TEST_PREFIX=${_TEST_PREFIX}"
-D "TEST_SUFFIX=${_TEST_SUFFIX}" -D "TEST_SUFFIX=${_TEST_SUFFIX}"
-D "TEST_LIST=${_TEST_LIST}" -D "TEST_LIST=${_TEST_LIST}"
-D "TEST_REPORTER=${_REPORTER}"
-D "TEST_OUTPUT_DIR=${_OUTPUT_DIR}"
-D "TEST_OUTPUT_PREFIX=${_OUTPUT_PREFIX}"
-D "TEST_OUTPUT_SUFFIX=${_OUTPUT_SUFFIX}"
-D "CTEST_FILE=${ctest_tests_file}" -D "CTEST_FILE=${ctest_tests_file}"
-P "${_CATCH_DISCOVER_TESTS_SCRIPT}" -P "${_CATCH_DISCOVER_TESTS_SCRIPT}"
VERBATIM VERBATIM

View File

@ -6,6 +6,10 @@ set(suffix "${TEST_SUFFIX}")
set(spec ${TEST_SPEC}) set(spec ${TEST_SPEC})
set(extra_args ${TEST_EXTRA_ARGS}) set(extra_args ${TEST_EXTRA_ARGS})
set(properties ${TEST_PROPERTIES}) set(properties ${TEST_PROPERTIES})
set(reporter ${TEST_REPORTER})
set(output_dir ${TEST_OUTPUT_DIR})
set(output_prefix ${TEST_OUTPUT_PREFIX})
set(output_suffix ${TEST_OUTPUT_SUFFIX})
set(script) set(script)
set(suite) set(suite)
set(tests) set(tests)
@ -48,6 +52,43 @@ endif()
string(REPLACE "\n" ";" output "${output}") string(REPLACE "\n" ";" output "${output}")
# Run test executable to get list of available reporters
execute_process(
COMMAND ${TEST_EXECUTOR} "${TEST_EXECUTABLE}" ${spec} --list-reporters
OUTPUT_VARIABLE reporters_output
RESULT_VARIABLE reporters_result
)
if(${reporters_result} EQUAL 0)
message(WARNING
"Test executable '${TEST_EXECUTABLE}' contains no reporters!\n"
)
elseif(${reporters_result} LESS 0)
message(FATAL_ERROR
"Error running test executable '${TEST_EXECUTABLE}':\n"
" Result: ${reporters_result}\n"
" Output: ${reporters_output}\n"
)
endif()
string(FIND "${reporters_output}" "${reporter}" reporter_is_valid)
if(reporter AND ${reporter_is_valid} EQUAL -1)
message(FATAL_ERROR
"\"${reporter}\" is not a valid reporter!\n"
)
endif()
# Prepare reporter
if(reporter)
set(reporter_arg "--reporter ${reporter}")
endif()
# Prepare output dir
if(output_dir AND NOT IS_ABSOLUTE ${output_dir})
set(output_dir "${TEST_WORKING_DIR}/${output_dir}")
if(NOT EXISTS ${output_dir})
file(MAKE_DIRECTORY ${output_dir})
endif()
endif()
# Parse output # Parse output
foreach(line ${output}) foreach(line ${output})
set(test ${line}) set(test ${line})
@ -56,6 +97,12 @@ foreach(line ${output})
foreach(char , [ ]) foreach(char , [ ])
string(REPLACE ${char} "\\${char}" test_name ${test_name}) string(REPLACE ${char} "\\${char}" test_name ${test_name})
endforeach(char) endforeach(char)
# ...add output dir
if(output_dir)
string(REGEX REPLACE "[^A-Za-z0-9_]" "_" test_name_clean ${test_name})
set(output_dir_arg "--out ${output_dir}/${output_prefix}${test_name_clean}${output_suffix}")
endif()
# ...and add to script # ...and add to script
add_command(add_test add_command(add_test
"${prefix}${test}${suffix}" "${prefix}${test}${suffix}"
@ -63,6 +110,8 @@ foreach(line ${output})
"${TEST_EXECUTABLE}" "${TEST_EXECUTABLE}"
"${test_name}" "${test_name}"
${extra_args} ${extra_args}
"${reporter_arg}"
"${output_dir_arg}"
) )
add_command(set_tests_properties add_command(set_tests_properties
"${prefix}${test}${suffix}" "${prefix}${test}${suffix}"

View File

@ -97,6 +97,10 @@ catch_discover_tests(target
[TEST_SUFFIX suffix] [TEST_SUFFIX suffix]
[PROPERTIES name1 value1...] [PROPERTIES name1 value1...]
[TEST_LIST var] [TEST_LIST var]
[REPORTER reporter]
[OUTPUT_DIR dir]
[OUTPUT_PREFIX prefix]
[OUTPUT_SUFFIX suffix]
) )
``` ```
@ -143,6 +147,32 @@ default `<target>_TESTS`. This can be useful when the same test
executable is being used in multiple calls to `catch_discover_tests()`. executable is being used in multiple calls to `catch_discover_tests()`.
Note that this variable is only available in CTest. Note that this variable is only available in CTest.
* `REPORTER reporter`
Use the specified reporter when running the test case. The reporter will
be passed to the test runner as `--reporter reporter`.
* `OUTPUT_DIR dir`
If specified, the parameter is passed along as
`--out dir/<test_name>` to test executable. The actual file name is the
same as the test name. This should be used instead of
`EXTRA_ARGS --out foo` to avoid race conditions writing the result output
when using parallel test execution.
* `OUTPUT_PREFIX prefix`
May be used in conjunction with `OUTPUT_DIR`.
If specified, `prefix` is added to each output file name, like so
`--out dir/prefix<test_name>`.
* `OUTPUT_SUFFIX suffix`
May be used in conjunction with `OUTPUT_DIR`.
If specified, `suffix` is added to each output file name, like so
`--out dir/<test_name>suffix`. This can be used to add a file extension to
the output file name e.g. ".xml".
### `ParseAndAddCatchTests.cmake` ### `ParseAndAddCatchTests.cmake`