From b753f05d745a1fb505d3ca0de3a131235727f19a Mon Sep 17 00:00:00 2001 From: Justin Wilson Date: Wed, 22 Feb 2017 04:17:25 -0600 Subject: [PATCH] Add reporter for Automake (#826) This allows for integration with Automake's default log compiler. See #826 for more details. --- CMakeLists.txt | 1 + docs/build-systems.md | 5 + include/reporters/catch_reporter_automake.hpp | 60 +++++++ .../Baselines/automake.std.approved.txt | 168 ++++++++++++++++++ 4 files changed, 234 insertions(+) create mode 100644 include/reporters/catch_reporter_automake.hpp create mode 100644 projects/SelfTest/Baselines/automake.std.approved.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 605fce0f..7ee391df 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -210,6 +210,7 @@ CheckFileList(INTERNAL_HEADERS ${HEADER_DIR}/internal) # Please keep these ordered alphabetically set(REPORTER_HEADERS + ${HEADER_DIR}/reporters/catch_reporter_automake.hpp ${HEADER_DIR}/reporters/catch_reporter_bases.hpp ${HEADER_DIR}/reporters/catch_reporter_compact.hpp ${HEADER_DIR}/reporters/catch_reporter_console.hpp diff --git a/docs/build-systems.md b/docs/build-systems.md index a14c93e0..04271f64 100644 --- a/docs/build-systems.md +++ b/docs/build-systems.md @@ -43,6 +43,11 @@ e.g.: Being specific to TeamCity this is the best reporter to use with it - but it is completely unsuitable for any other purpose. It is a streaming format (it writes as it goes) - although test results don't appear in the TeamCity interface until the completion of a suite (usually the whole test run). +## Automake Reporter +```-r automake``` + +The Automake Reporter writes out the [meta tags](https://www.gnu.org/software/automake/manual/html_node/Log-files-generation-and-test-results-recording.html#Log-files-generation-and-test-results-recording) expected by automake via `make check`. + # Low-level tools ## CMake diff --git a/include/reporters/catch_reporter_automake.hpp b/include/reporters/catch_reporter_automake.hpp new file mode 100644 index 00000000..b18396b5 --- /dev/null +++ b/include/reporters/catch_reporter_automake.hpp @@ -0,0 +1,60 @@ +/* + * Created by Justin R. Wilson on 2/19/2017. + * Copyright 2017 Justin R. Wilson. All rights reserved. + * + * Distributed under the Boost Software License, Version 1.0. (See accompanying + * file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + */ +#ifndef TWOBLUECUBES_CATCH_REPORTER_AUTOMAKE_HPP_INCLUDED +#define TWOBLUECUBES_CATCH_REPORTER_AUTOMAKE_HPP_INCLUDED + +// Don't #include any Catch headers here - we can assume they are already +// included before this header. +// This is not good practice in general but is necessary in this case so this +// file can be distributed as a single header that works with the main +// Catch single header. + +namespace Catch { + + struct AutomakeReporter : StreamingReporterBase { + AutomakeReporter( ReporterConfig const& _config ) + : StreamingReporterBase( _config ) + {} + + static std::string getDescription() { + return "Reports test results in the format of Automake .trs files"; + } + + virtual void assertionStarting( AssertionInfo const& ) CATCH_OVERRIDE {} + + virtual bool assertionEnded( AssertionStats const& _assertionStats ) CATCH_OVERRIDE { return true; } + + virtual void testCaseEnded( TestCaseStats const& _testCaseStats ) CATCH_OVERRIDE { + // Possible values to emit are PASS, XFAIL, SKIP, FAIL, XPASS and ERROR. + stream << ":test-result: "; + if (_testCaseStats.totals.assertions.allPassed()) { + stream << "PASS"; + } else if (_testCaseStats.totals.assertions.allOk()) { + stream << "XFAIL"; + } else { + stream << "FAIL"; + } + stream << ' ' << _testCaseStats.testInfo.name << '\n'; + StreamingReporterBase::testCaseEnded( _testCaseStats ); + } + + virtual void skipTest( TestCaseInfo const& testInfo ) CATCH_OVERRIDE { + stream << ":test-result: SKIP " << testInfo.name << '\n'; + } + + }; + +#ifdef CATCH_IMPL + AutomakeReporter::~AutomakeReporter() {} +#endif + + INTERNAL_CATCH_REGISTER_REPORTER( "automake", AutomakeReporter) + +} // end namespace Catch + +#endif // TWOBLUECUBES_CATCH_REPORTER_AUTOMAKE_HPP_INCLUDED diff --git a/projects/SelfTest/Baselines/automake.std.approved.txt b/projects/SelfTest/Baselines/automake.std.approved.txt new file mode 100644 index 00000000..85299f70 --- /dev/null +++ b/projects/SelfTest/Baselines/automake.std.approved.txt @@ -0,0 +1,168 @@ +:test-result: PASS # A test name that starts with a # +:test-result: PASS #542 +:test-result: PASS #809 +:test-result: FAIL 'Not' checks that should fail +:test-result: PASS 'Not' checks that should succeed +:test-result: PASS (unimplemented) static bools can be evaluated +:test-result: FAIL A METHOD_AS_TEST_CASE based test run that fails +:test-result: PASS A METHOD_AS_TEST_CASE based test run that succeeds +:test-result: FAIL A TEST_CASE_METHOD based test run that fails +:test-result: PASS A TEST_CASE_METHOD based test run that succeeds +:test-result: FAIL A couple of nested sections followed by a failure +:test-result: FAIL A failing expression with a non streamable type is still captured +:test-result: PASS AllOf matcher +:test-result: PASS An empty test with no assertions +:test-result: PASS An expression with side-effects should only be evaluated once +:test-result: FAIL An unchecked exception reports the line of the last assertion +:test-result: PASS Anonymous test case 1 +:test-result: PASS AnyOf matcher +:test-result: PASS Approximate PI +:test-result: PASS Approximate comparisons with different epsilons +:test-result: PASS Approximate comparisons with floats +:test-result: PASS Approximate comparisons with ints +:test-result: PASS Approximate comparisons with mixed numeric types +:test-result: PASS Assertions then sections +:test-result: PASS Character pretty printing +:test-result: PASS Comparing function pointers +:test-result: PASS Comparing member function pointers +:test-result: PASS Comparisons between ints where one side is computed +:test-result: PASS Comparisons between unsigned ints and negative signed ints match c++ standard behaviour +:test-result: PASS Comparisons with int literals don't warn when mixing signed/ unsigned +:test-result: FAIL Contains string matcher +:test-result: FAIL Custom exceptions can be translated when testing for nothrow +:test-result: FAIL Custom exceptions can be translated when testing for throwing as something else +:test-result: FAIL Custom std-exceptions can be custom translated +:test-result: PASS Demonstrate that a non-const == is not used +:test-result: FAIL EndsWith string matcher +:test-result: XFAIL Equality checks that should fail +:test-result: PASS Equality checks that should succeed +:test-result: PASS Equals +:test-result: FAIL Equals string matcher +:test-result: PASS Exception messages can be tested for +:test-result: FAIL Expected exceptions that don't throw or unexpected exceptions fail the test +:test-result: FAIL FAIL aborts the test +:test-result: FAIL FAIL does not require an argument +:test-result: PASS Factorials are computed +:test-result: PASS Generator over a range of pairs +:test-result: PASS Generators over two ranges +:test-result: PASS Greater-than inequalities with different epsilons +:test-result: PASS INFO and WARN do not abort tests +:test-result: FAIL INFO gets logged on failure +:test-result: FAIL INFO gets logged on failure, even if captured before successful assertions +:test-result: XFAIL Inequality checks that should fail +:test-result: PASS Inequality checks that should succeed +:test-result: PASS Less-than inequalities with different epsilons +:test-result: PASS Long strings can be wrapped +:test-result: PASS Long text is truncted +:test-result: PASS ManuallyRegistered +:test-result: PASS Matchers can be (AllOf) composed with the && operator +:test-result: PASS Matchers can be (AnyOf) composed with the || operator +:test-result: PASS Matchers can be composed with both && and || +:test-result: FAIL Matchers can be composed with both && and || - failing +:test-result: PASS Matchers can be negated (Not) with the ! operator +:test-result: FAIL Matchers can be negated (Not) with the ! operator - failing +:test-result: FAIL Mismatching exception messages failing the test +:test-result: PASS Nice descriptive name +:test-result: FAIL Non-std exceptions can be translated +:test-result: PASS NotImplemented exception +:test-result: PASS Objects that evaluated in boolean contexts can be checked +:test-result: PASS Operators at different namespace levels not hijacked by Koenig lookup +:test-result: FAIL Ordering comparison checks that should fail +:test-result: PASS Ordering comparison checks that should succeed +:test-result: FAIL Output from all sections is reported +:test-result: PASS Parse test names and tags +:test-result: PASS Parsing a std::pair +:test-result: PASS Pointers can be compared to null +:test-result: PASS Pointers can be converted to strings +:test-result: PASS Process can be configured on command line +:test-result: FAIL SCOPED_INFO is reset for each loop +:test-result: PASS SUCCEED counts as a test pass +:test-result: PASS SUCCESS does not require an argument +:test-result: PASS Scenario: BDD tests requiring Fixtures to provide commonly-accessed data or methods +:test-result: PASS Scenario: Do that thing with the thing +:test-result: PASS Scenario: This is a really long scenario name to see how the list command deals with wrapping +:test-result: PASS Scenario: Vector resizing affects size and capacity +A string sent directly to stdout +A string sent directly to stderr +:test-result: PASS Sends stuff to stdout and stderr +:test-result: PASS Some simple comparisons between doubles +Message from section one +Message from section two +:test-result: PASS Standard output from all sections is reported +:test-result: FAIL StartsWith string matcher +:test-result: PASS String matchers +hello +hello +:test-result: PASS Strings can be rendered with colour +:test-result: FAIL Tabs and newlines show in output +:test-result: PASS Tag alias can be registered against tag patterns +:test-result: PASS Test case with one argument +:test-result: PASS Test enum bit values +:test-result: PASS Text can be formatted using the Text class +:test-result: PASS The NO_FAIL macro reports a failure but does not fail the test +:test-result: FAIL This test 'should' fail but doesn't +:test-result: PASS Tracker +:test-result: FAIL Unexpected exceptions can be translated +:test-result: PASS Use a custom approx +:test-result: PASS Variadic macros +:test-result: PASS When checked exceptions are thrown they can be expected or unexpected +:test-result: FAIL When unchecked exceptions are thrown directly they are always failures +:test-result: FAIL When unchecked exceptions are thrown during a CHECK the test should continue +:test-result: FAIL When unchecked exceptions are thrown during a REQUIRE the test should abort fail +:test-result: FAIL When unchecked exceptions are thrown from functions they are always failures +:test-result: FAIL When unchecked exceptions are thrown from sections they are always failures +:test-result: PASS When unchecked exceptions are thrown, but caught, they do not affect the test +:test-result: PASS Where the LHS is not a simple value +:test-result: PASS Where there is more to the expression after the RHS +:test-result: PASS X/level/0/a +:test-result: PASS X/level/0/b +:test-result: PASS X/level/1/a +:test-result: PASS X/level/1/b +:test-result: PASS XmlEncode +:test-result: PASS atomic if +:test-result: PASS boolean member +:test-result: PASS checkedElse +:test-result: FAIL checkedElse, failing +:test-result: PASS checkedIf +:test-result: FAIL checkedIf, failing +:test-result: PASS comparisons between const int variables +:test-result: PASS comparisons between int variables +:test-result: PASS even more nested SECTION tests +:test-result: PASS first tag +spanner:test-result: PASS has printf +:test-result: FAIL just failure +:test-result: PASS just info +:test-result: FAIL looped SECTION tests +:test-result: FAIL looped tests +:test-result: FAIL more nested SECTION tests +:test-result: PASS nested SECTION tests +:test-result: PASS non streamable - with conv. op +:test-result: PASS not allowed +:test-result: PASS null strings +:test-result: PASS pair > -> toString +:test-result: PASS pointer to class +:test-result: PASS random SECTION tests +:test-result: PASS replaceInPlace +:test-result: PASS second tag +:test-result: FAIL send a single char to INFO +:test-result: FAIL sends information to INFO +:test-result: PASS std::pair -> toString +:test-result: PASS std::pair -> toString +:test-result: PASS std::vector > -> toString +:test-result: FAIL string literals of different sizes can be compared +:test-result: PASS toString on const wchar_t const pointer returns the string contents +:test-result: PASS toString on const wchar_t pointer returns the string contents +:test-result: PASS toString on wchar_t const pointer returns the string contents +:test-result: PASS toString on wchar_t returns the string contents +:test-result: PASS toString( has_maker ) +:test-result: PASS toString( has_maker_and_toString ) +:test-result: PASS toString( has_toString ) +:test-result: PASS toString( vectors -> toString +:test-result: PASS vector -> toString +:test-result: PASS vectors can be sized and resized +:test-result: PASS xmlentitycheck