diff --git a/tests/SelfTest/TestRegistrations.cpp b/tests/SelfTest/TestRegistrations.cpp index cb8376bd..45a01c02 100644 --- a/tests/SelfTest/TestRegistrations.cpp +++ b/tests/SelfTest/TestRegistrations.cpp @@ -1,14 +1,21 @@ -/* - * Distributed under the Boost Software License, Version 1.0. (See accompanying - * file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - */ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 #include #include +#include +#include +#include + // Some example tag aliases -CATCH_REGISTER_TAG_ALIAS( "[@nhf]", "[failing]~[.]" ) -CATCH_REGISTER_TAG_ALIAS( "[@tricky]", "[tricky]~[.]" ) +CATCH_REGISTER_TAG_ALIAS("[@nhf]", "[failing]~[.]") +CATCH_REGISTER_TAG_ALIAS("[@tricky]", "[tricky]~[.]") #ifdef __clang__ # pragma clang diagnostic ignored "-Wpadded" @@ -16,11 +23,155 @@ CATCH_REGISTER_TAG_ALIAS( "[@tricky]", "[tricky]~[.]" ) # pragma clang diagnostic ignored "-Wc++98-compat" #endif +/** + * Event listener that internally counts and validates received events. + * + * Currently only performs validation by counting received events, rather + * than performing full matching. This means that it won't fail if the *Ended + * events are provided in wrong order, as long as they come in the right amount + * and with the right nesting. + */ +class ValidatingTestListener : public Catch::EventListenerBase { + struct EventCounter { + int starting = 0; + int ended = 0; -struct TestListener : Catch::EventListenerBase { - using EventListenerBase::EventListenerBase; + bool hasActiveEvent() const { + return starting > ended; + } + bool hasSingleActiveEvent() const { + return starting - 1 == ended; + } + bool allEventsEnded() const { + return starting == ended; + } + }; + +public: + ValidatingTestListener(Catch::ReporterConfig const& config) : + EventListenerBase(config) { + m_preferences.shouldReportAllAssertions = true; + } + + void testRunStarting( Catch::TestRunInfo const& ) override { + CATCH_ENFORCE( m_testRunCounter.starting == 0, + "Test run can only start once" ); + ++m_testRunCounter.starting; + } + void testCaseStarting(Catch::TestCaseInfo const&) override { + CATCH_ENFORCE( m_testRunCounter.hasActiveEvent(), + "Test case can only be started if the test run has already started" ); + CATCH_ENFORCE( m_testCaseCounter.allEventsEnded(), + "Test case cannot start if there is an unfinished one" ); + + ++m_testCaseCounter.starting; + + // Reset the part tracking for partial test case events + m_lastSeenPartNumber = -1; + } + + void testCasePartialStarting(Catch::TestCaseInfo const&, + uint64_t partNumber) override { + CATCH_ENFORCE( m_testCaseCounter.hasSingleActiveEvent(), + "Test case can only be partially started if the test case has fully started already" ); + CATCH_ENFORCE( m_lastSeenPartNumber + 1 == partNumber, + "Partial test case started out of order" ); + + ++m_testCasePartialCounter.starting; + m_lastSeenPartNumber = partNumber; + } + + void sectionStarting(Catch::SectionInfo const&) override { + CATCH_ENFORCE( m_testCaseCounter.hasSingleActiveEvent(), + "Section can only start in a test case" ); + CATCH_ENFORCE( m_testCasePartialCounter.hasSingleActiveEvent(), + "Section can only start in a test case" ); + + ++m_sectionCounter.starting; + } + + void assertionStarting(Catch::AssertionInfo const&) override { + CATCH_ENFORCE( m_testCaseCounter.hasSingleActiveEvent(), + "Assertion can only start if test case is started" ); + + ++m_assertionCounter.starting; + } + void assertionEnded(Catch::AssertionStats const&) override { + // todo: + // * Check that assertions are balanced + // * Check that assertions has started + ++m_assertionCounter.ended; + } + + void sectionEnded(Catch::SectionStats const&) override { + CATCH_ENFORCE( m_sectionCounter.hasActiveEvent(), + "Section ended without corresponding start" ); + // TODO: Check that all assertions ended + + ++m_sectionCounter.ended; + } + + + void testCasePartialEnded(Catch::TestCaseStats const&, + uint64_t partNumber) override { + CATCH_ENFORCE( m_lastSeenPartNumber == partNumber, + "Partial test case ended out of order" ); + CATCH_ENFORCE( m_testCasePartialCounter.hasSingleActiveEvent(), + "Partial test case ended without corresponding start" ); + CATCH_ENFORCE( m_sectionCounter.allEventsEnded(), + "Partial test case ended with unbalanced sections" ); + // TODO: Check that all assertions ended + + ++m_testCasePartialCounter.ended; + } + + + void testCaseEnded(Catch::TestCaseStats const&) override { + CATCH_ENFORCE( m_testCaseCounter.hasSingleActiveEvent(), + "Test case end is not matched with test case start" ); + CATCH_ENFORCE( m_testCasePartialCounter.allEventsEnded(), + "A partial test case has not ended" ); + CATCH_ENFORCE( m_sectionCounter.allEventsEnded(), + "Test case ended with unbalanced sections" ); + + // TODO: Check that all assertions ended + + ++m_testCaseCounter.ended; + } + void testRunEnded( Catch::TestRunStats const& ) override { + CATCH_ENFORCE( m_testRunCounter.hasSingleActiveEvent(), + "Test run end is not matched with test run start" ); + CATCH_ENFORCE( m_testRunCounter.ended == 0, + "Test run can only end once" ); + + ++m_testRunCounter.ended; + } + + ~ValidatingTestListener() override; + +private: + EventCounter m_testRunCounter; + EventCounter m_testCaseCounter; + EventCounter m_testCasePartialCounter; + uint64_t m_lastSeenPartNumber = 0; + EventCounter m_sectionCounter; + EventCounter m_assertionCounter; }; -#include -CATCH_REGISTER_LISTENER( TestListener ) +ValidatingTestListener::~ValidatingTestListener() { + // Throwing from noexcept destructor terminates, but we don't mind + // because this is test-only check and we don't need to try and recover + // from assumption violation here. + + CATCH_ENFORCE( m_testRunCounter.ended < 2, + "Test run should be started at most once" ); + CATCH_ENFORCE( m_testRunCounter.allEventsEnded(), + "The test run has not finished" ); + CATCH_ENFORCE( m_testCaseCounter.allEventsEnded(), + "A test case did not finish" ); + + // TODO: other counters being balanced? +} + +CATCH_REGISTER_LISTENER( ValidatingTestListener )