From f696ab836b9c7d9ea2f80b64597541cdc67f6821 Mon Sep 17 00:00:00 2001
From: John Bytheway <jbytheway@gmail.com>
Date: Thu, 9 Apr 2020 20:17:05 -0400
Subject: [PATCH] Change random test shuffling technique

Previously a random test ordering was obtained by applying std::shuffle
to the tests in declaration order.  This has two problems:

- It depends on the declaration order, so the order in which the tests
  will be run will be platform-specific.
- When trying to debug accidental inter-test dependencies, it is helpful
  to be able to find a minimal subset of tests which exhibits the issue.
  However, any change to the set of tests being run will completely
  change the test ordering, making it difficult or impossible to reduce
  the set of tests being run in any reasonably efficient manner.

Therefore, change the randomization approach to resolve both these
issues.

Generate a random value based on the user-provided RNG seed.  Convert
every test case to an integer by hashing a combination of that value
with the test name.  Sort the test cases by this integer.

The test names and RNG are platform-independent, so this should be
consistent across platforms.  Also, removing one test does not change
the integer value associated with the remaining tests, so they remain in
the same order.

To hash, use the FNV-1a hash, except with the basis being our randomly
selected value rather than the fixed basis set in the algorithm.  Cannot
use std::hash, because it is important that the result be
platform-independent.
---
 .../catch_test_case_registry_impl.cpp         | 57 ++++++++++++++++---
 1 file changed, 48 insertions(+), 9 deletions(-)

diff --git a/include/internal/catch_test_case_registry_impl.cpp b/include/internal/catch_test_case_registry_impl.cpp
index b254ca08..94e35e3a 100644
--- a/include/internal/catch_test_case_registry_impl.cpp
+++ b/include/internal/catch_test_case_registry_impl.cpp
@@ -15,27 +15,66 @@
 #include "catch_string_manip.h"
 #include "catch_test_case_info.h"
 
+#include <algorithm>
+#include <iterator>
+#include <random>
 #include <sstream>
 
 namespace Catch {
 
+    struct HashTest {
+        explicit HashTest( Catch::SimplePcg32& rng ) {
+            basis = rng();
+            basis <<= 32;
+            basis |= rng();
+        }
+
+        uint64_t basis;
+
+        uint64_t operator()( TestCase const& t ) const {
+            // Modified FNV-1a hash
+            static constexpr uint64_t prime = 1099511628211;
+            uint64_t hash = basis;
+            for( const char c : t.name ) {
+                hash ^= c;
+                hash *= prime;
+            }
+            return hash;
+        }
+    };
+
     std::vector<TestCase> sortTests( IConfig const& config, std::vector<TestCase> const& unsortedTestCases ) {
-
-        std::vector<TestCase> sorted = unsortedTestCases;
-
         switch( config.runOrder() ) {
-            case RunTests::InLexicographicalOrder:
+            case RunTests::InLexicographicalOrder: {
+                std::vector<TestCase> sorted = unsortedTestCases;
                 std::sort( sorted.begin(), sorted.end() );
-                break;
-            case RunTests::InRandomOrder:
+                return sorted;
+            }
+            case RunTests::InRandomOrder: {
                 seedRng( config );
-                std::shuffle( sorted.begin(), sorted.end(), rng() );
-                break;
+                HashTest h( rng() );
+                std::vector<std::tuple<uint64_t, std::string, TestCase const*>> indexed_tests;
+                indexed_tests.reserve( unsortedTestCases.size() );
+                std::transform( unsortedTestCases.begin(), unsortedTestCases.end(),
+                    std::back_inserter( indexed_tests ),
+                    [&]( TestCase const& t ) {
+                      return std::make_tuple( h(t), t.name, &t );
+                    } );
+                std::sort( indexed_tests.begin(), indexed_tests.end() );
+                std::vector<TestCase> sorted;
+                sorted.reserve( unsortedTestCases.size() );
+                std::transform( indexed_tests.begin(), indexed_tests.end(),
+                    std::back_inserter( sorted ),
+                    []( std::tuple<uint64_t, std::string, TestCase const*> const& t ) {
+                        return *std::get<2>( t );
+                    } );
+                return sorted;
+            }
             case RunTests::InDeclarationOrder:
                 // already in declaration order
                 break;
         }
-        return sorted;
+        return unsortedTestCases;
     }
 
     bool isThrowSafe( TestCase const& testCase, IConfig const& config ) {