3rd-party/catch2
1
0
Fork 0
mirror of https://github.com/catchorg/Catch2.git synced 2024-12-24 12:03:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

Randomize test for subset invariant random ordering of tests

Browse Source 
Also removed the iterative checking that seeds 1-100 do not create
the same output, because it used too much runtime.
This commit is contained in:
Martin Hořeňovský 2020-04-14 16:38:07 +02:00
parent da9e3eec65
commit a2fc7cf8c0
No known key found for this signature in database
GPG Key ID: DE48307B8B0D381A
1 changed files with 16 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
projects/TestScripts/testRandomOrder.py
View File

@ -1,7 +1,16 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""
This test script verifies that the random ordering of tests inside
Catch2 is invariant in regards to subsetting. This is done by running
the binary 3 times, once with all tests selected, and twice with smaller
subsets of tests selected, and verifying that the selected tests are in
the same relative order.
"""
import subprocess import subprocess
import sys import sys
import random
def list_tests(self_test_exe, tags, rng_seed): def list_tests(self_test_exe, tags, rng_seed):
cmd = [self_test_exe, '--list-test-names-only', '--order', 'rand', cmd = [self_test_exe, '--list-test-names-only', '--order', 'rand',
@ -34,20 +43,17 @@ def check_is_sublist_of(shorter, longer):
def main(): def main():
self_test_exe, = sys.argv[1:] self_test_exe, = sys.argv[1:]
list_one_tag = list_tests(self_test_exe, ['generators'], 1) # We want a random seed for the test, but want to avoid 0,
list_two_tags = list_tests(self_test_exe, ['generators', 'matchers'], 1) # because it has special meaning
list_all = list_tests(self_test_exe, [], 1) seed = random.randint(1, 2 ** 32 - 1)
list_one_tag = list_tests(self_test_exe, ['generators'], seed)
list_two_tags = list_tests(self_test_exe, ['generators', 'matchers'], seed)
list_all = list_tests(self_test_exe, [], seed)
# First, verify that restricting to a subset yields the same order # First, verify that restricting to a subset yields the same order
check_is_sublist_of(list_two_tags, list_all) check_is_sublist_of(list_two_tags, list_all)
check_is_sublist_of(list_one_tag, list_two_tags) check_is_sublist_of(list_one_tag, list_two_tags)
# Second, verify that different seeds yield different orders
num_seeds = 100
seeds = range(1, num_seeds + 1) # Avoid zero since that's special
lists = {tuple(list_tests(self_test_exe, [], seed)) for seed in seeds}
assert len(lists) == num_seeds, (
'Got {} distict lists from {} seeds'.format(len(lists), num_seeds))
if __name__ == '__main__': if __name__ == '__main__':
sys.exit(main()) sys.exit(main())