From 26df0781a58782f66dbc6093916ea008688c7f7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ho=C5=99e=C5=88ovsk=C3=BD?= Date: Wed, 8 Feb 2017 14:14:01 +0100 Subject: [PATCH 1/5] Added a script for running synthetic compile time benchmark --- scripts/benchmarkCompile.py | 128 ++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100755 scripts/benchmarkCompile.py diff --git a/scripts/benchmarkCompile.py b/scripts/benchmarkCompile.py new file mode 100755 index 00000000..26df15b5 --- /dev/null +++ b/scripts/benchmarkCompile.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 + +import time, subprocess, sys, os, shutil, glob, random +import argparse +from statistics import median, stdev + +compiler_path = '' + +main_file = r''' +#define CATCH_CONFIG_MAIN +#include "catch.hpp" +''' +main_name = 'catch-main.cpp' + +dir_name = 'benchmark-dir' + +files = 20 +test_cases_in_file = 20 +sections_in_file = 4 +assertions_per_section = 5 + +checks = [ + 'a != b', 'a != c', 'a != d', 'a != e', 'b != c', 'b != d', 'b != e', 'c != d', 'c != e', 'd != e', 'a + a == a', + 'a + b == b', 'a + c == c', 'a + d == d', 'a + e == e', 'b + a == b', 'b + b == c', 'b + c == d', + 'b + d == e', 'c + a == c', 'c + b == d', 'c + c == e', 'd + a == d', 'd + b == e', 'e + a == e', + 'a + a + a == a', 'b + c == a + d', 'c + a + a == a + b + b + a', + 'a < b', 'b < c', 'c < d', 'd < e', 'a >= a', 'd >= b', +] + +def create_temp_dir(): + if os.path.exists(dir_name): + shutil.rmtree(dir_name) + os.mkdir(dir_name) + +def copy_catch(path_to_catch): + shutil.copy(path_to_catch, dir_name) + +def create_catch_main(): + with open(main_name, 'w') as f: + f.write(main_file) + +def compile_main(): + start_t = time.perf_counter() + subprocess.check_call([compiler_path, main_name, '-c']) + end_t = time.perf_counter() + return end_t - start_t + +def compile_files(): + cpp_files = glob.glob('*.cpp') + start_t = time.perf_counter() + subprocess.check_call([compiler_path] + cpp_files + ['-c']) + end_t = time.perf_counter() + return end_t - start_t + +def link_files(): + obj_files = glob.glob('*.o') + start_t = time.perf_counter() + subprocess.check_call([compiler_path] + obj_files) + end_t = time.perf_counter() + return end_t - start_t + +def benchmark(func): + return median([func() for i in range(10)]) + +def char_range(start, end): + for c in range(ord(start), ord(end)): + yield chr(c) + +def generate_sections(fd): + for i in range(sections_in_file): + fd.write(' SECTION("Section {}") {{\n'.format(i)) + fd.write('\n'.join(' CHECK({});'.format(check) for check in random.sample(checks, assertions_per_section))) + fd.write(' }\n') + + +def generate_file(file_no): + with open('tests{}.cpp'.format(file_no), 'w') as f: + f.write('#include "catch.hpp"\n\n') + for i in range(test_cases_in_file): + f.write('TEST_CASE("File {} test {}", "[.compile]"){{\n'.format(file_no, i)) + for i, c in enumerate(char_range('a', 'f')): + f.write(' int {} = {};\n'.format(c, i)) + generate_sections(f) + f.write('}\n\n') + + +def generate_files(): + create_catch_main() + for i in range(files): + generate_file(i) + + +options = ['all', 'main', 'files', 'link'] + +parser = argparse.ArgumentParser(description='Benchmarks Catch\'s compile times against some synthetic tests') +# Add first arg -- benchmark type +parser.add_argument('benchmark_kind', nargs='?', default='all', choices=options, help='What kind of benchmark to run, default: all') + +# Args to allow changing header/compiler +parser.add_argument('-I', '--catch-header', default='catch.hpp', help = 'Path to catch.hpp, default: catch.hpp') +parser.add_argument('-c', '--compiler', default='g++', help = 'Compiler to use, default: g++') + +# Allow creating files only, without running the whole thing +parser.add_argument('-g', '--generate-only', action='store_true', help='Generate test files and quit') + +args = parser.parse_args() + +compiler_path = args.compiler +catch_path = args.catch_header + +create_temp_dir() +copy_catch(catch_path) +os.chdir(dir_name) +# now create the fake test files +generate_files() +# Early exit +if args.generate_only: + print('Finished generating files') + exit(1) + + +print('Time needed for ...') +if args.benchmark_kind in ('all', 'main'): + print(' ... compiling main: {:.2f} s'.format(benchmark(compile_main))) +if args.benchmark_kind in ('all', 'files'): + print(' ... compiling test files: {:.2f} s'.format(benchmark(compile_files))) +if args.benchmark_kind in ('all', 'link'): + print(' ... linking everything: {:.2f} s'.format(benchmark(link_files))) From 0837132ce30503c90915df6b6a8bbcefd1b8b714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ho=C5=99e=C5=88ovsk=C3=BD?= Date: Sun, 12 Feb 2017 12:25:43 +0100 Subject: [PATCH 2/5] Make the benchmarking script Python 2 compatible Ended up using `time.time()`, even if it supposedly has worse accuracy, because Python running under WSL supports `time.clock()` very badly. --- scripts/benchmarkCompile.py | 39 +++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/scripts/benchmarkCompile.py b/scripts/benchmarkCompile.py index 26df15b5..6f647511 100755 --- a/scripts/benchmarkCompile.py +++ b/scripts/benchmarkCompile.py @@ -1,8 +1,17 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python + +from __future__ import print_function import time, subprocess, sys, os, shutil, glob, random import argparse -from statistics import median, stdev + +def median(lst): + lst = sorted(lst) + mid, odd = divmod(len(lst), 2) + if odd: + return lst[mid] + else: + return (lst[mid - 1] + lst[mid]) / 2.0 compiler_path = '' @@ -40,23 +49,23 @@ def create_catch_main(): f.write(main_file) def compile_main(): - start_t = time.perf_counter() + start_t = time.time() subprocess.check_call([compiler_path, main_name, '-c']) - end_t = time.perf_counter() + end_t = time.time() return end_t - start_t def compile_files(): cpp_files = glob.glob('*.cpp') - start_t = time.perf_counter() + start_t = time.time() subprocess.check_call([compiler_path] + cpp_files + ['-c']) - end_t = time.perf_counter() + end_t = time.time() return end_t - start_t def link_files(): obj_files = glob.glob('*.o') - start_t = time.perf_counter() + start_t = time.time() subprocess.check_call([compiler_path] + obj_files) - end_t = time.perf_counter() + end_t = time.time() return end_t - start_t def benchmark(func): @@ -101,20 +110,20 @@ parser.add_argument('-I', '--catch-header', default='catch.hpp', help = 'Path to parser.add_argument('-c', '--compiler', default='g++', help = 'Compiler to use, default: g++') # Allow creating files only, without running the whole thing -parser.add_argument('-g', '--generate-only', action='store_true', help='Generate test files and quit') +parser.add_argument('-g', '--generate-files', action='store_true', help='Generate test files and quit') args = parser.parse_args() compiler_path = args.compiler catch_path = args.catch_header -create_temp_dir() -copy_catch(catch_path) os.chdir(dir_name) -# now create the fake test files -generate_files() -# Early exit -if args.generate_only: +if args.generate_files: + create_temp_dir() + copy_catch(catch_path) + # now create the fake test files + generate_files() + # Early exit print('Finished generating files') exit(1) From 204911393519ab42c2ab6aa48b3aaa8c1158526c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ho=C5=99e=C5=88ovsk=C3=BD?= Date: Tue, 14 Feb 2017 15:34:00 +0100 Subject: [PATCH 3/5] Benchmark script: use median AND mean of compile time --- scripts/benchmarkCompile.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/benchmarkCompile.py b/scripts/benchmarkCompile.py index 6f647511..a3fc32cd 100755 --- a/scripts/benchmarkCompile.py +++ b/scripts/benchmarkCompile.py @@ -13,6 +13,9 @@ def median(lst): else: return (lst[mid - 1] + lst[mid]) / 2.0 +def mean(lst): + return float(sum(lst)) / max(len(lst), 1) + compiler_path = '' main_file = r''' @@ -69,7 +72,8 @@ def link_files(): return end_t - start_t def benchmark(func): - return median([func() for i in range(10)]) + results = [func() for i in range(10)] + return mean(results), median(results) def char_range(start, end): for c in range(ord(start), ord(end)): @@ -130,8 +134,8 @@ if args.generate_files: print('Time needed for ...') if args.benchmark_kind in ('all', 'main'): - print(' ... compiling main: {:.2f} s'.format(benchmark(compile_main))) + print(' ... compiling main, mean: {:.2f}, median: {:.2f} s'.format(*benchmark(compile_main))) if args.benchmark_kind in ('all', 'files'): - print(' ... compiling test files: {:.2f} s'.format(benchmark(compile_files))) + print(' ... compiling test files, mean: {:.2f}, median: {:.2f} s'.format(*benchmark(compile_files))) if args.benchmark_kind in ('all', 'link'): - print(' ... linking everything: {:.2f} s'.format(benchmark(link_files))) + print(' ... linking everything, mean: {:.2f}, median: {:.2f} s'.format(*benchmark(link_files))) From 6da5e0862a19455f41214a8f82e3dc901127d318 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ho=C5=99e=C5=88ovsk=C3=BD?= Date: Tue, 14 Feb 2017 15:34:17 +0100 Subject: [PATCH 4/5] Benchmark script: allow passing flags to compiler --- scripts/benchmarkCompile.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/benchmarkCompile.py b/scripts/benchmarkCompile.py index a3fc32cd..7e49485c 100755 --- a/scripts/benchmarkCompile.py +++ b/scripts/benchmarkCompile.py @@ -17,6 +17,7 @@ def mean(lst): return float(sum(lst)) / max(len(lst), 1) compiler_path = '' +flags = [] main_file = r''' #define CATCH_CONFIG_MAIN @@ -53,21 +54,21 @@ def create_catch_main(): def compile_main(): start_t = time.time() - subprocess.check_call([compiler_path, main_name, '-c']) + subprocess.check_call([compiler_path, main_name, '-c'] + flags) end_t = time.time() return end_t - start_t def compile_files(): cpp_files = glob.glob('*.cpp') start_t = time.time() - subprocess.check_call([compiler_path] + cpp_files + ['-c']) + subprocess.check_call([compiler_path, '-c'] + flags + cpp_files) end_t = time.time() return end_t - start_t def link_files(): obj_files = glob.glob('*.o') start_t = time.time() - subprocess.check_call([compiler_path] + obj_files) + subprocess.check_call([compiler_path] + flags + obj_files) end_t = time.time() return end_t - start_t @@ -113,6 +114,8 @@ parser.add_argument('benchmark_kind', nargs='?', default='all', choices=options, parser.add_argument('-I', '--catch-header', default='catch.hpp', help = 'Path to catch.hpp, default: catch.hpp') parser.add_argument('-c', '--compiler', default='g++', help = 'Compiler to use, default: g++') +parser.add_argument('-f', '--flags', nargs='*', help = 'Flags to be passed to the compiler') + # Allow creating files only, without running the whole thing parser.add_argument('-g', '--generate-files', action='store_true', help='Generate test files and quit') @@ -131,6 +134,8 @@ if args.generate_files: print('Finished generating files') exit(1) +if args.flags: + flags = args.flags print('Time needed for ...') if args.benchmark_kind in ('all', 'main'): From 7b13a8f85a0fb7d99759591b01539d6d0202f383 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ho=C5=99e=C5=88ovsk=C3=BD?= Date: Tue, 14 Feb 2017 15:35:12 +0100 Subject: [PATCH 5/5] Move debug break out of tests, speeds up compilation time This is hidden behind CATCH_CONFIG_FAST_COMPILE --- include/internal/catch_capture.hpp | 11 ++++++++++- include/internal/catch_result_builder.hpp | 9 +++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/include/internal/catch_capture.hpp b/include/internal/catch_capture.hpp index 04be7016..30a5b8da 100644 --- a/include/internal/catch_capture.hpp +++ b/include/internal/catch_capture.hpp @@ -18,6 +18,14 @@ #include "catch_compiler_capabilities.h" +#if defined(CATCH_CONFIG_FAST_COMPILE) +/////////////////////////////////////////////////////////////////////////////// +// We can speedup compilation significantly by breaking into debugger lower in +// the callstack, because then we don't have to expand CATCH_BREAK_INTO_DEBUGGER +// macro in each assertion +#define INTERNAL_CATCH_REACT( resultBuilder ) \ + resultBuilder.react(); +#else /////////////////////////////////////////////////////////////////////////////// // In the event of a failure works out if the debugger needs to be invoked // and/or an exception thrown and takes appropriate action. @@ -25,7 +33,8 @@ // source code rather than in Catch library code #define INTERNAL_CATCH_REACT( resultBuilder ) \ if( resultBuilder.shouldDebugBreak() ) CATCH_BREAK_INTO_DEBUGGER(); \ - resultBuilder.react(); + resultBuilder.react(); +#endif /////////////////////////////////////////////////////////////////////////////// diff --git a/include/internal/catch_result_builder.hpp b/include/internal/catch_result_builder.hpp index 7bb2cdc2..4e093347 100644 --- a/include/internal/catch_result_builder.hpp +++ b/include/internal/catch_result_builder.hpp @@ -99,6 +99,15 @@ namespace Catch { } void ResultBuilder::react() { +#if defined(CATCH_CONFIG_FAST_COMPILE) + if (m_shouldDebugBreak) { + /////////////////////////////////////////////////////////////////// + // To inspect the state during test, you need to go one level up the callstack + // To go back to the test and change execution, jump over the throw statement + /////////////////////////////////////////////////////////////////// + CATCH_BREAK_INTO_DEBUGGER(); + } +#endif if( m_shouldThrow ) throw Catch::TestFailureException(); }