• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2
3from __future__ import print_function
4
5import time, subprocess, sys, os, shutil, glob, random
6import argparse
7
8def median(lst):
9    lst = sorted(lst)
10    mid, odd = divmod(len(lst), 2)
11    if odd:
12        return lst[mid]
13    else:
14        return (lst[mid - 1] + lst[mid]) / 2.0
15
16def mean(lst):
17    return float(sum(lst)) / max(len(lst), 1)
18
19compiler_path = ''
20flags = []
21
22main_file = r'''
23#define CATCH_CONFIG_MAIN
24#include "catch.hpp"
25'''
26main_name = 'catch-main.cpp'
27
28dir_name = 'benchmark-dir'
29
30files = 20
31test_cases_in_file = 20
32sections_in_file = 4
33assertions_per_section = 5
34
35checks = [
36    'a != b', 'a != c', 'a != d', 'a != e', 'b != c', 'b != d', 'b != e', 'c != d', 'c != e', 'd != e', 'a + a == a',
37    'a + b == b', 'a + c == c', 'a + d == d', 'a + e == e', 'b + a == b', 'b + b == c', 'b + c == d',
38    'b + d == e', 'c + a == c', 'c + b == d', 'c + c == e', 'd + a == d', 'd + b == e', 'e + a == e',
39    'a + a + a == a', 'b + c == a + d', 'c + a + a == a + b + b + a',
40    'a < b', 'b < c', 'c < d', 'd < e', 'a >= a', 'd >= b',
41]
42
43def create_temp_dir():
44    if os.path.exists(dir_name):
45        shutil.rmtree(dir_name)
46    os.mkdir(dir_name)
47
48def copy_catch(path_to_catch):
49    shutil.copy(path_to_catch, dir_name)
50
51def create_catch_main():
52    with open(main_name, 'w') as f:
53        f.write(main_file)
54
55def compile_main():
56    start_t = time.time()
57    subprocess.check_call([compiler_path, main_name, '-c'] + flags)
58    end_t = time.time()
59    return end_t - start_t
60
61def compile_files():
62    cpp_files = glob.glob('tests*.cpp')
63    start_t = time.time()
64    subprocess.check_call([compiler_path, '-c'] + flags + cpp_files)
65    end_t = time.time()
66    return end_t - start_t
67
68def link_files():
69    obj_files = glob.glob('*.o')
70    start_t = time.time()
71    subprocess.check_call([compiler_path] + flags + obj_files)
72    end_t = time.time()
73    return end_t - start_t
74
75def benchmark(func):
76    results = [func() for i in range(10)]
77    return mean(results), median(results)
78
79def char_range(start, end):
80    for c in range(ord(start), ord(end)):
81        yield chr(c)
82
83def generate_sections(fd):
84    for i in range(sections_in_file):
85        fd.write('    SECTION("Section {}") {{\n'.format(i))
86        fd.write('\n'.join('        CHECK({});'.format(check) for check in random.sample(checks, assertions_per_section)))
87        fd.write('    }\n')
88
89
90def generate_file(file_no):
91    with open('tests{}.cpp'.format(file_no), 'w') as f:
92        f.write('#include "catch.hpp"\n\n')
93        for i in range(test_cases_in_file):
94            f.write('TEST_CASE("File {} test {}", "[.compile]"){{\n'.format(file_no, i))
95            for i, c in enumerate(char_range('a', 'f')):
96                f.write('    int {} = {};\n'.format(c, i))
97            generate_sections(f)
98            f.write('}\n\n')
99
100
101def generate_files():
102    create_catch_main()
103    for i in range(files):
104        generate_file(i)
105
106
107options = ['all', 'main', 'files', 'link']
108
109parser = argparse.ArgumentParser(description='Benchmarks Catch\'s compile times against some synthetic tests')
110# Add first arg -- benchmark type
111parser.add_argument('benchmark_kind', nargs='?', default='all', choices=options, help='What kind of benchmark to run, default: all')
112
113# Args to allow changing header/compiler
114parser.add_argument('-I', '--catch-header', default='catch.hpp', help = 'Path to catch.hpp, default: catch.hpp')
115parser.add_argument('-c', '--compiler', default='g++', help = 'Compiler to use, default: g++')
116
117parser.add_argument('-f', '--flags', help = 'Flags to be passed to the compiler. Pass as "," separated list')
118
119# Allow creating files only, without running the whole thing
120parser.add_argument('-g', '--generate-files', action='store_true', help='Generate test files and quit')
121
122args = parser.parse_args()
123
124compiler_path = args.compiler
125catch_path = args.catch_header
126
127if args.generate_files:
128    create_temp_dir()
129    copy_catch(catch_path)
130    os.chdir(dir_name)
131    # now create the fake test files
132    generate_files()
133    # Early exit
134    print('Finished generating files')
135    exit(1)
136
137os.chdir(dir_name)
138
139if args.flags:
140    flags = args.flags.split(',')
141
142print('Time needed for ...')
143if args.benchmark_kind in ('all', 'main'):
144    print('        ... compiling main, mean: {:.2f}, median: {:.2f} s'.format(*benchmark(compile_main)))
145if args.benchmark_kind in ('all', 'files'):
146    print('        ... compiling test files, mean: {:.2f}, median: {:.2f} s'.format(*benchmark(compile_files)))
147if args.benchmark_kind in ('all', 'link'):
148    print('        ... linking everything, mean: {:.2f}, median: {:.2f} s'.format(*benchmark(link_files)))
149