#!/usr/bin/env python3 # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS-IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import argparse import re import textwrap from collections import defaultdict from timeit import default_timer as timer import tempfile import os import shutil import itertools import traceback from typing import Dict, List, Tuple, Optional, Any, TypeVar, Callable, Iterable import numpy import subprocess import yaml from numpy import floor, log10 import scipy import multiprocessing import json import statsmodels.stats.api as stats from generate_benchmark import generate_benchmark import git from functools import lru_cache as memoize class CommandFailedException(Exception): def __init__(self, command: List[str], stdout: str, stderr: str, error_code: str): self.command = command self.stdout = stdout self.stderr = stderr self.error_code = error_code def __str__(self): return textwrap.dedent('''\ Ran command: {command} Exit code {error_code} Stdout: {stdout} Stderr: {stderr} ''').format(command=self.command, error_code=self.error_code, stdout=self.stdout, stderr=self.stderr) def run_command(executable: str, args: List[Any]=[], cwd: str=None, env: Dict[str, str]=None) -> Tuple[str, str]: args = [str(arg) for arg in args] command = [executable] + args try: p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, cwd=cwd, env=env) (stdout, stderr) = p.communicate() except Exception as e: raise Exception("While executing: %s" % command) if p.returncode != 0: raise CommandFailedException(command, stdout, stderr, p.returncode) return (stdout, stderr) compile_flags = ['-O2', '-DNDEBUG'] make_args = ['-j', multiprocessing.cpu_count() + 1] def parse_results(result_lines: List[str]) -> Dict[str, float]: """ Parses results from the format: ['Dimension name1 = 123', 'Long dimension name2 = 23.45'] Into a dict {'Dimension name1': 123.0, 'Dimension name2': 23.45} """ result_dict = dict() for line in result_lines: line_splits = line.split('=') metric = line_splits[0].strip() value = float(line_splits[1].strip()) result_dict[metric] = value return result_dict # We memoize the result since this might be called repeatedly and it's somewhat expensive. @memoize(maxsize=None) def determine_compiler_name(compiler_executable_name: str) -> str: tmpdir = tempfile.gettempdir() + '/fruit-determine-compiler-version-dir' ensure_empty_dir(tmpdir) with open(tmpdir + '/CMakeLists.txt', 'w') as file: file.write('message("@@@${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}@@@")\n') modified_env = os.environ.copy() modified_env['CXX'] = compiler_executable_name # By converting to a list, we force all output to be read (so the command execution is guaranteed to be complete after this line). # Otherwise, subsequent calls to determine_compiler_name might have trouble deleting the temporary directory because the cmake # process is still writing files in there. _, stderr = run_command('cmake', args=['.'], cwd=tmpdir, env=modified_env) cmake_output = stderr.splitlines() for line in cmake_output: re_result = re.search('@@@(.*)@@@', line) if re_result: pretty_name = re_result.group(1) # CMake calls GCC 'GNU', change it into 'GCC'. return pretty_name.replace('GNU ', 'GCC ') raise Exception('Unable to determine compiler. CMake output was: \n', cmake_output) # Returns a pair (sha256_hash, version_name), where version_name will be None if no version tag was found at HEAD. @memoize(maxsize=None) def git_repo_info(repo_path: str) -> Tuple[str, str]: repo = git.Repo(repo_path) head_tags = [tag.name for tag in repo.tags if tag.commit == repo.head.commit and re.match('v[0-9].*', tag.name)] if head_tags == []: head_tag = None else: # There should be only 1 version at any given commit. [head_tag] = head_tags # Remove the 'v' prefix. head_tag = head_tag[1:] return (repo.head.commit.hexsha, head_tag) # Some benchmark parameters, e.g. 'compiler_name' are synthesized automatically from other dimensions (e.g. 'compiler' dimension) or from the environment. # We put the compiler name/version in the results because the same 'compiler' value might refer to different compiler versions # (e.g. if GCC 6.0.0 is installed when benchmarks are run, then it's updated to GCC 6.0.1 and finally the results are formatted, we # want the formatted results to say "GCC 6.0.0" instead of "GCC 6.0.1"). def add_synthetic_benchmark_parameters(original_benchmark_parameters: Dict[str, Any], path_to_code_under_test: Optional[str]): benchmark_params = original_benchmark_parameters.copy() benchmark_params['compiler_name'] = determine_compiler_name(original_benchmark_parameters['compiler']) if path_to_code_under_test is not None: sha256_hash, version_name = git_repo_info(path_to_code_under_test) benchmark_params['di_library_git_commit_hash'] = sha256_hash if version_name is not None: benchmark_params['di_library_version_name'] = version_name return benchmark_params class Benchmark: def prepare(self) -> None: ... def run(self) -> Dict[str, float]: ... def describe(self) -> str: ... class SimpleNewDeleteRunTimeBenchmark(Benchmark): def __init__(self, benchmark_definition: Dict[str, Any], fruit_benchmark_sources_dir: str): self.benchmark_definition = add_synthetic_benchmark_parameters(benchmark_definition, path_to_code_under_test=None) self.fruit_benchmark_sources_dir = fruit_benchmark_sources_dir def prepare(self): cxx_std = self.benchmark_definition['cxx_std'] num_classes = self.benchmark_definition['num_classes'] compiler_executable_name = self.benchmark_definition['compiler'] self.tmpdir = tempfile.gettempdir() + '/fruit-benchmark-dir' ensure_empty_dir(self.tmpdir) run_command(compiler_executable_name, args=compile_flags + [ '-std=%s' % cxx_std, '-DMULTIPLIER=%s' % num_classes, self.fruit_benchmark_sources_dir + '/extras/benchmark/new_delete_benchmark.cpp', '-o', self.tmpdir + '/main', ]) def run(self): loop_factor = self.benchmark_definition['loop_factor'] stdout, _ = run_command(self.tmpdir + '/main', args = [int(5000000 * loop_factor)]) return parse_results(stdout.splitlines()) def describe(self): return self.benchmark_definition class FruitSingleFileCompileTimeBenchmark(Benchmark): def __init__(self, benchmark_definition: Dict[str, Any], fruit_sources_dir: str, fruit_build_dir: str, fruit_benchmark_sources_dir: str): self.benchmark_definition = add_synthetic_benchmark_parameters(benchmark_definition, path_to_code_under_test=fruit_sources_dir) self.fruit_sources_dir = fruit_sources_dir self.fruit_build_dir = fruit_build_dir self.fruit_benchmark_sources_dir = fruit_benchmark_sources_dir num_bindings = self.benchmark_definition['num_bindings'] assert (num_bindings % 5) == 0, num_bindings def prepare(self): pass def run(self): start = timer() cxx_std = self.benchmark_definition['cxx_std'] num_bindings = self.benchmark_definition['num_bindings'] compiler_executable_name = self.benchmark_definition['compiler'] run_command(compiler_executable_name, args = compile_flags + [ '-std=%s' % cxx_std, '-DMULTIPLIER=%s' % (num_bindings // 5), '-I', self.fruit_sources_dir + '/include', '-I', self.fruit_build_dir + '/include', '-ftemplate-depth=1000', '-c', self.fruit_benchmark_sources_dir + '/extras/benchmark/compile_time_benchmark.cpp', '-o', '/dev/null', ]) end = timer() return {"compile_time": end - start} def describe(self): return self.benchmark_definition def ensure_empty_dir(dirname: str): # We start by creating the directory instead of just calling rmtree with ignore_errors=True because that would ignore # all errors, so we might otherwise go ahead even if the directory wasn't properly deleted. os.makedirs(dirname, exist_ok=True) shutil.rmtree(dirname) os.makedirs(dirname) class GenericGeneratedSourcesBenchmark(Benchmark): def __init__(self, di_library, benchmark_definition, path_to_code_under_test=None, **other_args): self.di_library = di_library self.benchmark_definition = add_synthetic_benchmark_parameters(benchmark_definition, path_to_code_under_test=path_to_code_under_test) self.other_args = other_args self.arbitrary_file = None def prepare_compile_benchmark(self): num_classes = self.benchmark_definition['num_classes'] cxx_std = self.benchmark_definition['cxx_std'] compiler_executable_name = self.benchmark_definition['compiler'] benchmark_generation_flags = {flag_name: True for flag_name in self.benchmark_definition['benchmark_generation_flags']} self.tmpdir = tempfile.gettempdir() + '/fruit-benchmark-dir' ensure_empty_dir(self.tmpdir) num_classes_with_no_deps = int(num_classes * 0.1) return generate_benchmark( compiler=compiler_executable_name, num_components_with_no_deps=num_classes_with_no_deps, num_components_with_deps=num_classes - num_classes_with_no_deps, num_deps=10, output_dir=self.tmpdir, cxx_std=cxx_std, di_library=self.di_library, **benchmark_generation_flags, **self.other_args) def run_make_build(self): run_command('make', args=make_args, cwd=self.tmpdir) def prepare_incremental_compile_benchmark(self): files = self.prepare_compile_benchmark() self.run_make_build() files = list(sorted(file for file in files if file.endswith('.h'))) # 5 files, equally spaced (but not at beginning/end) in the sorted sequence. num_files_changed = 5 self.arbitrary_files = [files[i * (len(files) // (num_files_changed + 2))] for i in range(1, num_files_changed + 1)] def prepare_compile_memory_benchmark(self): self.prepare_compile_benchmark() self.run_compile_memory_benchmark() def prepare_runtime_benchmark(self): self.prepare_compile_benchmark() self.run_make_build() def prepare_startup_benchmark(self): self.prepare_compile_benchmark() self.run_make_build() run_command('strip', args=[self.tmpdir + '/main']) def prepare_executable_size_benchmark(self): self.prepare_runtime_benchmark() run_command('strip', args=[self.tmpdir + '/main']) def run_compile_benchmark(self): run_command('make', args=make_args + ['clean'], cwd=self.tmpdir) start = timer() self.run_make_build() end = timer() result = {'compile_time': end - start} return result def run_incremental_compile_benchmark(self): run_command('touch', args=self.arbitrary_files, cwd=self.tmpdir) start = timer() self.run_make_build() end = timer() result = {'incremental_compile_time': end - start} return result def run_compile_memory_benchmark(self): run_command('make', args=make_args + ['clean'], cwd=self.tmpdir) run_command('make', args=make_args + ['main_ram.txt'], cwd=self.tmpdir) with open(self.tmpdir + '/main_ram.txt') as f: ram_usages = [int(n)*1024 for n in f.readlines()] return { 'total_max_ram_usage': sum(ram_usages), 'max_ram_usage': max(ram_usages), } def run_runtime_benchmark(self): num_classes = self.benchmark_definition['num_classes'] loop_factor = self.benchmark_definition['loop_factor'] results, _ = run_command(self.tmpdir + '/main', args = [ # 40M loops with 100 classes, 40M with 1000 int(4 * 1000 * 1000 * 1000 * loop_factor / num_classes), ]) return parse_results(results.splitlines()) def run_startup_benchmark(self): n = 1000 start = timer() for i in range(0, n): run_command(self.tmpdir + '/main', args = []) end = timer() result = {'startup_time': (end - start) / n} return result def run_executable_size_benchmark(self): wc_result, _ = run_command('wc', args=['-c', self.tmpdir + '/main']) num_bytes = wc_result.splitlines()[0].split(' ')[0] return {'num_bytes': float(num_bytes)} def describe(self): return self.benchmark_definition class CompileTimeBenchmark(GenericGeneratedSourcesBenchmark): def __init__(self, **kwargs): super().__init__(generate_runtime_bench_code=False, **kwargs) def prepare(self): self.prepare_compile_benchmark() def run(self): return self.run_compile_benchmark() class IncrementalCompileTimeBenchmark(GenericGeneratedSourcesBenchmark): def __init__(self, **kwargs): super().__init__(generate_runtime_bench_code=False, **kwargs) def prepare(self): self.prepare_incremental_compile_benchmark() def run(self): return self.run_incremental_compile_benchmark() class CompileMemoryBenchmark(GenericGeneratedSourcesBenchmark): def __init__(self, **kwargs): super().__init__(generate_runtime_bench_code=False, **kwargs) def prepare(self): self.prepare_compile_memory_benchmark() def run(self): return self.run_compile_memory_benchmark() class StartupTimeBenchmark(GenericGeneratedSourcesBenchmark): def __init__(self, **kwargs): super().__init__(generate_runtime_bench_code=False, **kwargs) def prepare(self): self.prepare_startup_benchmark() def run(self): return self.run_startup_benchmark() class RunTimeBenchmark(GenericGeneratedSourcesBenchmark): def __init__(self, **kwargs): super().__init__(generate_runtime_bench_code=True, **kwargs) def prepare(self): self.prepare_runtime_benchmark() def run(self): return self.run_runtime_benchmark() # This is not really a 'benchmark', but we consider it as such to reuse the benchmark infrastructure. class ExecutableSizeBenchmark(GenericGeneratedSourcesBenchmark): def __init__(self, **kwargs): super().__init__(generate_runtime_bench_code=False, **kwargs) def prepare(self): self.prepare_executable_size_benchmark() def run(self): return self.run_executable_size_benchmark() # This is not really a 'benchmark', but we consider it as such to reuse the benchmark infrastructure. class ExecutableSizeBenchmarkWithoutExceptionsAndRtti(ExecutableSizeBenchmark): def __init__(self, **kwargs): super().__init__(use_exceptions=False, use_rtti=False, **kwargs) class FruitCompileTimeBenchmark(CompileTimeBenchmark): def __init__(self, fruit_sources_dir, **kwargs): super().__init__(di_library='fruit', path_to_code_under_test=fruit_sources_dir, fruit_sources_dir=fruit_sources_dir, **kwargs) class FruitIncrementalCompileTimeBenchmark(IncrementalCompileTimeBenchmark): def __init__(self, fruit_sources_dir, **kwargs): super().__init__(di_library='fruit', path_to_code_under_test=fruit_sources_dir, fruit_sources_dir=fruit_sources_dir, **kwargs) class FruitCompileMemoryBenchmark(CompileMemoryBenchmark): def __init__(self, fruit_sources_dir, **kwargs): super().__init__(di_library='fruit', path_to_code_under_test=fruit_sources_dir, fruit_sources_dir=fruit_sources_dir, **kwargs) class FruitRunTimeBenchmark(RunTimeBenchmark): def __init__(self, fruit_sources_dir, **kwargs): super().__init__(di_library='fruit', path_to_code_under_test=fruit_sources_dir, fruit_sources_dir=fruit_sources_dir, **kwargs) class FruitStartupTimeBenchmark(StartupTimeBenchmark): def __init__(self, fruit_sources_dir, **kwargs): super().__init__(di_library='fruit', path_to_code_under_test=fruit_sources_dir, fruit_sources_dir=fruit_sources_dir, **kwargs) class FruitStartupTimeWithNormalizedComponentBenchmark(FruitStartupTimeBenchmark): def __init__(self, **kwargs): super().__init__(use_normalized_component=True, **kwargs) # This is not really a 'benchmark', but we consider it as such to reuse the benchmark infrastructure. class FruitExecutableSizeBenchmark(ExecutableSizeBenchmark): def __init__(self, fruit_sources_dir, **kwargs): super().__init__(di_library='fruit', path_to_code_under_test=fruit_sources_dir, fruit_sources_dir=fruit_sources_dir, **kwargs) # This is not really a 'benchmark', but we consider it as such to reuse the benchmark infrastructure. class FruitExecutableSizeBenchmarkWithoutExceptionsAndRtti(ExecutableSizeBenchmarkWithoutExceptionsAndRtti): def __init__(self, fruit_sources_dir, **kwargs): super().__init__(di_library='fruit', path_to_code_under_test=fruit_sources_dir, fruit_sources_dir=fruit_sources_dir, **kwargs) class BoostDiCompileTimeBenchmark(CompileTimeBenchmark): def __init__(self, boost_di_sources_dir, **kwargs): super().__init__(di_library='boost_di', path_to_code_under_test=boost_di_sources_dir, boost_di_sources_dir=boost_di_sources_dir, **kwargs) class BoostDiIncrementalCompileTimeBenchmark(IncrementalCompileTimeBenchmark): def __init__(self, boost_di_sources_dir, **kwargs): super().__init__(di_library='boost_di', path_to_code_under_test=boost_di_sources_dir, boost_di_sources_dir=boost_di_sources_dir, **kwargs) class BoostDiCompileMemoryBenchmark(CompileMemoryBenchmark): def __init__(self, boost_di_sources_dir, **kwargs): super().__init__(di_library='boost_di', path_to_code_under_test=boost_di_sources_dir, boost_di_sources_dir=boost_di_sources_dir, **kwargs) class BoostDiRunTimeBenchmark(RunTimeBenchmark): def __init__(self, boost_di_sources_dir, **kwargs): super().__init__(di_library='boost_di', path_to_code_under_test=boost_di_sources_dir, boost_di_sources_dir=boost_di_sources_dir, **kwargs) class BoostDiStartupTimeBenchmark(StartupTimeBenchmark): def __init__(self, boost_di_sources_dir, **kwargs): super().__init__(di_library='boost_di', path_to_code_under_test=boost_di_sources_dir, boost_di_sources_dir=boost_di_sources_dir, **kwargs) # This is not really a 'benchmark', but we consider it as such to reuse the benchmark infrastructure. class BoostDiExecutableSizeBenchmark(ExecutableSizeBenchmark): def __init__(self, boost_di_sources_dir, **kwargs): super().__init__(di_library='boost_di', path_to_code_under_test=boost_di_sources_dir, boost_di_sources_dir=boost_di_sources_dir, **kwargs) # This is not really a 'benchmark', but we consider it as such to reuse the benchmark infrastructure. class BoostDiExecutableSizeBenchmarkWithoutExceptionsAndRtti(ExecutableSizeBenchmarkWithoutExceptionsAndRtti): def __init__(self, boost_di_sources_dir, **kwargs): super().__init__(di_library='boost_di', path_to_code_under_test=boost_di_sources_dir, boost_di_sources_dir=boost_di_sources_dir, **kwargs) class SimpleDiCompileTimeBenchmark(CompileTimeBenchmark): def __init__(self, **kwargs): super().__init__(di_library='none', **kwargs) class SimpleDiIncrementalCompileTimeBenchmark(IncrementalCompileTimeBenchmark): def __init__(self, **kwargs): super().__init__(di_library='none', **kwargs) class SimpleDiCompileMemoryBenchmark(CompileMemoryBenchmark): def __init__(self, **kwargs): super().__init__(di_library='none', **kwargs) class SimpleDiRunTimeBenchmark(RunTimeBenchmark): def __init__(self, **kwargs): super().__init__(di_library='none', **kwargs) class SimpleDiStartupTimeBenchmark(StartupTimeBenchmark): def __init__(self, **kwargs): super().__init__(di_library='none', **kwargs) # This is not really a 'benchmark', but we consider it as such to reuse the benchmark infrastructure. class SimpleDiExecutableSizeBenchmark(ExecutableSizeBenchmark): def __init__(self, **kwargs): super().__init__(di_library='none', **kwargs) # This is not really a 'benchmark', but we consider it as such to reuse the benchmark infrastructure. class SimpleDiExecutableSizeBenchmarkWithoutExceptionsAndRtti(ExecutableSizeBenchmarkWithoutExceptionsAndRtti): def __init__(self, **kwargs): super().__init__(di_library='none', **kwargs) class SimpleDiWithInterfacesCompileTimeBenchmark(SimpleDiCompileTimeBenchmark): def __init__(self, **kwargs): super().__init__(use_interfaces=True, **kwargs) class SimpleDiWithInterfacesIncrementalCompileTimeBenchmark(SimpleDiIncrementalCompileTimeBenchmark): def __init__(self, **kwargs): super().__init__(use_interfaces=True, **kwargs) class SimpleDiWithInterfacesCompileMemoryBenchmark(SimpleDiCompileMemoryBenchmark): def __init__(self, **kwargs): super().__init__(use_interfaces=True, **kwargs) class SimpleDiWithInterfacesRunTimeBenchmark(SimpleDiRunTimeBenchmark): def __init__(self, **kwargs): super().__init__(use_interfaces=True, **kwargs) class SimpleDiWithInterfacesStartupTimeBenchmark(SimpleDiStartupTimeBenchmark): def __init__(self, **kwargs): super().__init__(use_interfaces=True, **kwargs) # This is not really a 'benchmark', but we consider it as such to reuse the benchmark infrastructure. class SimpleDiWithInterfacesExecutableSizeBenchmark(SimpleDiExecutableSizeBenchmark): def __init__(self, **kwargs): super().__init__(use_interfaces=True, **kwargs) # This is not really a 'benchmark', but we consider it as such to reuse the benchmark infrastructure. class SimpleDiWithInterfacesExecutableSizeBenchmarkWithoutExceptionsAndRtti(SimpleDiExecutableSizeBenchmarkWithoutExceptionsAndRtti): def __init__(self, **kwargs): super().__init__(use_interfaces=True, **kwargs) class SimpleDiWithInterfacesAndNewDeleteCompileTimeBenchmark(SimpleDiWithInterfacesCompileTimeBenchmark): def __init__(self, **kwargs): super().__init__(use_new_delete=True, **kwargs) class SimpleDiWithInterfacesAndNewDeleteIncrementalCompileTimeBenchmark(SimpleDiWithInterfacesIncrementalCompileTimeBenchmark): def __init__(self, **kwargs): super().__init__(use_new_delete=True, **kwargs) class SimpleDiWithInterfacesAndNewDeleteCompileMemoryBenchmark(SimpleDiWithInterfacesCompileMemoryBenchmark): def __init__(self, **kwargs): super().__init__(use_new_delete=True, **kwargs) class SimpleDiWithInterfacesAndNewDeleteRunTimeBenchmark(SimpleDiWithInterfacesRunTimeBenchmark): def __init__(self, **kwargs): super().__init__(use_new_delete=True, **kwargs) class SimpleDiWithInterfacesAndNewDeleteStartupTimeBenchmark(SimpleDiWithInterfacesStartupTimeBenchmark): def __init__(self, **kwargs): super().__init__(use_new_delete=True, **kwargs) # This is not really a 'benchmark', but we consider it as such to reuse the benchmark infrastructure. class SimpleDiWithInterfacesAndNewDeleteExecutableSizeBenchmark(SimpleDiWithInterfacesExecutableSizeBenchmark): def __init__(self, **kwargs): super().__init__(use_new_delete=True, **kwargs) # This is not really a 'benchmark', but we consider it as such to reuse the benchmark infrastructure. class SimpleDiWithInterfacesAndNewDeleteExecutableSizeBenchmarkWithoutExceptionsAndRtti(SimpleDiWithInterfacesExecutableSizeBenchmarkWithoutExceptionsAndRtti): def __init__(self, **kwargs): super().__init__(use_new_delete=True, **kwargs) def round_to_significant_digits(n: float, num_significant_digits: int) -> float: if n <= 0: # We special-case this, otherwise the log10 below will fail. return 0 return round(n, num_significant_digits - int(floor(log10(n))) - 1) def run_benchmark(benchmark: Benchmark, max_runs: int, timeout_hours: int, output_file: str, min_runs: int=3) -> None: def run_benchmark_once(): print('Running benchmark... ', end='', flush=True) result = benchmark.run() print(result) for dimension, value in result.items(): results_by_dimension[dimension] += [value] results_by_dimension = defaultdict(lambda: []) print('Preparing for benchmark... ', end='', flush=True) benchmark.prepare() print('Done.') start_time = timer() # Run at least min_runs times for i in range(min_runs): run_benchmark_once() # Then consider running a few more times to get the desired precision. while True: if timer() - start_time > timeout_hours * 3600: print("Warning: timed out, couldn't determine a result with the desired precision.") break for dimension, results in results_by_dimension.items(): if all(result == results[0] for result in results): # If all results are exactly the same the code below misbehaves. We don't need to run again in this case. continue confidence_interval = stats.DescrStatsW(results).tconfint_mean(0.05) confidence_interval_2dig = (round_to_significant_digits(confidence_interval[0], 2), round_to_significant_digits(confidence_interval[1], 2)) if abs(confidence_interval_2dig[0] - confidence_interval_2dig[1]) > numpy.finfo(float).eps * 10: if len(results) < max_runs: print("Running again to get more precision on the metric %s. Current confidence interval: [%.3g, %.3g]" % ( dimension, confidence_interval[0], confidence_interval[1])) break else: print("Warning: couldn't determine a precise result for the metric %s. Confidence interval: [%.3g, %.3g]" % ( dimension, confidence_interval[0], confidence_interval[1])) else: # We've reached sufficient precision in all metrics, or we've reached the max number of runs. break run_benchmark_once() # We've reached the desired precision in all dimensions or reached the maximum number of runs. Record the results. rounded_confidence_intervals_by_dimension = {} confidence_intervals_by_dimension = {} for dimension, results in results_by_dimension.items(): confidence_interval = stats.DescrStatsW(results).tconfint_mean(0.05) confidence_interval_2dig = (round_to_significant_digits(confidence_interval[0], 2), round_to_significant_digits(confidence_interval[1], 2)) rounded_confidence_intervals_by_dimension[dimension] = confidence_interval_2dig confidence_intervals_by_dimension[dimension] = (confidence_interval, confidence_interval_2dig) with open(output_file, 'a') as f: json.dump({"benchmark": benchmark.describe(), "results": confidence_intervals_by_dimension}, f) print(file=f) print('Benchmark finished. Result: ', rounded_confidence_intervals_by_dimension) print() def expand_benchmark_definition(benchmark_definition: Dict[str, Any]) -> List[Dict[str, Tuple[Any]]]: """ Takes a benchmark definition, e.g.: [{name: 'foo', compiler: ['g++-5', 'g++-6']}, {name: ['bar', 'baz'], compiler: ['g++-5'], cxx_std: 'c++14'}] And expands it into the individual benchmarks to run, in the example above: [{name: 'foo', compiler: 'g++-5'}, {name: 'foo', compiler: 'g++-6'}, {name: 'bar', compiler: 'g++-5', cxx_std: 'c++14'}, {name: 'baz', compiler: 'g++-5', cxx_std: 'c++14'}] """ dict_keys = sorted(benchmark_definition.keys()) # Turn non-list values into single-item lists. benchmark_definition = {dict_key: value if isinstance(value, list) else [value] for dict_key, value in benchmark_definition.items()} # Compute the cartesian product of the value lists value_combinations = itertools.product(*(benchmark_definition[dict_key] for dict_key in dict_keys)) # Then turn the result back into a dict. return [dict(zip(dict_keys, value_combination)) for value_combination in value_combinations] def expand_benchmark_definitions(benchmark_definitions: List[Dict[str, Any]]): return list(itertools.chain(*[expand_benchmark_definition(benchmark_definition) for benchmark_definition in benchmark_definitions])) T = TypeVar('T') K = TypeVar('K') def group_by(l: List[T], element_to_key: Callable[[T], K]) -> Iterable[Tuple[K, List[T]]]: """Takes a list and returns a list of sublists, where the elements are grouped using the provided function""" result = defaultdict(list) # type: Dict[K, List[T]] for elem in l: result[element_to_key(elem)].append(elem) return result.items() def main(): # This configures numpy/scipy to raise an exception in case of errors, instead of printing a warning and going ahead. numpy.seterr(all='raise') scipy.seterr(all='raise') parser = argparse.ArgumentParser(description='Runs a set of benchmarks defined in a YAML file.') parser.add_argument('--fruit-benchmark-sources-dir', help='Path to the fruit sources (used for benchmarking code only)') parser.add_argument('--fruit-sources-dir', help='Path to the fruit sources') parser.add_argument('--boost-di-sources-dir', help='Path to the Boost.DI sources') parser.add_argument('--output-file', help='The output file where benchmark results will be stored (1 per line, with each line in JSON format). These can then be formatted by e.g. the format_bench_results script.') parser.add_argument('--benchmark-definition', help='The YAML file that defines the benchmarks (see fruit_wiki_benchs_fruit.yml for an example).') parser.add_argument('--continue-benchmark', help='If this is \'true\', continues a previous benchmark run instead of starting from scratch (taking into account the existing benchmark results in the file specified with --output-file).') args = parser.parse_args() if args.output_file is None: raise Exception('You must specify --output_file') if args.continue_benchmark == 'true': try: with open(args.output_file, 'r') as f: previous_run_completed_benchmarks = [json.loads(line)['benchmark'] for line in f.readlines()] except FileNotFoundError: previous_run_completed_benchmarks = [] else: previous_run_completed_benchmarks = [] run_command('rm', args=['-f', args.output_file]) fruit_build_dir = tempfile.gettempdir() + '/fruit-benchmark-build-dir' with open(args.benchmark_definition, 'r') as f: yaml_file_content = yaml.full_load(f) global_definitions = yaml_file_content['global'] benchmark_definitions = expand_benchmark_definitions(yaml_file_content['benchmarks']) benchmark_index = 0 for (compiler_executable_name, additional_cmake_args), benchmark_definitions_with_current_config \ in group_by(benchmark_definitions, lambda benchmark_definition: (benchmark_definition['compiler'], tuple(benchmark_definition['additional_cmake_args']))): print('Preparing for benchmarks with the compiler %s, with additional CMake args %s' % (compiler_executable_name, additional_cmake_args)) try: # We compute this here (and memoize the result) so that the benchmark's describe() will retrieve the cached # value instantly. determine_compiler_name(compiler_executable_name) # Build Fruit in fruit_build_dir, so that fruit_build_dir points to a built Fruit (useful for e.g. the config header). shutil.rmtree(fruit_build_dir, ignore_errors=True) os.makedirs(fruit_build_dir) modified_env = os.environ.copy() modified_env['CXX'] = compiler_executable_name run_command('cmake', args=[ args.fruit_sources_dir, '-DCMAKE_BUILD_TYPE=Release', *additional_cmake_args, ], cwd=fruit_build_dir, env=modified_env) run_command('make', args=make_args, cwd=fruit_build_dir) except Exception as e: print('Exception while preparing for benchmarks with the compiler %s, with additional CMake args %s.\n%s\nGoing ahead with the rest.' % (compiler_executable_name, additional_cmake_args, traceback.format_exc())) continue for benchmark_definition in benchmark_definitions_with_current_config: benchmark_index += 1 print('%s/%s: %s' % (benchmark_index, len(benchmark_definitions), benchmark_definition)) benchmark_name = benchmark_definition['name'] if (benchmark_name in {'boost_di_compile_time', 'boost_di_run_time', 'boost_di_executable_size'} and args.boost_di_sources_dir is None): raise Exception('Error: you need to specify the --boost-di-sources-dir flag in order to run Boost.DI benchmarks.') if benchmark_name == 'new_delete_run_time': benchmark = SimpleNewDeleteRunTimeBenchmark( benchmark_definition, fruit_benchmark_sources_dir=args.fruit_benchmark_sources_dir) elif benchmark_name == 'fruit_single_file_compile_time': benchmark = FruitSingleFileCompileTimeBenchmark( benchmark_definition, fruit_sources_dir=args.fruit_sources_dir, fruit_benchmark_sources_dir=args.fruit_benchmark_sources_dir, fruit_build_dir=fruit_build_dir) elif benchmark_name.startswith('fruit_'): benchmark_class = { 'fruit_compile_time': FruitCompileTimeBenchmark, 'fruit_incremental_compile_time': FruitIncrementalCompileTimeBenchmark, 'fruit_compile_memory': FruitCompileMemoryBenchmark, 'fruit_run_time': FruitRunTimeBenchmark, 'fruit_startup_time': FruitStartupTimeBenchmark, 'fruit_startup_time_with_normalized_component': FruitStartupTimeWithNormalizedComponentBenchmark, 'fruit_executable_size': FruitExecutableSizeBenchmark, 'fruit_executable_size_without_exceptions_and_rtti': FruitExecutableSizeBenchmarkWithoutExceptionsAndRtti, }[benchmark_name] benchmark = benchmark_class( benchmark_definition=benchmark_definition, fruit_sources_dir=args.fruit_sources_dir, fruit_build_dir=fruit_build_dir) elif benchmark_name.startswith('boost_di_'): benchmark_class = { 'boost_di_compile_time': BoostDiCompileTimeBenchmark, 'boost_di_incremental_compile_time': BoostDiIncrementalCompileTimeBenchmark, 'boost_di_compile_memory': BoostDiCompileMemoryBenchmark, 'boost_di_run_time': BoostDiRunTimeBenchmark, 'boost_di_startup_time': BoostDiStartupTimeBenchmark, 'boost_di_executable_size': BoostDiExecutableSizeBenchmark, 'boost_di_executable_size_without_exceptions_and_rtti': BoostDiExecutableSizeBenchmarkWithoutExceptionsAndRtti, }[benchmark_name] benchmark = benchmark_class( benchmark_definition=benchmark_definition, boost_di_sources_dir=args.boost_di_sources_dir) elif benchmark_name.startswith('simple_di_'): benchmark_class = { 'simple_di_compile_time': SimpleDiCompileTimeBenchmark, 'simple_di_incremental_compile_time': SimpleDiIncrementalCompileTimeBenchmark, 'simple_di_compile_memory': SimpleDiCompileMemoryBenchmark, 'simple_di_run_time': SimpleDiRunTimeBenchmark, 'simple_di_startup_time': SimpleDiStartupTimeBenchmark, 'simple_di_executable_size': SimpleDiExecutableSizeBenchmark, 'simple_di_executable_size_without_exceptions_and_rtti': SimpleDiExecutableSizeBenchmarkWithoutExceptionsAndRtti, 'simple_di_with_interfaces_compile_time': SimpleDiWithInterfacesCompileTimeBenchmark, 'simple_di_with_interfaces_incremental_compile_time': SimpleDiWithInterfacesIncrementalCompileTimeBenchmark, 'simple_di_with_interfaces_compile_memory': SimpleDiWithInterfacesCompileMemoryBenchmark, 'simple_di_with_interfaces_run_time': SimpleDiWithInterfacesRunTimeBenchmark, 'simple_di_with_interfaces_startup_time': SimpleDiWithInterfacesStartupTimeBenchmark, 'simple_di_with_interfaces_executable_size': SimpleDiWithInterfacesExecutableSizeBenchmark, 'simple_di_with_interfaces_executable_size_without_exceptions_and_rtti': SimpleDiWithInterfacesExecutableSizeBenchmarkWithoutExceptionsAndRtti, 'simple_di_with_interfaces_and_new_delete_compile_time': SimpleDiWithInterfacesAndNewDeleteCompileTimeBenchmark, 'simple_di_with_interfaces_and_new_delete_incremental_compile_time': SimpleDiWithInterfacesAndNewDeleteIncrementalCompileTimeBenchmark, 'simple_di_with_interfaces_and_new_delete_compile_memory': SimpleDiWithInterfacesAndNewDeleteCompileMemoryBenchmark, 'simple_di_with_interfaces_and_new_delete_run_time': SimpleDiWithInterfacesAndNewDeleteRunTimeBenchmark, 'simple_di_with_interfaces_and_new_delete_startup_time': SimpleDiWithInterfacesAndNewDeleteStartupTimeBenchmark, 'simple_di_with_interfaces_and_new_delete_executable_size': SimpleDiWithInterfacesAndNewDeleteExecutableSizeBenchmark, 'simple_di_with_interfaces_and_new_delete_executable_size_without_exceptions_and_rtti': SimpleDiWithInterfacesAndNewDeleteExecutableSizeBenchmarkWithoutExceptionsAndRtti, }[benchmark_name] benchmark = benchmark_class( benchmark_definition=benchmark_definition) else: raise Exception("Unrecognized benchmark: %s" % benchmark_name) if benchmark.describe() in previous_run_completed_benchmarks: print("Skipping benchmark that was already run previously (due to --continue-benchmark):", benchmark.describe()) continue try: run_benchmark(benchmark, output_file=args.output_file, max_runs=global_definitions['max_runs'], timeout_hours=global_definitions['max_hours_per_combination']) except Exception as e: print('Exception while running benchmark: %s.\n%s\nGoing ahead with the rest.' % (benchmark.describe(), traceback.format_exc())) if __name__ == "__main__": main()