1#!/usr/bin/env python3 2# Copyright 2016 Google Inc. All Rights Reserved. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS-IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16import argparse 17import re 18import textwrap 19from collections import defaultdict 20from timeit import default_timer as timer 21import tempfile 22import os 23import shutil 24import itertools 25import traceback 26from typing import Dict, List, Tuple, Optional, Any, TypeVar, Callable, Iterable 27 28import numpy 29import subprocess 30import yaml 31from numpy import floor, log10 32import scipy 33import multiprocessing 34import json 35import statsmodels.stats.api as stats 36from generate_benchmark import generate_benchmark 37import git 38from functools import lru_cache as memoize 39 40class CommandFailedException(Exception): 41 def __init__(self, command: List[str], stdout: str, stderr: str, error_code: str): 42 self.command = command 43 self.stdout = stdout 44 self.stderr = stderr 45 self.error_code = error_code 46 47 def __str__(self): 48 return textwrap.dedent('''\ 49 Ran command: {command} 50 Exit code {error_code} 51 Stdout: 52 {stdout} 53 54 Stderr: 55 {stderr} 56 ''').format(command=self.command, error_code=self.error_code, stdout=self.stdout, stderr=self.stderr) 57 58def run_command(executable: str, args: List[Any]=[], cwd: str=None, env: Dict[str, str]=None) -> Tuple[str, str]: 59 args = [str(arg) for arg in args] 60 command = [executable] + args 61 try: 62 p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, cwd=cwd, 63 env=env) 64 (stdout, stderr) = p.communicate() 65 except Exception as e: 66 raise Exception("While executing: %s" % command) 67 if p.returncode != 0: 68 raise CommandFailedException(command, stdout, stderr, p.returncode) 69 return (stdout, stderr) 70 71compile_flags = ['-O2', '-DNDEBUG'] 72 73make_args = ['-j', multiprocessing.cpu_count() + 1] 74 75def parse_results(result_lines: List[str]) -> Dict[str, float]: 76 """ 77 Parses results from the format: 78 ['Dimension name1 = 123', 79 'Long dimension name2 = 23.45'] 80 81 Into a dict {'Dimension name1': 123.0, 'Dimension name2': 23.45} 82 """ 83 result_dict = dict() 84 for line in result_lines: 85 line_splits = line.split('=') 86 metric = line_splits[0].strip() 87 value = float(line_splits[1].strip()) 88 result_dict[metric] = value 89 return result_dict 90 91 92# We memoize the result since this might be called repeatedly and it's somewhat expensive. 93@memoize(maxsize=None) 94def determine_compiler_name(compiler_executable_name: str) -> str: 95 tmpdir = tempfile.gettempdir() + '/fruit-determine-compiler-version-dir' 96 ensure_empty_dir(tmpdir) 97 with open(tmpdir + '/CMakeLists.txt', 'w') as file: 98 file.write('message("@@@${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}@@@")\n') 99 modified_env = os.environ.copy() 100 modified_env['CXX'] = compiler_executable_name 101 # By converting to a list, we force all output to be read (so the command execution is guaranteed to be complete after this line). 102 # Otherwise, subsequent calls to determine_compiler_name might have trouble deleting the temporary directory because the cmake 103 # process is still writing files in there. 104 _, stderr = run_command('cmake', args=['.'], cwd=tmpdir, env=modified_env) 105 cmake_output = stderr.splitlines() 106 for line in cmake_output: 107 re_result = re.search('@@@(.*)@@@', line) 108 if re_result: 109 pretty_name = re_result.group(1) 110 # CMake calls GCC 'GNU', change it into 'GCC'. 111 return pretty_name.replace('GNU ', 'GCC ') 112 raise Exception('Unable to determine compiler. CMake output was: \n', cmake_output) 113 114 115# Returns a pair (sha256_hash, version_name), where version_name will be None if no version tag was found at HEAD. 116@memoize(maxsize=None) 117def git_repo_info(repo_path: str) -> Tuple[str, str]: 118 repo = git.Repo(repo_path) 119 head_tags = [tag.name for tag in repo.tags if tag.commit == repo.head.commit and re.match('v[0-9].*', tag.name)] 120 if head_tags == []: 121 head_tag = None 122 else: 123 # There should be only 1 version at any given commit. 124 [head_tag] = head_tags 125 # Remove the 'v' prefix. 126 head_tag = head_tag[1:] 127 return (repo.head.commit.hexsha, head_tag) 128 129 130# Some benchmark parameters, e.g. 'compiler_name' are synthesized automatically from other dimensions (e.g. 'compiler' dimension) or from the environment. 131# We put the compiler name/version in the results because the same 'compiler' value might refer to different compiler versions 132# (e.g. if GCC 6.0.0 is installed when benchmarks are run, then it's updated to GCC 6.0.1 and finally the results are formatted, we 133# want the formatted results to say "GCC 6.0.0" instead of "GCC 6.0.1"). 134def add_synthetic_benchmark_parameters(original_benchmark_parameters: Dict[str, Any], path_to_code_under_test: Optional[str]): 135 benchmark_params = original_benchmark_parameters.copy() 136 benchmark_params['compiler_name'] = determine_compiler_name(original_benchmark_parameters['compiler']) 137 if path_to_code_under_test is not None: 138 sha256_hash, version_name = git_repo_info(path_to_code_under_test) 139 benchmark_params['di_library_git_commit_hash'] = sha256_hash 140 if version_name is not None: 141 benchmark_params['di_library_version_name'] = version_name 142 return benchmark_params 143 144 145class Benchmark: 146 def prepare(self) -> None: ... 147 def run(self) -> Dict[str, float]: ... 148 def describe(self) -> str: ... 149 150class SimpleNewDeleteRunTimeBenchmark(Benchmark): 151 def __init__(self, benchmark_definition: Dict[str, Any], fruit_benchmark_sources_dir: str): 152 self.benchmark_definition = add_synthetic_benchmark_parameters(benchmark_definition, path_to_code_under_test=None) 153 self.fruit_benchmark_sources_dir = fruit_benchmark_sources_dir 154 155 def prepare(self): 156 cxx_std = self.benchmark_definition['cxx_std'] 157 num_classes = self.benchmark_definition['num_classes'] 158 compiler_executable_name = self.benchmark_definition['compiler'] 159 160 self.tmpdir = tempfile.gettempdir() + '/fruit-benchmark-dir' 161 ensure_empty_dir(self.tmpdir) 162 run_command(compiler_executable_name, 163 args=compile_flags + [ 164 '-std=%s' % cxx_std, 165 '-DMULTIPLIER=%s' % num_classes, 166 self.fruit_benchmark_sources_dir + '/extras/benchmark/new_delete_benchmark.cpp', 167 '-o', 168 self.tmpdir + '/main', 169 ]) 170 171 def run(self): 172 loop_factor = self.benchmark_definition['loop_factor'] 173 stdout, _ = run_command(self.tmpdir + '/main', args = [int(5000000 * loop_factor)]) 174 return parse_results(stdout.splitlines()) 175 176 def describe(self): 177 return self.benchmark_definition 178 179 180class FruitSingleFileCompileTimeBenchmark(Benchmark): 181 def __init__(self, benchmark_definition: Dict[str, Any], fruit_sources_dir: str, fruit_build_dir: str, fruit_benchmark_sources_dir: str): 182 self.benchmark_definition = add_synthetic_benchmark_parameters(benchmark_definition, path_to_code_under_test=fruit_sources_dir) 183 self.fruit_sources_dir = fruit_sources_dir 184 self.fruit_build_dir = fruit_build_dir 185 self.fruit_benchmark_sources_dir = fruit_benchmark_sources_dir 186 num_bindings = self.benchmark_definition['num_bindings'] 187 assert (num_bindings % 5) == 0, num_bindings 188 189 def prepare(self): 190 pass 191 192 def run(self): 193 start = timer() 194 cxx_std = self.benchmark_definition['cxx_std'] 195 num_bindings = self.benchmark_definition['num_bindings'] 196 compiler_executable_name = self.benchmark_definition['compiler'] 197 198 run_command(compiler_executable_name, 199 args = compile_flags + [ 200 '-std=%s' % cxx_std, 201 '-DMULTIPLIER=%s' % (num_bindings // 5), 202 '-I', self.fruit_sources_dir + '/include', 203 '-I', self.fruit_build_dir + '/include', 204 '-ftemplate-depth=1000', 205 '-c', 206 self.fruit_benchmark_sources_dir + '/extras/benchmark/compile_time_benchmark.cpp', 207 '-o', 208 '/dev/null', 209 ]) 210 end = timer() 211 return {"compile_time": end - start} 212 213 def describe(self): 214 return self.benchmark_definition 215 216 217def ensure_empty_dir(dirname: str): 218 # We start by creating the directory instead of just calling rmtree with ignore_errors=True because that would ignore 219 # all errors, so we might otherwise go ahead even if the directory wasn't properly deleted. 220 os.makedirs(dirname, exist_ok=True) 221 shutil.rmtree(dirname) 222 os.makedirs(dirname) 223 224 225class GenericGeneratedSourcesBenchmark(Benchmark): 226 def __init__(self, 227 di_library, 228 benchmark_definition, 229 path_to_code_under_test=None, 230 **other_args): 231 self.di_library = di_library 232 self.benchmark_definition = add_synthetic_benchmark_parameters(benchmark_definition, path_to_code_under_test=path_to_code_under_test) 233 self.other_args = other_args 234 self.arbitrary_file = None 235 236 def prepare_compile_benchmark(self): 237 num_classes = self.benchmark_definition['num_classes'] 238 cxx_std = self.benchmark_definition['cxx_std'] 239 compiler_executable_name = self.benchmark_definition['compiler'] 240 benchmark_generation_flags = {flag_name: True for flag_name in self.benchmark_definition['benchmark_generation_flags']} 241 242 self.tmpdir = tempfile.gettempdir() + '/fruit-benchmark-dir' 243 ensure_empty_dir(self.tmpdir) 244 num_classes_with_no_deps = int(num_classes * 0.1) 245 return generate_benchmark( 246 compiler=compiler_executable_name, 247 num_components_with_no_deps=num_classes_with_no_deps, 248 num_components_with_deps=num_classes - num_classes_with_no_deps, 249 num_deps=10, 250 output_dir=self.tmpdir, 251 cxx_std=cxx_std, 252 di_library=self.di_library, 253 **benchmark_generation_flags, 254 **self.other_args) 255 256 def run_make_build(self): 257 run_command('make', args=make_args, cwd=self.tmpdir) 258 259 def prepare_incremental_compile_benchmark(self): 260 files = self.prepare_compile_benchmark() 261 self.run_make_build() 262 files = list(sorted(file for file in files if file.endswith('.h'))) 263 # 5 files, equally spaced (but not at beginning/end) in the sorted sequence. 264 num_files_changed = 5 265 self.arbitrary_files = [files[i * (len(files) // (num_files_changed + 2))] 266 for i in range(1, num_files_changed + 1)] 267 268 def prepare_compile_memory_benchmark(self): 269 self.prepare_compile_benchmark() 270 self.run_compile_memory_benchmark() 271 272 def prepare_runtime_benchmark(self): 273 self.prepare_compile_benchmark() 274 self.run_make_build() 275 276 def prepare_startup_benchmark(self): 277 self.prepare_compile_benchmark() 278 self.run_make_build() 279 run_command('strip', args=[self.tmpdir + '/main']) 280 281 def prepare_executable_size_benchmark(self): 282 self.prepare_runtime_benchmark() 283 run_command('strip', args=[self.tmpdir + '/main']) 284 285 def run_compile_benchmark(self): 286 run_command('make', 287 args=make_args + ['clean'], 288 cwd=self.tmpdir) 289 start = timer() 290 self.run_make_build() 291 end = timer() 292 result = {'compile_time': end - start} 293 return result 294 295 def run_incremental_compile_benchmark(self): 296 run_command('touch', args=self.arbitrary_files, cwd=self.tmpdir) 297 start = timer() 298 self.run_make_build() 299 end = timer() 300 result = {'incremental_compile_time': end - start} 301 return result 302 303 def run_compile_memory_benchmark(self): 304 run_command('make', args=make_args + ['clean'], cwd=self.tmpdir) 305 run_command('make', args=make_args + ['main_ram.txt'], cwd=self.tmpdir) 306 with open(self.tmpdir + '/main_ram.txt') as f: 307 ram_usages = [int(n)*1024 for n in f.readlines()] 308 return { 309 'total_max_ram_usage': sum(ram_usages), 310 'max_ram_usage': max(ram_usages), 311 } 312 313 def run_runtime_benchmark(self): 314 num_classes = self.benchmark_definition['num_classes'] 315 loop_factor = self.benchmark_definition['loop_factor'] 316 317 results, _ = run_command(self.tmpdir + '/main', 318 args = [ 319 # 40M loops with 100 classes, 40M with 1000 320 int(4 * 1000 * 1000 * 1000 * loop_factor / num_classes), 321 ]) 322 return parse_results(results.splitlines()) 323 324 def run_startup_benchmark(self): 325 n = 1000 326 start = timer() 327 for i in range(0, n): 328 run_command(self.tmpdir + '/main', args = []) 329 end = timer() 330 result = {'startup_time': (end - start) / n} 331 return result 332 333 def run_executable_size_benchmark(self): 334 wc_result, _ = run_command('wc', args=['-c', self.tmpdir + '/main']) 335 num_bytes = wc_result.splitlines()[0].split(' ')[0] 336 return {'num_bytes': float(num_bytes)} 337 338 def describe(self): 339 return self.benchmark_definition 340 341 342class CompileTimeBenchmark(GenericGeneratedSourcesBenchmark): 343 def __init__(self, **kwargs): 344 super().__init__(generate_runtime_bench_code=False, 345 **kwargs) 346 347 def prepare(self): 348 self.prepare_compile_benchmark() 349 350 def run(self): 351 return self.run_compile_benchmark() 352 353class IncrementalCompileTimeBenchmark(GenericGeneratedSourcesBenchmark): 354 def __init__(self, **kwargs): 355 super().__init__(generate_runtime_bench_code=False, 356 **kwargs) 357 358 def prepare(self): 359 self.prepare_incremental_compile_benchmark() 360 361 def run(self): 362 return self.run_incremental_compile_benchmark() 363 364class CompileMemoryBenchmark(GenericGeneratedSourcesBenchmark): 365 def __init__(self, **kwargs): 366 super().__init__(generate_runtime_bench_code=False, 367 **kwargs) 368 369 def prepare(self): 370 self.prepare_compile_memory_benchmark() 371 372 def run(self): 373 return self.run_compile_memory_benchmark() 374 375class StartupTimeBenchmark(GenericGeneratedSourcesBenchmark): 376 def __init__(self, **kwargs): 377 super().__init__(generate_runtime_bench_code=False, 378 **kwargs) 379 380 def prepare(self): 381 self.prepare_startup_benchmark() 382 383 def run(self): 384 return self.run_startup_benchmark() 385 386class RunTimeBenchmark(GenericGeneratedSourcesBenchmark): 387 def __init__(self, **kwargs): 388 super().__init__(generate_runtime_bench_code=True, 389 **kwargs) 390 391 def prepare(self): 392 self.prepare_runtime_benchmark() 393 394 def run(self): 395 return self.run_runtime_benchmark() 396 397# This is not really a 'benchmark', but we consider it as such to reuse the benchmark infrastructure. 398class ExecutableSizeBenchmark(GenericGeneratedSourcesBenchmark): 399 def __init__(self, **kwargs): 400 super().__init__(generate_runtime_bench_code=False, 401 **kwargs) 402 403 def prepare(self): 404 self.prepare_executable_size_benchmark() 405 406 def run(self): 407 return self.run_executable_size_benchmark() 408 409# This is not really a 'benchmark', but we consider it as such to reuse the benchmark infrastructure. 410class ExecutableSizeBenchmarkWithoutExceptionsAndRtti(ExecutableSizeBenchmark): 411 def __init__(self, **kwargs): 412 super().__init__(use_exceptions=False, 413 use_rtti=False, 414 **kwargs) 415 416class FruitCompileTimeBenchmark(CompileTimeBenchmark): 417 def __init__(self, fruit_sources_dir, **kwargs): 418 super().__init__(di_library='fruit', 419 path_to_code_under_test=fruit_sources_dir, 420 fruit_sources_dir=fruit_sources_dir, 421 **kwargs) 422 423class FruitIncrementalCompileTimeBenchmark(IncrementalCompileTimeBenchmark): 424 def __init__(self, fruit_sources_dir, **kwargs): 425 super().__init__(di_library='fruit', 426 path_to_code_under_test=fruit_sources_dir, 427 fruit_sources_dir=fruit_sources_dir, 428 **kwargs) 429 430class FruitCompileMemoryBenchmark(CompileMemoryBenchmark): 431 def __init__(self, fruit_sources_dir, **kwargs): 432 super().__init__(di_library='fruit', 433 path_to_code_under_test=fruit_sources_dir, 434 fruit_sources_dir=fruit_sources_dir, 435 **kwargs) 436 437class FruitRunTimeBenchmark(RunTimeBenchmark): 438 def __init__(self, fruit_sources_dir, **kwargs): 439 super().__init__(di_library='fruit', 440 path_to_code_under_test=fruit_sources_dir, 441 fruit_sources_dir=fruit_sources_dir, 442 **kwargs) 443 444class FruitStartupTimeBenchmark(StartupTimeBenchmark): 445 def __init__(self, fruit_sources_dir, **kwargs): 446 super().__init__(di_library='fruit', 447 path_to_code_under_test=fruit_sources_dir, 448 fruit_sources_dir=fruit_sources_dir, 449 **kwargs) 450 451class FruitStartupTimeWithNormalizedComponentBenchmark(FruitStartupTimeBenchmark): 452 def __init__(self, **kwargs): 453 super().__init__(use_normalized_component=True, 454 **kwargs) 455 456# This is not really a 'benchmark', but we consider it as such to reuse the benchmark infrastructure. 457class FruitExecutableSizeBenchmark(ExecutableSizeBenchmark): 458 def __init__(self, fruit_sources_dir, **kwargs): 459 super().__init__(di_library='fruit', 460 path_to_code_under_test=fruit_sources_dir, 461 fruit_sources_dir=fruit_sources_dir, 462 **kwargs) 463 464# This is not really a 'benchmark', but we consider it as such to reuse the benchmark infrastructure. 465class FruitExecutableSizeBenchmarkWithoutExceptionsAndRtti(ExecutableSizeBenchmarkWithoutExceptionsAndRtti): 466 def __init__(self, fruit_sources_dir, **kwargs): 467 super().__init__(di_library='fruit', 468 path_to_code_under_test=fruit_sources_dir, 469 fruit_sources_dir=fruit_sources_dir, 470 **kwargs) 471 472class BoostDiCompileTimeBenchmark(CompileTimeBenchmark): 473 def __init__(self, boost_di_sources_dir, **kwargs): 474 super().__init__(di_library='boost_di', 475 path_to_code_under_test=boost_di_sources_dir, 476 boost_di_sources_dir=boost_di_sources_dir, 477 **kwargs) 478 479class BoostDiIncrementalCompileTimeBenchmark(IncrementalCompileTimeBenchmark): 480 def __init__(self, boost_di_sources_dir, **kwargs): 481 super().__init__(di_library='boost_di', 482 path_to_code_under_test=boost_di_sources_dir, 483 boost_di_sources_dir=boost_di_sources_dir, 484 **kwargs) 485 486class BoostDiCompileMemoryBenchmark(CompileMemoryBenchmark): 487 def __init__(self, boost_di_sources_dir, **kwargs): 488 super().__init__(di_library='boost_di', 489 path_to_code_under_test=boost_di_sources_dir, 490 boost_di_sources_dir=boost_di_sources_dir, 491 **kwargs) 492 493class BoostDiRunTimeBenchmark(RunTimeBenchmark): 494 def __init__(self, boost_di_sources_dir, **kwargs): 495 super().__init__(di_library='boost_di', 496 path_to_code_under_test=boost_di_sources_dir, 497 boost_di_sources_dir=boost_di_sources_dir, 498 **kwargs) 499 500class BoostDiStartupTimeBenchmark(StartupTimeBenchmark): 501 def __init__(self, boost_di_sources_dir, **kwargs): 502 super().__init__(di_library='boost_di', 503 path_to_code_under_test=boost_di_sources_dir, 504 boost_di_sources_dir=boost_di_sources_dir, 505 **kwargs) 506 507# This is not really a 'benchmark', but we consider it as such to reuse the benchmark infrastructure. 508class BoostDiExecutableSizeBenchmark(ExecutableSizeBenchmark): 509 def __init__(self, boost_di_sources_dir, **kwargs): 510 super().__init__(di_library='boost_di', 511 path_to_code_under_test=boost_di_sources_dir, 512 boost_di_sources_dir=boost_di_sources_dir, 513 **kwargs) 514 515# This is not really a 'benchmark', but we consider it as such to reuse the benchmark infrastructure. 516class BoostDiExecutableSizeBenchmarkWithoutExceptionsAndRtti(ExecutableSizeBenchmarkWithoutExceptionsAndRtti): 517 def __init__(self, boost_di_sources_dir, **kwargs): 518 super().__init__(di_library='boost_di', 519 path_to_code_under_test=boost_di_sources_dir, 520 boost_di_sources_dir=boost_di_sources_dir, 521 **kwargs) 522 523class SimpleDiCompileTimeBenchmark(CompileTimeBenchmark): 524 def __init__(self, **kwargs): 525 super().__init__(di_library='none', 526 **kwargs) 527 528class SimpleDiIncrementalCompileTimeBenchmark(IncrementalCompileTimeBenchmark): 529 def __init__(self, **kwargs): 530 super().__init__(di_library='none', 531 **kwargs) 532 533class SimpleDiCompileMemoryBenchmark(CompileMemoryBenchmark): 534 def __init__(self, **kwargs): 535 super().__init__(di_library='none', 536 **kwargs) 537 538class SimpleDiRunTimeBenchmark(RunTimeBenchmark): 539 def __init__(self, **kwargs): 540 super().__init__(di_library='none', 541 **kwargs) 542 543class SimpleDiStartupTimeBenchmark(StartupTimeBenchmark): 544 def __init__(self, **kwargs): 545 super().__init__(di_library='none', 546 **kwargs) 547 548# This is not really a 'benchmark', but we consider it as such to reuse the benchmark infrastructure. 549class SimpleDiExecutableSizeBenchmark(ExecutableSizeBenchmark): 550 def __init__(self, **kwargs): 551 super().__init__(di_library='none', 552 **kwargs) 553 554# This is not really a 'benchmark', but we consider it as such to reuse the benchmark infrastructure. 555class SimpleDiExecutableSizeBenchmarkWithoutExceptionsAndRtti(ExecutableSizeBenchmarkWithoutExceptionsAndRtti): 556 def __init__(self, **kwargs): 557 super().__init__(di_library='none', 558 **kwargs) 559 560class SimpleDiWithInterfacesCompileTimeBenchmark(SimpleDiCompileTimeBenchmark): 561 def __init__(self, **kwargs): 562 super().__init__(use_interfaces=True, **kwargs) 563 564class SimpleDiWithInterfacesIncrementalCompileTimeBenchmark(SimpleDiIncrementalCompileTimeBenchmark): 565 def __init__(self, **kwargs): 566 super().__init__(use_interfaces=True, **kwargs) 567 568class SimpleDiWithInterfacesCompileMemoryBenchmark(SimpleDiCompileMemoryBenchmark): 569 def __init__(self, **kwargs): 570 super().__init__(use_interfaces=True, **kwargs) 571 572class SimpleDiWithInterfacesRunTimeBenchmark(SimpleDiRunTimeBenchmark): 573 def __init__(self, **kwargs): 574 super().__init__(use_interfaces=True, **kwargs) 575 576class SimpleDiWithInterfacesStartupTimeBenchmark(SimpleDiStartupTimeBenchmark): 577 def __init__(self, **kwargs): 578 super().__init__(use_interfaces=True, **kwargs) 579 580# This is not really a 'benchmark', but we consider it as such to reuse the benchmark infrastructure. 581class SimpleDiWithInterfacesExecutableSizeBenchmark(SimpleDiExecutableSizeBenchmark): 582 def __init__(self, **kwargs): 583 super().__init__(use_interfaces=True, **kwargs) 584 585# This is not really a 'benchmark', but we consider it as such to reuse the benchmark infrastructure. 586class SimpleDiWithInterfacesExecutableSizeBenchmarkWithoutExceptionsAndRtti(SimpleDiExecutableSizeBenchmarkWithoutExceptionsAndRtti): 587 def __init__(self, **kwargs): 588 super().__init__(use_interfaces=True, **kwargs) 589 590class SimpleDiWithInterfacesAndNewDeleteCompileTimeBenchmark(SimpleDiWithInterfacesCompileTimeBenchmark): 591 def __init__(self, **kwargs): 592 super().__init__(use_new_delete=True, **kwargs) 593 594class SimpleDiWithInterfacesAndNewDeleteIncrementalCompileTimeBenchmark(SimpleDiWithInterfacesIncrementalCompileTimeBenchmark): 595 def __init__(self, **kwargs): 596 super().__init__(use_new_delete=True, **kwargs) 597 598class SimpleDiWithInterfacesAndNewDeleteCompileMemoryBenchmark(SimpleDiWithInterfacesCompileMemoryBenchmark): 599 def __init__(self, **kwargs): 600 super().__init__(use_new_delete=True, **kwargs) 601 602class SimpleDiWithInterfacesAndNewDeleteRunTimeBenchmark(SimpleDiWithInterfacesRunTimeBenchmark): 603 def __init__(self, **kwargs): 604 super().__init__(use_new_delete=True, **kwargs) 605 606class SimpleDiWithInterfacesAndNewDeleteStartupTimeBenchmark(SimpleDiWithInterfacesStartupTimeBenchmark): 607 def __init__(self, **kwargs): 608 super().__init__(use_new_delete=True, **kwargs) 609 610# This is not really a 'benchmark', but we consider it as such to reuse the benchmark infrastructure. 611class SimpleDiWithInterfacesAndNewDeleteExecutableSizeBenchmark(SimpleDiWithInterfacesExecutableSizeBenchmark): 612 def __init__(self, **kwargs): 613 super().__init__(use_new_delete=True, **kwargs) 614 615# This is not really a 'benchmark', but we consider it as such to reuse the benchmark infrastructure. 616class SimpleDiWithInterfacesAndNewDeleteExecutableSizeBenchmarkWithoutExceptionsAndRtti(SimpleDiWithInterfacesExecutableSizeBenchmarkWithoutExceptionsAndRtti): 617 def __init__(self, **kwargs): 618 super().__init__(use_new_delete=True, **kwargs) 619 620 621def round_to_significant_digits(n: float, num_significant_digits: int) -> float: 622 if n <= 0: 623 # We special-case this, otherwise the log10 below will fail. 624 return 0 625 return round(n, num_significant_digits - int(floor(log10(n))) - 1) 626 627def run_benchmark(benchmark: Benchmark, max_runs: int, timeout_hours: int, output_file: str, min_runs: int=3) -> None: 628 def run_benchmark_once(): 629 print('Running benchmark... ', end='', flush=True) 630 result = benchmark.run() 631 print(result) 632 for dimension, value in result.items(): 633 results_by_dimension[dimension] += [value] 634 635 results_by_dimension = defaultdict(lambda: []) 636 print('Preparing for benchmark... ', end='', flush=True) 637 benchmark.prepare() 638 print('Done.') 639 640 start_time = timer() 641 642 # Run at least min_runs times 643 for i in range(min_runs): 644 run_benchmark_once() 645 646 # Then consider running a few more times to get the desired precision. 647 while True: 648 if timer() - start_time > timeout_hours * 3600: 649 print("Warning: timed out, couldn't determine a result with the desired precision.") 650 break 651 652 for dimension, results in results_by_dimension.items(): 653 if all(result == results[0] for result in results): 654 # If all results are exactly the same the code below misbehaves. We don't need to run again in this case. 655 continue 656 confidence_interval = stats.DescrStatsW(results).tconfint_mean(0.05) 657 confidence_interval_2dig = (round_to_significant_digits(confidence_interval[0], 2), 658 round_to_significant_digits(confidence_interval[1], 2)) 659 if abs(confidence_interval_2dig[0] - confidence_interval_2dig[1]) > numpy.finfo(float).eps * 10: 660 if len(results) < max_runs: 661 print("Running again to get more precision on the metric %s. Current confidence interval: [%.3g, %.3g]" % ( 662 dimension, confidence_interval[0], confidence_interval[1])) 663 break 664 else: 665 print("Warning: couldn't determine a precise result for the metric %s. Confidence interval: [%.3g, %.3g]" % ( 666 dimension, confidence_interval[0], confidence_interval[1])) 667 else: 668 # We've reached sufficient precision in all metrics, or we've reached the max number of runs. 669 break 670 671 run_benchmark_once() 672 673 # We've reached the desired precision in all dimensions or reached the maximum number of runs. Record the results. 674 rounded_confidence_intervals_by_dimension = {} 675 confidence_intervals_by_dimension = {} 676 for dimension, results in results_by_dimension.items(): 677 confidence_interval = stats.DescrStatsW(results).tconfint_mean(0.05) 678 confidence_interval_2dig = (round_to_significant_digits(confidence_interval[0], 2), 679 round_to_significant_digits(confidence_interval[1], 2)) 680 rounded_confidence_intervals_by_dimension[dimension] = confidence_interval_2dig 681 confidence_intervals_by_dimension[dimension] = (confidence_interval, confidence_interval_2dig) 682 with open(output_file, 'a') as f: 683 json.dump({"benchmark": benchmark.describe(), "results": confidence_intervals_by_dimension}, f) 684 print(file=f) 685 print('Benchmark finished. Result: ', rounded_confidence_intervals_by_dimension) 686 print() 687 688 689def expand_benchmark_definition(benchmark_definition: Dict[str, Any]) -> List[Dict[str, Tuple[Any]]]: 690 """ 691 Takes a benchmark definition, e.g.: 692 [{name: 'foo', compiler: ['g++-5', 'g++-6']}, 693 {name: ['bar', 'baz'], compiler: ['g++-5'], cxx_std: 'c++14'}] 694 695 And expands it into the individual benchmarks to run, in the example above: 696 [{name: 'foo', compiler: 'g++-5'}, 697 {name: 'foo', compiler: 'g++-6'}, 698 {name: 'bar', compiler: 'g++-5', cxx_std: 'c++14'}, 699 {name: 'baz', compiler: 'g++-5', cxx_std: 'c++14'}] 700 """ 701 dict_keys = sorted(benchmark_definition.keys()) 702 # Turn non-list values into single-item lists. 703 benchmark_definition = {dict_key: value if isinstance(value, list) else [value] 704 for dict_key, value in benchmark_definition.items()} 705 # Compute the cartesian product of the value lists 706 value_combinations = itertools.product(*(benchmark_definition[dict_key] for dict_key in dict_keys)) 707 # Then turn the result back into a dict. 708 return [dict(zip(dict_keys, value_combination)) 709 for value_combination in value_combinations] 710 711 712def expand_benchmark_definitions(benchmark_definitions: List[Dict[str, Any]]): 713 return list(itertools.chain(*[expand_benchmark_definition(benchmark_definition) for benchmark_definition in benchmark_definitions])) 714 715T = TypeVar('T') 716K = TypeVar('K') 717 718def group_by(l: List[T], element_to_key: Callable[[T], K]) -> Iterable[Tuple[K, List[T]]]: 719 """Takes a list and returns a list of sublists, where the elements are grouped using the provided function""" 720 result = defaultdict(list) # type: Dict[K, List[T]] 721 for elem in l: 722 result[element_to_key(elem)].append(elem) 723 return result.items() 724 725def main(): 726 # This configures numpy/scipy to raise an exception in case of errors, instead of printing a warning and going ahead. 727 numpy.seterr(all='raise') 728 scipy.seterr(all='raise') 729 730 parser = argparse.ArgumentParser(description='Runs a set of benchmarks defined in a YAML file.') 731 parser.add_argument('--fruit-benchmark-sources-dir', help='Path to the fruit sources (used for benchmarking code only)') 732 parser.add_argument('--fruit-sources-dir', help='Path to the fruit sources') 733 parser.add_argument('--boost-di-sources-dir', help='Path to the Boost.DI sources') 734 parser.add_argument('--output-file', 735 help='The output file where benchmark results will be stored (1 per line, with each line in JSON format). These can then be formatted by e.g. the format_bench_results script.') 736 parser.add_argument('--benchmark-definition', help='The YAML file that defines the benchmarks (see fruit_wiki_benchs_fruit.yml for an example).') 737 parser.add_argument('--continue-benchmark', help='If this is \'true\', continues a previous benchmark run instead of starting from scratch (taking into account the existing benchmark results in the file specified with --output-file).') 738 args = parser.parse_args() 739 740 if args.output_file is None: 741 raise Exception('You must specify --output_file') 742 if args.continue_benchmark == 'true': 743 try: 744 with open(args.output_file, 'r') as f: 745 previous_run_completed_benchmarks = [json.loads(line)['benchmark'] for line in f.readlines()] 746 except FileNotFoundError: 747 previous_run_completed_benchmarks = [] 748 else: 749 previous_run_completed_benchmarks = [] 750 run_command('rm', args=['-f', args.output_file]) 751 752 fruit_build_dir = tempfile.gettempdir() + '/fruit-benchmark-build-dir' 753 754 with open(args.benchmark_definition, 'r') as f: 755 yaml_file_content = yaml.full_load(f) 756 global_definitions = yaml_file_content['global'] 757 benchmark_definitions = expand_benchmark_definitions(yaml_file_content['benchmarks']) 758 759 benchmark_index = 0 760 761 for (compiler_executable_name, additional_cmake_args), benchmark_definitions_with_current_config \ 762 in group_by(benchmark_definitions, 763 lambda benchmark_definition: 764 (benchmark_definition['compiler'], tuple(benchmark_definition['additional_cmake_args']))): 765 766 print('Preparing for benchmarks with the compiler %s, with additional CMake args %s' % (compiler_executable_name, additional_cmake_args)) 767 try: 768 # We compute this here (and memoize the result) so that the benchmark's describe() will retrieve the cached 769 # value instantly. 770 determine_compiler_name(compiler_executable_name) 771 772 # Build Fruit in fruit_build_dir, so that fruit_build_dir points to a built Fruit (useful for e.g. the config header). 773 shutil.rmtree(fruit_build_dir, ignore_errors=True) 774 os.makedirs(fruit_build_dir) 775 modified_env = os.environ.copy() 776 modified_env['CXX'] = compiler_executable_name 777 run_command('cmake', 778 args=[ 779 args.fruit_sources_dir, 780 '-DCMAKE_BUILD_TYPE=Release', 781 *additional_cmake_args, 782 ], 783 cwd=fruit_build_dir, 784 env=modified_env) 785 run_command('make', args=make_args, cwd=fruit_build_dir) 786 except Exception as e: 787 print('Exception while preparing for benchmarks with the compiler %s, with additional CMake args %s.\n%s\nGoing ahead with the rest.' % (compiler_executable_name, additional_cmake_args, traceback.format_exc())) 788 continue 789 790 for benchmark_definition in benchmark_definitions_with_current_config: 791 benchmark_index += 1 792 print('%s/%s: %s' % (benchmark_index, len(benchmark_definitions), benchmark_definition)) 793 benchmark_name = benchmark_definition['name'] 794 795 if (benchmark_name in {'boost_di_compile_time', 'boost_di_run_time', 'boost_di_executable_size'} 796 and args.boost_di_sources_dir is None): 797 raise Exception('Error: you need to specify the --boost-di-sources-dir flag in order to run Boost.DI benchmarks.') 798 799 if benchmark_name == 'new_delete_run_time': 800 benchmark = SimpleNewDeleteRunTimeBenchmark( 801 benchmark_definition, 802 fruit_benchmark_sources_dir=args.fruit_benchmark_sources_dir) 803 elif benchmark_name == 'fruit_single_file_compile_time': 804 benchmark = FruitSingleFileCompileTimeBenchmark( 805 benchmark_definition, 806 fruit_sources_dir=args.fruit_sources_dir, 807 fruit_benchmark_sources_dir=args.fruit_benchmark_sources_dir, 808 fruit_build_dir=fruit_build_dir) 809 elif benchmark_name.startswith('fruit_'): 810 benchmark_class = { 811 'fruit_compile_time': FruitCompileTimeBenchmark, 812 'fruit_incremental_compile_time': FruitIncrementalCompileTimeBenchmark, 813 'fruit_compile_memory': FruitCompileMemoryBenchmark, 814 'fruit_run_time': FruitRunTimeBenchmark, 815 'fruit_startup_time': FruitStartupTimeBenchmark, 816 'fruit_startup_time_with_normalized_component': FruitStartupTimeWithNormalizedComponentBenchmark, 817 'fruit_executable_size': FruitExecutableSizeBenchmark, 818 'fruit_executable_size_without_exceptions_and_rtti': FruitExecutableSizeBenchmarkWithoutExceptionsAndRtti, 819 }[benchmark_name] 820 benchmark = benchmark_class( 821 benchmark_definition=benchmark_definition, 822 fruit_sources_dir=args.fruit_sources_dir, 823 fruit_build_dir=fruit_build_dir) 824 elif benchmark_name.startswith('boost_di_'): 825 benchmark_class = { 826 'boost_di_compile_time': BoostDiCompileTimeBenchmark, 827 'boost_di_incremental_compile_time': BoostDiIncrementalCompileTimeBenchmark, 828 'boost_di_compile_memory': BoostDiCompileMemoryBenchmark, 829 'boost_di_run_time': BoostDiRunTimeBenchmark, 830 'boost_di_startup_time': BoostDiStartupTimeBenchmark, 831 'boost_di_executable_size': BoostDiExecutableSizeBenchmark, 832 'boost_di_executable_size_without_exceptions_and_rtti': BoostDiExecutableSizeBenchmarkWithoutExceptionsAndRtti, 833 }[benchmark_name] 834 benchmark = benchmark_class( 835 benchmark_definition=benchmark_definition, 836 boost_di_sources_dir=args.boost_di_sources_dir) 837 elif benchmark_name.startswith('simple_di_'): 838 benchmark_class = { 839 'simple_di_compile_time': SimpleDiCompileTimeBenchmark, 840 'simple_di_incremental_compile_time': SimpleDiIncrementalCompileTimeBenchmark, 841 'simple_di_compile_memory': SimpleDiCompileMemoryBenchmark, 842 'simple_di_run_time': SimpleDiRunTimeBenchmark, 843 'simple_di_startup_time': SimpleDiStartupTimeBenchmark, 844 'simple_di_executable_size': SimpleDiExecutableSizeBenchmark, 845 'simple_di_executable_size_without_exceptions_and_rtti': SimpleDiExecutableSizeBenchmarkWithoutExceptionsAndRtti, 846 'simple_di_with_interfaces_compile_time': SimpleDiWithInterfacesCompileTimeBenchmark, 847 'simple_di_with_interfaces_incremental_compile_time': SimpleDiWithInterfacesIncrementalCompileTimeBenchmark, 848 'simple_di_with_interfaces_compile_memory': SimpleDiWithInterfacesCompileMemoryBenchmark, 849 'simple_di_with_interfaces_run_time': SimpleDiWithInterfacesRunTimeBenchmark, 850 'simple_di_with_interfaces_startup_time': SimpleDiWithInterfacesStartupTimeBenchmark, 851 'simple_di_with_interfaces_executable_size': SimpleDiWithInterfacesExecutableSizeBenchmark, 852 'simple_di_with_interfaces_executable_size_without_exceptions_and_rtti': SimpleDiWithInterfacesExecutableSizeBenchmarkWithoutExceptionsAndRtti, 853 'simple_di_with_interfaces_and_new_delete_compile_time': SimpleDiWithInterfacesAndNewDeleteCompileTimeBenchmark, 854 'simple_di_with_interfaces_and_new_delete_incremental_compile_time': SimpleDiWithInterfacesAndNewDeleteIncrementalCompileTimeBenchmark, 855 'simple_di_with_interfaces_and_new_delete_compile_memory': SimpleDiWithInterfacesAndNewDeleteCompileMemoryBenchmark, 856 'simple_di_with_interfaces_and_new_delete_run_time': SimpleDiWithInterfacesAndNewDeleteRunTimeBenchmark, 857 'simple_di_with_interfaces_and_new_delete_startup_time': SimpleDiWithInterfacesAndNewDeleteStartupTimeBenchmark, 858 'simple_di_with_interfaces_and_new_delete_executable_size': SimpleDiWithInterfacesAndNewDeleteExecutableSizeBenchmark, 859 'simple_di_with_interfaces_and_new_delete_executable_size_without_exceptions_and_rtti': SimpleDiWithInterfacesAndNewDeleteExecutableSizeBenchmarkWithoutExceptionsAndRtti, 860 }[benchmark_name] 861 benchmark = benchmark_class( 862 benchmark_definition=benchmark_definition) 863 else: 864 raise Exception("Unrecognized benchmark: %s" % benchmark_name) 865 866 if benchmark.describe() in previous_run_completed_benchmarks: 867 print("Skipping benchmark that was already run previously (due to --continue-benchmark):", benchmark.describe()) 868 continue 869 870 try: 871 run_benchmark(benchmark, 872 output_file=args.output_file, 873 max_runs=global_definitions['max_runs'], 874 timeout_hours=global_definitions['max_hours_per_combination']) 875 except Exception as e: 876 print('Exception while running benchmark: %s.\n%s\nGoing ahead with the rest.' % (benchmark.describe(), traceback.format_exc())) 877 878 879if __name__ == "__main__": 880 main() 881