1#!/usr/bin/env python 2# Copyright 2014 the V8 project authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6# for py2/py3 compatibility 7from __future__ import print_function 8 9import argparse 10import os 11import subprocess 12import sys 13 14BOTS = { 15 '--linux32': 'v8_linux32_perf_try', 16 '--linux64': 'v8_linux64_perf_try', 17 '--nexus5': 'v8_nexus5_perf_try', 18 '--nexus7': 'v8_nexus7_perf_try', 19 '--pixel2': 'v8_pixel2_perf_try', 20} 21 22DEFAULT_BOTS = [ 23 'v8_linux32_perf_try', 24 'v8_linux64_perf_try', 25] 26 27PUBLIC_BENCHMARKS = [ 28 'arewefastyet', 29 'ares6', 30 'blazor', 31 'compile', 32 'embenchen', 33 'emscripten', 34 'jetstream', 35 'jsbench', 36 'jstests', 37 'kraken_orig', 38 'massive', 39 'memory', 40 'octane', 41 'octane-noopt', 42 'octane-pr', 43 'octane-tf', 44 'octane-tf-pr', 45 'sunspider', 46 'unity', 47 'wasm', 48 'web-tooling-benchmark', 49] 50 51V8_BASE = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) 52 53def main(): 54 parser = argparse.ArgumentParser(description='') 55 parser.add_argument('benchmarks', nargs='+', help='The benchmarks to run.') 56 parser.add_argument('--extra-flags', default='', 57 help='Extra flags to be passed to the executable.') 58 parser.add_argument('-r', '--revision', type=str, default=None, 59 help='Revision (use full hash!) to use for the try job; ' 60 'default: the revision will be determined by the ' 61 'try server; see its waterfall for more info') 62 parser.add_argument('-v', '--verbose', action='store_true', 63 help='Print debug information') 64 parser.add_argument('-c', '--confidence-level', type=float, 65 help='Repeatedly runs each benchmark until specified ' 66 'confidence level is reached. The value is interpreted ' 67 'as the number of standard deviations from the mean that ' 68 'all values must lie within. Typical values are 1, 2 and ' 69 '3 and correspond to 68%%, 95%% and 99.7%% probability ' 70 'that the measured value is within 0.1%% of the true ' 71 'value. Larger values result in more retries and thus ' 72 'longer runtime, but also provide more reliable results.') 73 for option in sorted(BOTS): 74 parser.add_argument( 75 option, dest='bots', action='append_const', const=BOTS[option], 76 help='Add %s trybot.' % BOTS[option]) 77 options = parser.parse_args() 78 if not options.bots: 79 print('No trybots specified. Using default %s.' % ','.join(DEFAULT_BOTS)) 80 options.bots = DEFAULT_BOTS 81 82 if not options.benchmarks: 83 print('Please specify the benchmarks to run as arguments.') 84 return 1 85 86 for benchmark in options.benchmarks: 87 if benchmark not in PUBLIC_BENCHMARKS: 88 print ('%s not found in our benchmark list. The respective trybot might ' 89 'fail, unless you run something this script isn\'t aware of. ' 90 'Available public benchmarks: %s' % (benchmark, PUBLIC_BENCHMARKS)) 91 print('Proceed anyways? [Y/n] ', end=' ') 92 answer = sys.stdin.readline().strip() 93 if answer != "" and answer != "Y" and answer != "y": 94 return 1 95 96 assert '"' not in options.extra_flags and '\'' not in options.extra_flags, ( 97 'Invalid flag specification.') 98 99 # Ensure depot_tools are updated. 100 subprocess.check_output( 101 'update_depot_tools', shell=True, stderr=subprocess.STDOUT, cwd=V8_BASE) 102 103 cmd = ['git cl try', '-B', 'luci.v8-internal.try'] 104 cmd += ['-b %s' % bot for bot in options.bots] 105 if options.revision: 106 cmd.append('-r %s' % options.revision) 107 benchmarks = ['"%s"' % benchmark for benchmark in options.benchmarks] 108 cmd.append('-p \'testfilter=[%s]\'' % ','.join(benchmarks)) 109 if options.extra_flags: 110 cmd.append('-p \'extra_flags="%s"\'' % options.extra_flags) 111 if options.confidence_level: 112 cmd.append('-p confidence_level=%f' % options.confidence_level) 113 if options.verbose: 114 cmd.append('-vv') 115 print('Running %s' % ' '.join(cmd)) 116 subprocess.check_call(' '.join(cmd), shell=True, cwd=V8_BASE) 117 118if __name__ == '__main__': # pragma: no cover 119 sys.exit(main()) 120