1#!/usr/bin/python 2 3# Copyright 2017 Google Inc. 4# 5# Use of this source code is governed by a BSD-style license that can be 6# found in the LICENSE file. 7 8 9from __future__ import print_function 10import os 11import sys 12import subprocess 13import multiprocessing 14 15from argparse import ArgumentParser 16 17 18README = """ 19Simply run 20\033[36m 21 python {0} TEST_GIT_BRANCH 22\033[0m 23to see if TEST_GIT_BRANCH has performance regressions against main in 8888. 24 25To compare a specific config with svg and skp resources included, add --config 26and --extraarg option. For exampe, 27\033[36m 28 python {0} TEST_GIT_BRANCH --config gl \\ 29 --extraarg "--svgs ~/Desktop/bots/svgs --skps ~/Desktop/bots/skps" 30\033[0m 31For more options, please see 32 33 python {0} --help 34""".format(__file__) 35 36 37CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) 38AB_SCRIPT = "ab.py" 39 40 41def parse_args(): 42 if len(sys.argv) <= 1 or sys.argv[1] == '-h' or sys.argv[1] == '--help': 43 print(README) 44 45 parser = ArgumentParser( 46 description='Noiselessly (hence calm) becnhmark a git branch against ' + 47 'another baseline branch (e.g., main) using multiple ' + 48 ' nanobench runs.' 49 ) 50 51 default_threads = max(1, multiprocessing.cpu_count() / 2); 52 default_skiadir = os.path.normpath(CURRENT_DIR + "/../../") 53 54 config_help = ( 55 'nanobench config; we currently support only one config ' 56 'at a time (default: %(default)s)') 57 reps_help = ( 58 'initial repititions of the nanobench run; this may be ' 59 'overridden when we have many threads (default: %(default)s)') 60 extraarg_help = ( 61 'nanobench args (example: --svgs ~/Desktop/bots/svgs --skps ' 62 '~/Desktop/bots/skps)') 63 baseline_help = ( 64 'baseline branch to compare against (default: %(default)s)') 65 basearg_help = ( 66 'nanobench arg for the baseline branch; if not given, we use ' 67 ' the same arg for both the test branch and the baseline branch') 68 threads_help = ( 69 'number of threads to be used (default: %(default)s); ' 70 'for GPU config, this will always be 1') 71 no_compile_help = ( 72 'whether NOT to compile nanobench and copy it to WRITEDIR ' 73 '(i.e., reuse previous nanobench compiled)') 74 skip_base_help = ( 75 'whether NOT to run nanobench on baseline branch ' 76 '(i.e., reuse previous baseline measurements)') 77 noinit_help = ( 78 'whether to skip initial nanobench runs (default: %(default)s)') 79 branch_help = ( 80 "the test branch to benchmark; if it's 'modified', we'll benchmark the " 81 "current modified code against 'git stash'.") 82 83 definitions = [ 84 # argname, type, default value, help 85 ['--config', str, '8888', config_help], 86 ['--skiadir', str, default_skiadir, 'default: %(default)s'], 87 ['--ninjadir', str, 'out/Release', 'default: %(default)s'], 88 ['--writedir', str, '/var/tmp', 'default: %(default)s'], 89 ['--extraarg', str, '', extraarg_help], 90 ['--baseline', str, 'main', baseline_help], 91 ['--basearg', str, '', basearg_help], 92 ['--reps', int, 2, reps_help], 93 ['--threads', int, default_threads, threads_help], 94 ] 95 96 for d in definitions: 97 parser.add_argument(d[0], type=d[1], default=d[2], help=d[3]) 98 99 parser.add_argument('branch', type=str, help=branch_help) 100 parser.add_argument('--no-compile', dest='no_compile', action="store_true", 101 help=no_compile_help) 102 parser.add_argument('--skip-base', dest='skipbase', action="store_true", 103 help=skip_base_help) 104 parser.add_argument('--noinit', dest='noinit', action="store_true", 105 help=noinit_help) 106 parser.add_argument('--concise', dest='concise', action="store_true", 107 help="If set, no verbose thread info will be printed.") 108 parser.set_defaults(no_compile=False); 109 parser.set_defaults(skipbase=False); 110 parser.set_defaults(noinit=False); 111 parser.set_defaults(concise=False); 112 113 # Additional args for bots 114 BHELP = "bot specific options" 115 parser.add_argument('--githash', type=str, help=BHELP) 116 parser.add_argument('--keys', type=str, default=[], nargs='+', help=BHELP) 117 118 args = parser.parse_args() 119 if not args.basearg: 120 args.basearg = args.extraarg 121 122 return args 123 124 125def nano_path(args, branch): 126 return args.writedir + '/nanobench_' + branch 127 128 129def compile_branch(args, branch): 130 print("Compiling branch %s" % args.branch) 131 132 commands = [ 133 ['git', 'checkout', branch], 134 ['ninja', '-C', args.ninjadir, 'nanobench'], 135 ['cp', args.ninjadir + '/nanobench', nano_path(args, branch)] 136 ] 137 for command in commands: 138 subprocess.check_call(command, cwd=args.skiadir) 139 140 141def compile_modified(args): 142 print("Compiling modified code") 143 subprocess.check_call( 144 ['ninja', '-C', args.ninjadir, 'nanobench'], cwd=args.skiadir) 145 subprocess.check_call( 146 ['cp', args.ninjadir + '/nanobench', nano_path(args, args.branch)], 147 cwd=args.skiadir) 148 149 print("Compiling stashed code") 150 stash_output = subprocess.check_output(['git', 'stash'], cwd=args.skiadir) 151 if 'No local changes to save' in stash_output: 152 subprocess.check_call(['git', 'reset', 'HEAD^', '--soft']) 153 subprocess.check_call(['git', 'stash']) 154 155 subprocess.check_call(['gclient', 'sync'], cwd=args.skiadir) 156 subprocess.check_call( 157 ['ninja', '-C', args.ninjadir, 'nanobench'], cwd=args.skiadir) 158 subprocess.check_call( 159 ['cp', args.ninjadir + '/nanobench', nano_path(args, args.baseline)], 160 cwd=args.skiadir) 161 subprocess.check_call(['git', 'stash', 'pop'], cwd=args.skiadir) 162 163def compile_nanobench(args): 164 if args.branch == 'modified': 165 compile_modified(args) 166 else: 167 compile_branch(args, args.branch) 168 compile_branch(args, args.baseline) 169 170 171def main(): 172 args = parse_args() 173 174 # copy in case that it will be gone after git branch switching 175 orig_ab_name = CURRENT_DIR + "/" + AB_SCRIPT 176 temp_ab_name = args.writedir + "/" + AB_SCRIPT 177 subprocess.check_call(['cp', orig_ab_name, temp_ab_name]) 178 179 if not args.no_compile: 180 compile_nanobench(args) 181 182 command = [ 183 'python', 184 temp_ab_name, 185 args.writedir, 186 args.branch + ("_A" if args.branch == args.baseline else ""), 187 args.baseline + ("_B" if args.branch == args.baseline else ""), 188 nano_path(args, args.branch), 189 nano_path(args, args.baseline), 190 args.extraarg, 191 args.basearg, 192 str(args.reps), 193 "true" if args.skipbase else "false", 194 args.config, 195 str(args.threads if args.config in ["8888", "565"] else 1), 196 "true" if args.noinit else "false" 197 ] 198 199 if args.githash: 200 command += ['--githash', args.githash] 201 if args.keys: 202 command += (['--keys'] + args.keys) 203 204 if args.concise: 205 command.append("--concise") 206 207 p = subprocess.Popen(command, cwd=args.skiadir) 208 try: 209 p.wait() 210 except KeyboardInterrupt: 211 try: 212 p.terminate() 213 except OSError as e: 214 print(e) 215 216 217if __name__ == "__main__": 218 main() 219