1# Copyright (c) 2013~2015 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4"""SuiteRunner defines the interface from crosperf to test script.""" 5 6from __future__ import print_function 7 8import os 9import time 10import shlex 11 12from cros_utils import command_executer 13import test_flag 14 15TEST_THAT_PATH = '/usr/bin/test_that' 16AUTOTEST_DIR = '~/trunk/src/third_party/autotest/files' 17CHROME_MOUNT_DIR = '/tmp/chrome_root' 18 19 20def GetProfilerArgs(profiler_args): 21 # Remove "--" from in front of profiler args. 22 args_list = shlex.split(profiler_args) 23 new_list = [] 24 for arg in args_list: 25 if arg[0:2] == '--': 26 arg = arg[2:] 27 new_list.append(arg) 28 args_list = new_list 29 30 # Remove "perf_options=" from middle of profiler args. 31 new_list = [] 32 for arg in args_list: 33 idx = arg.find('perf_options=') 34 if idx != -1: 35 prefix = arg[0:idx] 36 suffix = arg[idx + len('perf_options=') + 1:-1] 37 new_arg = prefix + "'" + suffix + "'" 38 new_list.append(new_arg) 39 else: 40 new_list.append(arg) 41 args_list = new_list 42 43 return ' '.join(args_list) 44 45 46class SuiteRunner(object): 47 """This defines the interface from crosperf to test script.""" 48 49 def __init__(self, 50 logger_to_use=None, 51 log_level='verbose', 52 cmd_exec=None, 53 cmd_term=None): 54 self.logger = logger_to_use 55 self.log_level = log_level 56 self._ce = cmd_exec or command_executer.GetCommandExecuter( 57 self.logger, log_level=self.log_level) 58 self._ct = cmd_term or command_executer.CommandTerminator() 59 60 def Run(self, machine, label, benchmark, test_args, profiler_args): 61 for i in range(0, benchmark.retries + 1): 62 self.PinGovernorExecutionFrequencies(machine, label.chromeos_root) 63 if benchmark.suite == 'telemetry': 64 self.DecreaseWaitTime(machine, label.chromeos_root) 65 ret_tup = self.Telemetry_Run(machine, label, benchmark, profiler_args) 66 elif benchmark.suite == 'telemetry_Crosperf': 67 self.DecreaseWaitTime(machine, label.chromeos_root) 68 ret_tup = self.Telemetry_Crosperf_Run(machine, label, benchmark, 69 test_args, profiler_args) 70 else: 71 ret_tup = self.Test_That_Run(machine, label, benchmark, test_args, 72 profiler_args) 73 if ret_tup[0] != 0: 74 self.logger.LogOutput('benchmark %s failed. Retries left: %s' % 75 (benchmark.name, benchmark.retries - i)) 76 elif i > 0: 77 self.logger.LogOutput( 78 'benchmark %s succeded after %s retries' % (benchmark.name, i)) 79 break 80 else: 81 self.logger.LogOutput( 82 'benchmark %s succeded on first try' % benchmark.name) 83 break 84 return ret_tup 85 86 def PinGovernorExecutionFrequencies(self, machine_name, chromeos_root): 87 """Set min and max frequencies to max static frequency.""" 88 # pyformat: disable 89 set_cpu_freq = ( 90 'set -e && ' 91 # Disable Turbo in Intel pstate driver 92 'if [[ -e /sys/devices/system/cpu/intel_pstate/no_turbo ]]; then ' 93 ' if grep -q 0 /sys/devices/system/cpu/intel_pstate/no_turbo; then ' 94 ' echo -n 1 > /sys/devices/system/cpu/intel_pstate/no_turbo; ' 95 ' fi; ' 96 'fi; ' 97 # Set governor to performance for each cpu 98 'for f in /sys/devices/system/cpu/cpu*/cpufreq; do ' 99 'cd $f; ' 100 'echo performance > scaling_governor; ' 101 # Uncomment rest of lines to enable setting frequency by crosperf 102 #'val=0; ' 103 #'if [[ -e scaling_available_frequencies ]]; then ' 104 # pylint: disable=line-too-long 105 #' val=`cat scaling_available_frequencies | tr " " "\\n" | sort -n -b -r`; ' 106 #'else ' 107 #' val=`cat scaling_max_freq | tr " " "\\n" | sort -n -b -r`; fi ; ' 108 #'set -- $val; ' 109 #'highest=$1; ' 110 #'if [[ $# -gt 1 ]]; then ' 111 #' case $highest in *1000) highest=$2;; esac; ' 112 #'fi ;' 113 #'echo $highest > scaling_max_freq; ' 114 #'echo $highest > scaling_min_freq; ' 115 'done' 116 ) 117 # pyformat: enable 118 if self.log_level == 'average': 119 self.logger.LogOutput( 120 'Pinning governor execution frequencies for %s' % machine_name) 121 ret = self._ce.CrosRunCommand( 122 set_cpu_freq, machine=machine_name, chromeos_root=chromeos_root) 123 self.logger.LogFatalIf( 124 ret, 'Could not pin frequencies on machine: %s' % machine_name) 125 126 def DecreaseWaitTime(self, machine_name, chromeos_root): 127 """Change the ten seconds wait time for pagecycler to two seconds.""" 128 FILE = '/usr/local/telemetry/src/tools/perf/page_sets/page_cycler_story.py' 129 ret = self._ce.CrosRunCommand( 130 'ls ' + FILE, machine=machine_name, chromeos_root=chromeos_root) 131 self.logger.LogFatalIf( 132 ret, 'Could not find {} on machine: {}'.format(FILE, machine_name)) 133 134 if not ret: 135 sed_command = 'sed -i "s/_TTI_WAIT_TIME = 10/_TTI_WAIT_TIME = 2/g" ' 136 ret = self._ce.CrosRunCommand( 137 sed_command + FILE, machine=machine_name, chromeos_root=chromeos_root) 138 self.logger.LogFatalIf( 139 ret, 'Could not modify {} on machine: {}'.format(FILE, machine_name)) 140 141 def RestartUI(self, machine_name, chromeos_root): 142 command = 'stop ui; sleep 5; start ui' 143 self._ce.CrosRunCommand( 144 command, machine=machine_name, chromeos_root=chromeos_root) 145 146 def Test_That_Run(self, machine, label, benchmark, test_args, profiler_args): 147 """Run the test_that test..""" 148 options = '' 149 if label.board: 150 options += ' --board=%s' % label.board 151 if test_args: 152 options += ' %s' % test_args 153 if profiler_args: 154 self.logger.LogFatal('test_that does not support profiler.') 155 command = 'rm -rf /usr/local/autotest/results/*' 156 self._ce.CrosRunCommand( 157 command, machine=machine, chromeos_root=label.chromeos_root) 158 159 # We do this because some tests leave the machine in weird states. 160 # Rebooting between iterations has proven to help with this. 161 # But the beep is anoying, we will try restart ui. 162 self.RestartUI(machine, label.chromeos_root) 163 164 autotest_dir = AUTOTEST_DIR 165 if label.autotest_path != '': 166 autotest_dir = label.autotest_path 167 168 autotest_dir_arg = '--autotest_dir %s' % autotest_dir 169 # For non-telemetry tests, specify an autotest directory only if the 170 # specified directory is different from default (crosbug.com/679001). 171 if autotest_dir == AUTOTEST_DIR: 172 autotest_dir_arg = '' 173 174 command = (('%s %s --fast ' 175 '%s %s %s') % (TEST_THAT_PATH, autotest_dir_arg, options, 176 machine, benchmark.test_name)) 177 if self.log_level != 'verbose': 178 self.logger.LogOutput('Running test.') 179 self.logger.LogOutput('CMD: %s' % command) 180 # Use --no-ns-pid so that cros_sdk does not create a different 181 # process namespace and we can kill process created easily by 182 # their process group. 183 return self._ce.ChrootRunCommandWOutput( 184 label.chromeos_root, 185 command, 186 command_terminator=self._ct, 187 cros_sdk_options='--no-ns-pid') 188 189 def RemoveTelemetryTempFile(self, machine, chromeos_root): 190 filename = 'telemetry@%s' % machine 191 fullname = os.path.join(chromeos_root, 'chroot', 'tmp', filename) 192 if os.path.exists(fullname): 193 os.remove(fullname) 194 195 def Telemetry_Crosperf_Run(self, machine, label, benchmark, test_args, 196 profiler_args): 197 if not os.path.isdir(label.chrome_src): 198 self.logger.LogFatal('Cannot find chrome src dir to' 199 ' run telemetry: %s' % label.chrome_src) 200 201 # Check for and remove temporary file that may have been left by 202 # previous telemetry runs (and which might prevent this run from 203 # working). 204 self.RemoveTelemetryTempFile(machine, label.chromeos_root) 205 206 # For telemetry runs, we can use the autotest copy from the source 207 # location. No need to have one under /build/<board>. 208 autotest_dir_arg = '--autotest_dir %s' % AUTOTEST_DIR 209 if label.autotest_path != '': 210 autotest_dir_arg = '--autotest_dir %s' % label.autotest_path 211 212 profiler_args = GetProfilerArgs(profiler_args) 213 fast_arg = '' 214 if not profiler_args: 215 # --fast works unless we are doing profiling (autotest limitation). 216 # --fast avoids unnecessary copies of syslogs. 217 fast_arg = '--fast' 218 args_string = '' 219 if test_args: 220 # Strip double quotes off args (so we can wrap them in single 221 # quotes, to pass through to Telemetry). 222 if test_args[0] == '"' and test_args[-1] == '"': 223 test_args = test_args[1:-1] 224 args_string = "test_args='%s'" % test_args 225 226 cmd = ('{} {} {} --board={} --args="{} run_local={} test={} ' 227 '{}" {} telemetry_Crosperf'.format( 228 TEST_THAT_PATH, autotest_dir_arg, fast_arg, label.board, 229 args_string, benchmark.run_local, benchmark.test_name, 230 profiler_args, machine)) 231 232 # Use --no-ns-pid so that cros_sdk does not create a different 233 # process namespace and we can kill process created easily by their 234 # process group. 235 chrome_root_options = ('--no-ns-pid ' 236 '--chrome_root={} --chrome_root_mount={} ' 237 "FEATURES=\"-usersandbox\" " 238 'CHROME_ROOT={}'.format(label.chrome_src, 239 CHROME_MOUNT_DIR, 240 CHROME_MOUNT_DIR)) 241 if self.log_level != 'verbose': 242 self.logger.LogOutput('Running test.') 243 self.logger.LogOutput('CMD: %s' % cmd) 244 return self._ce.ChrootRunCommandWOutput( 245 label.chromeos_root, 246 cmd, 247 command_terminator=self._ct, 248 cros_sdk_options=chrome_root_options) 249 250 def Telemetry_Run(self, machine, label, benchmark, profiler_args): 251 telemetry_run_path = '' 252 if not os.path.isdir(label.chrome_src): 253 self.logger.LogFatal('Cannot find chrome src dir to' ' run telemetry.') 254 else: 255 telemetry_run_path = os.path.join(label.chrome_src, 'src/tools/perf') 256 if not os.path.exists(telemetry_run_path): 257 self.logger.LogFatal('Cannot find %s directory.' % telemetry_run_path) 258 259 if profiler_args: 260 self.logger.LogFatal('Telemetry does not support the perf profiler.') 261 262 # Check for and remove temporary file that may have been left by 263 # previous telemetry runs (and which might prevent this run from 264 # working). 265 if not test_flag.GetTestMode(): 266 self.RemoveTelemetryTempFile(machine, label.chromeos_root) 267 268 rsa_key = os.path.join( 269 label.chromeos_root, 270 'src/scripts/mod_for_test_scripts/ssh_keys/testing_rsa') 271 272 cmd = ('cd {0} && ' 273 './run_measurement ' 274 '--browser=cros-chrome ' 275 '--output-format=csv ' 276 '--remote={1} ' 277 '--identity {2} ' 278 '{3} {4}'.format(telemetry_run_path, machine, rsa_key, 279 benchmark.test_name, benchmark.test_args)) 280 if self.log_level != 'verbose': 281 self.logger.LogOutput('Running test.') 282 self.logger.LogOutput('CMD: %s' % cmd) 283 return self._ce.RunCommandWOutput(cmd, print_to_console=False) 284 285 def CommandTerminator(self): 286 return self._ct 287 288 def Terminate(self): 289 self._ct.Terminate() 290 291 292class MockSuiteRunner(object): 293 """Mock suite runner for test.""" 294 295 def __init__(self): 296 self._true = True 297 298 def Run(self, *_args): 299 if self._true: 300 return [0, '', ''] 301 else: 302 return [0, '', ''] 303