1# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import logging 6import os 7import StringIO 8 9from autotest_lib.client.common_lib import error, utils 10from autotest_lib.client.common_lib.cros import dev_server 11 12 13TELEMETRY_RUN_BENCHMARKS_SCRIPT = 'tools/perf/run_benchmark' 14TELEMETRY_RUN_TESTS_SCRIPT = 'tools/telemetry/run_tests' 15TELEMETRY_TIMEOUT_MINS = 120 16 17DUT_CHROME_ROOT = '/usr/local/telemetry/src' 18DUT_COMMON_SSH_OPTIONS = ['-o StrictHostKeyChecking=no', 19 '-o UserKnownHostsFile=/dev/null', 20 '-o BatchMode=yes', 21 '-o ConnectTimeout=30', 22 '-o ServerAliveInterval=900', 23 '-o ServerAliveCountMax=3', 24 '-o ConnectionAttempts=4', 25 '-o Protocol=2'] 26DUT_SSH_OPTIONS = ' '.join(DUT_COMMON_SSH_OPTIONS + ['-x', '-a', '-l root']) 27DUT_SCP_OPTIONS = ' '.join(DUT_COMMON_SSH_OPTIONS) 28DUT_RSYNC_OPTIONS = ' '.join(['--rsh="/usr/bin/ssh %s"' % DUT_SSH_OPTIONS, 29 '-L', '--timeout=1800', '-az', 30 '--no-o', '--no-g']) 31# Prevent double quotes from being unfolded. 32DUT_RSYNC_OPTIONS = utils.sh_escape(DUT_RSYNC_OPTIONS) 33 34# Result Statuses 35SUCCESS_STATUS = 'SUCCESS' 36WARNING_STATUS = 'WARNING' 37FAILED_STATUS = 'FAILED' 38 39# A list of benchmarks with that the telemetry test harness can run on dut. 40ON_DUT_WHITE_LIST = ['dromaeo.domcoreattr', 41 'dromaeo.domcoremodify', 42 'dromaeo.domcorequery', 43 'dromaeo.domcoretraverse', 44 'image_decoding.image_decoding_measurement', 45 'jetstream', 46 'kraken', 47 'memory.top_7_stress', 48 'octane', 49 'page_cycler.typical_25', 50 'page_cycler_v2.typical_25', 51 'robohornet_pro', 52 'smoothness.top_25_smooth', 53 'smoothness.tough_animation_cases', 54 'smoothness.tough_canvas_cases', 55 'smoothness.tough_filters_cases', 56 'smoothness.tough_pinch_zoom_cases', 57 'smoothness.tough_scrolling_cases', 58 'smoothness.tough_webgl_cases', 59 'speedometer', 60 'startup.cold.blank_page', 61 'sunspider', 62 'tab_switching.top_10', 63 'tab_switching.typical_25', 64 'webrtc.peerconnection', 65 'webrtc.stress'] 66 67# BLACK LIST 68# 'session_restore.cold.typical_25', # profile generator not implemented on 69 # CrOS. 70 71class TelemetryResult(object): 72 """Class to represent the results of a telemetry run. 73 74 This class represents the results of a telemetry run, whether it ran 75 successful, failed or had warnings. 76 """ 77 78 79 def __init__(self, exit_code=0, stdout='', stderr=''): 80 """Initializes this TelemetryResultObject instance. 81 82 @param status: Status of the telemtry run. 83 @param stdout: Stdout of the telemetry run. 84 @param stderr: Stderr of the telemetry run. 85 """ 86 if exit_code == 0: 87 self.status = SUCCESS_STATUS 88 else: 89 self.status = FAILED_STATUS 90 91 self._stdout = stdout 92 self._stderr = stderr 93 self.output = '\n'.join([stdout, stderr]) 94 95 96class TelemetryRunner(object): 97 """Class responsible for telemetry for a given build. 98 99 This class will extract and install telemetry on the devserver and is 100 responsible for executing the telemetry benchmarks and returning their 101 output to the caller. 102 """ 103 104 def __init__(self, host, local=False, telemetry_on_dut=True): 105 """Initializes this telemetry runner instance. 106 107 If telemetry is not installed for this build, it will be. 108 109 Basically, the following commands on the local pc on which test_that 110 will be executed, depending on the 4 possible combinations of 111 local x telemetry_on_dut: 112 113 local=True, telemetry_on_dut=False: 114 run_benchmark --browser=cros-chrome --remote=[dut] [test] 115 116 local=True, telemetry_on_dut=True: 117 ssh [dut] run_benchmark --browser=system [test] 118 119 local=False, telemetry_on_dut=False: 120 ssh [devserver] run_benchmark --browser=cros-chrome --remote=[dut] [test] 121 122 local=False, telemetry_on_dut=True: 123 ssh [devserver] ssh [dut] run_benchmark --browser=system [test] 124 125 @param host: Host where the test will be run. 126 @param local: If set, no devserver will be used, test will be run 127 locally. 128 If not set, "ssh [devserver] " will be appended to test 129 commands. 130 @param telemetry_on_dut: If set, telemetry itself (the test harness) 131 will run on dut. 132 It decides browser=[system|cros-chrome] 133 """ 134 self._host = host 135 self._devserver = None 136 self._telemetry_path = None 137 self._telemetry_on_dut = telemetry_on_dut 138 # TODO (llozano crbug.com/324964). Remove conditional code. 139 # Use a class hierarchy instead. 140 if local: 141 self._setup_local_telemetry() 142 else: 143 self._setup_devserver_telemetry() 144 145 logging.debug('Telemetry Path: %s', self._telemetry_path) 146 147 148 def _setup_devserver_telemetry(self): 149 """Setup Telemetry to use the devserver.""" 150 logging.debug('Setting up telemetry for devserver testing') 151 logging.debug('Grabbing build from AFE.') 152 info = self._host.host_info_store.get() 153 if not info.build: 154 logging.error('Unable to locate build label for host: %s.', 155 self._host.hostname) 156 raise error.AutotestError('Failed to grab build for host %s.' % 157 self._host.hostname) 158 159 logging.debug('Setting up telemetry for build: %s', info.build) 160 161 self._devserver = dev_server.ImageServer.resolve( 162 info.build, hostname=self._host.hostname) 163 self._devserver.stage_artifacts(info.build, ['autotest_packages']) 164 self._telemetry_path = self._devserver.setup_telemetry(build=info.build) 165 166 167 def _setup_local_telemetry(self): 168 """Setup Telemetry to use local path to its sources. 169 170 First look for chrome source root, either externally mounted, or inside 171 the chroot. Prefer chrome-src-internal source tree to chrome-src. 172 """ 173 TELEMETRY_DIR = 'src' 174 CHROME_LOCAL_SRC = '/var/cache/chromeos-cache/distfiles/target/' 175 CHROME_EXTERNAL_SRC = os.path.expanduser('~/chrome_root/') 176 177 logging.debug('Setting up telemetry for local testing') 178 179 sources_list = ('chrome-src-internal', 'chrome-src') 180 dir_list = [CHROME_EXTERNAL_SRC] 181 dir_list.extend( 182 [os.path.join(CHROME_LOCAL_SRC, x) for x in sources_list]) 183 if 'CHROME_ROOT' in os.environ: 184 dir_list.insert(0, os.environ['CHROME_ROOT']) 185 186 telemetry_src = '' 187 for dir in dir_list: 188 if os.path.exists(dir): 189 telemetry_src = os.path.join(dir, TELEMETRY_DIR) 190 break 191 else: 192 raise error.TestError('Telemetry source directory not found.') 193 194 self._devserver = None 195 self._telemetry_path = telemetry_src 196 197 198 def _get_telemetry_cmd(self, script, test_or_benchmark, *args): 199 """Build command to execute telemetry based on script and benchmark. 200 201 @param script: Telemetry script we want to run. For example: 202 [path_to_telemetry_src]/src/tools/telemetry/run_tests. 203 @param test_or_benchmark: Name of the test or benchmark we want to run, 204 with the page_set (if required) as part of 205 the string. 206 @param args: additional list of arguments to pass to the script. 207 208 @returns Full telemetry command to execute the script. 209 """ 210 telemetry_cmd = [] 211 if self._devserver: 212 devserver_hostname = self._devserver.hostname 213 telemetry_cmd.extend(['ssh', devserver_hostname]) 214 215 if self._telemetry_on_dut: 216 telemetry_cmd.extend( 217 ['ssh', 218 DUT_SSH_OPTIONS, 219 self._host.hostname, 220 'python', 221 script, 222 '--verbose', 223 '--output-format=chartjson', 224 '--output-dir=%s' % DUT_CHROME_ROOT, 225 '--browser=system']) 226 else: 227 telemetry_cmd.extend( 228 ['python', 229 script, 230 '--verbose', 231 '--browser=cros-chrome', 232 '--output-format=chartjson', 233 '--output-dir=%s' % self._telemetry_path, 234 '--remote=%s' % self._host.hostname]) 235 telemetry_cmd.extend(args) 236 telemetry_cmd.append(test_or_benchmark) 237 238 return ' '.join(telemetry_cmd) 239 240 241 def _scp_telemetry_results_cmd(self, perf_results_dir): 242 """Build command to copy the telemetry results from the devserver. 243 244 @param perf_results_dir: directory path where test output is to be 245 collected. 246 @returns SCP command to copy the results json to the specified directory. 247 """ 248 scp_cmd = [] 249 devserver_hostname = '' 250 if perf_results_dir: 251 if self._devserver: 252 devserver_hostname = self._devserver.hostname + ':' 253 if self._telemetry_on_dut: 254 src = ('root@%s:%s/results-chart.json' % 255 (self._host.hostname, DUT_CHROME_ROOT)) 256 scp_cmd.extend(['scp', DUT_SCP_OPTIONS, src, perf_results_dir]) 257 else: 258 src = ('%s%s/results-chart.json' % 259 (devserver_hostname, self._telemetry_path)) 260 scp_cmd.extend(['scp', src, perf_results_dir]) 261 262 return ' '.join(scp_cmd) 263 264 265 def _run_cmd(self, cmd): 266 """Execute an command in a external shell and capture the output. 267 268 @param cmd: String of is a valid shell command. 269 270 @returns The standard out, standard error and the integer exit code of 271 the executed command. 272 """ 273 logging.debug('Running: %s', cmd) 274 275 output = StringIO.StringIO() 276 error_output = StringIO.StringIO() 277 exit_code = 0 278 try: 279 result = utils.run(cmd, stdout_tee=output, 280 stderr_tee=error_output, 281 timeout=TELEMETRY_TIMEOUT_MINS*60) 282 exit_code = result.exit_status 283 except error.CmdError as e: 284 logging.debug('Error occurred executing.') 285 exit_code = e.result_obj.exit_status 286 287 stdout = output.getvalue() 288 stderr = error_output.getvalue() 289 logging.debug('Completed with exit code: %d.\nstdout:%s\n' 290 'stderr:%s', exit_code, stdout, stderr) 291 return stdout, stderr, exit_code 292 293 294 def _run_telemetry(self, script, test_or_benchmark, *args): 295 """Runs telemetry on a dut. 296 297 @param script: Telemetry script we want to run. For example: 298 [path_to_telemetry_src]/src/tools/telemetry/run_tests. 299 @param test_or_benchmark: Name of the test or benchmark we want to run, 300 with the page_set (if required) as part of the 301 string. 302 @param args: additional list of arguments to pass to the script. 303 304 @returns A TelemetryResult Instance with the results of this telemetry 305 execution. 306 """ 307 # TODO (sbasi crbug.com/239933) add support for incognito mode. 308 309 telemetry_cmd = self._get_telemetry_cmd(script, 310 test_or_benchmark, 311 *args) 312 logging.debug('Running Telemetry: %s', telemetry_cmd) 313 314 stdout, stderr, exit_code = self._run_cmd(telemetry_cmd) 315 316 return TelemetryResult(exit_code=exit_code, stdout=stdout, 317 stderr=stderr) 318 319 320 def _run_scp(self, perf_results_dir): 321 """Runs telemetry on a dut. 322 323 @param perf_results_dir: The local directory that results are being 324 collected. 325 """ 326 scp_cmd = self._scp_telemetry_results_cmd(perf_results_dir) 327 logging.debug('Retrieving Results: %s', scp_cmd) 328 329 self._run_cmd(scp_cmd) 330 331 332 def _run_test(self, script, test, *args): 333 """Runs a telemetry test on a dut. 334 335 @param script: Which telemetry test script we want to run. Can be 336 telemetry's base test script or the Chrome OS specific 337 test script. 338 @param test: Telemetry test we want to run. 339 @param args: additional list of arguments to pass to the script. 340 341 @returns A TelemetryResult Instance with the results of this telemetry 342 execution. 343 """ 344 logging.debug('Running telemetry test: %s', test) 345 telemetry_script = os.path.join(self._telemetry_path, script) 346 result = self._run_telemetry(telemetry_script, test, *args) 347 if result.status is FAILED_STATUS: 348 raise error.TestFail('Telemetry test %s failed.' % test) 349 return result 350 351 352 def run_telemetry_test(self, test, *args): 353 """Runs a telemetry test on a dut. 354 355 @param test: Telemetry test we want to run. 356 @param args: additional list of arguments to pass to the telemetry 357 execution script. 358 359 @returns A TelemetryResult Instance with the results of this telemetry 360 execution. 361 """ 362 return self._run_test(TELEMETRY_RUN_TESTS_SCRIPT, test, *args) 363 364 365 def run_telemetry_benchmark(self, benchmark, perf_value_writer=None, 366 *args): 367 """Runs a telemetry benchmark on a dut. 368 369 @param benchmark: Benchmark we want to run. 370 @param perf_value_writer: Should be an instance with the function 371 output_perf_value(), if None, no perf value 372 will be written. Typically this will be the 373 job object from an autotest test. 374 @param args: additional list of arguments to pass to the telemetry 375 execution script. 376 377 @returns A TelemetryResult Instance with the results of this telemetry 378 execution. 379 """ 380 logging.debug('Running telemetry benchmark: %s', benchmark) 381 382 if benchmark not in ON_DUT_WHITE_LIST: 383 self._telemetry_on_dut = False 384 385 if self._telemetry_on_dut: 386 telemetry_script = os.path.join(DUT_CHROME_ROOT, 387 TELEMETRY_RUN_BENCHMARKS_SCRIPT) 388 self._ensure_deps(self._host, benchmark) 389 else: 390 telemetry_script = os.path.join(self._telemetry_path, 391 TELEMETRY_RUN_BENCHMARKS_SCRIPT) 392 393 result = self._run_telemetry(telemetry_script, benchmark, *args) 394 395 if result.status is WARNING_STATUS: 396 raise error.TestWarn('Telemetry Benchmark: %s' 397 ' exited with Warnings.' % benchmark) 398 if result.status is FAILED_STATUS: 399 raise error.TestFail('Telemetry Benchmark: %s' 400 ' failed to run.' % benchmark) 401 if perf_value_writer: 402 self._run_scp(perf_value_writer.resultsdir) 403 return result 404 405 def _ensure_deps(self, dut, test_name): 406 """ 407 Ensure the dependencies are locally available on DUT. 408 409 @param dut: The autotest host object representing DUT. 410 @param test_name: Name of the telemetry test. 411 """ 412 # Get DEPs using host's telemetry. 413 format_string = ('python %s/tools/perf/fetch_benchmark_deps.py %s') 414 command = format_string % (self._telemetry_path, test_name) 415 stdout = StringIO.StringIO() 416 stderr = StringIO.StringIO() 417 418 if self._devserver: 419 devserver_hostname = self._devserver.url().split( 420 'http://')[1].split(':')[0] 421 command = 'ssh %s %s' % (devserver_hostname, command) 422 423 logging.info('Getting DEPs: %s', command) 424 try: 425 result = utils.run(command, stdout_tee=stdout, 426 stderr_tee=stderr) 427 except error.CmdError as e: 428 logging.debug('Error occurred getting DEPs: %s\n %s\n', 429 stdout.getvalue(), stderr.getvalue()) 430 raise error.TestFail('Error occurred while getting DEPs.') 431 432 # Download DEPs to DUT. 433 # send_file() relies on rsync over ssh. Couldn't be better. 434 stdout_str = stdout.getvalue() 435 stdout.close() 436 stderr.close() 437 for dep in stdout_str.split(): 438 src = os.path.join(self._telemetry_path, dep) 439 dst = os.path.join(DUT_CHROME_ROOT, dep) 440 if self._devserver: 441 logging.info('Copying: %s -> %s', src, dst) 442 utils.run('ssh %s rsync %s %s %s:%s' % 443 (devserver_hostname, DUT_RSYNC_OPTIONS, src, 444 self._host.hostname, dst)) 445 else: 446 if not os.path.isfile(src): 447 raise error.TestFail('Error occurred while saving DEPs.') 448 logging.info('Copying: %s -> %s', src, dst) 449 dut.send_file(src, dst) 450