1# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import logging 6import os 7import StringIO 8 9from autotest_lib.client.common_lib import error, utils 10from autotest_lib.client.common_lib.cros import dev_server 11 12 13TELEMETRY_RUN_BENCHMARKS_SCRIPT = 'tools/perf/run_benchmark' 14TELEMETRY_RUN_TESTS_SCRIPT = 'tools/telemetry/run_tests' 15TELEMETRY_RUN_GPU_TESTS_SCRIPT = 'content/test/gpu/run_gpu_integration_test.py' 16TELEMETRY_TIMEOUT_MINS = 150 17 18DUT_CHROME_ROOT = '/usr/local/telemetry/src' 19 20# Result Statuses 21SUCCESS_STATUS = 'SUCCESS' 22WARNING_STATUS = 'WARNING' 23FAILED_STATUS = 'FAILED' 24 25# A list of benchmarks with that the telemetry test harness can run on dut. 26ON_DUT_WHITE_LIST = ['cros_ui_smoothness', 27 'dromaeo.domcoreattr', 28 'dromaeo.domcoremodify', 29 'dromaeo.domcorequery', 30 'dromaeo.domcoretraverse', 31 'image_decoding.image_decoding_measurement', 32 'jetstream', 33 'kraken', 34 'memory.top_7_stress', 35 'octane', 36 'page_cycler.typical_25', 37 'page_cycler_v2.typical_25', 38 'robohornet_pro', 39 'smoothness.top_25_smooth', 40 'smoothness.tough_animation_cases', 41 'smoothness.tough_canvas_cases', 42 'smoothness.tough_filters_cases', 43 'smoothness.tough_pinch_zoom_cases', 44 'smoothness.tough_scrolling_cases', 45 'smoothness.tough_webgl_cases', 46 'speedometer', 47 'sunspider', 48 'tab_switching.top_10', 49 'tab_switching.typical_25', 50 'webrtc.peerconnection', 51 'webrtc.stress'] 52 53# BLACK LIST 54# 'session_restore.cold.typical_25', # profile generator not implemented on 55 # CrOS. 56 57class TelemetryResult(object): 58 """Class to represent the results of a telemetry run. 59 60 This class represents the results of a telemetry run, whether it ran 61 successful, failed or had warnings. 62 """ 63 64 65 def __init__(self, exit_code=0, stdout='', stderr=''): 66 """Initializes this TelemetryResultObject instance. 67 68 @param status: Status of the telemtry run. 69 @param stdout: Stdout of the telemetry run. 70 @param stderr: Stderr of the telemetry run. 71 """ 72 if exit_code == 0: 73 self.status = SUCCESS_STATUS 74 else: 75 self.status = FAILED_STATUS 76 77 self._stdout = stdout 78 self._stderr = stderr 79 self.output = '\n'.join([stdout, stderr]) 80 81 82class TelemetryRunner(object): 83 """Class responsible for telemetry for a given build. 84 85 This class will extract and install telemetry on the devserver and is 86 responsible for executing the telemetry benchmarks and returning their 87 output to the caller. 88 """ 89 90 def __init__(self, host, local=False, telemetry_on_dut=True): 91 """Initializes this telemetry runner instance. 92 93 If telemetry is not installed for this build, it will be. 94 95 Basically, the following commands on the local pc on which test_that 96 will be executed, depending on the 4 possible combinations of 97 local x telemetry_on_dut: 98 99 local=True, telemetry_on_dut=False: 100 run_benchmark --browser=cros-chrome --remote=[dut] [test] 101 102 local=True, telemetry_on_dut=True: 103 ssh [dut] run_benchmark --browser=system [test] 104 105 local=False, telemetry_on_dut=False: 106 ssh [devserver] run_benchmark --browser=cros-chrome --remote=[dut] [test] 107 108 local=False, telemetry_on_dut=True: 109 ssh [devserver] ssh [dut] run_benchmark --browser=system [test] 110 111 @param host: Host where the test will be run. 112 @param local: If set, no devserver will be used, test will be run 113 locally. 114 If not set, "ssh [devserver] " will be appended to test 115 commands. 116 @param telemetry_on_dut: If set, telemetry itself (the test harness) 117 will run on dut. 118 It decides browser=[system|cros-chrome] 119 """ 120 self._host = host 121 self._devserver = None 122 self._telemetry_path = None 123 self._telemetry_on_dut = telemetry_on_dut 124 # TODO (llozano crbug.com/324964). Remove conditional code. 125 # Use a class hierarchy instead. 126 if local: 127 self._setup_local_telemetry() 128 else: 129 self._setup_devserver_telemetry() 130 131 logging.debug('Telemetry Path: %s', self._telemetry_path) 132 133 134 def _setup_devserver_telemetry(self): 135 """Setup Telemetry to use the devserver.""" 136 logging.debug('Setting up telemetry for devserver testing') 137 logging.debug('Grabbing build from AFE.') 138 info = self._host.host_info_store.get() 139 if not info.build: 140 logging.error('Unable to locate build label for host: %s.', 141 self._host.host_port) 142 raise error.AutotestError('Failed to grab build for host %s.' % 143 self._host.host_port) 144 145 logging.debug('Setting up telemetry for build: %s', info.build) 146 147 self._devserver = dev_server.ImageServer.resolve( 148 info.build, hostname=self._host.hostname) 149 self._devserver.stage_artifacts(info.build, ['autotest_packages']) 150 self._telemetry_path = self._devserver.setup_telemetry(build=info.build) 151 152 153 def _setup_local_telemetry(self): 154 """Setup Telemetry to use local path to its sources. 155 156 First look for chrome source root, either externally mounted, or inside 157 the chroot. Prefer chrome-src-internal source tree to chrome-src. 158 """ 159 TELEMETRY_DIR = 'src' 160 CHROME_LOCAL_SRC = '/var/cache/chromeos-cache/distfiles/target/' 161 CHROME_EXTERNAL_SRC = os.path.expanduser('~/chrome_root/') 162 163 logging.debug('Setting up telemetry for local testing') 164 165 sources_list = ('chrome-src-internal', 'chrome-src') 166 dir_list = [CHROME_EXTERNAL_SRC] 167 dir_list.extend( 168 [os.path.join(CHROME_LOCAL_SRC, x) for x in sources_list]) 169 if 'CHROME_ROOT' in os.environ: 170 dir_list.insert(0, os.environ['CHROME_ROOT']) 171 172 telemetry_src = '' 173 for dir in dir_list: 174 if os.path.exists(dir): 175 telemetry_src = os.path.join(dir, TELEMETRY_DIR) 176 break 177 else: 178 raise error.TestError('Telemetry source directory not found.') 179 180 self._devserver = None 181 self._telemetry_path = telemetry_src 182 183 184 def _get_telemetry_cmd(self, script, test_or_benchmark, *args): 185 """Build command to execute telemetry based on script and benchmark. 186 187 @param script: Telemetry script we want to run. For example: 188 [path_to_telemetry_src]/src/tools/telemetry/run_tests. 189 @param test_or_benchmark: Name of the test or benchmark we want to run, 190 with the page_set (if required) as part of 191 the string. 192 @param args: additional list of arguments to pass to the script. 193 194 @returns Full telemetry command to execute the script. 195 """ 196 telemetry_cmd = [] 197 if self._devserver: 198 devserver_hostname = self._devserver.hostname 199 telemetry_cmd.extend(['ssh', devserver_hostname]) 200 201 if self._telemetry_on_dut: 202 telemetry_cmd.extend( 203 [self._host.ssh_command(alive_interval=900, 204 connection_attempts=4), 205 'python', 206 script, 207 '--verbose', 208 '--output-format=chartjson', 209 '--output-dir=%s' % DUT_CHROME_ROOT, 210 '--browser=system']) 211 else: 212 telemetry_cmd.extend( 213 ['python', 214 script, 215 '--verbose', 216 '--browser=cros-chrome', 217 '--output-format=chartjson', 218 '--output-dir=%s' % self._telemetry_path, 219 '--remote=%s' % self._host.host_port]) 220 telemetry_cmd.extend(args) 221 telemetry_cmd.append(test_or_benchmark) 222 223 return ' '.join(telemetry_cmd) 224 225 226 def _scp_telemetry_results_cmd(self, perf_results_dir): 227 """Build command to copy the telemetry results from the devserver. 228 229 @param perf_results_dir: directory path where test output is to be 230 collected. 231 @returns SCP command to copy the results json to the specified directory. 232 """ 233 if not perf_results_dir: 234 return '' 235 236 scp_cmd = ['scp'] 237 if self._telemetry_on_dut: 238 scp_cmd.append(self._host.make_ssh_options(alive_interval=900, 239 connection_attempts=4)) 240 if not self._host.is_default_port: 241 scp_cmd.append('-P %d' % self._host.port) 242 src = 'root@%s:%s/results-chart.json' % (self._host.hostname, 243 DUT_CHROME_ROOT) 244 else: 245 devserver_hostname = '' 246 if self._devserver: 247 devserver_hostname = self._devserver.hostname + ':' 248 src = '%s%s/results-chart.json' % (devserver_hostname, 249 self._telemetry_path) 250 251 scp_cmd.extend([src, perf_results_dir]) 252 return ' '.join(scp_cmd) 253 254 255 def _run_cmd(self, cmd): 256 """Execute an command in a external shell and capture the output. 257 258 @param cmd: String of is a valid shell command. 259 260 @returns The standard out, standard error and the integer exit code of 261 the executed command. 262 """ 263 logging.debug('Running: %s', cmd) 264 265 output = StringIO.StringIO() 266 error_output = StringIO.StringIO() 267 exit_code = 0 268 try: 269 result = utils.run(cmd, stdout_tee=output, 270 stderr_tee=error_output, 271 timeout=TELEMETRY_TIMEOUT_MINS*60) 272 exit_code = result.exit_status 273 except error.CmdError as e: 274 logging.debug('Error occurred executing.') 275 exit_code = e.result_obj.exit_status 276 277 stdout = output.getvalue() 278 stderr = error_output.getvalue() 279 logging.debug('Completed with exit code: %d.\nstdout:%s\n' 280 'stderr:%s', exit_code, stdout, stderr) 281 return stdout, stderr, exit_code 282 283 284 def _run_telemetry(self, script, test_or_benchmark, *args): 285 """Runs telemetry on a dut. 286 287 @param script: Telemetry script we want to run. For example: 288 [path_to_telemetry_src]/src/tools/telemetry/run_tests. 289 @param test_or_benchmark: Name of the test or benchmark we want to run, 290 with the page_set (if required) as part of the 291 string. 292 @param args: additional list of arguments to pass to the script. 293 294 @returns A TelemetryResult Instance with the results of this telemetry 295 execution. 296 """ 297 # TODO (sbasi crbug.com/239933) add support for incognito mode. 298 299 telemetry_cmd = self._get_telemetry_cmd(script, 300 test_or_benchmark, 301 *args) 302 logging.debug('Running Telemetry: %s', telemetry_cmd) 303 304 stdout, stderr, exit_code = self._run_cmd(telemetry_cmd) 305 306 return TelemetryResult(exit_code=exit_code, stdout=stdout, 307 stderr=stderr) 308 309 310 def _run_scp(self, perf_results_dir): 311 """Runs telemetry on a dut. 312 313 @param perf_results_dir: The local directory that results are being 314 collected. 315 """ 316 scp_cmd = self._scp_telemetry_results_cmd(perf_results_dir) 317 logging.debug('Retrieving Results: %s', scp_cmd) 318 _, _, exit_code = self._run_cmd(scp_cmd) 319 if exit_code != 0: 320 raise error.TestFail('Unable to retrieve results.') 321 322 323 def _run_test(self, script, test, *args): 324 """Runs a telemetry test on a dut. 325 326 @param script: Which telemetry test script we want to run. Can be 327 telemetry's base test script or the Chrome OS specific 328 test script. 329 @param test: Telemetry test we want to run. 330 @param args: additional list of arguments to pass to the script. 331 332 @returns A TelemetryResult Instance with the results of this telemetry 333 execution. 334 """ 335 logging.debug('Running telemetry test: %s', test) 336 telemetry_script = os.path.join(self._telemetry_path, script) 337 result = self._run_telemetry(telemetry_script, test, *args) 338 if result.status is FAILED_STATUS: 339 raise error.TestFail('Telemetry test %s failed.' % test) 340 return result 341 342 343 def run_telemetry_test(self, test, *args): 344 """Runs a telemetry test on a dut. 345 346 @param test: Telemetry test we want to run. 347 @param args: additional list of arguments to pass to the telemetry 348 execution script. 349 350 @returns A TelemetryResult Instance with the results of this telemetry 351 execution. 352 """ 353 return self._run_test(TELEMETRY_RUN_TESTS_SCRIPT, test, *args) 354 355 356 def run_telemetry_benchmark(self, benchmark, perf_value_writer=None, 357 *args): 358 """Runs a telemetry benchmark on a dut. 359 360 @param benchmark: Benchmark we want to run. 361 @param perf_value_writer: Should be an instance with the function 362 output_perf_value(), if None, no perf value 363 will be written. Typically this will be the 364 job object from an autotest test. 365 @param args: additional list of arguments to pass to the telemetry 366 execution script. 367 368 @returns A TelemetryResult Instance with the results of this telemetry 369 execution. 370 """ 371 logging.debug('Running telemetry benchmark: %s', benchmark) 372 373 if benchmark not in ON_DUT_WHITE_LIST: 374 self._telemetry_on_dut = False 375 376 if self._telemetry_on_dut: 377 telemetry_script = os.path.join(DUT_CHROME_ROOT, 378 TELEMETRY_RUN_BENCHMARKS_SCRIPT) 379 self._ensure_deps(self._host, benchmark) 380 else: 381 telemetry_script = os.path.join(self._telemetry_path, 382 TELEMETRY_RUN_BENCHMARKS_SCRIPT) 383 384 result = self._run_telemetry(telemetry_script, benchmark, *args) 385 386 if result.status is WARNING_STATUS: 387 raise error.TestWarn('Telemetry Benchmark: %s' 388 ' exited with Warnings.' % benchmark) 389 if result.status is FAILED_STATUS: 390 raise error.TestFail('Telemetry Benchmark: %s' 391 ' failed to run.' % benchmark) 392 if perf_value_writer: 393 self._run_scp(perf_value_writer.resultsdir) 394 return result 395 396 397 def run_gpu_integration_test(self, test, *args): 398 """Runs a gpu test on a dut. 399 400 @param test: Gpu test we want to run. 401 @param args: additional list of arguments to pass to the telemetry 402 execution script. 403 404 @returns A TelemetryResult instance with the results of this telemetry 405 execution. 406 """ 407 script = os.path.join(DUT_CHROME_ROOT, 408 TELEMETRY_RUN_GPU_TESTS_SCRIPT) 409 cmd = [] 410 if self._devserver: 411 devserver_hostname = self._devserver.hostname 412 cmd.extend(['ssh', devserver_hostname]) 413 414 cmd.extend( 415 [self._host.ssh_command(alive_interval=900, connection_attempts=4), 416 'python', script]) 417 cmd.extend(args) 418 cmd.append(test) 419 cmd = ' '.join(cmd) 420 stdout, stderr, exit_code = self._run_cmd(cmd) 421 422 return TelemetryResult(exit_code=exit_code, stdout=stdout, 423 stderr=stderr) 424 425 426 def _ensure_deps(self, dut, test_name): 427 """ 428 Ensure the dependencies are locally available on DUT. 429 430 @param dut: The autotest host object representing DUT. 431 @param test_name: Name of the telemetry test. 432 """ 433 # Get DEPs using host's telemetry. 434 format_string = ('python %s/tools/perf/fetch_benchmark_deps.py %s') 435 command = format_string % (self._telemetry_path, test_name) 436 stdout = StringIO.StringIO() 437 stderr = StringIO.StringIO() 438 439 if self._devserver: 440 devserver_hostname = self._devserver.url().split( 441 'http://')[1].split(':')[0] 442 command = 'ssh %s %s' % (devserver_hostname, command) 443 444 logging.info('Getting DEPs: %s', command) 445 try: 446 result = utils.run(command, stdout_tee=stdout, 447 stderr_tee=stderr) 448 except error.CmdError as e: 449 logging.debug('Error occurred getting DEPs: %s\n %s\n', 450 stdout.getvalue(), stderr.getvalue()) 451 raise error.TestFail('Error occurred while getting DEPs.') 452 453 # Download DEPs to DUT. 454 # send_file() relies on rsync over ssh. Couldn't be better. 455 stdout_str = stdout.getvalue() 456 stdout.close() 457 stderr.close() 458 for dep in stdout_str.split(): 459 src = os.path.join(self._telemetry_path, dep) 460 dst = os.path.join(DUT_CHROME_ROOT, dep) 461 if self._devserver: 462 logging.info('Copying: %s -> %s', src, dst) 463 rsync_cmd = utils.sh_escape('rsync %s %s %s:%s' % 464 (self._host.rsync_options(), src, 465 self._host.hostname, dst)) 466 utils.run('ssh %s "%s"' % (devserver_hostname, rsync_cmd)) 467 else: 468 if not os.path.isfile(src): 469 raise error.TestFail('Error occurred while saving DEPs.') 470 logging.info('Copying: %s -> %s', src, dst) 471 dut.send_file(src, dst) 472