1# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import logging 6import os 7import StringIO 8 9from autotest_lib.client.common_lib import error, utils 10from autotest_lib.client.common_lib.cros import dev_server 11 12 13TELEMETRY_RUN_BENCHMARKS_SCRIPT = 'tools/perf/run_benchmark' 14TELEMETRY_RUN_TESTS_SCRIPT = 'tools/telemetry/run_tests' 15TELEMETRY_RUN_GPU_TESTS_SCRIPT = 'content/test/gpu/run_gpu_integration_test.py' 16TELEMETRY_TIMEOUT_MINS = 120 17 18DUT_CHROME_ROOT = '/usr/local/telemetry/src' 19DUT_COMMON_SSH_OPTIONS = ['-o StrictHostKeyChecking=no', 20 '-o UserKnownHostsFile=/dev/null', 21 '-o BatchMode=yes', 22 '-o ConnectTimeout=30', 23 '-o ServerAliveInterval=900', 24 '-o ServerAliveCountMax=3', 25 '-o ConnectionAttempts=4', 26 '-o Protocol=2'] 27DUT_SSH_OPTIONS = ' '.join(DUT_COMMON_SSH_OPTIONS + ['-x', '-a', '-l root']) 28DUT_SCP_OPTIONS = ' '.join(DUT_COMMON_SSH_OPTIONS) 29DUT_RSYNC_OPTIONS = ' '.join(['--rsh="/usr/bin/ssh %s"' % DUT_SSH_OPTIONS, 30 '-L', '--timeout=1800', '-az', 31 '--no-o', '--no-g']) 32# Prevent double quotes from being unfolded. 33DUT_RSYNC_OPTIONS = utils.sh_escape(DUT_RSYNC_OPTIONS) 34 35# Result Statuses 36SUCCESS_STATUS = 'SUCCESS' 37WARNING_STATUS = 'WARNING' 38FAILED_STATUS = 'FAILED' 39 40# A list of benchmarks with that the telemetry test harness can run on dut. 41ON_DUT_WHITE_LIST = ['dromaeo.domcoreattr', 42 'dromaeo.domcoremodify', 43 'dromaeo.domcorequery', 44 'dromaeo.domcoretraverse', 45 'image_decoding.image_decoding_measurement', 46 'jetstream', 47 'kraken', 48 'memory.top_7_stress', 49 'octane', 50 'page_cycler.typical_25', 51 'page_cycler_v2.typical_25', 52 'robohornet_pro', 53 'smoothness.top_25_smooth', 54 'smoothness.tough_animation_cases', 55 'smoothness.tough_canvas_cases', 56 'smoothness.tough_filters_cases', 57 'smoothness.tough_pinch_zoom_cases', 58 'smoothness.tough_scrolling_cases', 59 'smoothness.tough_webgl_cases', 60 'speedometer', 61 'startup.cold.blank_page', 62 'sunspider', 63 'tab_switching.top_10', 64 'tab_switching.typical_25', 65 'webrtc.peerconnection', 66 'webrtc.stress'] 67 68# BLACK LIST 69# 'session_restore.cold.typical_25', # profile generator not implemented on 70 # CrOS. 71 72class TelemetryResult(object): 73 """Class to represent the results of a telemetry run. 74 75 This class represents the results of a telemetry run, whether it ran 76 successful, failed or had warnings. 77 """ 78 79 80 def __init__(self, exit_code=0, stdout='', stderr=''): 81 """Initializes this TelemetryResultObject instance. 82 83 @param status: Status of the telemtry run. 84 @param stdout: Stdout of the telemetry run. 85 @param stderr: Stderr of the telemetry run. 86 """ 87 if exit_code == 0: 88 self.status = SUCCESS_STATUS 89 else: 90 self.status = FAILED_STATUS 91 92 self._stdout = stdout 93 self._stderr = stderr 94 self.output = '\n'.join([stdout, stderr]) 95 96 97class TelemetryRunner(object): 98 """Class responsible for telemetry for a given build. 99 100 This class will extract and install telemetry on the devserver and is 101 responsible for executing the telemetry benchmarks and returning their 102 output to the caller. 103 """ 104 105 def __init__(self, host, local=False, telemetry_on_dut=True): 106 """Initializes this telemetry runner instance. 107 108 If telemetry is not installed for this build, it will be. 109 110 Basically, the following commands on the local pc on which test_that 111 will be executed, depending on the 4 possible combinations of 112 local x telemetry_on_dut: 113 114 local=True, telemetry_on_dut=False: 115 run_benchmark --browser=cros-chrome --remote=[dut] [test] 116 117 local=True, telemetry_on_dut=True: 118 ssh [dut] run_benchmark --browser=system [test] 119 120 local=False, telemetry_on_dut=False: 121 ssh [devserver] run_benchmark --browser=cros-chrome --remote=[dut] [test] 122 123 local=False, telemetry_on_dut=True: 124 ssh [devserver] ssh [dut] run_benchmark --browser=system [test] 125 126 @param host: Host where the test will be run. 127 @param local: If set, no devserver will be used, test will be run 128 locally. 129 If not set, "ssh [devserver] " will be appended to test 130 commands. 131 @param telemetry_on_dut: If set, telemetry itself (the test harness) 132 will run on dut. 133 It decides browser=[system|cros-chrome] 134 """ 135 self._host = host 136 self._devserver = None 137 self._telemetry_path = None 138 self._telemetry_on_dut = telemetry_on_dut 139 # TODO (llozano crbug.com/324964). Remove conditional code. 140 # Use a class hierarchy instead. 141 if local: 142 self._setup_local_telemetry() 143 else: 144 self._setup_devserver_telemetry() 145 146 logging.debug('Telemetry Path: %s', self._telemetry_path) 147 148 149 def _setup_devserver_telemetry(self): 150 """Setup Telemetry to use the devserver.""" 151 logging.debug('Setting up telemetry for devserver testing') 152 logging.debug('Grabbing build from AFE.') 153 info = self._host.host_info_store.get() 154 if not info.build: 155 logging.error('Unable to locate build label for host: %s.', 156 self._host.hostname) 157 raise error.AutotestError('Failed to grab build for host %s.' % 158 self._host.hostname) 159 160 logging.debug('Setting up telemetry for build: %s', info.build) 161 162 self._devserver = dev_server.ImageServer.resolve( 163 info.build, hostname=self._host.hostname) 164 self._devserver.stage_artifacts(info.build, ['autotest_packages']) 165 self._telemetry_path = self._devserver.setup_telemetry(build=info.build) 166 167 168 def _setup_local_telemetry(self): 169 """Setup Telemetry to use local path to its sources. 170 171 First look for chrome source root, either externally mounted, or inside 172 the chroot. Prefer chrome-src-internal source tree to chrome-src. 173 """ 174 TELEMETRY_DIR = 'src' 175 CHROME_LOCAL_SRC = '/var/cache/chromeos-cache/distfiles/target/' 176 CHROME_EXTERNAL_SRC = os.path.expanduser('~/chrome_root/') 177 178 logging.debug('Setting up telemetry for local testing') 179 180 sources_list = ('chrome-src-internal', 'chrome-src') 181 dir_list = [CHROME_EXTERNAL_SRC] 182 dir_list.extend( 183 [os.path.join(CHROME_LOCAL_SRC, x) for x in sources_list]) 184 if 'CHROME_ROOT' in os.environ: 185 dir_list.insert(0, os.environ['CHROME_ROOT']) 186 187 telemetry_src = '' 188 for dir in dir_list: 189 if os.path.exists(dir): 190 telemetry_src = os.path.join(dir, TELEMETRY_DIR) 191 break 192 else: 193 raise error.TestError('Telemetry source directory not found.') 194 195 self._devserver = None 196 self._telemetry_path = telemetry_src 197 198 199 def _get_telemetry_cmd(self, script, test_or_benchmark, *args): 200 """Build command to execute telemetry based on script and benchmark. 201 202 @param script: Telemetry script we want to run. For example: 203 [path_to_telemetry_src]/src/tools/telemetry/run_tests. 204 @param test_or_benchmark: Name of the test or benchmark we want to run, 205 with the page_set (if required) as part of 206 the string. 207 @param args: additional list of arguments to pass to the script. 208 209 @returns Full telemetry command to execute the script. 210 """ 211 telemetry_cmd = [] 212 if self._devserver: 213 devserver_hostname = self._devserver.hostname 214 telemetry_cmd.extend(['ssh', devserver_hostname]) 215 216 if self._telemetry_on_dut: 217 telemetry_cmd.extend( 218 ['ssh', 219 DUT_SSH_OPTIONS, 220 self._host.hostname, 221 'python', 222 script, 223 '--verbose', 224 '--output-format=chartjson', 225 '--output-dir=%s' % DUT_CHROME_ROOT, 226 '--browser=system']) 227 else: 228 telemetry_cmd.extend( 229 ['python', 230 script, 231 '--verbose', 232 '--browser=cros-chrome', 233 '--output-format=chartjson', 234 '--output-dir=%s' % self._telemetry_path, 235 '--remote=%s' % self._host.hostname]) 236 telemetry_cmd.extend(args) 237 telemetry_cmd.append(test_or_benchmark) 238 239 return ' '.join(telemetry_cmd) 240 241 242 def _scp_telemetry_results_cmd(self, perf_results_dir): 243 """Build command to copy the telemetry results from the devserver. 244 245 @param perf_results_dir: directory path where test output is to be 246 collected. 247 @returns SCP command to copy the results json to the specified directory. 248 """ 249 scp_cmd = [] 250 devserver_hostname = '' 251 if perf_results_dir: 252 if self._devserver: 253 devserver_hostname = self._devserver.hostname + ':' 254 if self._telemetry_on_dut: 255 src = ('root@%s:%s/results-chart.json' % 256 (self._host.hostname, DUT_CHROME_ROOT)) 257 scp_cmd.extend(['scp', DUT_SCP_OPTIONS, src, perf_results_dir]) 258 else: 259 src = ('%s%s/results-chart.json' % 260 (devserver_hostname, self._telemetry_path)) 261 scp_cmd.extend(['scp', src, perf_results_dir]) 262 263 return ' '.join(scp_cmd) 264 265 266 def _run_cmd(self, cmd): 267 """Execute an command in a external shell and capture the output. 268 269 @param cmd: String of is a valid shell command. 270 271 @returns The standard out, standard error and the integer exit code of 272 the executed command. 273 """ 274 logging.debug('Running: %s', cmd) 275 276 output = StringIO.StringIO() 277 error_output = StringIO.StringIO() 278 exit_code = 0 279 try: 280 result = utils.run(cmd, stdout_tee=output, 281 stderr_tee=error_output, 282 timeout=TELEMETRY_TIMEOUT_MINS*60) 283 exit_code = result.exit_status 284 except error.CmdError as e: 285 logging.debug('Error occurred executing.') 286 exit_code = e.result_obj.exit_status 287 288 stdout = output.getvalue() 289 stderr = error_output.getvalue() 290 logging.debug('Completed with exit code: %d.\nstdout:%s\n' 291 'stderr:%s', exit_code, stdout, stderr) 292 return stdout, stderr, exit_code 293 294 295 def _run_telemetry(self, script, test_or_benchmark, *args): 296 """Runs telemetry on a dut. 297 298 @param script: Telemetry script we want to run. For example: 299 [path_to_telemetry_src]/src/tools/telemetry/run_tests. 300 @param test_or_benchmark: Name of the test or benchmark we want to run, 301 with the page_set (if required) as part of the 302 string. 303 @param args: additional list of arguments to pass to the script. 304 305 @returns A TelemetryResult Instance with the results of this telemetry 306 execution. 307 """ 308 # TODO (sbasi crbug.com/239933) add support for incognito mode. 309 310 telemetry_cmd = self._get_telemetry_cmd(script, 311 test_or_benchmark, 312 *args) 313 logging.debug('Running Telemetry: %s', telemetry_cmd) 314 315 stdout, stderr, exit_code = self._run_cmd(telemetry_cmd) 316 317 return TelemetryResult(exit_code=exit_code, stdout=stdout, 318 stderr=stderr) 319 320 321 def _run_scp(self, perf_results_dir): 322 """Runs telemetry on a dut. 323 324 @param perf_results_dir: The local directory that results are being 325 collected. 326 """ 327 scp_cmd = self._scp_telemetry_results_cmd(perf_results_dir) 328 logging.debug('Retrieving Results: %s', scp_cmd) 329 330 self._run_cmd(scp_cmd) 331 332 333 def _run_test(self, script, test, *args): 334 """Runs a telemetry test on a dut. 335 336 @param script: Which telemetry test script we want to run. Can be 337 telemetry's base test script or the Chrome OS specific 338 test script. 339 @param test: Telemetry test we want to run. 340 @param args: additional list of arguments to pass to the script. 341 342 @returns A TelemetryResult Instance with the results of this telemetry 343 execution. 344 """ 345 logging.debug('Running telemetry test: %s', test) 346 telemetry_script = os.path.join(self._telemetry_path, script) 347 result = self._run_telemetry(telemetry_script, test, *args) 348 if result.status is FAILED_STATUS: 349 raise error.TestFail('Telemetry test %s failed.' % test) 350 return result 351 352 353 def run_telemetry_test(self, test, *args): 354 """Runs a telemetry test on a dut. 355 356 @param test: Telemetry test we want to run. 357 @param args: additional list of arguments to pass to the telemetry 358 execution script. 359 360 @returns A TelemetryResult Instance with the results of this telemetry 361 execution. 362 """ 363 return self._run_test(TELEMETRY_RUN_TESTS_SCRIPT, test, *args) 364 365 366 def run_telemetry_benchmark(self, benchmark, perf_value_writer=None, 367 *args): 368 """Runs a telemetry benchmark on a dut. 369 370 @param benchmark: Benchmark we want to run. 371 @param perf_value_writer: Should be an instance with the function 372 output_perf_value(), if None, no perf value 373 will be written. Typically this will be the 374 job object from an autotest test. 375 @param args: additional list of arguments to pass to the telemetry 376 execution script. 377 378 @returns A TelemetryResult Instance with the results of this telemetry 379 execution. 380 """ 381 logging.debug('Running telemetry benchmark: %s', benchmark) 382 383 if benchmark not in ON_DUT_WHITE_LIST: 384 self._telemetry_on_dut = False 385 386 if self._telemetry_on_dut: 387 telemetry_script = os.path.join(DUT_CHROME_ROOT, 388 TELEMETRY_RUN_BENCHMARKS_SCRIPT) 389 self._ensure_deps(self._host, benchmark) 390 else: 391 telemetry_script = os.path.join(self._telemetry_path, 392 TELEMETRY_RUN_BENCHMARKS_SCRIPT) 393 394 result = self._run_telemetry(telemetry_script, benchmark, *args) 395 396 if result.status is WARNING_STATUS: 397 raise error.TestWarn('Telemetry Benchmark: %s' 398 ' exited with Warnings.' % benchmark) 399 if result.status is FAILED_STATUS: 400 raise error.TestFail('Telemetry Benchmark: %s' 401 ' failed to run.' % benchmark) 402 if perf_value_writer: 403 self._run_scp(perf_value_writer.resultsdir) 404 return result 405 406 407 def run_gpu_integration_test(self, test, *args): 408 """Runs a gpu test on a dut. 409 410 @param test: Gpu test we want to run. 411 @param args: additional list of arguments to pass to the telemetry 412 execution script. 413 414 @returns A TelemetryResult instance with the results of this telemetry 415 execution. 416 """ 417 script = os.path.join(DUT_CHROME_ROOT, 418 TELEMETRY_RUN_GPU_TESTS_SCRIPT) 419 cmd = [] 420 if self._devserver: 421 devserver_hostname = self._devserver.hostname 422 cmd.extend(['ssh', devserver_hostname]) 423 424 cmd.extend( 425 ['ssh', 426 DUT_SSH_OPTIONS, 427 self._host.hostname, 428 'python', 429 script]) 430 431 cmd.extend(args) 432 cmd.append(test) 433 cmd = ' '.join(cmd) 434 stdout, stderr, exit_code = self._run_cmd(cmd) 435 436 return TelemetryResult(exit_code=exit_code, stdout=stdout, 437 stderr=stderr) 438 439 440 def _ensure_deps(self, dut, test_name): 441 """ 442 Ensure the dependencies are locally available on DUT. 443 444 @param dut: The autotest host object representing DUT. 445 @param test_name: Name of the telemetry test. 446 """ 447 # Get DEPs using host's telemetry. 448 format_string = ('python %s/tools/perf/fetch_benchmark_deps.py %s') 449 command = format_string % (self._telemetry_path, test_name) 450 stdout = StringIO.StringIO() 451 stderr = StringIO.StringIO() 452 453 if self._devserver: 454 devserver_hostname = self._devserver.url().split( 455 'http://')[1].split(':')[0] 456 command = 'ssh %s %s' % (devserver_hostname, command) 457 458 logging.info('Getting DEPs: %s', command) 459 try: 460 result = utils.run(command, stdout_tee=stdout, 461 stderr_tee=stderr) 462 except error.CmdError as e: 463 logging.debug('Error occurred getting DEPs: %s\n %s\n', 464 stdout.getvalue(), stderr.getvalue()) 465 raise error.TestFail('Error occurred while getting DEPs.') 466 467 # Download DEPs to DUT. 468 # send_file() relies on rsync over ssh. Couldn't be better. 469 stdout_str = stdout.getvalue() 470 stdout.close() 471 stderr.close() 472 for dep in stdout_str.split(): 473 src = os.path.join(self._telemetry_path, dep) 474 dst = os.path.join(DUT_CHROME_ROOT, dep) 475 if self._devserver: 476 logging.info('Copying: %s -> %s', src, dst) 477 utils.run('ssh %s rsync %s %s %s:%s' % 478 (devserver_hostname, DUT_RSYNC_OPTIONS, src, 479 self._host.hostname, dst)) 480 else: 481 if not os.path.isfile(src): 482 raise error.TestFail('Error occurred while saving DEPs.') 483 logging.info('Copying: %s -> %s', src, dst) 484 dut.send_file(src, dst) 485