1# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import json 6import logging 7import os 8import StringIO 9 10from autotest_lib.client.common_lib import error, utils 11from autotest_lib.client.common_lib.cros import dev_server 12 13 14TELEMETRY_RUN_BENCHMARKS_SCRIPT = 'tools/perf/run_benchmark' 15TELEMETRY_RUN_TESTS_SCRIPT = 'tools/telemetry/run_tests' 16TELEMETRY_RUN_GPU_TESTS_SCRIPT = 'content/test/gpu/run_gpu_integration_test.py' 17TELEMETRY_TIMEOUT_MINS = 150 18 19DUT_CHROME_ROOT = '/usr/local/telemetry/src' 20 21# Result Statuses 22SUCCESS_STATUS = 'SUCCESS' 23WARNING_STATUS = 'WARNING' 24FAILED_STATUS = 'FAILED' 25 26# A list of telemetry tests that cannot run on dut. 27ON_DUT_BLACKLIST = [ 28 'loading.desktop', # crbug/882299 29 'rendering.desktop', # crbug/882291 30 'system_health.memory_desktop', # crbug/874386 31] 32 33class TelemetryResult(object): 34 """Class to represent the results of a telemetry run. 35 36 This class represents the results of a telemetry run, whether it ran 37 successful, failed or had warnings. 38 """ 39 40 41 def __init__(self, exit_code=0, stdout='', stderr=''): 42 """Initializes this TelemetryResultObject instance. 43 44 @param status: Status of the telemtry run. 45 @param stdout: Stdout of the telemetry run. 46 @param stderr: Stderr of the telemetry run. 47 """ 48 if exit_code == 0: 49 self.status = SUCCESS_STATUS 50 else: 51 self.status = FAILED_STATUS 52 53 self._stdout = stdout 54 self._stderr = stderr 55 self.output = '\n'.join([stdout, stderr]) 56 57 58class TelemetryRunner(object): 59 """Class responsible for telemetry for a given build. 60 61 This class will extract and install telemetry on the devserver and is 62 responsible for executing the telemetry benchmarks and returning their 63 output to the caller. 64 """ 65 66 def __init__(self, host, local=False, telemetry_on_dut=True): 67 """Initializes this telemetry runner instance. 68 69 If telemetry is not installed for this build, it will be. 70 71 Basically, the following commands on the local pc on which test_that 72 will be executed, depending on the 4 possible combinations of 73 local x telemetry_on_dut: 74 75 local=True, telemetry_on_dut=False: 76 run_benchmark --browser=cros-chrome --remote=[dut] [test] 77 78 local=True, telemetry_on_dut=True: 79 ssh [dut] run_benchmark --browser=system [test] 80 81 local=False, telemetry_on_dut=False: 82 ssh [devserver] run_benchmark --browser=cros-chrome --remote=[dut] [test] 83 84 local=False, telemetry_on_dut=True: 85 ssh [devserver] ssh [dut] run_benchmark --browser=system [test] 86 87 @param host: Host where the test will be run. 88 @param local: If set, no devserver will be used, test will be run 89 locally. 90 If not set, "ssh [devserver] " will be appended to test 91 commands. 92 @param telemetry_on_dut: If set, telemetry itself (the test harness) 93 will run on dut. 94 It decides browser=[system|cros-chrome] 95 """ 96 self._host = host 97 self._devserver = None 98 self._telemetry_path = None 99 self._telemetry_on_dut = telemetry_on_dut 100 # TODO (llozano crbug.com/324964). Remove conditional code. 101 # Use a class hierarchy instead. 102 if local: 103 self._setup_local_telemetry() 104 else: 105 self._setup_devserver_telemetry() 106 107 logging.debug('Telemetry Path: %s', self._telemetry_path) 108 109 110 def _setup_devserver_telemetry(self): 111 """Setup Telemetry to use the devserver.""" 112 logging.debug('Setting up telemetry for devserver testing') 113 logging.debug('Grabbing build from AFE.') 114 info = self._host.host_info_store.get() 115 if not info.build: 116 logging.error('Unable to locate build label for host: %s.', 117 self._host.host_port) 118 raise error.AutotestError('Failed to grab build for host %s.' % 119 self._host.host_port) 120 121 logging.debug('Setting up telemetry for build: %s', info.build) 122 123 self._devserver = dev_server.ImageServer.resolve( 124 info.build, hostname=self._host.hostname) 125 self._devserver.stage_artifacts(info.build, ['autotest_packages']) 126 self._telemetry_path = self._devserver.setup_telemetry(build=info.build) 127 128 129 def _setup_local_telemetry(self): 130 """Setup Telemetry to use local path to its sources. 131 132 First look for chrome source root, either externally mounted, or inside 133 the chroot. Prefer chrome-src-internal source tree to chrome-src. 134 """ 135 TELEMETRY_DIR = 'src' 136 CHROME_LOCAL_SRC = '/var/cache/chromeos-cache/distfiles/target/' 137 CHROME_EXTERNAL_SRC = os.path.expanduser('~/chrome_root/') 138 139 logging.debug('Setting up telemetry for local testing') 140 141 sources_list = ('chrome-src-internal', 'chrome-src') 142 dir_list = [CHROME_EXTERNAL_SRC] 143 dir_list.extend( 144 [os.path.join(CHROME_LOCAL_SRC, x) for x in sources_list]) 145 if 'CHROME_ROOT' in os.environ: 146 dir_list.insert(0, os.environ['CHROME_ROOT']) 147 148 telemetry_src = '' 149 for dir in dir_list: 150 if os.path.exists(dir): 151 telemetry_src = os.path.join(dir, TELEMETRY_DIR) 152 break 153 else: 154 raise error.TestError('Telemetry source directory not found.') 155 156 self._devserver = None 157 self._telemetry_path = telemetry_src 158 159 160 def _get_telemetry_cmd(self, script, test_or_benchmark, *args): 161 """Build command to execute telemetry based on script and benchmark. 162 163 @param script: Telemetry script we want to run. For example: 164 [path_to_telemetry_src]/src/tools/telemetry/run_tests. 165 @param test_or_benchmark: Name of the test or benchmark we want to run, 166 with the page_set (if required) as part of 167 the string. 168 @param args: additional list of arguments to pass to the script. 169 170 @returns Full telemetry command to execute the script. 171 """ 172 telemetry_cmd = [] 173 if self._devserver: 174 devserver_hostname = self._devserver.hostname 175 telemetry_cmd.extend(['ssh', devserver_hostname]) 176 177 if self._telemetry_on_dut: 178 telemetry_cmd.extend( 179 [self._host.ssh_command(alive_interval=900, 180 connection_attempts=4), 181 'python', 182 script, 183 '--verbose', 184 '--output-format=chartjson', 185 '--output-dir=%s' % DUT_CHROME_ROOT, 186 '--browser=system']) 187 else: 188 telemetry_cmd.extend( 189 ['python', 190 script, 191 '--verbose', 192 '--browser=cros-chrome', 193 '--output-format=chartjson', 194 '--output-dir=%s' % self._telemetry_path, 195 '--remote=%s' % self._host.host_port]) 196 telemetry_cmd.extend(args) 197 telemetry_cmd.append(test_or_benchmark) 198 199 return ' '.join(telemetry_cmd) 200 201 202 def _scp_telemetry_results_cmd(self, perf_results_dir): 203 """Build command to copy the telemetry results from the devserver. 204 205 @param perf_results_dir: directory path where test output is to be 206 collected. 207 @returns SCP command to copy the results json to the specified directory. 208 """ 209 if not perf_results_dir: 210 return '' 211 212 scp_cmd = ['scp'] 213 if self._telemetry_on_dut: 214 scp_cmd.append(self._host.make_ssh_options(alive_interval=900, 215 connection_attempts=4)) 216 if not self._host.is_default_port: 217 scp_cmd.append('-P %d' % self._host.port) 218 src = 'root@%s:%s/results-chart.json' % (self._host.hostname, 219 DUT_CHROME_ROOT) 220 else: 221 devserver_hostname = '' 222 if self._devserver: 223 devserver_hostname = self._devserver.hostname + ':' 224 src = '%s%s/results-chart.json' % (devserver_hostname, 225 self._telemetry_path) 226 227 scp_cmd.extend([src, perf_results_dir]) 228 return ' '.join(scp_cmd) 229 230 231 def _run_cmd(self, cmd): 232 """Execute an command in a external shell and capture the output. 233 234 @param cmd: String of is a valid shell command. 235 236 @returns The standard out, standard error and the integer exit code of 237 the executed command. 238 """ 239 logging.debug('Running: %s', cmd) 240 241 output = StringIO.StringIO() 242 error_output = StringIO.StringIO() 243 exit_code = 0 244 try: 245 result = utils.run(cmd, stdout_tee=output, 246 stderr_tee=error_output, 247 timeout=TELEMETRY_TIMEOUT_MINS*60) 248 exit_code = result.exit_status 249 except error.CmdError as e: 250 logging.debug('Error occurred executing.') 251 exit_code = e.result_obj.exit_status 252 253 stdout = output.getvalue() 254 stderr = error_output.getvalue() 255 logging.debug('Completed with exit code: %d.\nstdout:%s\n' 256 'stderr:%s', exit_code, stdout, stderr) 257 return stdout, stderr, exit_code 258 259 260 def _run_telemetry(self, script, test_or_benchmark, *args): 261 """Runs telemetry on a dut. 262 263 @param script: Telemetry script we want to run. For example: 264 [path_to_telemetry_src]/src/tools/telemetry/run_tests. 265 @param test_or_benchmark: Name of the test or benchmark we want to run, 266 with the page_set (if required) as part of the 267 string. 268 @param args: additional list of arguments to pass to the script. 269 270 @returns A TelemetryResult Instance with the results of this telemetry 271 execution. 272 """ 273 # TODO (sbasi crbug.com/239933) add support for incognito mode. 274 275 telemetry_cmd = self._get_telemetry_cmd(script, 276 test_or_benchmark, 277 *args) 278 logging.debug('Running Telemetry: %s', telemetry_cmd) 279 280 stdout, stderr, exit_code = self._run_cmd(telemetry_cmd) 281 282 return TelemetryResult(exit_code=exit_code, stdout=stdout, 283 stderr=stderr) 284 285 286 def _run_scp(self, perf_results_dir): 287 """Runs telemetry on a dut. 288 289 @param perf_results_dir: The local directory that results are being 290 collected. 291 """ 292 scp_cmd = self._scp_telemetry_results_cmd(perf_results_dir) 293 logging.debug('Retrieving Results: %s', scp_cmd) 294 _, _, exit_code = self._run_cmd(scp_cmd) 295 if exit_code != 0: 296 raise error.TestFail('Unable to retrieve results.') 297 298 299 def _run_test(self, script, test, *args): 300 """Runs a telemetry test on a dut. 301 302 @param script: Which telemetry test script we want to run. Can be 303 telemetry's base test script or the Chrome OS specific 304 test script. 305 @param test: Telemetry test we want to run. 306 @param args: additional list of arguments to pass to the script. 307 308 @returns A TelemetryResult Instance with the results of this telemetry 309 execution. 310 """ 311 logging.debug('Running telemetry test: %s', test) 312 telemetry_script = os.path.join(self._telemetry_path, script) 313 result = self._run_telemetry(telemetry_script, test, *args) 314 if result.status is FAILED_STATUS: 315 raise error.TestFail('Telemetry test %s failed.' % test) 316 return result 317 318 319 def run_telemetry_test(self, test, *args): 320 """Runs a telemetry test on a dut. 321 322 @param test: Telemetry test we want to run. 323 @param args: additional list of arguments to pass to the telemetry 324 execution script. 325 326 @returns A TelemetryResult Instance with the results of this telemetry 327 execution. 328 """ 329 return self._run_test(TELEMETRY_RUN_TESTS_SCRIPT, test, *args) 330 331 332 def run_telemetry_benchmark(self, benchmark, perf_value_writer=None, 333 *args): 334 """Runs a telemetry benchmark on a dut. 335 336 @param benchmark: Benchmark we want to run. 337 @param perf_value_writer: Should be an instance with the function 338 output_perf_value(), if None, no perf value 339 will be written. Typically this will be the 340 job object from an autotest test. 341 @param args: additional list of arguments to pass to the telemetry 342 execution script. 343 344 @returns A TelemetryResult Instance with the results of this telemetry 345 execution. 346 """ 347 logging.debug('Running telemetry benchmark: %s', benchmark) 348 349 if benchmark in ON_DUT_BLACKLIST: 350 self._telemetry_on_dut = False 351 352 if self._telemetry_on_dut: 353 telemetry_script = os.path.join(DUT_CHROME_ROOT, 354 TELEMETRY_RUN_BENCHMARKS_SCRIPT) 355 self._ensure_deps(self._host, benchmark) 356 else: 357 telemetry_script = os.path.join(self._telemetry_path, 358 TELEMETRY_RUN_BENCHMARKS_SCRIPT) 359 360 result = self._run_telemetry(telemetry_script, benchmark, *args) 361 362 if result.status is WARNING_STATUS: 363 raise error.TestWarn('Telemetry Benchmark: %s' 364 ' exited with Warnings.' % benchmark) 365 if result.status is FAILED_STATUS: 366 raise error.TestFail('Telemetry Benchmark: %s' 367 ' failed to run.' % benchmark) 368 if perf_value_writer: 369 self._run_scp(perf_value_writer.resultsdir) 370 return result 371 372 373 def run_gpu_integration_test(self, test, *args): 374 """Runs a gpu test on a dut. 375 376 @param test: Gpu test we want to run. 377 @param args: additional list of arguments to pass to the telemetry 378 execution script. 379 380 @returns A TelemetryResult instance with the results of this telemetry 381 execution. 382 """ 383 script = os.path.join(DUT_CHROME_ROOT, 384 TELEMETRY_RUN_GPU_TESTS_SCRIPT) 385 cmd = [] 386 if self._devserver: 387 devserver_hostname = self._devserver.hostname 388 cmd.extend(['ssh', devserver_hostname]) 389 390 cmd.extend( 391 [self._host.ssh_command(alive_interval=900, connection_attempts=4), 392 'python', script]) 393 cmd.extend(args) 394 cmd.append(test) 395 cmd = ' '.join(cmd) 396 stdout, stderr, exit_code = self._run_cmd(cmd) 397 398 if exit_code: 399 raise error.TestFail('Gpu Integration Test: %s' 400 ' failed to run.' % test) 401 402 return TelemetryResult(exit_code=exit_code, stdout=stdout, 403 stderr=stderr) 404 405 406 def _ensure_deps(self, dut, test_name): 407 """ 408 Ensure the dependencies are locally available on DUT. 409 410 @param dut: The autotest host object representing DUT. 411 @param test_name: Name of the telemetry test. 412 """ 413 # Get DEPs using host's telemetry. 414 # Example output, fetch_benchmark_deps.py --output-deps=deps octane: 415 # {'octane': ['tools/perf/page_sets/data/octane_002.wprgo']} 416 perf_path = os.path.join(self._telemetry_path, 'tools', 'perf') 417 deps_path = os.path.join(perf_path, 'fetch_benchmark_deps_result.json') 418 fetch_path = os.path.join(perf_path, 'fetch_benchmark_deps.py') 419 format_fetch = ('python %s --output-deps=%s %s') 420 command_fetch = format_fetch % (fetch_path, deps_path, test_name) 421 command_get = 'cat %s' % deps_path 422 423 if self._devserver: 424 devserver_hostname = self._devserver.url().split( 425 'http://')[1].split(':')[0] 426 command_fetch = 'ssh %s %s' % (devserver_hostname, command_fetch) 427 command_get = 'ssh %s %s' % (devserver_hostname, command_get) 428 429 logging.info('Getting DEPs: %s', command_fetch) 430 _, _, exit_code = self._run_cmd(command_fetch) 431 if exit_code != 0: 432 raise error.TestFail('Error occurred while fetching DEPs.') 433 stdout, _, exit_code = self._run_cmd(command_get) 434 if exit_code != 0: 435 raise error.TestFail('Error occurred while getting DEPs.') 436 437 # Download DEPs to DUT. 438 # send_file() relies on rsync over ssh. Couldn't be better. 439 deps = json.loads(stdout) 440 for dep in deps[test_name]: 441 src = os.path.join(self._telemetry_path, dep) 442 dst = os.path.join(DUT_CHROME_ROOT, dep) 443 if self._devserver: 444 logging.info('Copying: %s -> %s', src, dst) 445 rsync_cmd = utils.sh_escape('rsync %s %s %s:%s' % 446 (self._host.rsync_options(), src, 447 self._host.hostname, dst)) 448 utils.run('ssh %s "%s"' % (devserver_hostname, rsync_cmd)) 449 else: 450 if not os.path.isfile(src): 451 raise error.TestFail('Error occurred while saving DEPs.') 452 logging.info('Copying: %s -> %s', src, dst) 453 dut.send_file(src, dst) 454