# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import logging import os import StringIO from autotest_lib.client.common_lib import error, utils from autotest_lib.client.common_lib.cros import dev_server TELEMETRY_RUN_BENCHMARKS_SCRIPT = 'tools/perf/run_benchmark' TELEMETRY_RUN_TESTS_SCRIPT = 'tools/telemetry/run_tests' TELEMETRY_RUN_GPU_TESTS_SCRIPT = 'content/test/gpu/run_gpu_integration_test.py' TELEMETRY_TIMEOUT_MINS = 120 DUT_CHROME_ROOT = '/usr/local/telemetry/src' DUT_COMMON_SSH_OPTIONS = ['-o StrictHostKeyChecking=no', '-o UserKnownHostsFile=/dev/null', '-o BatchMode=yes', '-o ConnectTimeout=30', '-o ServerAliveInterval=900', '-o ServerAliveCountMax=3', '-o ConnectionAttempts=4', '-o Protocol=2'] DUT_SSH_OPTIONS = ' '.join(DUT_COMMON_SSH_OPTIONS + ['-x', '-a', '-l root']) DUT_SCP_OPTIONS = ' '.join(DUT_COMMON_SSH_OPTIONS) DUT_RSYNC_OPTIONS = ' '.join(['--rsh="/usr/bin/ssh %s"' % DUT_SSH_OPTIONS, '-L', '--timeout=1800', '-az', '--no-o', '--no-g']) # Prevent double quotes from being unfolded. DUT_RSYNC_OPTIONS = utils.sh_escape(DUT_RSYNC_OPTIONS) # Result Statuses SUCCESS_STATUS = 'SUCCESS' WARNING_STATUS = 'WARNING' FAILED_STATUS = 'FAILED' # A list of benchmarks with that the telemetry test harness can run on dut. ON_DUT_WHITE_LIST = ['dromaeo.domcoreattr', 'dromaeo.domcoremodify', 'dromaeo.domcorequery', 'dromaeo.domcoretraverse', 'image_decoding.image_decoding_measurement', 'jetstream', 'kraken', 'memory.top_7_stress', 'octane', 'page_cycler.typical_25', 'page_cycler_v2.typical_25', 'robohornet_pro', 'smoothness.top_25_smooth', 'smoothness.tough_animation_cases', 'smoothness.tough_canvas_cases', 'smoothness.tough_filters_cases', 'smoothness.tough_pinch_zoom_cases', 'smoothness.tough_scrolling_cases', 'smoothness.tough_webgl_cases', 'speedometer', 'startup.cold.blank_page', 'sunspider', 'tab_switching.top_10', 'tab_switching.typical_25', 'webrtc.peerconnection', 'webrtc.stress'] # BLACK LIST # 'session_restore.cold.typical_25', # profile generator not implemented on # CrOS. class TelemetryResult(object): """Class to represent the results of a telemetry run. This class represents the results of a telemetry run, whether it ran successful, failed or had warnings. """ def __init__(self, exit_code=0, stdout='', stderr=''): """Initializes this TelemetryResultObject instance. @param status: Status of the telemtry run. @param stdout: Stdout of the telemetry run. @param stderr: Stderr of the telemetry run. """ if exit_code == 0: self.status = SUCCESS_STATUS else: self.status = FAILED_STATUS self._stdout = stdout self._stderr = stderr self.output = '\n'.join([stdout, stderr]) class TelemetryRunner(object): """Class responsible for telemetry for a given build. This class will extract and install telemetry on the devserver and is responsible for executing the telemetry benchmarks and returning their output to the caller. """ def __init__(self, host, local=False, telemetry_on_dut=True): """Initializes this telemetry runner instance. If telemetry is not installed for this build, it will be. Basically, the following commands on the local pc on which test_that will be executed, depending on the 4 possible combinations of local x telemetry_on_dut: local=True, telemetry_on_dut=False: run_benchmark --browser=cros-chrome --remote=[dut] [test] local=True, telemetry_on_dut=True: ssh [dut] run_benchmark --browser=system [test] local=False, telemetry_on_dut=False: ssh [devserver] run_benchmark --browser=cros-chrome --remote=[dut] [test] local=False, telemetry_on_dut=True: ssh [devserver] ssh [dut] run_benchmark --browser=system [test] @param host: Host where the test will be run. @param local: If set, no devserver will be used, test will be run locally. If not set, "ssh [devserver] " will be appended to test commands. @param telemetry_on_dut: If set, telemetry itself (the test harness) will run on dut. It decides browser=[system|cros-chrome] """ self._host = host self._devserver = None self._telemetry_path = None self._telemetry_on_dut = telemetry_on_dut # TODO (llozano crbug.com/324964). Remove conditional code. # Use a class hierarchy instead. if local: self._setup_local_telemetry() else: self._setup_devserver_telemetry() logging.debug('Telemetry Path: %s', self._telemetry_path) def _setup_devserver_telemetry(self): """Setup Telemetry to use the devserver.""" logging.debug('Setting up telemetry for devserver testing') logging.debug('Grabbing build from AFE.') info = self._host.host_info_store.get() if not info.build: logging.error('Unable to locate build label for host: %s.', self._host.hostname) raise error.AutotestError('Failed to grab build for host %s.' % self._host.hostname) logging.debug('Setting up telemetry for build: %s', info.build) self._devserver = dev_server.ImageServer.resolve( info.build, hostname=self._host.hostname) self._devserver.stage_artifacts(info.build, ['autotest_packages']) self._telemetry_path = self._devserver.setup_telemetry(build=info.build) def _setup_local_telemetry(self): """Setup Telemetry to use local path to its sources. First look for chrome source root, either externally mounted, or inside the chroot. Prefer chrome-src-internal source tree to chrome-src. """ TELEMETRY_DIR = 'src' CHROME_LOCAL_SRC = '/var/cache/chromeos-cache/distfiles/target/' CHROME_EXTERNAL_SRC = os.path.expanduser('~/chrome_root/') logging.debug('Setting up telemetry for local testing') sources_list = ('chrome-src-internal', 'chrome-src') dir_list = [CHROME_EXTERNAL_SRC] dir_list.extend( [os.path.join(CHROME_LOCAL_SRC, x) for x in sources_list]) if 'CHROME_ROOT' in os.environ: dir_list.insert(0, os.environ['CHROME_ROOT']) telemetry_src = '' for dir in dir_list: if os.path.exists(dir): telemetry_src = os.path.join(dir, TELEMETRY_DIR) break else: raise error.TestError('Telemetry source directory not found.') self._devserver = None self._telemetry_path = telemetry_src def _get_telemetry_cmd(self, script, test_or_benchmark, *args): """Build command to execute telemetry based on script and benchmark. @param script: Telemetry script we want to run. For example: [path_to_telemetry_src]/src/tools/telemetry/run_tests. @param test_or_benchmark: Name of the test or benchmark we want to run, with the page_set (if required) as part of the string. @param args: additional list of arguments to pass to the script. @returns Full telemetry command to execute the script. """ telemetry_cmd = [] if self._devserver: devserver_hostname = self._devserver.hostname telemetry_cmd.extend(['ssh', devserver_hostname]) if self._telemetry_on_dut: telemetry_cmd.extend( ['ssh', DUT_SSH_OPTIONS, self._host.hostname, 'python', script, '--verbose', '--output-format=chartjson', '--output-dir=%s' % DUT_CHROME_ROOT, '--browser=system']) else: telemetry_cmd.extend( ['python', script, '--verbose', '--browser=cros-chrome', '--output-format=chartjson', '--output-dir=%s' % self._telemetry_path, '--remote=%s' % self._host.hostname]) telemetry_cmd.extend(args) telemetry_cmd.append(test_or_benchmark) return ' '.join(telemetry_cmd) def _scp_telemetry_results_cmd(self, perf_results_dir): """Build command to copy the telemetry results from the devserver. @param perf_results_dir: directory path where test output is to be collected. @returns SCP command to copy the results json to the specified directory. """ scp_cmd = [] devserver_hostname = '' if perf_results_dir: if self._devserver: devserver_hostname = self._devserver.hostname + ':' if self._telemetry_on_dut: src = ('root@%s:%s/results-chart.json' % (self._host.hostname, DUT_CHROME_ROOT)) scp_cmd.extend(['scp', DUT_SCP_OPTIONS, src, perf_results_dir]) else: src = ('%s%s/results-chart.json' % (devserver_hostname, self._telemetry_path)) scp_cmd.extend(['scp', src, perf_results_dir]) return ' '.join(scp_cmd) def _run_cmd(self, cmd): """Execute an command in a external shell and capture the output. @param cmd: String of is a valid shell command. @returns The standard out, standard error and the integer exit code of the executed command. """ logging.debug('Running: %s', cmd) output = StringIO.StringIO() error_output = StringIO.StringIO() exit_code = 0 try: result = utils.run(cmd, stdout_tee=output, stderr_tee=error_output, timeout=TELEMETRY_TIMEOUT_MINS*60) exit_code = result.exit_status except error.CmdError as e: logging.debug('Error occurred executing.') exit_code = e.result_obj.exit_status stdout = output.getvalue() stderr = error_output.getvalue() logging.debug('Completed with exit code: %d.\nstdout:%s\n' 'stderr:%s', exit_code, stdout, stderr) return stdout, stderr, exit_code def _run_telemetry(self, script, test_or_benchmark, *args): """Runs telemetry on a dut. @param script: Telemetry script we want to run. For example: [path_to_telemetry_src]/src/tools/telemetry/run_tests. @param test_or_benchmark: Name of the test or benchmark we want to run, with the page_set (if required) as part of the string. @param args: additional list of arguments to pass to the script. @returns A TelemetryResult Instance with the results of this telemetry execution. """ # TODO (sbasi crbug.com/239933) add support for incognito mode. telemetry_cmd = self._get_telemetry_cmd(script, test_or_benchmark, *args) logging.debug('Running Telemetry: %s', telemetry_cmd) stdout, stderr, exit_code = self._run_cmd(telemetry_cmd) return TelemetryResult(exit_code=exit_code, stdout=stdout, stderr=stderr) def _run_scp(self, perf_results_dir): """Runs telemetry on a dut. @param perf_results_dir: The local directory that results are being collected. """ scp_cmd = self._scp_telemetry_results_cmd(perf_results_dir) logging.debug('Retrieving Results: %s', scp_cmd) self._run_cmd(scp_cmd) def _run_test(self, script, test, *args): """Runs a telemetry test on a dut. @param script: Which telemetry test script we want to run. Can be telemetry's base test script or the Chrome OS specific test script. @param test: Telemetry test we want to run. @param args: additional list of arguments to pass to the script. @returns A TelemetryResult Instance with the results of this telemetry execution. """ logging.debug('Running telemetry test: %s', test) telemetry_script = os.path.join(self._telemetry_path, script) result = self._run_telemetry(telemetry_script, test, *args) if result.status is FAILED_STATUS: raise error.TestFail('Telemetry test %s failed.' % test) return result def run_telemetry_test(self, test, *args): """Runs a telemetry test on a dut. @param test: Telemetry test we want to run. @param args: additional list of arguments to pass to the telemetry execution script. @returns A TelemetryResult Instance with the results of this telemetry execution. """ return self._run_test(TELEMETRY_RUN_TESTS_SCRIPT, test, *args) def run_telemetry_benchmark(self, benchmark, perf_value_writer=None, *args): """Runs a telemetry benchmark on a dut. @param benchmark: Benchmark we want to run. @param perf_value_writer: Should be an instance with the function output_perf_value(), if None, no perf value will be written. Typically this will be the job object from an autotest test. @param args: additional list of arguments to pass to the telemetry execution script. @returns A TelemetryResult Instance with the results of this telemetry execution. """ logging.debug('Running telemetry benchmark: %s', benchmark) if benchmark not in ON_DUT_WHITE_LIST: self._telemetry_on_dut = False if self._telemetry_on_dut: telemetry_script = os.path.join(DUT_CHROME_ROOT, TELEMETRY_RUN_BENCHMARKS_SCRIPT) self._ensure_deps(self._host, benchmark) else: telemetry_script = os.path.join(self._telemetry_path, TELEMETRY_RUN_BENCHMARKS_SCRIPT) result = self._run_telemetry(telemetry_script, benchmark, *args) if result.status is WARNING_STATUS: raise error.TestWarn('Telemetry Benchmark: %s' ' exited with Warnings.' % benchmark) if result.status is FAILED_STATUS: raise error.TestFail('Telemetry Benchmark: %s' ' failed to run.' % benchmark) if perf_value_writer: self._run_scp(perf_value_writer.resultsdir) return result def run_gpu_integration_test(self, test, *args): """Runs a gpu test on a dut. @param test: Gpu test we want to run. @param args: additional list of arguments to pass to the telemetry execution script. @returns A TelemetryResult instance with the results of this telemetry execution. """ script = os.path.join(DUT_CHROME_ROOT, TELEMETRY_RUN_GPU_TESTS_SCRIPT) cmd = [] if self._devserver: devserver_hostname = self._devserver.hostname cmd.extend(['ssh', devserver_hostname]) cmd.extend( ['ssh', DUT_SSH_OPTIONS, self._host.hostname, 'python', script]) cmd.extend(args) cmd.append(test) cmd = ' '.join(cmd) stdout, stderr, exit_code = self._run_cmd(cmd) return TelemetryResult(exit_code=exit_code, stdout=stdout, stderr=stderr) def _ensure_deps(self, dut, test_name): """ Ensure the dependencies are locally available on DUT. @param dut: The autotest host object representing DUT. @param test_name: Name of the telemetry test. """ # Get DEPs using host's telemetry. format_string = ('python %s/tools/perf/fetch_benchmark_deps.py %s') command = format_string % (self._telemetry_path, test_name) stdout = StringIO.StringIO() stderr = StringIO.StringIO() if self._devserver: devserver_hostname = self._devserver.url().split( 'http://')[1].split(':')[0] command = 'ssh %s %s' % (devserver_hostname, command) logging.info('Getting DEPs: %s', command) try: result = utils.run(command, stdout_tee=stdout, stderr_tee=stderr) except error.CmdError as e: logging.debug('Error occurred getting DEPs: %s\n %s\n', stdout.getvalue(), stderr.getvalue()) raise error.TestFail('Error occurred while getting DEPs.') # Download DEPs to DUT. # send_file() relies on rsync over ssh. Couldn't be better. stdout_str = stdout.getvalue() stdout.close() stderr.close() for dep in stdout_str.split(): src = os.path.join(self._telemetry_path, dep) dst = os.path.join(DUT_CHROME_ROOT, dep) if self._devserver: logging.info('Copying: %s -> %s', src, dst) utils.run('ssh %s rsync %s %s %s:%s' % (devserver_hostname, DUT_RSYNC_OPTIONS, src, self._host.hostname, dst)) else: if not os.path.isfile(src): raise error.TestFail('Error occurred while saving DEPs.') logging.info('Copying: %s -> %s', src, dst) dut.send_file(src, dst)