• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (c) 2013~2015 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4"""SuiteRunner defines the interface from crosperf to test script."""
5
6from __future__ import print_function
7
8import os
9import time
10import shlex
11
12from cros_utils import command_executer
13import test_flag
14
15TEST_THAT_PATH = '/usr/bin/test_that'
16AUTOTEST_DIR = '~/trunk/src/third_party/autotest/files'
17CHROME_MOUNT_DIR = '/tmp/chrome_root'
18
19
20def GetProfilerArgs(profiler_args):
21  # Remove "--" from in front of profiler args.
22  args_list = shlex.split(profiler_args)
23  new_list = []
24  for arg in args_list:
25    if arg[0:2] == '--':
26      arg = arg[2:]
27    new_list.append(arg)
28  args_list = new_list
29
30  # Remove "perf_options=" from middle of profiler args.
31  new_list = []
32  for arg in args_list:
33    idx = arg.find('perf_options=')
34    if idx != -1:
35      prefix = arg[0:idx]
36      suffix = arg[idx + len('perf_options=') + 1:-1]
37      new_arg = prefix + "'" + suffix + "'"
38      new_list.append(new_arg)
39    else:
40      new_list.append(arg)
41  args_list = new_list
42
43  return ' '.join(args_list)
44
45
46class SuiteRunner(object):
47  """This defines the interface from crosperf to test script."""
48
49  def __init__(self,
50               logger_to_use=None,
51               log_level='verbose',
52               cmd_exec=None,
53               cmd_term=None):
54    self.logger = logger_to_use
55    self.log_level = log_level
56    self._ce = cmd_exec or command_executer.GetCommandExecuter(
57        self.logger, log_level=self.log_level)
58    self._ct = cmd_term or command_executer.CommandTerminator()
59
60  def Run(self, machine, label, benchmark, test_args, profiler_args):
61    for i in range(0, benchmark.retries + 1):
62      self.PinGovernorExecutionFrequencies(machine, label.chromeos_root)
63      if benchmark.suite == 'telemetry':
64        self.DecreaseWaitTime(machine, label.chromeos_root)
65        ret_tup = self.Telemetry_Run(machine, label, benchmark, profiler_args)
66      elif benchmark.suite == 'telemetry_Crosperf':
67        self.DecreaseWaitTime(machine, label.chromeos_root)
68        ret_tup = self.Telemetry_Crosperf_Run(machine, label, benchmark,
69                                              test_args, profiler_args)
70      else:
71        ret_tup = self.Test_That_Run(machine, label, benchmark, test_args,
72                                     profiler_args)
73      if ret_tup[0] != 0:
74        self.logger.LogOutput('benchmark %s failed. Retries left: %s' %
75                              (benchmark.name, benchmark.retries - i))
76      elif i > 0:
77        self.logger.LogOutput('benchmark %s succeded after %s retries' %
78                              (benchmark.name, i))
79        break
80      else:
81        self.logger.LogOutput(
82            'benchmark %s succeded on first try' % benchmark.name)
83        break
84    return ret_tup
85
86  def PinGovernorExecutionFrequencies(self, machine_name, chromeos_root):
87    """Set min and max frequencies to max static frequency."""
88    # pyformat: disable
89    set_cpu_freq = (
90        'set -e && '
91        # Disable Turbo in Intel pstate driver
92        'if [[ -e /sys/devices/system/cpu/intel_pstate/no_turbo ]]; then '
93        'echo -n 1 > /sys/devices/system/cpu/intel_pstate/no_turbo; fi; '
94        # Set governor to performance for each cpu
95        'for f in /sys/devices/system/cpu/cpu*/cpufreq; do '
96        'cd $f; '
97        'echo performance > scaling_governor; '
98        # Uncomment rest of lines to enable setting frequency by crosperf
99        #'val=0; '
100        #'if [[ -e scaling_available_frequencies ]]; then '
101        # pylint: disable=line-too-long
102        #'  val=`cat scaling_available_frequencies | tr " " "\\n" | sort -n -b -r`; '
103        #'else '
104        #'  val=`cat scaling_max_freq | tr " " "\\n" | sort -n -b -r`; fi ; '
105        #'set -- $val; '
106        #'highest=$1; '
107        #'if [[ $# -gt 1 ]]; then '
108        #'  case $highest in *1000) highest=$2;; esac; '
109        #'fi ;'
110        #'echo $highest > scaling_max_freq; '
111        #'echo $highest > scaling_min_freq; '
112        'done'
113    )
114    # pyformat: enable
115    if self.log_level == 'average':
116      self.logger.LogOutput(
117          'Pinning governor execution frequencies for %s' % machine_name)
118    ret = self._ce.CrosRunCommand(
119        set_cpu_freq, machine=machine_name, chromeos_root=chromeos_root)
120    self.logger.LogFatalIf(
121        ret, 'Could not pin frequencies on machine: %s' % machine_name)
122
123  def DecreaseWaitTime(self, machine_name, chromeos_root):
124    """Change the ten seconds wait time for pagecycler to two seconds."""
125    FILE = '/usr/local/telemetry/src/tools/perf/page_sets/page_cycler_story.py'
126    ret = self._ce.CrosRunCommand(
127        'ls ' + FILE, machine=machine_name, chromeos_root=chromeos_root)
128    self.logger.LogFatalIf(ret, 'Could not find {} on machine: {}'.format(
129        FILE, machine_name))
130
131    if not ret:
132      sed_command = 'sed -i "s/_TTI_WAIT_TIME = 10/_TTI_WAIT_TIME = 2/g" '
133      ret = self._ce.CrosRunCommand(
134          sed_command + FILE, machine=machine_name, chromeos_root=chromeos_root)
135      self.logger.LogFatalIf(ret, 'Could not modify {} on machine: {}'.format(
136          FILE, machine_name))
137
138  def RebootMachine(self, machine_name, chromeos_root):
139    command = 'reboot && exit'
140    self._ce.CrosRunCommand(
141        command, machine=machine_name, chromeos_root=chromeos_root)
142    time.sleep(60)
143    # Whenever we reboot the machine, we need to restore the governor settings.
144    self.PinGovernorExecutionFrequencies(machine_name, chromeos_root)
145
146  def Test_That_Run(self, machine, label, benchmark, test_args, profiler_args):
147    """Run the test_that test.."""
148    options = ''
149    if label.board:
150      options += ' --board=%s' % label.board
151    if test_args:
152      options += ' %s' % test_args
153    if profiler_args:
154      self.logger.LogFatal('test_that does not support profiler.')
155    command = 'rm -rf /usr/local/autotest/results/*'
156    self._ce.CrosRunCommand(
157        command, machine=machine, chromeos_root=label.chromeos_root)
158
159    # We do this because some tests leave the machine in weird states.
160    # Rebooting between iterations has proven to help with this.
161    self.RebootMachine(machine, label.chromeos_root)
162
163    autotest_dir = AUTOTEST_DIR
164    if label.autotest_path != '':
165      autotest_dir = label.autotest_path
166
167    autotest_dir_arg = '--autotest_dir %s' % autotest_dir
168    # For non-telemetry tests, specify an autotest directory only if the
169    # specified directory is different from default (crosbug.com/679001).
170    if autotest_dir == AUTOTEST_DIR:
171      autotest_dir_arg = ''
172
173    command = (('%s %s --fast '
174                '%s %s %s') % (TEST_THAT_PATH, autotest_dir_arg, options,
175                               machine, benchmark.test_name))
176    if self.log_level != 'verbose':
177      self.logger.LogOutput('Running test.')
178      self.logger.LogOutput('CMD: %s' % command)
179    # Use --no-ns-pid so that cros_sdk does not create a different
180    # process namespace and we can kill process created easily by
181    # their process group.
182    return self._ce.ChrootRunCommandWOutput(
183        label.chromeos_root,
184        command,
185        command_terminator=self._ct,
186        cros_sdk_options='--no-ns-pid')
187
188  def RemoveTelemetryTempFile(self, machine, chromeos_root):
189    filename = 'telemetry@%s' % machine
190    fullname = os.path.join(chromeos_root, 'chroot', 'tmp', filename)
191    if os.path.exists(fullname):
192      os.remove(fullname)
193
194  def Telemetry_Crosperf_Run(self, machine, label, benchmark, test_args,
195                             profiler_args):
196    if not os.path.isdir(label.chrome_src):
197      self.logger.LogFatal('Cannot find chrome src dir to'
198                           ' run telemetry: %s' % label.chrome_src)
199
200    # Check for and remove temporary file that may have been left by
201    # previous telemetry runs (and which might prevent this run from
202    # working).
203    self.RemoveTelemetryTempFile(machine, label.chromeos_root)
204
205    # For telemetry runs, we can use the autotest copy from the source
206    # location. No need to have one under /build/<board>.
207    autotest_dir_arg = '--autotest_dir %s' % AUTOTEST_DIR
208    if label.autotest_path != '':
209      autotest_dir_arg = '--autotest_dir %s' % label.autotest_path
210
211    profiler_args = GetProfilerArgs(profiler_args)
212    fast_arg = ''
213    if not profiler_args:
214      # --fast works unless we are doing profiling (autotest limitation).
215      # --fast avoids unnecessary copies of syslogs.
216      fast_arg = '--fast'
217    args_string = ''
218    if test_args:
219      # Strip double quotes off args (so we can wrap them in single
220      # quotes, to pass through to Telemetry).
221      if test_args[0] == '"' and test_args[-1] == '"':
222        test_args = test_args[1:-1]
223      args_string = "test_args='%s'" % test_args
224
225    cmd = ('{} {} {} --board={} --args="{} run_local={} test={} '
226           '{}" {} telemetry_Crosperf'.format(
227               TEST_THAT_PATH, autotest_dir_arg, fast_arg, label.board,
228               args_string, benchmark.run_local, benchmark.test_name,
229               profiler_args, machine))
230
231    # Use --no-ns-pid so that cros_sdk does not create a different
232    # process namespace and we can kill process created easily by their
233    # process group.
234    chrome_root_options = ('--no-ns-pid '
235                           '--chrome_root={} --chrome_root_mount={} '
236                           "FEATURES=\"-usersandbox\" "
237                           'CHROME_ROOT={}'.format(label.chrome_src,
238                                                   CHROME_MOUNT_DIR,
239                                                   CHROME_MOUNT_DIR))
240    if self.log_level != 'verbose':
241      self.logger.LogOutput('Running test.')
242      self.logger.LogOutput('CMD: %s' % cmd)
243    return self._ce.ChrootRunCommandWOutput(
244        label.chromeos_root,
245        cmd,
246        command_terminator=self._ct,
247        cros_sdk_options=chrome_root_options)
248
249  def Telemetry_Run(self, machine, label, benchmark, profiler_args):
250    telemetry_run_path = ''
251    if not os.path.isdir(label.chrome_src):
252      self.logger.LogFatal('Cannot find chrome src dir to' ' run telemetry.')
253    else:
254      telemetry_run_path = os.path.join(label.chrome_src, 'src/tools/perf')
255      if not os.path.exists(telemetry_run_path):
256        self.logger.LogFatal('Cannot find %s directory.' % telemetry_run_path)
257
258    if profiler_args:
259      self.logger.LogFatal('Telemetry does not support the perf profiler.')
260
261    # Check for and remove temporary file that may have been left by
262    # previous telemetry runs (and which might prevent this run from
263    # working).
264    if not test_flag.GetTestMode():
265      self.RemoveTelemetryTempFile(machine, label.chromeos_root)
266
267    rsa_key = os.path.join(
268        label.chromeos_root,
269        'src/scripts/mod_for_test_scripts/ssh_keys/testing_rsa')
270
271    cmd = ('cd {0} && '
272           './run_measurement '
273           '--browser=cros-chrome '
274           '--output-format=csv '
275           '--remote={1} '
276           '--identity {2} '
277           '{3} {4}'.format(telemetry_run_path, machine, rsa_key,
278                            benchmark.test_name, benchmark.test_args))
279    if self.log_level != 'verbose':
280      self.logger.LogOutput('Running test.')
281      self.logger.LogOutput('CMD: %s' % cmd)
282    return self._ce.RunCommandWOutput(cmd, print_to_console=False)
283
284  def CommandTerminator(self):
285    return self._ct
286
287  def Terminate(self):
288    self._ct.Terminate()
289
290
291class MockSuiteRunner(object):
292  """Mock suite runner for test."""
293
294  def __init__(self):
295    self._true = True
296
297  def Run(self, *_args):
298    if self._true:
299      return [0, '', '']
300    else:
301      return [0, '', '']
302