• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python2
2
3# Script to test different toolchains against ChromeOS benchmarks.
4"""Toolchain team nightly performance test script (local builds)."""
5
6from __future__ import print_function
7
8import argparse
9import datetime
10import os
11import sys
12import build_chromeos
13import setup_chromeos
14from cros_utils import command_executer
15from cros_utils import misc
16from cros_utils import logger
17
18CROSTC_ROOT = '/usr/local/google/crostc'
19MAIL_PROGRAM = '~/var/bin/mail-sheriff'
20PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives')
21NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports')
22
23
24class GCCConfig(object):
25  """GCC configuration class."""
26
27  def __init__(self, githash):
28    self.githash = githash
29
30
31class ToolchainConfig(object):
32  """Toolchain configuration class."""
33
34  def __init__(self, gcc_config=None):
35    self.gcc_config = gcc_config
36
37
38class ChromeOSCheckout(object):
39  """Main class for checking out, building and testing ChromeOS."""
40
41  def __init__(self, board, chromeos_root):
42    self._board = board
43    self._chromeos_root = chromeos_root
44    self._ce = command_executer.GetCommandExecuter()
45    self._l = logger.GetLogger()
46    self._build_num = None
47
48  def _DeleteChroot(self):
49    command = 'cd %s; cros_sdk --delete' % self._chromeos_root
50    return self._ce.RunCommand(command)
51
52  def _DeleteCcahe(self):
53    # crosbug.com/34956
54    command = 'sudo rm -rf %s' % os.path.join(self._chromeos_root, '.cache')
55    return self._ce.RunCommand(command)
56
57  def _GetBuildNumber(self):
58    """Get the build number of the ChromeOS image from the chroot.
59
60    This function assumes a ChromeOS image has been built in the chroot.
61    It translates the 'latest' symlink in the
62    <chroot>/src/build/images/<board> directory, to find the actual
63    ChromeOS build number for the image that was built.  For example, if
64    src/build/image/lumpy/latest ->  R37-5982.0.2014_06_23_0454-a1, then
65    This function would parse it out and assign 'R37-5982' to self._build_num.
66    This is used to determine the official, vanilla build to use for
67    comparison tests.
68    """
69    # Get the path to 'latest'
70    sym_path = os.path.join(
71        misc.GetImageDir(self._chromeos_root, self._board), 'latest')
72    # Translate the symlink to its 'real' path.
73    real_path = os.path.realpath(sym_path)
74    # Break up the path and get the last piece
75    # (e.g. 'R37-5982.0.2014_06_23_0454-a1"
76    path_pieces = real_path.split('/')
77    last_piece = path_pieces[-1]
78    # Break this piece into the image number + other pieces, and get the
79    # image number [ 'R37-5982', '0', '2014_06_23_0454-a1']
80    image_parts = last_piece.split('.')
81    self._build_num = image_parts[0]
82
83  def _BuildLabelName(self, config):
84    pieces = config.split('/')
85    compiler_version = pieces[-1]
86    label = compiler_version + '_tot_afdo'
87    return label
88
89  def _BuildAndImage(self, label=''):
90    if (not label or
91        not misc.DoesLabelExist(self._chromeos_root, self._board, label)):
92      build_chromeos_args = [
93          build_chromeos.__file__, '--chromeos_root=%s' % self._chromeos_root,
94          '--board=%s' % self._board, '--rebuild'
95      ]
96      if self._public:
97        build_chromeos_args.append('--env=USE=-chrome_internal')
98
99      ret = build_chromeos.Main(build_chromeos_args)
100      if ret != 0:
101        raise RuntimeError("Couldn't build ChromeOS!")
102
103      if not self._build_num:
104        self._GetBuildNumber()
105      # Check to see if we need to create the symbolic link for the vanilla
106      # image, and do so if appropriate.
107      if not misc.DoesLabelExist(self._chromeos_root, self._board, 'vanilla'):
108        build_name = '%s-release/%s.0.0' % (self._board, self._build_num)
109        full_vanilla_path = os.path.join(os.getcwd(), self._chromeos_root,
110                                         'chroot/tmp', build_name)
111        misc.LabelLatestImage(self._chromeos_root, self._board, label,
112                              full_vanilla_path)
113      else:
114        misc.LabelLatestImage(self._chromeos_root, self._board, label)
115    return label
116
117  def _SetupBoard(self, env_dict, usepkg_flag, clobber_flag):
118    env_string = misc.GetEnvStringFromDict(env_dict)
119    command = ('%s %s' % (env_string, misc.GetSetupBoardCommand(
120        self._board, usepkg=usepkg_flag, force=clobber_flag)))
121    ret = self._ce.ChrootRunCommand(self._chromeos_root, command)
122    error_str = "Could not setup board: '%s'" % command
123    assert ret == 0, error_str
124
125  def _UnInstallToolchain(self):
126    command = ('sudo CLEAN_DELAY=0 emerge -C cross-%s/gcc' %
127               misc.GetCtargetFromBoard(self._board, self._chromeos_root))
128    ret = self._ce.ChrootRunCommand(self._chromeos_root, command)
129    if ret != 0:
130      raise RuntimeError("Couldn't uninstall the toolchain!")
131
132  def _CheckoutChromeOS(self):
133    # TODO(asharif): Setup a fixed ChromeOS version (quarterly snapshot).
134    if not os.path.exists(self._chromeos_root):
135      setup_chromeos_args = ['--dir=%s' % self._chromeos_root]
136      if self._public:
137        setup_chromeos_args.append('--public')
138      ret = setup_chromeos.Main(setup_chromeos_args)
139      if ret != 0:
140        raise RuntimeError("Couldn't run setup_chromeos!")
141
142  def _BuildToolchain(self, config):
143    # Call setup_board for basic, vanilla setup.
144    self._SetupBoard({}, usepkg_flag=True, clobber_flag=False)
145    # Now uninstall the vanilla compiler and setup/build our custom
146    # compiler.
147    self._UnInstallToolchain()
148    envdict = {
149        'USE': 'git_gcc',
150        'GCC_GITHASH': config.gcc_config.githash,
151        'EMERGE_DEFAULT_OPTS': '--exclude=gcc'
152    }
153    self._SetupBoard(envdict, usepkg_flag=False, clobber_flag=False)
154
155
156class ToolchainComparator(ChromeOSCheckout):
157  """Main class for running tests and generating reports."""
158
159  def __init__(self,
160               board,
161               remotes,
162               configs,
163               clean,
164               public,
165               force_mismatch,
166               noschedv2=False):
167    self._board = board
168    self._remotes = remotes
169    self._chromeos_root = 'chromeos'
170    self._configs = configs
171    self._clean = clean
172    self._public = public
173    self._force_mismatch = force_mismatch
174    self._ce = command_executer.GetCommandExecuter()
175    self._l = logger.GetLogger()
176    timestamp = datetime.datetime.strftime(datetime.datetime.now(),
177                                           '%Y-%m-%d_%H:%M:%S')
178    self._reports_dir = os.path.join(
179        NIGHTLY_TESTS_DIR,
180        '%s.%s' % (timestamp, board),)
181    self._noschedv2 = noschedv2
182    ChromeOSCheckout.__init__(self, board, self._chromeos_root)
183
184  def _FinishSetup(self):
185    # Get correct .boto file
186    current_dir = os.getcwd()
187    src = '/usr/local/google/home/mobiletc-prebuild/.boto'
188    dest = os.path.join(current_dir, self._chromeos_root,
189                        'src/private-overlays/chromeos-overlay/'
190                        'googlestorage_account.boto')
191    # Copy the file to the correct place
192    copy_cmd = 'cp %s %s' % (src, dest)
193    retv = self._ce.RunCommand(copy_cmd)
194    if retv != 0:
195      raise RuntimeError("Couldn't copy .boto file for google storage.")
196
197    # Fix protections on ssh key
198    command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target'
199               '/chrome-src-internal/src/third_party/chromite/ssh_keys'
200               '/testing_rsa')
201    retv = self._ce.ChrootRunCommand(self._chromeos_root, command)
202    if retv != 0:
203      raise RuntimeError('chmod for testing_rsa failed')
204
205  def _TestLabels(self, labels):
206    experiment_file = 'toolchain_experiment.txt'
207    image_args = ''
208    if self._force_mismatch:
209      image_args = '--force-mismatch'
210    experiment_header = """
211    board: %s
212    remote: %s
213    retries: 1
214    """ % (self._board, self._remotes)
215    experiment_tests = """
216    benchmark: all_toolchain_perf {
217      suite: telemetry_Crosperf
218      iterations: 3
219    }
220    """
221
222    with open(experiment_file, 'w') as f:
223      f.write(experiment_header)
224      f.write(experiment_tests)
225      for label in labels:
226        # TODO(asharif): Fix crosperf so it accepts labels with symbols
227        crosperf_label = label
228        crosperf_label = crosperf_label.replace('-', '_')
229        crosperf_label = crosperf_label.replace('+', '_')
230        crosperf_label = crosperf_label.replace('.', '')
231
232        # Use the official build instead of building vanilla ourselves.
233        if label == 'vanilla':
234          build_name = '%s-release/%s.0.0' % (self._board, self._build_num)
235
236          # Now add 'official build' to test file.
237          official_image = """
238          official_image {
239            chromeos_root: %s
240            build: %s
241          }
242          """ % (self._chromeos_root, build_name)
243          f.write(official_image)
244
245        else:
246          experiment_image = """
247          %s {
248            chromeos_image: %s
249            image_args: %s
250          }
251          """ % (crosperf_label, os.path.join(
252              misc.GetImageDir(self._chromeos_root, self._board), label,
253              'chromiumos_test_image.bin'), image_args)
254          f.write(experiment_image)
255
256    crosperf = os.path.join(os.path.dirname(__file__), 'crosperf', 'crosperf')
257    noschedv2_opts = '--noschedv2' if self._noschedv2 else ''
258    command = ('{crosperf} --no_email=True --results_dir={r_dir} '
259               '--json_report=True {noschedv2_opts} {exp_file}').format(
260                   crosperf=crosperf,
261                   r_dir=self._reports_dir,
262                   noschedv2_opts=noschedv2_opts,
263                   exp_file=experiment_file)
264
265    ret = self._ce.RunCommand(command)
266    if ret != 0:
267      raise RuntimeError('Crosperf execution error!')
268    else:
269      # Copy json report to pending archives directory.
270      command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR)
271      ret = self._ce.RunCommand(command)
272    return
273
274  def _SendEmail(self):
275    """Find email msesage generated by crosperf and send it."""
276    filename = os.path.join(self._reports_dir, 'msg_body.html')
277    if (os.path.exists(filename) and
278        os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
279      command = ('cat %s | %s -s "Nightly test results, %s" -team -html' %
280                 (filename, MAIL_PROGRAM, self._board))
281      self._ce.RunCommand(command)
282
283  def DoAll(self):
284    self._CheckoutChromeOS()
285    labels = []
286    labels.append('vanilla')
287    for config in self._configs:
288      label = self._BuildLabelName(config.gcc_config.githash)
289      if not misc.DoesLabelExist(self._chromeos_root, self._board, label):
290        self._BuildToolchain(config)
291        label = self._BuildAndImage(label)
292      labels.append(label)
293    self._FinishSetup()
294    self._TestLabels(labels)
295    self._SendEmail()
296    if self._clean:
297      ret = self._DeleteChroot()
298      if ret != 0:
299        return ret
300      ret = self._DeleteCcahe()
301      if ret != 0:
302        return ret
303    return 0
304
305
306def Main(argv):
307  """The main function."""
308  # Common initializations
309  ###  command_executer.InitCommandExecuter(True)
310  command_executer.InitCommandExecuter()
311  parser = argparse.ArgumentParser()
312  parser.add_argument(
313      '--remote', dest='remote', help='Remote machines to run tests on.')
314  parser.add_argument(
315      '--board', dest='board', default='x86-alex', help='The target board.')
316  parser.add_argument(
317      '--githashes',
318      dest='githashes',
319      default='master',
320      help='The gcc githashes to test.')
321  parser.add_argument(
322      '--clean',
323      dest='clean',
324      default=False,
325      action='store_true',
326      help='Clean the chroot after testing.')
327  parser.add_argument(
328      '--public',
329      dest='public',
330      default=False,
331      action='store_true',
332      help='Use the public checkout/build.')
333  parser.add_argument(
334      '--force-mismatch',
335      dest='force_mismatch',
336      default='',
337      help='Force the image regardless of board mismatch')
338  parser.add_argument(
339      '--noschedv2',
340      dest='noschedv2',
341      action='store_true',
342      default=False,
343      help='Pass --noschedv2 to crosperf.')
344  options = parser.parse_args(argv)
345  if not options.board:
346    print('Please give a board.')
347    return 1
348  if not options.remote:
349    print('Please give at least one remote machine.')
350    return 1
351  toolchain_configs = []
352  for githash in options.githashes.split(','):
353    gcc_config = GCCConfig(githash=githash)
354    toolchain_config = ToolchainConfig(gcc_config=gcc_config)
355    toolchain_configs.append(toolchain_config)
356  fc = ToolchainComparator(options.board, options.remote, toolchain_configs,
357                           options.clean, options.public,
358                           options.force_mismatch, options.noschedv2)
359  return fc.DoAll()
360
361
362if __name__ == '__main__':
363  retval = Main(sys.argv[1:])
364  sys.exit(retval)
365