• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python2
2#
3# Copyright 2016 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6"""Script for running nightly compiler tests on ChromeOS.
7
8This script launches a buildbot to build ChromeOS with the latest compiler on
9a particular board; then it finds and downloads the trybot image and the
10corresponding official image, and runs crosperf performance tests comparing
11the two.  It then generates a report, emails it to the c-compiler-chrome, as
12well as copying the images into the seven-day reports directory.
13"""
14
15# Script to test different toolchains against ChromeOS benchmarks.
16
17from __future__ import print_function
18
19import argparse
20import datetime
21import os
22import re
23import sys
24import time
25
26from cros_utils import command_executer
27from cros_utils import logger
28
29from cros_utils import buildbot_utils
30
31# CL that uses LLVM-Next to build the images (includes chrome).
32USE_LLVM_NEXT_PATCH = '513590'
33
34CROSTC_ROOT = '/usr/local/google/crostc'
35ROLE_ACCOUNT = 'mobiletc-prebuild'
36TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
37MAIL_PROGRAM = '~/var/bin/mail-sheriff'
38PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives')
39NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports')
40
41IMAGE_DIR = '{board}-{image_type}'
42IMAGE_VERSION_STR = r'{chrome_version}-{tip}\.{branch}\.{branch_branch}'
43IMAGE_FS = IMAGE_DIR + '/' + IMAGE_VERSION_STR
44TRYBOT_IMAGE_FS = IMAGE_FS + '-{build_id}'
45PFQ_IMAGE_FS = IMAGE_FS + '-rc1'
46IMAGE_RE_GROUPS = {
47    'board': r'(?P<board>\S+)',
48    'image_type': r'(?P<image_type>\S+)',
49    'chrome_version': r'(?P<chrome_version>R\d+)',
50    'tip': r'(?P<tip>\d+)',
51    'branch': r'(?P<branch>\d+)',
52    'branch_branch': r'(?P<branch_branch>\d+)',
53    'build_id': r'(?P<build_id>b\d+)'
54}
55TRYBOT_IMAGE_RE = TRYBOT_IMAGE_FS.format(**IMAGE_RE_GROUPS)
56
57
58class ToolchainComparator(object):
59  """Class for doing the nightly tests work."""
60
61  def __init__(self,
62               board,
63               remotes,
64               chromeos_root,
65               weekday,
66               patches,
67               noschedv2=False):
68    self._board = board
69    self._remotes = remotes
70    self._chromeos_root = chromeos_root
71    self._base_dir = os.getcwd()
72    self._ce = command_executer.GetCommandExecuter()
73    self._l = logger.GetLogger()
74    self._build = '%s-release-tryjob' % board
75    self._patches = patches.split(',') if patches else []
76    self._patches_string = '_'.join(str(p) for p in self._patches)
77    self._noschedv2 = noschedv2
78
79    if not weekday:
80      self._weekday = time.strftime('%a')
81    else:
82      self._weekday = weekday
83    timestamp = datetime.datetime.strftime(datetime.datetime.now(),
84                                           '%Y-%m-%d_%H:%M:%S')
85    self._reports_dir = os.path.join(
86        NIGHTLY_TESTS_DIR,
87        '%s.%s' % (timestamp, board),
88    )
89
90  def _GetVanillaImageName(self, trybot_image):
91    """Given a trybot artifact name, get latest vanilla image name.
92
93    Args:
94      trybot_image: artifact name such as
95          'daisy-release-tryjob/R40-6394.0.0-b1389'
96
97    Returns:
98      Latest official image name, e.g. 'daisy-release/R57-9089.0.0'.
99    """
100    # We need to filter out -tryjob in the trybot_image.
101    trybot = re.sub('-tryjob', '', trybot_image)
102    mo = re.search(TRYBOT_IMAGE_RE, trybot)
103    assert mo
104    dirname = IMAGE_DIR.replace('\\', '').format(**mo.groupdict())
105    return buildbot_utils.GetLatestImage(self._chromeos_root, dirname)
106
107  def _GetNonAFDOImageName(self, trybot_image):
108    """Given a trybot artifact name, get corresponding non-AFDO image name.
109
110    We get the non-AFDO image from the PFQ builders. This image
111    is not generated for all the boards and, the closest PFQ image
112    was the one build for the previous ChromeOS version (the chrome
113    used in the current version is the one validated in the previous
114    version).
115    The previous ChromeOS does not always exist either. So, we try
116    a couple of versions before.
117
118    Args:
119      trybot_image: artifact name such as
120          'daisy-release-tryjob/R40-6394.0.0-b1389'
121
122    Returns:
123      Corresponding chrome PFQ image name, e.g.
124      'daisy-chrome-pfq/R40-6393.0.0-rc1'.
125    """
126    trybot = re.sub('-tryjob', '', trybot_image)
127    mo = re.search(TRYBOT_IMAGE_RE, trybot)
128    assert mo
129    image_dict = mo.groupdict()
130    image_dict['image_type'] = 'chrome-pfq'
131    for _ in xrange(2):
132      image_dict['tip'] = str(int(image_dict['tip']) - 1)
133      nonafdo_image = PFQ_IMAGE_FS.replace('\\', '').format(**image_dict)
134      if buildbot_utils.DoesImageExist(self._chromeos_root, nonafdo_image):
135        return nonafdo_image
136    return ''
137
138  def _TestImages(self, trybot_image, vanilla_image, nonafdo_image):
139    """Create crosperf experiment file.
140
141    Given the names of the trybot, vanilla and non-AFDO images, create the
142    appropriate crosperf experiment file and launch crosperf on it.
143    """
144    experiment_file_dir = os.path.join(self._chromeos_root, '..', self._weekday)
145    experiment_file_name = '%s_toolchain_experiment.txt' % self._board
146
147    compiler_string = 'llvm'
148    if USE_LLVM_NEXT_PATCH in self._patches_string:
149      experiment_file_name = '%s_llvm_next_experiment.txt' % self._board
150      compiler_string = 'llvm_next'
151
152    experiment_file = os.path.join(experiment_file_dir, experiment_file_name)
153    experiment_header = """
154    board: %s
155    remote: %s
156    retries: 1
157    """ % (self._board, self._remotes)
158    experiment_tests = """
159    benchmark: all_toolchain_perf {
160      suite: telemetry_Crosperf
161      iterations: 0
162      run_local: False
163    }
164
165    benchmark: page_cycler_v2.typical_25 {
166      suite: telemetry_Crosperf
167      iterations: 0
168      run_local: False
169      retries: 0
170    }
171    """
172
173    with open(experiment_file, 'w') as f:
174      f.write(experiment_header)
175      f.write(experiment_tests)
176
177      # Now add vanilla to test file.
178      official_image = """
179          vanilla_image {
180            chromeos_root: %s
181            build: %s
182            compiler: llvm
183          }
184          """ % (self._chromeos_root, vanilla_image)
185      f.write(official_image)
186
187      # Now add non-AFDO image to test file.
188      if nonafdo_image:
189        official_nonafdo_image = """
190          nonafdo_image {
191            chromeos_root: %s
192            build: %s
193            compiler: llvm
194          }
195          """ % (self._chromeos_root, nonafdo_image)
196        f.write(official_nonafdo_image)
197
198      label_string = '%s_trybot_image' % compiler_string
199
200      # Reuse autotest files from vanilla image for trybot images
201      autotest_files = os.path.join('/tmp', vanilla_image, 'autotest_files')
202      experiment_image = """
203          %s {
204            chromeos_root: %s
205            build: %s
206            autotest_path: %s
207            compiler: %s
208          }
209          """ % (label_string, self._chromeos_root, trybot_image,
210                 autotest_files, compiler_string)
211      f.write(experiment_image)
212
213    crosperf = os.path.join(TOOLCHAIN_DIR, 'crosperf', 'crosperf')
214    noschedv2_opts = '--noschedv2' if self._noschedv2 else ''
215    command = ('{crosperf} --no_email=True --results_dir={r_dir} '
216               '--json_report=True {noschedv2_opts} {exp_file}').format(
217                   crosperf=crosperf,
218                   r_dir=self._reports_dir,
219                   noschedv2_opts=noschedv2_opts,
220                   exp_file=experiment_file)
221
222    ret = self._ce.RunCommand(command)
223    if ret != 0:
224      raise RuntimeError('Crosperf execution error!')
225    else:
226      # Copy json report to pending archives directory.
227      command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR)
228      ret = self._ce.RunCommand(command)
229    return
230
231  def _SendEmail(self):
232    """Find email message generated by crosperf and send it."""
233    filename = os.path.join(self._reports_dir, 'msg_body.html')
234    if (os.path.exists(filename) and
235        os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
236      email_title = 'buildbot llvm test results'
237      if USE_LLVM_NEXT_PATCH in self._patches_string:
238        email_title = 'buildbot llvm_next test results'
239      command = ('cat %s | %s -s "%s, %s" -team -html' %
240                 (filename, MAIL_PROGRAM, email_title, self._board))
241      self._ce.RunCommand(command)
242
243  def DoAll(self):
244    """Main function inside ToolchainComparator class.
245
246    Launch trybot, get image names, create crosperf experiment file, run
247    crosperf, and copy images into seven-day report directories.
248    """
249    buildbucket_id, trybot_image = buildbot_utils.GetTrybotImage(
250        self._chromeos_root,
251        self._build,
252        self._patches,
253        tryjob_flags=['--notests'],
254        build_toolchain=True)
255
256    print('trybot_url: \
257          http://cros-goldeneye/chromeos/healthmonitoring/buildDetails?buildbucketId=%s'
258          % buildbucket_id)
259    if len(trybot_image) == 0:
260      self._l.LogError('Unable to find trybot_image!')
261      return 2
262
263    vanilla_image = self._GetVanillaImageName(trybot_image)
264    nonafdo_image = self._GetNonAFDOImageName(trybot_image)
265
266    print('trybot_image: %s' % trybot_image)
267    print('vanilla_image: %s' % vanilla_image)
268    print('nonafdo_image: %s' % nonafdo_image)
269
270    self._TestImages(trybot_image, vanilla_image, nonafdo_image)
271    self._SendEmail()
272    return 0
273
274
275def Main(argv):
276  """The main function."""
277
278  # Common initializations
279  command_executer.InitCommandExecuter()
280  parser = argparse.ArgumentParser()
281  parser.add_argument(
282      '--remote', dest='remote', help='Remote machines to run tests on.')
283  parser.add_argument(
284      '--board', dest='board', default='x86-zgb', help='The target board.')
285  parser.add_argument(
286      '--chromeos_root',
287      dest='chromeos_root',
288      help='The chromeos root from which to run tests.')
289  parser.add_argument(
290      '--weekday',
291      default='',
292      dest='weekday',
293      help='The day of the week for which to run tests.')
294  parser.add_argument(
295      '--patch',
296      dest='patches',
297      help='The patches to use for the testing, '
298      "seprate the patch numbers with ',' "
299      'for more than one patches.')
300  parser.add_argument(
301      '--noschedv2',
302      dest='noschedv2',
303      action='store_true',
304      default=False,
305      help='Pass --noschedv2 to crosperf.')
306
307  options = parser.parse_args(argv[1:])
308  if not options.board:
309    print('Please give a board.')
310    return 1
311  if not options.remote:
312    print('Please give at least one remote machine.')
313    return 1
314  if not options.chromeos_root:
315    print('Please specify the ChromeOS root directory.')
316    return 1
317
318  fc = ToolchainComparator(options.board, options.remote, options.chromeos_root,
319                           options.weekday, options.patches, options.noschedv2)
320  return fc.DoAll()
321
322
323if __name__ == '__main__':
324  retval = Main(sys.argv)
325  sys.exit(retval)
326