• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2016 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4"""Utilities for launching and accessing ChromeOS buildbots."""
5
6from __future__ import print_function
7
8import base64
9import json
10import os
11import time
12import urllib2
13
14# pylint: disable=no-name-in-module
15from oauth2client.service_account import ServiceAccountCredentials
16
17from cros_utils import command_executer
18from cros_utils import logger
19from cros_utils import buildbot_json
20
21INITIAL_SLEEP_TIME = 7200  # 2 hours; wait time before polling buildbot.
22SLEEP_TIME = 600  # 10 minutes; time between polling of buildbot.
23TIME_OUT = 28800  # Decide the build is dead or will never finish
24# after this time (8 hours).
25OK_STATUS = [  # List of result status values that are 'ok'.
26    # This was obtained from:
27    #   https://chromium.googlesource.com/chromium/tools/build/+/
28    #       master/third_party/buildbot_8_4p1/buildbot/status/results.py
29    0,  # "success"
30    1,  # "warnings"
31    6,  # "retry"
32]
33
34
35class BuildbotTimeout(Exception):
36  """Exception to throw when a buildbot operation timesout."""
37  pass
38
39
40def ParseReportLog(url, build):
41  """Scrape the trybot image name off the Reports log page.
42
43  This takes the URL for a trybot Reports Stage web page,
44  and a trybot build type, such as 'daisy-release'.  It
45  opens the web page and parses it looking for the trybot
46  artifact name (e.g. something like
47  'trybot-daisy-release/R40-6394.0.0-b1389'). It returns the
48  artifact name, if found.
49  """
50  trybot_image = ''
51  url += '/text'
52  newurl = url.replace('uberchromegw', 'chromegw')
53  webpage = urllib2.urlopen(newurl)
54  data = webpage.read()
55  lines = data.split('\n')
56  for l in lines:
57    if l.find('Artifacts') > 0 and l.find('trybot') > 0:
58      trybot_name = 'trybot-%s' % build
59      start_pos = l.find(trybot_name)
60      end_pos = l.find('@https://storage')
61      trybot_image = l[start_pos:end_pos]
62
63  return trybot_image
64
65
66def GetBuildData(buildbot_queue, build_id):
67  """Find the Reports stage web page for a trybot build.
68
69  This takes the name of a buildbot_queue, such as 'daisy-release'
70  and a build id (the build number), and uses the json buildbot api to
71  find the Reports stage web page for that build, if it exists.
72  """
73  builder = buildbot_json.Buildbot(
74      'http://chromegw/p/tryserver.chromiumos/').builders[buildbot_queue]
75  build_data = builder.builds[build_id].data
76  logs = build_data['logs']
77  for l in logs:
78    fname = l[1]
79    if 'steps/Report/' in fname:
80      return fname
81
82  return ''
83
84
85def FindBuildRecordFromLog(description, build_info):
86  """Find the right build record in the build logs.
87
88  Get the first build record from build log with a reason field
89  that matches 'description'. ('description' is a special tag we
90  created when we launched the buildbot, so we could find it at this
91  point.)
92  """
93  for build_log in build_info:
94    if description in build_log['reason']:
95      return build_log
96  return {}
97
98
99def GetBuildInfo(file_dir, waterfall_builder):
100  """Get all the build records for the trybot builds."""
101
102  builder = ''
103  if waterfall_builder.endswith('-release'):
104    builder = 'release'
105  elif waterfall_builder.endswith('-gcc-toolchain'):
106    builder = 'gcc_toolchain'
107  elif waterfall_builder.endswith('-llvm-toolchain'):
108    builder = 'llvm_toolchain'
109  elif waterfall_builder.endswith('-llvm-next-toolchain'):
110    builder = 'llvm_next_toolchain'
111
112  sa_file = os.path.expanduser(
113      os.path.join(file_dir, 'cros_utils',
114                   'chromeos-toolchain-credentials.json'))
115  scopes = ['https://www.googleapis.com/auth/userinfo.email']
116
117  credentials = ServiceAccountCredentials.from_json_keyfile_name(
118      sa_file, scopes=scopes)
119  url = (
120      'https://luci-milo.appspot.com/prpc/milo.Buildbot/GetBuildbotBuildsJSON')
121
122  # NOTE: If we want to get build logs for the main waterfall builders, the
123  # 'master' field below should be 'chromeos' instead of 'chromiumos.tryserver'.
124  # Builder would be 'amd64-gcc-toolchain' or 'arm-llvm-toolchain', etc.
125
126  body = json.dumps({
127      'master': 'chromiumos.tryserver',
128      'builder': builder,
129      'include_current': True,
130      'limit': 100
131  })
132  access_token = credentials.get_access_token()
133  headers = {
134      'Accept': 'application/json',
135      'Content-Type': 'application/json',
136      'Authorization': 'Bearer %s' % access_token.access_token
137  }
138  r = urllib2.Request(url, body, headers)
139  u = urllib2.urlopen(r, timeout=60)
140  u.read(4)
141  o = json.load(u)
142  data = [base64.b64decode(item['data']) for item in o['builds']]
143  result = []
144  for d in data:
145    tmp = json.loads(d)
146    result.append(tmp)
147  return result
148
149
150def FindArchiveImage(chromeos_root, build, build_id):
151  """Returns name of the trybot artifact for board/build_id."""
152  ce = command_executer.GetCommandExecuter()
153  command = ('gsutil ls gs://chromeos-image-archive/trybot-%s/*b%s'
154             '/chromiumos_test_image.tar.xz' % (build, build_id))
155  _, out, _ = ce.ChrootRunCommandWOutput(
156      chromeos_root, command, print_to_console=False)
157  #
158  # If build_id is not unique, there may be multiple archive images
159  # to choose from; sort them & pick the first (newest).
160  #
161  # If there are multiple archive images found, out will look something
162  # like this:
163  #
164  # 'gs://.../R35-5692.0.0-b105/chromiumos_test_image.tar.xz
165  #  gs://.../R46-7339.0.0-b105/chromiumos_test_image.tar.xz'
166  #
167  out = out.rstrip('\n')
168  tmp_list = out.split('\n')
169  # After stripping the final '\n' and splitting on any other '\n', we get
170  # something like this:
171  #  tmp_list = [ 'gs://.../R35-5692.0.0-b105/chromiumos_test_image.tar.xz' ,
172  #               'gs://.../R46-7339.0.0-b105/chromiumos_test_image.tar.xz' ]
173  #
174  #  If we sort this in descending order, we should end up with the most
175  #  recent test image first, so that's what we do here.
176  #
177  if len(tmp_list) > 1:
178    tmp_list = sorted(tmp_list, reverse=True)
179  out = tmp_list[0]
180
181  trybot_image = ''
182  trybot_name = 'trybot-%s' % build
183  if out and out.find(trybot_name) > 0:
184    start_pos = out.find(trybot_name)
185    end_pos = out.find('/chromiumos_test_image')
186    trybot_image = out[start_pos:end_pos]
187
188  return trybot_image
189
190
191def GetTrybotImage(chromeos_root,
192                   buildbot_name,
193                   patch_list,
194                   build_tag,
195                   other_flags=None,
196                   build_toolchain=False,
197                   async=False):
198  """Launch buildbot and get resulting trybot artifact name.
199
200  This function launches a buildbot with the appropriate flags to
201  build the test ChromeOS image, with the current ToT mobile compiler.  It
202  checks every 10 minutes to see if the trybot has finished.  When the trybot
203  has finished, it parses the resulting report logs to find the trybot
204  artifact (if one was created), and returns that artifact name.
205
206  chromeos_root is the path to the ChromeOS root, needed for finding chromite
207  and launching the buildbot.
208
209  buildbot_name is the name of the buildbot queue, such as lumpy-release or
210  daisy-paladin.
211
212  patch_list a python list of the patches, if any, for the buildbot to use.
213
214  build_tag is a (unique) string to be used to look up the buildbot results
215  from among all the build records.
216  """
217  ce = command_executer.GetCommandExecuter()
218  cbuildbot_path = os.path.join(chromeos_root, 'chromite/cbuildbot')
219  base_dir = os.getcwd()
220  patch_arg = ''
221  if patch_list:
222    for p in patch_list:
223      patch_arg = patch_arg + ' -g ' + repr(p)
224  toolchain_flags = ''
225  if build_toolchain:
226    toolchain_flags += '--latest-toolchain'
227  os.chdir(cbuildbot_path)
228  if other_flags:
229    optional_flags = ' '.join(other_flags)
230  else:
231    optional_flags = ''
232
233  # Launch buildbot with appropriate flags.
234  build = buildbot_name
235  description = build_tag
236  command_prefix = ''
237  if not patch_arg:
238    command_prefix = 'yes | '
239  command = ('%s ./cbuildbot --remote --nochromesdk %s'
240             ' --remote-description=%s %s %s %s' % (command_prefix,
241                                                    optional_flags, description,
242                                                    toolchain_flags, patch_arg,
243                                                    build))
244  _, out, _ = ce.RunCommandWOutput(command)
245  if 'Tryjob submitted!' not in out:
246    logger.GetLogger().LogFatal('Error occurred while launching trybot job: '
247                                '%s' % command)
248
249  os.chdir(base_dir)
250
251  build_id = 0
252  build_status = None
253  # Wait for  buildbot to finish running (check every 10 minutes).  Wait
254  # 10 minutes before the first check to give the buildbot time to launch
255  # (so we don't start looking for build data before it's out there).
256  time.sleep(SLEEP_TIME)
257  done = False
258  pending = True
259  # pending_time is the time between when we submit the job and when the
260  # buildbot actually launches the build.  running_time is the time between
261  # when the buildbot job launches and when it finishes.  The job is
262  # considered 'pending' until we can find an entry for it in the buildbot
263  # logs.
264  pending_time = SLEEP_TIME
265  running_time = 0
266  long_slept = False
267  while not done:
268    done = True
269    build_info = GetBuildInfo(base_dir, build)
270    if not build_info:
271      if pending_time > TIME_OUT:
272        logger.GetLogger().LogFatal('Unable to get build logs for target %s.' %
273                                    build)
274      else:
275        pending_message = 'Unable to find build log; job may be pending.'
276        done = False
277
278    if done:
279      data_dict = FindBuildRecordFromLog(description, build_info)
280      if not data_dict:
281        # Trybot job may be pending (not actually launched yet).
282        if pending_time > TIME_OUT:
283          logger.GetLogger().LogFatal('Unable to find build record for trybot'
284                                      ' %s.' % description)
285        else:
286          pending_message = 'Unable to find build record; job may be pending.'
287          done = False
288
289      else:
290        # Now that we have actually found the entry for the build
291        # job in the build log, we know the job is actually
292        # runnning, not pending, so we flip the 'pending' flag.  We
293        # still have to wait for the buildbot job to finish running
294        # however.
295        pending = False
296        build_id = data_dict['number']
297
298        if async:
299          # Do not wait for trybot job to finish; return immediately
300          return build_id, ' '
301
302        if not long_slept:
303          # The trybot generally takes more than 2 hours to finish.
304          # Wait two hours before polling the status.
305          long_slept = True
306          time.sleep(INITIAL_SLEEP_TIME)
307          pending_time += INITIAL_SLEEP_TIME
308        if True == data_dict['finished']:
309          build_status = data_dict['results']
310        else:
311          done = False
312
313    if not done:
314      if pending:
315        logger.GetLogger().LogOutput(pending_message)
316        logger.GetLogger().LogOutput('Current pending time: %d minutes.' %
317                                     (pending_time / 60))
318        pending_time += SLEEP_TIME
319      else:
320        logger.GetLogger().LogOutput('{0} minutes passed.'.format(running_time /
321                                                                  60))
322        logger.GetLogger().LogOutput('Sleeping {0} seconds.'.format(SLEEP_TIME))
323        running_time += SLEEP_TIME
324
325      time.sleep(SLEEP_TIME)
326      if running_time > TIME_OUT:
327        done = True
328
329  trybot_image = ''
330
331  if build.endswith('-toolchain'):
332    # For rotating testers, we don't care about their build_status
333    # result, because if any HWTest failed it will be non-zero.
334    trybot_image = FindArchiveImage(chromeos_root, build, build_id)
335  else:
336    # The nightly performance tests do not run HWTests, so if
337    # their build_status is non-zero, we do care.  In this case
338    # non-zero means the image itself probably did not build.
339    if build_status in OK_STATUS:
340      trybot_image = FindArchiveImage(chromeos_root, build, build_id)
341  if not trybot_image:
342    logger.GetLogger().LogError('Trybot job %s failed with status %d;'
343                                ' no trybot image generated.' %
344                                (description, build_status))
345
346  logger.GetLogger().LogOutput("trybot_image is '%s'" % trybot_image)
347  logger.GetLogger().LogOutput('build_status is %d' % build_status)
348  return build_id, trybot_image
349
350
351def GetGSContent(chromeos_root, path):
352  """gsutil cat path"""
353
354  ce = command_executer.GetCommandExecuter()
355  command = ('gsutil cat gs://chromeos-image-archive/%s' % path)
356  _, out, _ = ce.ChrootRunCommandWOutput(
357      chromeos_root, command, print_to_console=False)
358  return out
359
360
361def DoesImageExist(chromeos_root, build):
362  """Check if the image for the given build exists."""
363
364  ce = command_executer.GetCommandExecuter()
365  command = ('gsutil ls gs://chromeos-image-archive/%s'
366             '/chromiumos_test_image.tar.xz' % (build))
367  ret = ce.ChrootRunCommand(chromeos_root, command, print_to_console=False)
368  return not ret
369
370
371def WaitForImage(chromeos_root, build):
372  """Wait for an image to be ready."""
373
374  elapsed_time = 0
375  while elapsed_time < TIME_OUT:
376    if DoesImageExist(chromeos_root, build):
377      return
378    logger.GetLogger().LogOutput('Image %s not ready, waiting for 10 minutes' %
379                                 build)
380    time.sleep(SLEEP_TIME)
381    elapsed_time += SLEEP_TIME
382
383  logger.GetLogger().LogOutput('Image %s not found, waited for %d hours' %
384                               (build, (TIME_OUT / 3600)))
385  raise BuildbotTimeout('Timeout while waiting for image %s' % build)
386