1 # Copyright 2016 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 """Utilities for launching and accessing ChromeOS buildbots.""" 5 6 from __future__ import print_function 7 8 import base64 9 import json 10 import os 11 import re 12 import time 13 import urllib2 14 15 # pylint: disable=no-name-in-module 16 from oauth2client.service_account import ServiceAccountCredentials 17 18 from cros_utils import command_executer 19 from cros_utils import logger 20 from cros_utils import buildbot_json 21 22 INITIAL_SLEEP_TIME = 7200 # 2 hours; wait time before polling buildbot. 23 SLEEP_TIME = 600 # 10 minutes; time between polling of buildbot. 24 TIME_OUT = 28800 # Decide the build is dead or will never finish 25 # after this time (8 hours). 26 OK_STATUS = [ # List of result status values that are 'ok'. 27 # This was obtained from: 28 # https://chromium.googlesource.com/chromium/tools/build/+/ 29 # master/third_party/buildbot_8_4p1/buildbot/status/results.py 30 0, # "success" 31 1, # "warnings" 32 6, # "retry" 33 ] 34 35 36 class BuildbotTimeout(Exception): 37 """Exception to throw when a buildbot operation timesout.""" 38 pass 39 40 41 def ParseReportLog(url, build): 42 """Scrape the trybot image name off the Reports log page. 43 44 This takes the URL for a trybot Reports Stage web page, 45 and a trybot build type, such as 'daisy-release'. It 46 opens the web page and parses it looking for the trybot 47 artifact name (e.g. something like 48 'trybot-daisy-release/R40-6394.0.0-b1389'). It returns the 49 artifact name, if found. 50 """ 51 trybot_image = '' 52 url += '/text' 53 newurl = url.replace('uberchromegw', 'chromegw') 54 webpage = urllib2.urlopen(newurl) 55 data = webpage.read() 56 lines = data.split('\n') 57 for l in lines: 58 if l.find('Artifacts') > 0 and l.find('trybot') > 0: 59 trybot_name = 'trybot-%s' % build 60 start_pos = l.find(trybot_name) 61 end_pos = l.find('@https://storage') 62 trybot_image = l[start_pos:end_pos] 63 64 return trybot_image 65 66 67 def GetBuildData(buildbot_queue, build_id): 68 """Find the Reports stage web page for a trybot build. 69 70 This takes the name of a buildbot_queue, such as 'daisy-release' 71 and a build id (the build number), and uses the json buildbot api to 72 find the Reports stage web page for that build, if it exists. 73 """ 74 builder = buildbot_json.Buildbot( 75 'http://chromegw/p/tryserver.chromiumos/').builders[buildbot_queue] 76 build_data = builder.builds[build_id].data 77 logs = build_data['logs'] 78 for l in logs: 79 fname = l[1] 80 if 'steps/Report/' in fname: 81 return fname 82 83 return '' 84 85 86 def FindBuildRecordFromLog(description, build_info): 87 """Find the right build record in the build logs. 88 89 Get the first build record from build log with a reason field 90 that matches 'description'. ('description' is a special tag we 91 created when we launched the buildbot, so we could find it at this 92 point.) 93 """ 94 for build_log in build_info: 95 property_list = build_log['properties'] 96 for prop in property_list: 97 if len(prop) < 2: 98 continue 99 pname = prop[0] 100 pvalue = prop[1] 101 if pname == 'name' and pvalue == description: 102 return build_log 103 return {} 104 105 106 def GetBuildInfo(file_dir, waterfall_builder): 107 """Get all the build records for the trybot builds.""" 108 109 builder = '' 110 if waterfall_builder.endswith('-release'): 111 builder = 'release' 112 elif waterfall_builder.endswith('-gcc-toolchain'): 113 builder = 'gcc_toolchain' 114 elif waterfall_builder.endswith('-llvm-toolchain'): 115 builder = 'llvm_toolchain' 116 elif waterfall_builder.endswith('-llvm-next-toolchain'): 117 builder = 'llvm_next_toolchain' 118 119 sa_file = os.path.expanduser( 120 os.path.join(file_dir, 'cros_utils', 121 'chromeos-toolchain-credentials.json')) 122 scopes = ['https://www.googleapis.com/auth/userinfo.email'] 123 124 credentials = ServiceAccountCredentials.from_json_keyfile_name( 125 sa_file, scopes=scopes) 126 url = ( 127 'https://luci-milo.appspot.com/prpc/milo.Buildbot/GetBuildbotBuildsJSON') 128 129 # NOTE: If we want to get build logs for the main waterfall builders, the 130 # 'master' field below should be 'chromeos' instead of 'chromiumos.tryserver'. 131 # Builder would be 'amd64-gcc-toolchain' or 'arm-llvm-toolchain', etc. 132 133 body = json.dumps({ 134 'master': 'chromiumos.tryserver', 135 'builder': builder, 136 'include_current': True, 137 'limit': 100 138 }) 139 access_token = credentials.get_access_token() 140 headers = { 141 'Accept': 'application/json', 142 'Content-Type': 'application/json', 143 'Authorization': 'Bearer %s' % access_token.access_token 144 } 145 r = urllib2.Request(url, body, headers) 146 u = urllib2.urlopen(r, timeout=60) 147 u.read(4) 148 o = json.load(u) 149 data = [base64.b64decode(item['data']) for item in o['builds']] 150 result = [] 151 for d in data: 152 tmp = json.loads(d) 153 result.append(tmp) 154 return result 155 156 157 def FindArchiveImage(chromeos_root, build, build_id): 158 """Returns name of the trybot artifact for board/build_id.""" 159 ce = command_executer.GetCommandExecuter() 160 command = ('gsutil ls gs://chromeos-image-archive/trybot-%s/*b%s' 161 '/chromiumos_test_image.tar.xz' % (build, build_id)) 162 _, out, _ = ce.ChrootRunCommandWOutput( 163 chromeos_root, command, print_to_console=False) 164 # 165 # If build_id is not unique, there may be multiple archive images 166 # to choose from; sort them & pick the first (newest). 167 # 168 # If there are multiple archive images found, out will look something 169 # like this: 170 # 171 # 'gs://.../R35-5692.0.0-b105/chromiumos_test_image.tar.xz 172 # gs://.../R46-7339.0.0-b105/chromiumos_test_image.tar.xz' 173 # 174 out = out.rstrip('\n') 175 tmp_list = out.split('\n') 176 # After stripping the final '\n' and splitting on any other '\n', we get 177 # something like this: 178 # tmp_list = [ 'gs://.../R35-5692.0.0-b105/chromiumos_test_image.tar.xz' , 179 # 'gs://.../R46-7339.0.0-b105/chromiumos_test_image.tar.xz' ] 180 # 181 # If we sort this in descending order, we should end up with the most 182 # recent test image first, so that's what we do here. 183 # 184 if len(tmp_list) > 1: 185 tmp_list = sorted(tmp_list, reverse=True) 186 out = tmp_list[0] 187 188 trybot_image = '' 189 trybot_name = 'trybot-%s' % build 190 if out and out.find(trybot_name) > 0: 191 start_pos = out.find(trybot_name) 192 end_pos = out.find('/chromiumos_test_image') 193 trybot_image = out[start_pos:end_pos] 194 195 return trybot_image 196 197 198 def GetTrybotImage(chromeos_root, 199 buildbot_name, 200 patch_list, 201 build_tag, 202 other_flags=None, 203 build_toolchain=False, 204 async=False): 205 """Launch buildbot and get resulting trybot artifact name. 206 207 This function launches a buildbot with the appropriate flags to 208 build the test ChromeOS image, with the current ToT mobile compiler. It 209 checks every 10 minutes to see if the trybot has finished. When the trybot 210 has finished, it parses the resulting report logs to find the trybot 211 artifact (if one was created), and returns that artifact name. 212 213 chromeos_root is the path to the ChromeOS root, needed for finding chromite 214 and launching the buildbot. 215 216 buildbot_name is the name of the buildbot queue, such as lumpy-release or 217 daisy-paladin. 218 219 patch_list a python list of the patches, if any, for the buildbot to use. 220 221 build_tag is a (unique) string to be used to look up the buildbot results 222 from among all the build records. 223 """ 224 ce = command_executer.GetCommandExecuter() 225 cbuildbot_path = os.path.join(chromeos_root, 'chromite/cbuildbot') 226 base_dir = os.getcwd() 227 patch_arg = '' 228 if patch_list: 229 for p in patch_list: 230 patch_arg = patch_arg + ' -g ' + repr(p) 231 toolchain_flags = '' 232 if build_toolchain: 233 toolchain_flags += '--latest-toolchain' 234 os.chdir(cbuildbot_path) 235 if other_flags: 236 optional_flags = ' '.join(other_flags) 237 else: 238 optional_flags = '' 239 240 # Launch buildbot with appropriate flags. 241 build = buildbot_name 242 description = build_tag 243 command_prefix = '' 244 if not patch_arg: 245 command_prefix = 'yes | ' 246 command = ('%s ./cbuildbot --remote --nochromesdk %s' 247 ' --remote-description=%s %s %s %s' % 248 (command_prefix, optional_flags, description, toolchain_flags, 249 patch_arg, build)) 250 _, out, _ = ce.RunCommandWOutput(command) 251 if 'Tryjob submitted!' not in out: 252 logger.GetLogger().LogFatal('Error occurred while launching trybot job: ' 253 '%s' % command) 254 255 os.chdir(base_dir) 256 257 build_id = 0 258 build_status = None 259 # Wait for buildbot to finish running (check every 10 minutes). Wait 260 # 10 minutes before the first check to give the buildbot time to launch 261 # (so we don't start looking for build data before it's out there). 262 time.sleep(SLEEP_TIME) 263 done = False 264 pending = True 265 # pending_time is the time between when we submit the job and when the 266 # buildbot actually launches the build. running_time is the time between 267 # when the buildbot job launches and when it finishes. The job is 268 # considered 'pending' until we can find an entry for it in the buildbot 269 # logs. 270 pending_time = SLEEP_TIME 271 running_time = 0 272 long_slept = False 273 while not done: 274 done = True 275 build_info = GetBuildInfo(base_dir, build) 276 if not build_info: 277 if pending_time > TIME_OUT: 278 logger.GetLogger().LogFatal( 279 'Unable to get build logs for target %s.' % build) 280 else: 281 pending_message = 'Unable to find build log; job may be pending.' 282 done = False 283 284 if done: 285 data_dict = FindBuildRecordFromLog(description, build_info) 286 if not data_dict: 287 # Trybot job may be pending (not actually launched yet). 288 if pending_time > TIME_OUT: 289 logger.GetLogger().LogFatal('Unable to find build record for trybot' 290 ' %s.' % description) 291 else: 292 pending_message = 'Unable to find build record; job may be pending.' 293 done = False 294 295 else: 296 # Now that we have actually found the entry for the build 297 # job in the build log, we know the job is actually 298 # runnning, not pending, so we flip the 'pending' flag. We 299 # still have to wait for the buildbot job to finish running 300 # however. 301 pending = False 302 build_id = data_dict['number'] 303 304 if async: 305 # Do not wait for trybot job to finish; return immediately 306 return build_id, ' ' 307 308 if not long_slept: 309 # The trybot generally takes more than 2 hours to finish. 310 # Wait two hours before polling the status. 311 long_slept = True 312 time.sleep(INITIAL_SLEEP_TIME) 313 pending_time += INITIAL_SLEEP_TIME 314 if True == data_dict['finished']: 315 build_status = data_dict['results'] 316 else: 317 done = False 318 319 if not done: 320 if pending: 321 logger.GetLogger().LogOutput(pending_message) 322 logger.GetLogger().LogOutput('Current pending time: %d minutes.' % 323 (pending_time / 60)) 324 pending_time += SLEEP_TIME 325 else: 326 logger.GetLogger().LogOutput( 327 '{0} minutes passed.'.format(running_time / 60)) 328 logger.GetLogger().LogOutput('Sleeping {0} seconds.'.format(SLEEP_TIME)) 329 running_time += SLEEP_TIME 330 331 time.sleep(SLEEP_TIME) 332 if running_time > TIME_OUT: 333 done = True 334 335 trybot_image = '' 336 337 if build.endswith('-toolchain'): 338 # For rotating testers, we don't care about their build_status 339 # result, because if any HWTest failed it will be non-zero. 340 trybot_image = FindArchiveImage(chromeos_root, build, build_id) 341 else: 342 # The nightly performance tests do not run HWTests, so if 343 # their build_status is non-zero, we do care. In this case 344 # non-zero means the image itself probably did not build. 345 if build_status in OK_STATUS: 346 trybot_image = FindArchiveImage(chromeos_root, build, build_id) 347 if not trybot_image: 348 logger.GetLogger().LogError('Trybot job %s failed with status %d;' 349 ' no trybot image generated.' % (description, 350 build_status)) 351 352 logger.GetLogger().LogOutput("trybot_image is '%s'" % trybot_image) 353 logger.GetLogger().LogOutput('build_status is %d' % build_status) 354 return build_id, trybot_image 355 356 357 def GetGSContent(chromeos_root, path): 358 """gsutil cat path""" 359 360 ce = command_executer.GetCommandExecuter() 361 command = ('gsutil cat gs://chromeos-image-archive/%s' % path) 362 _, out, _ = ce.ChrootRunCommandWOutput( 363 chromeos_root, command, print_to_console=False) 364 return out 365 366 367 def DoesImageExist(chromeos_root, build): 368 """Check if the image for the given build exists.""" 369 370 ce = command_executer.GetCommandExecuter() 371 command = ('gsutil ls gs://chromeos-image-archive/%s' 372 '/chromiumos_test_image.tar.xz' % (build)) 373 ret = ce.ChrootRunCommand(chromeos_root, command, print_to_console=False) 374 return not ret 375 376 377 def WaitForImage(chromeos_root, build): 378 """Wait for an image to be ready.""" 379 380 elapsed_time = 0 381 while elapsed_time < TIME_OUT: 382 if DoesImageExist(chromeos_root, build): 383 return 384 logger.GetLogger().LogOutput( 385 'Image %s not ready, waiting for 10 minutes' % build) 386 time.sleep(SLEEP_TIME) 387 elapsed_time += SLEEP_TIME 388 389 logger.GetLogger().LogOutput('Image %s not found, waited for %d hours' % 390 (build, (TIME_OUT / 3600))) 391 raise BuildbotTimeout('Timeout while waiting for image %s' % build) 392 393 394 def GetLatestImage(chromeos_root, path): 395 """Get latest image""" 396 397 fmt = re.compile(r'R([0-9]+)-([0-9]+).([0-9]+).([0-9]+)') 398 399 ce = command_executer.GetCommandExecuter() 400 command = ('gsutil ls gs://chromeos-image-archive/%s' % path) 401 _, out, _ = ce.ChrootRunCommandWOutput( 402 chromeos_root, command, print_to_console=False) 403 candidates = [l.split('/')[-2] for l in out.split()] 404 candidates = map(fmt.match, candidates) 405 candidates = [[int(r) for r in m.group(1, 2, 3, 4)] for m in candidates if m] 406 candidates.sort(reverse=True) 407 for c in candidates: 408 build = '%s/R%d-%d.%d.%d' % (path, c[0], c[1], c[2], c[3]) 409 if DoesImageExist(chromeos_root, build): 410 return build 411