1# -*- coding: utf-8 -*- 2# Copyright 2017 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Utilities for launching and accessing ChromeOS buildbots.""" 7 8from __future__ import division 9from __future__ import print_function 10 11import ast 12import json 13import os 14import re 15import time 16 17from cros_utils import command_executer 18from cros_utils import logger 19 20INITIAL_SLEEP_TIME = 7200 # 2 hours; wait time before polling buildbot. 21SLEEP_TIME = 600 # 10 minutes; time between polling of buildbot. 22 23# Some of our slower builders (llvm-next) are taking more 24# than 12 hours. So, increase this TIME_OUT to 15 hours. 25TIME_OUT = 15 * 60 * 60 # Decide the build is dead or will never finish 26 27 28class BuildbotTimeout(Exception): 29 """Exception to throw when a buildbot operation timesout.""" 30 31 32def RunCommandInPath(path, cmd): 33 ce = command_executer.GetCommandExecuter() 34 cwd = os.getcwd() 35 os.chdir(path) 36 status, stdout, stderr = ce.RunCommandWOutput(cmd, print_to_console=False) 37 os.chdir(cwd) 38 return status, stdout, stderr 39 40 41def PeekTrybotImage(chromeos_root, buildbucket_id): 42 """Get the artifact URL of a given tryjob. 43 44 Args: 45 buildbucket_id: buildbucket-id 46 chromeos_root: root dir of chrome os checkout 47 48 Returns: 49 (status, url) where status can be 'pass', 'fail', 'running', 50 and url looks like: 51 gs://chromeos-image-archive/trybot-elm-release-tryjob/R67-10468.0.0-b20789 52 """ 53 command = ('cros buildresult --report json --buildbucket-id %s' % 54 buildbucket_id) 55 rc, out, _ = RunCommandInPath(chromeos_root, command) 56 57 # Current implementation of cros buildresult returns fail when a job is still 58 # running. 59 if rc != 0: 60 return ('running', None) 61 62 results = json.loads(out)[buildbucket_id] 63 64 # Handle the case where the tryjob failed to launch correctly. 65 if results['artifacts_url'] is None: 66 return (results['status'], '') 67 68 return (results['status'], results['artifacts_url'].rstrip('/')) 69 70 71def ParseTryjobBuildbucketId(msg): 72 """Find the buildbucket-id in the messages from `cros tryjob`. 73 74 Args: 75 msg: messages from `cros tryjob` 76 77 Returns: 78 buildbucket-id, which will be passed to `cros buildresult` 79 """ 80 output_list = ast.literal_eval(msg) 81 output_dict = output_list[0] 82 if 'buildbucket_id' in output_dict: 83 return output_dict['buildbucket_id'] 84 return None 85 86 87def SubmitTryjob(chromeos_root, 88 buildbot_name, 89 patch_list, 90 tryjob_flags=None, 91 build_toolchain=False): 92 """Calls `cros tryjob ...` 93 94 Args: 95 chromeos_root: the path to the ChromeOS root, needed for finding chromite 96 and launching the buildbot. 97 buildbot_name: the name of the buildbot queue, such as lumpy-release or 98 daisy-paladin. 99 patch_list: a python list of the patches, if any, for the buildbot to use. 100 tryjob_flags: See cros tryjob --help for available options. 101 build_toolchain: builds and uses the latest toolchain, rather than the 102 prebuilt one in SDK. 103 104 Returns: 105 buildbucket id 106 """ 107 patch_arg = '' 108 if patch_list: 109 for p in patch_list: 110 patch_arg = patch_arg + ' -g ' + repr(p) 111 if not tryjob_flags: 112 tryjob_flags = [] 113 if build_toolchain: 114 tryjob_flags.append('--latest-toolchain') 115 tryjob_flags = ' '.join(tryjob_flags) 116 117 # Launch buildbot with appropriate flags. 118 build = buildbot_name 119 command = ('cros_sdk -- cros tryjob --yes --json --nochromesdk %s %s %s' % 120 (tryjob_flags, patch_arg, build)) 121 print('CMD: %s' % command) 122 _, out, _ = RunCommandInPath(chromeos_root, command) 123 buildbucket_id = ParseTryjobBuildbucketId(out) 124 print('buildbucket_id: %s' % repr(buildbucket_id)) 125 if not buildbucket_id: 126 logger.GetLogger().LogFatal('Error occurred while launching trybot job: ' 127 '%s' % command) 128 return buildbucket_id 129 130 131def GetTrybotImage(chromeos_root, 132 buildbot_name, 133 patch_list, 134 tryjob_flags=None, 135 build_toolchain=False, 136 asynchronous=False): 137 """Launch buildbot and get resulting trybot artifact name. 138 139 This function launches a buildbot with the appropriate flags to 140 build the test ChromeOS image, with the current ToT mobile compiler. It 141 checks every 10 minutes to see if the trybot has finished. When the trybot 142 has finished, it parses the resulting report logs to find the trybot 143 artifact (if one was created), and returns that artifact name. 144 145 Args: 146 chromeos_root: the path to the ChromeOS root, needed for finding chromite 147 and launching the buildbot. 148 buildbot_name: the name of the buildbot queue, such as lumpy-release or 149 daisy-paladin. 150 patch_list: a python list of the patches, if any, for the buildbot to use. 151 tryjob_flags: See cros tryjob --help for available options. 152 build_toolchain: builds and uses the latest toolchain, rather than the 153 prebuilt one in SDK. 154 asynchronous: don't wait for artifacts; just return the buildbucket id 155 156 Returns: 157 (buildbucket id, partial image url) e.g. 158 (8952271933586980528, trybot-elm-release-tryjob/R67-10480.0.0-b2373596) 159 """ 160 buildbucket_id = SubmitTryjob(chromeos_root, buildbot_name, patch_list, 161 tryjob_flags, build_toolchain) 162 if asynchronous: 163 return buildbucket_id, ' ' 164 165 # The trybot generally takes more than 2 hours to finish. 166 # Wait two hours before polling the status. 167 time.sleep(INITIAL_SLEEP_TIME) 168 elapsed = INITIAL_SLEEP_TIME 169 status = 'running' 170 image = '' 171 while True: 172 status, image = PeekTrybotImage(chromeos_root, buildbucket_id) 173 if status == 'running': 174 if elapsed > TIME_OUT: 175 logger.GetLogger().LogFatal( 176 'Unable to get build result for target %s.' % buildbot_name) 177 else: 178 wait_msg = 'Unable to find build result; job may be running.' 179 logger.GetLogger().LogOutput(wait_msg) 180 logger.GetLogger().LogOutput('{0} minutes elapsed.'.format(elapsed / 60)) 181 logger.GetLogger().LogOutput('Sleeping {0} seconds.'.format(SLEEP_TIME)) 182 time.sleep(SLEEP_TIME) 183 elapsed += SLEEP_TIME 184 else: 185 break 186 187 if not buildbot_name.endswith('-toolchain') and status == 'fail': 188 # For rotating testers, we don't care about their status 189 # result, because if any HWTest failed it will be non-zero. 190 # 191 # The nightly performance tests do not run HWTests, so if 192 # their status is non-zero, we do care. In this case 193 # non-zero means the image itself probably did not build. 194 image = '' 195 196 if not image: 197 logger.GetLogger().LogError('Trybot job (buildbucket id: %s) failed with' 198 'status %s; no trybot image generated. ' % 199 (buildbucket_id, status)) 200 else: 201 # Convert full gs path to what crosperf expects. For example, convert 202 # gs://chromeos-image-archive/trybot-elm-release-tryjob/R67-10468.0.0-b20789 203 # to 204 # trybot-elm-release-tryjob/R67-10468.0.0-b20789 205 image = '/'.join(image.split('/')[-2:]) 206 207 logger.GetLogger().LogOutput("image is '%s'" % image) 208 logger.GetLogger().LogOutput('status is %s' % status) 209 return buildbucket_id, image 210 211 212def DoesImageExist(chromeos_root, build): 213 """Check if the image for the given build exists.""" 214 215 ce = command_executer.GetCommandExecuter() 216 command = ('gsutil ls gs://chromeos-image-archive/%s' 217 '/chromiumos_test_image.tar.xz' % (build)) 218 ret = ce.ChrootRunCommand(chromeos_root, command, print_to_console=False) 219 return not ret 220 221 222def WaitForImage(chromeos_root, build): 223 """Wait for an image to be ready.""" 224 225 elapsed_time = 0 226 while elapsed_time < TIME_OUT: 227 if DoesImageExist(chromeos_root, build): 228 return 229 logger.GetLogger().LogOutput('Image %s not ready, waiting for 10 minutes' % 230 build) 231 time.sleep(SLEEP_TIME) 232 elapsed_time += SLEEP_TIME 233 234 logger.GetLogger().LogOutput('Image %s not found, waited for %d hours' % 235 (build, (TIME_OUT / 3600))) 236 raise BuildbotTimeout('Timeout while waiting for image %s' % build) 237 238 239def GetLatestImage(chromeos_root, path): 240 """Get latest image""" 241 242 fmt = re.compile(r'R([0-9]+)-([0-9]+).([0-9]+).([0-9]+)') 243 244 ce = command_executer.GetCommandExecuter() 245 command = ('gsutil ls gs://chromeos-image-archive/%s' % path) 246 ret, out, _ = ce.ChrootRunCommandWOutput( 247 chromeos_root, command, print_to_console=False) 248 if ret != 0: 249 raise RuntimeError('Failed to list buckets with command: %s.' % command) 250 candidates = [l.split('/')[-2] for l in out.split()] 251 candidates = [fmt.match(c) for c in candidates] 252 candidates = [[int(r) for r in m.group(1, 2, 3, 4)] for m in candidates if m] 253 candidates.sort(reverse=True) 254 for c in candidates: 255 build = '%s/R%d-%d.%d.%d' % (path, c[0], c[1], c[2], c[3]) 256 # Denylist "R79-12384.0.0" image released by mistake. 257 # TODO(crbug.com/992242): Remove the filter by 2019-09-05. 258 if c == [79, 12384, 0, 0]: 259 continue 260 261 if DoesImageExist(chromeos_root, build): 262 return build 263 264 265def GetLatestRecipeImage(chromeos_root, path): 266 """Get latest nightly test image from recipe bucket. 267 268 Image location example: 269 $ARCHIVE/lulu-llvm-next-nightly/R84-13037.0.0-31011-8883172717979984032 270 """ 271 272 fmt = re.compile(r'R([0-9]+)-([0-9]+).([0-9]+).([0-9]+)-([0-9]+)') 273 274 ce = command_executer.GetCommandExecuter() 275 command = ('gsutil ls gs://chromeos-image-archive/%s' % path) 276 ret, out, _ = ce.ChrootRunCommandWOutput( 277 chromeos_root, command, print_to_console=False) 278 if ret != 0: 279 raise RuntimeError('Failed to list buckets with command: %s.' % command) 280 candidates = [l.split('/')[-2] for l in out.split()] 281 candidates = [(fmt.match(c), c) for c in candidates] 282 candidates = [([int(r) 283 for r in m[0].group(1, 2, 3, 4, 5)], m[1]) 284 for m in candidates 285 if m] 286 candidates.sort(key=lambda x: x[0], reverse=True) 287 # Try to get ony last two days of images since nightly tests are run once 288 # another day. 289 for c in candidates[:2]: 290 build = '%s/%s' % (path, c[1]) 291 if DoesImageExist(chromeos_root, build): 292 return build 293