1# -*- coding: utf-8 -*- 2# Copyright 2017 The ChromiumOS Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Utilities for launching and accessing ChromeOS buildbots.""" 7 8 9import ast 10import json 11import os 12import re 13import time 14 15from cros_utils import command_executer 16from cros_utils import logger 17 18 19INITIAL_SLEEP_TIME = 7200 # 2 hours; wait time before polling buildbot. 20SLEEP_TIME = 600 # 10 minutes; time between polling of buildbot. 21 22# Some of our slower builders (llvm-next) are taking more 23# than 12 hours. So, increase this TIME_OUT to 15 hours. 24TIME_OUT = 15 * 60 * 60 # Decide the build is dead or will never finish 25 26 27class BuildbotTimeout(Exception): 28 """Exception to throw when a buildbot operation timesout.""" 29 30 31def RunCommandInPath(path, cmd): 32 ce = command_executer.GetCommandExecuter() 33 cwd = os.getcwd() 34 os.chdir(path) 35 status, stdout, stderr = ce.RunCommandWOutput(cmd, print_to_console=False) 36 os.chdir(cwd) 37 return status, stdout, stderr 38 39 40def PeekTrybotImage(chromeos_root, buildbucket_id): 41 """Get the artifact URL of a given tryjob. 42 43 Args: 44 buildbucket_id: buildbucket-id 45 chromeos_root: root dir of chrome os checkout 46 47 Returns: 48 (status, url) where status can be 'pass', 'fail', 'running', 49 and url looks like: 50 gs://chromeos-image-archive/trybot-elm-release-tryjob/R67-10468.0.0-b20789 51 """ 52 command = ( 53 "cros buildresult --report json --buildbucket-id %s" % buildbucket_id 54 ) 55 rc, out, _ = RunCommandInPath(chromeos_root, command) 56 57 # Current implementation of cros buildresult returns fail when a job is still 58 # running. 59 if rc != 0: 60 return ("running", None) 61 62 results = json.loads(out)[buildbucket_id] 63 64 # Handle the case where the tryjob failed to launch correctly. 65 if results["artifacts_url"] is None: 66 return (results["status"], "") 67 68 return (results["status"], results["artifacts_url"].rstrip("/")) 69 70 71def ParseTryjobBuildbucketId(msg): 72 """Find the buildbucket-id in the messages from `cros tryjob`. 73 74 Args: 75 msg: messages from `cros tryjob` 76 77 Returns: 78 buildbucket-id, which will be passed to `cros buildresult` 79 """ 80 output_list = ast.literal_eval(msg) 81 output_dict = output_list[0] 82 if "buildbucket_id" in output_dict: 83 return output_dict["buildbucket_id"] 84 return None 85 86 87def SubmitTryjob( 88 chromeos_root, 89 buildbot_name, 90 patch_list, 91 tryjob_flags=None, 92 build_toolchain=False, 93): 94 """Calls `cros tryjob ...` 95 96 Args: 97 chromeos_root: the path to the ChromeOS root, needed for finding chromite 98 and launching the buildbot. 99 buildbot_name: the name of the buildbot queue, such as lumpy-release or 100 daisy-paladin. 101 patch_list: a python list of the patches, if any, for the buildbot to use. 102 tryjob_flags: See cros tryjob --help for available options. 103 build_toolchain: builds and uses the latest toolchain, rather than the 104 prebuilt one in SDK. 105 106 Returns: 107 buildbucket id 108 """ 109 patch_arg = "" 110 if patch_list: 111 for p in patch_list: 112 patch_arg = patch_arg + " -g " + repr(p) 113 if not tryjob_flags: 114 tryjob_flags = [] 115 if build_toolchain: 116 tryjob_flags.append("--latest-toolchain") 117 tryjob_flags = " ".join(tryjob_flags) 118 119 # Launch buildbot with appropriate flags. 120 build = buildbot_name 121 command = "cros_sdk -- cros tryjob --yes --json --nochromesdk %s %s %s" % ( 122 tryjob_flags, 123 patch_arg, 124 build, 125 ) 126 print("CMD: %s" % command) 127 _, out, _ = RunCommandInPath(chromeos_root, command) 128 buildbucket_id = ParseTryjobBuildbucketId(out) 129 print("buildbucket_id: %s" % repr(buildbucket_id)) 130 if not buildbucket_id: 131 logger.GetLogger().LogFatal( 132 "Error occurred while launching trybot job: " "%s" % command 133 ) 134 return buildbucket_id 135 136 137def GetTrybotImage( 138 chromeos_root, 139 buildbot_name, 140 patch_list, 141 tryjob_flags=None, 142 build_toolchain=False, 143 asynchronous=False, 144): 145 """Launch buildbot and get resulting trybot artifact name. 146 147 This function launches a buildbot with the appropriate flags to 148 build the test ChromeOS image, with the current ToT mobile compiler. It 149 checks every 10 minutes to see if the trybot has finished. When the trybot 150 has finished, it parses the resulting report logs to find the trybot 151 artifact (if one was created), and returns that artifact name. 152 153 Args: 154 chromeos_root: the path to the ChromeOS root, needed for finding chromite 155 and launching the buildbot. 156 buildbot_name: the name of the buildbot queue, such as lumpy-release or 157 daisy-paladin. 158 patch_list: a python list of the patches, if any, for the buildbot to use. 159 tryjob_flags: See cros tryjob --help for available options. 160 build_toolchain: builds and uses the latest toolchain, rather than the 161 prebuilt one in SDK. 162 asynchronous: don't wait for artifacts; just return the buildbucket id 163 164 Returns: 165 (buildbucket id, partial image url) e.g. 166 (8952271933586980528, trybot-elm-release-tryjob/R67-10480.0.0-b2373596) 167 """ 168 buildbucket_id = SubmitTryjob( 169 chromeos_root, buildbot_name, patch_list, tryjob_flags, build_toolchain 170 ) 171 if asynchronous: 172 return buildbucket_id, " " 173 174 # The trybot generally takes more than 2 hours to finish. 175 # Wait two hours before polling the status. 176 time.sleep(INITIAL_SLEEP_TIME) 177 elapsed = INITIAL_SLEEP_TIME 178 status = "running" 179 image = "" 180 while True: 181 status, image = PeekTrybotImage(chromeos_root, buildbucket_id) 182 if status == "running": 183 if elapsed > TIME_OUT: 184 logger.GetLogger().LogFatal( 185 "Unable to get build result for target %s." % buildbot_name 186 ) 187 else: 188 wait_msg = "Unable to find build result; job may be running." 189 logger.GetLogger().LogOutput(wait_msg) 190 logger.GetLogger().LogOutput(f"{elapsed / 60} minutes elapsed.") 191 logger.GetLogger().LogOutput(f"Sleeping {SLEEP_TIME} seconds.") 192 time.sleep(SLEEP_TIME) 193 elapsed += SLEEP_TIME 194 else: 195 break 196 197 if not buildbot_name.endswith("-toolchain") and status == "fail": 198 # For rotating testers, we don't care about their status 199 # result, because if any HWTest failed it will be non-zero. 200 # 201 # The nightly performance tests do not run HWTests, so if 202 # their status is non-zero, we do care. In this case 203 # non-zero means the image itself probably did not build. 204 image = "" 205 206 if not image: 207 logger.GetLogger().LogError( 208 "Trybot job (buildbucket id: %s) failed with" 209 "status %s; no trybot image generated. " % (buildbucket_id, status) 210 ) 211 else: 212 # Convert full gs path to what crosperf expects. For example, convert 213 # gs://chromeos-image-archive/trybot-elm-release-tryjob/R67-10468.0.0-b20789 214 # to 215 # trybot-elm-release-tryjob/R67-10468.0.0-b20789 216 image = "/".join(image.split("/")[-2:]) 217 218 logger.GetLogger().LogOutput("image is '%s'" % image) 219 logger.GetLogger().LogOutput("status is %s" % status) 220 return buildbucket_id, image 221 222 223def DoesImageExist(chromeos_root, build): 224 """Check if the image for the given build exists.""" 225 226 ce = command_executer.GetCommandExecuter() 227 command = ( 228 "gsutil ls gs://chromeos-image-archive/%s" 229 "/chromiumos_test_image.tar.xz" % (build) 230 ) 231 ret = ce.ChrootRunCommand(chromeos_root, command, print_to_console=False) 232 return not ret 233 234 235def WaitForImage(chromeos_root, build): 236 """Wait for an image to be ready.""" 237 238 elapsed_time = 0 239 while elapsed_time < TIME_OUT: 240 if DoesImageExist(chromeos_root, build): 241 return 242 logger.GetLogger().LogOutput( 243 "Image %s not ready, waiting for 10 minutes" % build 244 ) 245 time.sleep(SLEEP_TIME) 246 elapsed_time += SLEEP_TIME 247 248 logger.GetLogger().LogOutput( 249 "Image %s not found, waited for %d hours" % (build, (TIME_OUT / 3600)) 250 ) 251 raise BuildbotTimeout("Timeout while waiting for image %s" % build) 252 253 254def GetLatestImage(chromeos_root, path): 255 """Get latest image""" 256 257 fmt = re.compile(r"R([0-9]+)-([0-9]+).([0-9]+).([0-9]+)") 258 259 ce = command_executer.GetCommandExecuter() 260 command = "gsutil ls gs://chromeos-image-archive/%s" % path 261 ret, out, _ = ce.ChrootRunCommandWOutput( 262 chromeos_root, command, print_to_console=False 263 ) 264 if ret != 0: 265 raise RuntimeError("Failed to list buckets with command: %s." % command) 266 candidates = [l.split("/")[-2] for l in out.split()] 267 candidates = [fmt.match(c) for c in candidates] 268 candidates = [ 269 [int(r) for r in m.group(1, 2, 3, 4)] for m in candidates if m 270 ] 271 candidates.sort(reverse=True) 272 for c in candidates: 273 build = "%s/R%d-%d.%d.%d" % (path, c[0], c[1], c[2], c[3]) 274 if DoesImageExist(chromeos_root, build): 275 return build 276 277 278def GetLatestRecipeImage(chromeos_root, path): 279 """Get latest nightly test image from recipe bucket. 280 281 Image location example: 282 $ARCHIVE/lulu-llvm-next-nightly/R84-13037.0.0-31011-8883172717979984032 283 """ 284 285 fmt = re.compile(r"R([0-9]+)-([0-9]+).([0-9]+).([0-9]+)-([0-9]+)") 286 287 ce = command_executer.GetCommandExecuter() 288 command = "gsutil ls gs://chromeos-image-archive/%s" % path 289 ret, out, _ = ce.ChrootRunCommandWOutput( 290 chromeos_root, command, print_to_console=False 291 ) 292 if ret != 0: 293 raise RuntimeError("Failed to list buckets with command: %s." % command) 294 candidates = [l.split("/")[-2] for l in out.split()] 295 candidates = [(fmt.match(c), c) for c in candidates] 296 candidates = [ 297 ([int(r) for r in m[0].group(1, 2, 3, 4, 5)], m[1]) 298 for m in candidates 299 if m 300 ] 301 candidates.sort(key=lambda x: x[0], reverse=True) 302 # Try to get ony last two days of images since nightly tests are run once 303 # another day. 304 for c in candidates[:2]: 305 build = "%s/%s" % (path, c[1]) 306 if DoesImageExist(chromeos_root, build): 307 return build 308