1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# Copyright 2019 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Runs a tryjob/tryjobs after updating the packages.""" 8 9from __future__ import print_function 10 11import argparse 12import datetime 13import json 14import os 15 16from assert_not_in_chroot import VerifyOutsideChroot 17from failure_modes import FailureModes 18from get_llvm_hash import GetLLVMHashAndVersionFromSVNOption 19from get_llvm_hash import is_svn_option 20from subprocess_helpers import ChrootRunCommand 21from subprocess_helpers import ExecCommandAndCaptureOutput 22import update_chromeos_llvm_next_hash 23 24 25def GetCommandLineArgs(): 26 """Parses the command line for the command line arguments. 27 28 Returns: 29 The log level to use when retrieving the LLVM hash or google3 LLVM version, 30 the chroot path to use for executing chroot commands, 31 a list of a package or packages to update their LLVM next hash, 32 and the LLVM version to use when retrieving the LLVM hash. 33 """ 34 35 # Default path to the chroot if a path is not specified. 36 cros_root = os.path.expanduser('~') 37 cros_root = os.path.join(cros_root, 'chromiumos') 38 39 # Create parser and add optional command-line arguments. 40 parser = argparse.ArgumentParser( 41 description='Runs a tryjob if successfully updated packages\'' 42 '"LLVM_NEXT_HASH".') 43 44 # Add argument for the absolute path to the file that contains information on 45 # the previous tested svn version. 46 parser.add_argument( 47 '--last_tested', 48 help='the absolute path to the file that contains the last tested ' 49 'svn version') 50 51 # Add argument for other change lists that want to run alongside the tryjob 52 # which has a change list of updating a package's git hash. 53 parser.add_argument( 54 '--extra_change_lists', 55 type=int, 56 nargs='+', 57 help='change lists that would like to be run alongside the change list ' 58 'of updating the packages') 59 60 # Add argument for custom options for the tryjob. 61 parser.add_argument( 62 '--options', 63 required=False, 64 nargs='+', 65 help='options to use for the tryjob testing') 66 67 # Add argument for builders for the tryjob. 68 parser.add_argument( 69 '--builders', 70 required=True, 71 nargs='+', 72 help='builders to use for the tryjob testing') 73 74 # Add argument for the description of the tryjob. 75 parser.add_argument( 76 '--description', 77 required=False, 78 nargs='+', 79 help='the description of the tryjob') 80 81 # Add argument for a specific chroot path. 82 parser.add_argument( 83 '--chroot_path', 84 default=cros_root, 85 help='the path to the chroot (default: %(default)s)') 86 87 # Add argument for whether to display command contents to `stdout`. 88 parser.add_argument( 89 '--verbose', 90 action='store_true', 91 help='display contents of a command to the terminal ' 92 '(default: %(default)s)') 93 94 # Add argument for the LLVM version to use. 95 parser.add_argument( 96 '--llvm_version', 97 type=is_svn_option, 98 required=True, 99 help='which git hash of LLVM to find ' 100 '{google3, ToT, <svn_version>} ' 101 '(default: finds the git hash of the google3 LLVM ' 102 'version)') 103 104 args_output = parser.parse_args() 105 106 return args_output 107 108 109def GetLastTestedSVNVersion(last_tested_file): 110 """Gets the lasted tested svn version from the file. 111 112 Args: 113 last_tested_file: The absolute path to the file that contains the last 114 tested svn version. 115 116 Returns: 117 The last tested svn version or 'None' if the file did not have a last tested 118 svn version (the file exists, but failed to convert the contents to an 119 integer) or the file does not exist. 120 """ 121 122 if not last_tested_file: 123 return None 124 125 last_svn_version = None 126 127 # Get the last tested svn version if the file exists. 128 try: 129 with open(last_tested_file) as file_obj: 130 # For now, the first line contains the last tested svn version. 131 return int(file_obj.read().rstrip()) 132 133 except IOError: 134 pass 135 except ValueError: 136 pass 137 138 return last_svn_version 139 140 141def GetTryJobCommand(change_list, extra_change_lists, options, builder): 142 """Constructs the 'tryjob' command. 143 144 Args: 145 change_list: The CL obtained from updating the packages. 146 extra_change_lists: Extra change lists that would like to be run alongside 147 the change list of updating the packages. 148 options: Options to be passed into the tryjob command. 149 builder: The builder to be passed into the tryjob command. 150 151 Returns: 152 The 'tryjob' command with the change list of updating the packages and 153 any extra information that was passed into the command line. 154 """ 155 156 tryjob_cmd = ['cros', 'tryjob', '--yes', '--json', '-g', '%d' % change_list] 157 158 if extra_change_lists: 159 for extra_cl in extra_change_lists: 160 tryjob_cmd.extend(['-g', '%d' % extra_cl]) 161 162 tryjob_cmd.append(builder) 163 164 if options: 165 tryjob_cmd.extend('--%s' % option for option in options) 166 167 return tryjob_cmd 168 169 170def GetCurrentTimeInUTC(): 171 """Returns the current time via `datetime.datetime.utcnow()`.""" 172 return datetime.datetime.utcnow() 173 174 175def RunTryJobs(cl_number, extra_change_lists, options, builders, chroot_path, 176 verbose): 177 """Runs a tryjob/tryjobs. 178 179 Args: 180 cl_number: The CL created by updating the packages. 181 extra_change_lists: Any extra change lists that would run alongside the CL 182 that was created by updating the packages ('cl_number'). 183 options: Any options to be passed into the 'tryjob' command. 184 builders: All the builders to run the 'tryjob' with. 185 chroot_path: The absolute path to the chroot. 186 verbose: Print command contents to `stdout`. 187 188 Returns: 189 A list that contains stdout contents of each tryjob, where stdout is 190 information (a hashmap) about the tryjob. The hashmap also contains stderr 191 if there was an error when running a tryjob. 192 193 Raises: 194 ValueError: Failed to submit a tryjob. 195 """ 196 197 # Contains the results of each tryjob. The results are retrieved from 'out' 198 # which is stdout of the command executer. 199 tryjob_results = [] 200 201 # For each builder passed into the command line: 202 # 203 # Run a tryjob with the change list number obtained from updating the 204 # packages and append additional changes lists and options obtained from the 205 # command line. 206 for cur_builder in builders: 207 tryjob_cmd = GetTryJobCommand(cl_number, extra_change_lists, options, 208 cur_builder) 209 210 out = ChrootRunCommand(chroot_path, tryjob_cmd, verbose=verbose) 211 212 tryjob_launch_time = GetCurrentTimeInUTC() 213 214 tryjob_contents = json.loads(out) 215 216 buildbucket_id = int(tryjob_contents[0]['buildbucket_id']) 217 218 new_tryjob = { 219 'launch_time': str(tryjob_launch_time), 220 'link': str(tryjob_contents[0]['url']), 221 'buildbucket_id': buildbucket_id, 222 'extra_cls': extra_change_lists, 223 'options': options, 224 'builder': [cur_builder] 225 } 226 227 tryjob_results.append(new_tryjob) 228 229 AddTryjobLinkToCL(tryjob_results, cl_number, chroot_path) 230 231 return tryjob_results 232 233 234def AddTryjobLinkToCL(tryjobs, cl, chroot_path): 235 """Adds the tryjob link(s) to the CL via `gerrit message <CL> <message>`.""" 236 237 # NOTE: Invoking `cros_sdk` does not make each tryjob link appear on its own 238 # line, so invoking the `gerrit` command directly instead of using `cros_sdk` 239 # to do it for us. 240 # 241 # FIXME: Need to figure out why `cros_sdk` does not add each tryjob link as a 242 # newline. 243 gerrit_abs_path = os.path.join(chroot_path, 'chromite/bin/gerrit') 244 245 tryjob_links = ['Started the following tryjobs:'] 246 tryjob_links.extend(tryjob['link'] for tryjob in tryjobs) 247 248 add_message_cmd = [ 249 gerrit_abs_path, 'message', 250 str(cl), '\n'.join(tryjob_links) 251 ] 252 253 ExecCommandAndCaptureOutput(add_message_cmd) 254 255 256def main(): 257 """Updates the packages' 'LLVM_NEXT_HASH' and submits tryjobs. 258 259 Raises: 260 AssertionError: The script was run inside the chroot. 261 """ 262 263 VerifyOutsideChroot() 264 265 args_output = GetCommandLineArgs() 266 267 last_svn_version = GetLastTestedSVNVersion(args_output.last_tested) 268 269 update_packages = [ 270 'sys-devel/llvm', 'sys-libs/compiler-rt', 'sys-libs/libcxx', 271 'sys-libs/libcxxabi', 'sys-libs/llvm-libunwind' 272 ] 273 274 patch_metadata_file = 'PATCHES.json' 275 276 svn_option = args_output.llvm_version 277 278 git_hash, svn_version = GetLLVMHashAndVersionFromSVNOption(svn_option) 279 280 # There is no need to run tryjobs when the SVN version matches the last tested 281 # SVN version. 282 if last_svn_version == svn_version: 283 print('svn version (%d) matches the last tested svn version (%d) in %s' % 284 (svn_version, last_svn_version, args_output.last_tested)) 285 return 286 287 update_chromeos_llvm_next_hash.verbose = args_output.verbose 288 289 change_list = update_chromeos_llvm_next_hash.UpdatePackages( 290 update_packages, git_hash, svn_version, args_output.chroot_path, 291 patch_metadata_file, FailureModes.DISABLE_PATCHES, svn_option) 292 293 print('Successfully updated packages to %d' % svn_version) 294 print('Gerrit URL: %s' % change_list.url) 295 print('Change list number: %d' % change_list.cl_number) 296 297 tryjob_results = RunTryJobs(change_list.cl_number, 298 args_output.extra_change_lists, 299 args_output.options, args_output.builders, 300 args_output.chroot_path, args_output.verbose) 301 302 print('Tryjobs:') 303 for tryjob in tryjob_results: 304 print(tryjob) 305 306 # Updated the packages and submitted tryjobs successfully, so the file will 307 # contain 'svn_version' which will now become the last tested svn version. 308 if args_output.last_tested: 309 with open(args_output.last_tested, 'w') as file_obj: 310 file_obj.write(str(svn_version)) 311 312 313if __name__ == '__main__': 314 main() 315