1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# 4# Copyright 2016 The Chromium OS Authors. All rights reserved. 5# Use of this source code is governed by a BSD-style license that can be 6# found in the LICENSE file. 7 8"""Script for running nightly compiler tests on ChromeOS. 9 10This script launches a buildbot to build ChromeOS with the latest compiler on 11a particular board; then it finds and downloads the trybot image and the 12corresponding official image, and runs crosperf performance tests comparing 13the two. It then generates a report, emails it to the c-compiler-chrome, as 14well as copying the images into the seven-day reports directory. 15""" 16 17# Script to test different toolchains against ChromeOS benchmarks. 18 19from __future__ import print_function 20 21import argparse 22import datetime 23import os 24import re 25import shutil 26import sys 27import time 28 29from cros_utils import command_executer 30from cros_utils import logger 31 32from cros_utils import buildbot_utils 33 34# CL that uses LLVM-Next to build the images (includes chrome). 35USE_LLVM_NEXT_PATCH = '513590' 36 37CROSTC_ROOT = '/usr/local/google/crostc' 38NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly-tests') 39ROLE_ACCOUNT = 'mobiletc-prebuild' 40TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__)) 41TMP_TOOLCHAIN_TEST = '/tmp/toolchain-tests' 42MAIL_PROGRAM = '~/var/bin/mail-sheriff' 43PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives') 44NIGHTLY_TESTS_RESULTS = os.path.join(CROSTC_ROOT, 'nightly_test_reports') 45 46IMAGE_DIR = '{board}-{image_type}' 47IMAGE_VERSION_STR = r'{chrome_version}-{tip}\.{branch}\.{branch_branch}' 48IMAGE_FS = IMAGE_DIR + '/' + IMAGE_VERSION_STR 49TRYBOT_IMAGE_FS = IMAGE_FS + '-{build_id}' 50IMAGE_RE_GROUPS = { 51 'board': r'(?P<board>\S+)', 52 'image_type': r'(?P<image_type>\S+)', 53 'chrome_version': r'(?P<chrome_version>R\d+)', 54 'tip': r'(?P<tip>\d+)', 55 'branch': r'(?P<branch>\d+)', 56 'branch_branch': r'(?P<branch_branch>\d+)', 57 'build_id': r'(?P<build_id>b\d+)' 58} 59TRYBOT_IMAGE_RE = TRYBOT_IMAGE_FS.format(**IMAGE_RE_GROUPS) 60 61RECIPE_IMAGE_FS = IMAGE_FS + '-{build_id}-{buildbucket_id}' 62RECIPE_IMAGE_RE_GROUPS = { 63 'board': r'(?P<board>\S+)', 64 'image_type': r'(?P<image_type>\S+)', 65 'chrome_version': r'(?P<chrome_version>R\d+)', 66 'tip': r'(?P<tip>\d+)', 67 'branch': r'(?P<branch>\d+)', 68 'branch_branch': r'(?P<branch_branch>\d+)', 69 'build_id': r'(?P<build_id>\d+)', 70 'buildbucket_id': r'(?P<buildbucket_id>\d+)' 71} 72RECIPE_IMAGE_RE = RECIPE_IMAGE_FS.format(**RECIPE_IMAGE_RE_GROUPS) 73 74TELEMETRY_AQUARIUM_UNSUPPORTED = ['bob', 'elm', 'veyron_tiger'] 75 76 77class ToolchainComparator(object): 78 """Class for doing the nightly tests work.""" 79 80 def __init__(self, 81 board, 82 remotes, 83 chromeos_root, 84 weekday, 85 patches, 86 recipe=False, 87 test=False, 88 noschedv2=False): 89 self._board = board 90 self._remotes = remotes 91 self._chromeos_root = chromeos_root 92 self._base_dir = os.getcwd() 93 self._ce = command_executer.GetCommandExecuter() 94 self._l = logger.GetLogger() 95 self._build = '%s-release-tryjob' % board 96 self._patches = patches.split(',') if patches else [] 97 self._patches_string = '_'.join(str(p) for p in self._patches) 98 self._recipe = recipe 99 self._test = test 100 self._noschedv2 = noschedv2 101 102 if not weekday: 103 self._weekday = time.strftime('%a') 104 else: 105 self._weekday = weekday 106 self._date = datetime.date.today().strftime('%Y/%m/%d') 107 timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H:%M:%S') 108 self._reports_dir = os.path.join( 109 TMP_TOOLCHAIN_TEST if self._test else NIGHTLY_TESTS_RESULTS, 110 '%s.%s' % (timestamp, board), 111 ) 112 113 def _GetVanillaImageName(self, trybot_image): 114 """Given a trybot artifact name, get latest vanilla image name. 115 116 Args: 117 trybot_image: artifact name such as 118 'daisy-release-tryjob/R40-6394.0.0-b1389' 119 for recipe images, name is in this format: 120 'lulu-llvm-next-nightly/R84-13037.0.0-31011-8883172717979984032/' 121 122 Returns: 123 Latest official image name, e.g. 'daisy-release/R57-9089.0.0'. 124 """ 125 # For board names with underscores, we need to fix the trybot image name 126 # to replace the hyphen (for the recipe builder) with the underscore. 127 # Currently the only such board we use is 'veyron_tiger'. 128 if trybot_image.find('veyron-tiger') != -1: 129 trybot_image = trybot_image.replace('veyron-tiger', 'veyron_tiger') 130 # We need to filter out -tryjob in the trybot_image. 131 if self._recipe: 132 trybot = re.sub('-llvm-next-nightly', '-release', trybot_image) 133 mo = re.search(RECIPE_IMAGE_RE, trybot) 134 else: 135 trybot = re.sub('-tryjob', '', trybot_image) 136 mo = re.search(TRYBOT_IMAGE_RE, trybot) 137 assert mo 138 dirname = IMAGE_DIR.replace('\\', '').format(**mo.groupdict()) 139 return buildbot_utils.GetLatestImage(self._chromeos_root, dirname) 140 141 def _TestImages(self, trybot_image, vanilla_image): 142 """Create crosperf experiment file. 143 144 Given the names of the trybot, vanilla and non-AFDO images, create the 145 appropriate crosperf experiment file and launch crosperf on it. 146 """ 147 if self._test: 148 experiment_file_dir = TMP_TOOLCHAIN_TEST 149 else: 150 experiment_file_dir = os.path.join(NIGHTLY_TESTS_DIR, self._weekday) 151 experiment_file_name = '%s_toolchain_experiment.txt' % self._board 152 153 compiler_string = 'llvm' 154 if USE_LLVM_NEXT_PATCH in self._patches_string: 155 experiment_file_name = '%s_llvm_next_experiment.txt' % self._board 156 compiler_string = 'llvm_next' 157 158 experiment_file = os.path.join(experiment_file_dir, experiment_file_name) 159 experiment_header = """ 160 board: %s 161 remote: %s 162 retries: 1 163 """ % (self._board, self._remotes) 164 experiment_tests = """ 165 benchmark: all_toolchain_perf { 166 suite: telemetry_Crosperf 167 iterations: 5 168 run_local: False 169 } 170 171 benchmark: loading.desktop { 172 suite: telemetry_Crosperf 173 test_args: --story-tag-filter=typical 174 iterations: 3 175 run_local: False 176 retries: 0 177 } 178 """ 179 telemetry_aquarium_tests = """ 180 benchmark: rendering.desktop { 181 run_local: False 182 suite: telemetry_Crosperf 183 test_args: --story-filter=aquarium$ 184 iterations: 5 185 } 186 187 benchmark: rendering.desktop { 188 run_local: False 189 suite: telemetry_Crosperf 190 test_args: --story-filter=aquarium_20k$ 191 iterations: 3 192 } 193 """ 194 195 with open(experiment_file, 'w', encoding='utf-8') as f: 196 f.write(experiment_header) 197 f.write(experiment_tests) 198 199 if self._board not in TELEMETRY_AQUARIUM_UNSUPPORTED: 200 f.write(telemetry_aquarium_tests) 201 202 # Now add vanilla to test file. 203 official_image = """ 204 vanilla_image { 205 chromeos_root: %s 206 build: %s 207 compiler: llvm 208 } 209 """ % (self._chromeos_root, vanilla_image) 210 f.write(official_image) 211 212 label_string = '%s_trybot_image' % compiler_string 213 214 # Reuse autotest files from vanilla image for trybot images 215 autotest_files = os.path.join('/tmp', vanilla_image, 'autotest_files') 216 experiment_image = """ 217 %s { 218 chromeos_root: %s 219 build: %s 220 autotest_path: %s 221 compiler: %s 222 } 223 """ % (label_string, self._chromeos_root, trybot_image, autotest_files, 224 compiler_string) 225 f.write(experiment_image) 226 227 crosperf = os.path.join(TOOLCHAIN_DIR, 'crosperf', 'crosperf') 228 noschedv2_opts = '--noschedv2' if self._noschedv2 else '' 229 command = ('{crosperf} --no_email={no_email} --results_dir={r_dir} ' 230 '--logging_level=verbose --json_report=True {noschedv2_opts} ' 231 '{exp_file}').format( 232 crosperf=crosperf, 233 no_email=not self._test, 234 r_dir=self._reports_dir, 235 noschedv2_opts=noschedv2_opts, 236 exp_file=experiment_file) 237 238 return self._ce.RunCommand(command) 239 240 def _SendEmail(self): 241 """Find email message generated by crosperf and send it.""" 242 filename = os.path.join(self._reports_dir, 'msg_body.html') 243 if (os.path.exists(filename) and 244 os.path.exists(os.path.expanduser(MAIL_PROGRAM))): 245 email_title = 'buildbot llvm test results' 246 if USE_LLVM_NEXT_PATCH in self._patches_string: 247 email_title = 'buildbot llvm_next test results' 248 command = ('cat %s | %s -s "%s, %s %s" -team -html' % 249 (filename, MAIL_PROGRAM, email_title, self._board, self._date)) 250 self._ce.RunCommand(command) 251 252 def _CopyJson(self): 253 # Make sure a destination directory exists. 254 os.makedirs(PENDING_ARCHIVES_DIR, exist_ok=True) 255 # Copy json report to pending archives directory. 256 command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR) 257 ret = self._ce.RunCommand(command) 258 # Failing to access json report means that crosperf terminated or all tests 259 # failed, raise an error. 260 if ret != 0: 261 raise RuntimeError( 262 'Crosperf failed to run tests, cannot copy json report!') 263 264 def DoAll(self): 265 """Main function inside ToolchainComparator class. 266 267 Launch trybot, get image names, create crosperf experiment file, run 268 crosperf, and copy images into seven-day report directories. 269 """ 270 if self._recipe: 271 print('Using recipe buckets to get latest image.') 272 # crbug.com/1077313: Some boards are not consistently 273 # spelled, having underscores in some places and dashes in others. 274 # The image directories consistenly use dashes, so convert underscores 275 # to dashes to work around this. 276 trybot_image = buildbot_utils.GetLatestRecipeImage( 277 self._chromeos_root, 278 '%s-llvm-next-nightly' % self._board.replace('_', '-')) 279 else: 280 # Launch tryjob and wait to get image location. 281 buildbucket_id, trybot_image = buildbot_utils.GetTrybotImage( 282 self._chromeos_root, 283 self._build, 284 self._patches, 285 tryjob_flags=['--notests'], 286 build_toolchain=True) 287 print('trybot_url: \ 288 http://cros-goldeneye/chromeos/healthmonitoring/buildDetails?buildbucketId=%s' 289 % buildbucket_id) 290 291 if not trybot_image: 292 self._l.LogError('Unable to find trybot_image!') 293 return 2 294 295 vanilla_image = self._GetVanillaImageName(trybot_image) 296 297 print('trybot_image: %s' % trybot_image) 298 print('vanilla_image: %s' % vanilla_image) 299 300 ret = self._TestImages(trybot_image, vanilla_image) 301 # Always try to send report email as crosperf will generate report when 302 # tests partially succeeded. 303 if not self._test: 304 self._SendEmail() 305 self._CopyJson() 306 # Non-zero ret here means crosperf tests partially failed, raise error here 307 # so that toolchain summary report can catch it. 308 if ret != 0: 309 raise RuntimeError('Crosperf tests partially failed!') 310 311 return 0 312 313 314def Main(argv): 315 """The main function.""" 316 317 # Common initializations 318 command_executer.InitCommandExecuter() 319 parser = argparse.ArgumentParser() 320 parser.add_argument( 321 '--remote', dest='remote', help='Remote machines to run tests on.') 322 parser.add_argument( 323 '--board', dest='board', default='x86-zgb', help='The target board.') 324 parser.add_argument( 325 '--chromeos_root', 326 dest='chromeos_root', 327 help='The chromeos root from which to run tests.') 328 parser.add_argument( 329 '--weekday', 330 default='', 331 dest='weekday', 332 help='The day of the week for which to run tests.') 333 parser.add_argument( 334 '--patch', 335 dest='patches', 336 help='The patches to use for the testing, ' 337 "seprate the patch numbers with ',' " 338 'for more than one patches.') 339 parser.add_argument( 340 '--noschedv2', 341 dest='noschedv2', 342 action='store_true', 343 default=False, 344 help='Pass --noschedv2 to crosperf.') 345 parser.add_argument( 346 '--recipe', 347 dest='recipe', 348 default=True, 349 help='Use images generated from recipe rather than' 350 'launching tryjob to get images.') 351 parser.add_argument( 352 '--test', 353 dest='test', 354 default=False, 355 help='Test this script on local desktop, ' 356 'disabling mobiletc checking and email sending.' 357 'Artifacts stored in /tmp/toolchain-tests') 358 359 options = parser.parse_args(argv[1:]) 360 if not options.board: 361 print('Please give a board.') 362 return 1 363 if not options.remote: 364 print('Please give at least one remote machine.') 365 return 1 366 if not options.chromeos_root: 367 print('Please specify the ChromeOS root directory.') 368 return 1 369 if options.test: 370 print('Cleaning local test directory for this script.') 371 if os.path.exists(TMP_TOOLCHAIN_TEST): 372 shutil.rmtree(TMP_TOOLCHAIN_TEST) 373 os.mkdir(TMP_TOOLCHAIN_TEST) 374 375 fc = ToolchainComparator(options.board, options.remote, options.chromeos_root, 376 options.weekday, options.patches, options.recipe, 377 options.test, options.noschedv2) 378 return fc.DoAll() 379 380 381if __name__ == '__main__': 382 retval = Main(sys.argv) 383 sys.exit(retval) 384