1# Copyright 2015 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5 6import math 7 8 9DEPS = [ 10 'core', 11 'ct', 12 'depot_tools/gsutil', 13 'flavor', 14 'recipe_engine/context', 15 'recipe_engine/file', 16 'recipe_engine/json', 17 'recipe_engine/path', 18 'recipe_engine/properties', 19 'recipe_engine/step', 20 'recipe_engine/time', 21 'run', 22 'skia_swarming', 23 'vars', 24] 25 26 27SKPS_VERSION_FILE = 'skps_version' 28CT_SKPS_ISOLATE = 'ct_skps.isolate' 29 30# Do not batch archive more slaves than this value. This is used to prevent 31# no output timeouts in the 'isolate tests' step. 32MAX_SLAVES_TO_BATCHARCHIVE = 100 33 34TOOL_TO_DEFAULT_SKPS_PER_SLAVE = { 35 'dm': 10000, 36 'nanobench': 1000, 37 'get_images_from_skps': 10000, 38} 39 40# The SKP repository to use. 41DEFAULT_SKPS_CHROMIUM_BUILD = 'c37e844a6f8708-eee762104c75bd' 42 43 44def RunSteps(api): 45 # Figure out which repository to use. 46 buildername = api.properties['buildername'] 47 if '1k' in buildername: 48 ct_page_type = '10k' 49 num_pages = 1000 50 elif '10k' in buildername: 51 ct_page_type = '10k' 52 num_pages = 10000 53 elif '100k' in buildername: 54 ct_page_type = '100k' 55 num_pages = 100000 56 elif '1m' in buildername: 57 ct_page_type = 'All' 58 num_pages = 1000000 59 else: 60 raise Exception('Do not recognise the buildername %s.' % buildername) 61 62 # Figure out which tool to use. 63 if 'DM' in buildername: 64 skia_tool = 'dm' 65 build_target = 'dm' 66 elif 'BENCH' in buildername: 67 skia_tool = 'nanobench' 68 build_target = 'nanobench' 69 elif 'IMG_DECODE' in buildername: 70 skia_tool = 'get_images_from_skps' 71 build_target = 'tools' 72 else: 73 raise Exception('Do not recognise the buildername %s.' % buildername) 74 75 api.core.setup() 76 api.flavor.compile(build_target) 77 78 # Required paths. 79 infrabots_dir = api.vars.skia_dir.join('infra', 'bots') 80 isolate_dir = infrabots_dir.join('ct') 81 isolate_path = isolate_dir.join(CT_SKPS_ISOLATE) 82 83 api.run.copy_build_products( 84 api.flavor.out_dir, 85 isolate_dir) 86 api.skia_swarming.setup( 87 infrabots_dir.join('tools', 'luci-go'), 88 swarming_rev='') 89 90 skps_chromium_build = api.properties.get( 91 'skps_chromium_build', DEFAULT_SKPS_CHROMIUM_BUILD) 92 93 # Set build properties to make finding SKPs convenient. 94 webpage_rankings_link = ( 95 'https://storage.cloud.google.com/%s/csv/top-1m.csv' 96 % api.ct.CT_GS_BUCKET) 97 api.step.active_result.presentation.properties['Webpage rankings'] = ( 98 webpage_rankings_link) 99 download_skps_link = ( 100 'https://pantheon.corp.google.com/storage/browser/%s/swarming/skps/%s/%s/' 101 % (api.ct.CT_GS_BUCKET, ct_page_type, skps_chromium_build)) 102 api.step.active_result.presentation.properties['Download SKPs by rank'] = ( 103 download_skps_link) 104 105 # Delete swarming_temp_dir to ensure it starts from a clean slate. 106 api.run.rmtree(api.skia_swarming.swarming_temp_dir) 107 108 num_per_slave = api.properties.get( 109 'num_per_slave', 110 min(TOOL_TO_DEFAULT_SKPS_PER_SLAVE[skia_tool], num_pages)) 111 ct_num_slaves = api.properties.get( 112 'ct_num_slaves', 113 int(math.ceil(float(num_pages) / num_per_slave))) 114 115 # Try to figure out if the SKPs we are going to isolate already exist 116 # locally by reading the SKPS_VERSION_FILE. 117 download_skps = True 118 expected_version_contents = { 119 "chromium_build": skps_chromium_build, 120 "page_type": ct_page_type, 121 "num_slaves": ct_num_slaves, 122 } 123 skps_dir = api.vars.checkout_root.join('skps', buildername) 124 version_file = skps_dir.join(SKPS_VERSION_FILE) 125 if api.path.exists(version_file): # pragma: nocover 126 version_file_contents = api.file.read_text( 127 "Read %s" % version_file, 128 version_file, 129 test_data=expected_version_contents) 130 actual_version_contents = api.json.loads(version_file_contents) 131 differences = (set(expected_version_contents.items()) ^ 132 set(actual_version_contents.items())) 133 download_skps = len(differences) != 0 134 if download_skps: 135 # Delete and recreate the skps dir. 136 api.run.rmtree(skps_dir) 137 api.file.ensure_directory( 138 'makedirs %s' % api.path.basename(skps_dir), skps_dir) 139 140 # If a blacklist file exists then specify SKPs to be blacklisted. 141 blacklists_dir = api.vars.skia_dir.join('infra', 'bots', 'ct', 'blacklists') 142 blacklist_file = blacklists_dir.join( 143 '%s_%s_%s.json' % (skia_tool, ct_page_type, skps_chromium_build)) 144 blacklist_skps = [] 145 if api.path.exists(blacklist_file): # pragma: nocover 146 blacklist_file_contents = api.file.read_text( 147 "Read %s" % blacklist_file, 148 blacklist_file) 149 blacklist_skps = api.json.loads(blacklist_file_contents)['blacklisted_skps'] 150 151 for slave_num in range(1, ct_num_slaves + 1): 152 if download_skps: 153 # Download SKPs. 154 api.ct.download_swarming_skps( 155 ct_page_type, slave_num, skps_chromium_build, 156 skps_dir, 157 start_range=((slave_num-1)*num_per_slave) + 1, 158 num_skps=num_per_slave) 159 160 # Create this slave's isolated.gen.json file to use for batcharchiving. 161 extra_variables = { 162 'SLAVE_NUM': str(slave_num), 163 'TOOL_NAME': skia_tool, 164 'GIT_HASH': api.vars.got_revision, 165 'CONFIGURATION': api.vars.configuration, 166 'BUILDER': buildername, 167 } 168 api.skia_swarming.create_isolated_gen_json( 169 isolate_path, isolate_dir, 'linux', 'ct-%s-%s' % (skia_tool, slave_num), 170 extra_variables, blacklist=blacklist_skps) 171 172 if download_skps: 173 # Since we had to download SKPs create an updated version file. 174 api.file.write_text("Create %s" % version_file, 175 version_file, 176 api.json.dumps(expected_version_contents)) 177 178 # Batcharchive everything on the isolate server for efficiency. 179 max_slaves_to_batcharchive = MAX_SLAVES_TO_BATCHARCHIVE 180 if '1m' in buildername: 181 # Break up the "isolate tests" step into batches with <100k files due to 182 # https://github.com/luci/luci-go/issues/9 183 max_slaves_to_batcharchive = 5 184 tasks_to_swarm_hashes = [] 185 for slave_start_num in xrange(1, ct_num_slaves+1, max_slaves_to_batcharchive): 186 m = min(max_slaves_to_batcharchive, ct_num_slaves) 187 batcharchive_output = api.skia_swarming.batcharchive( 188 targets=['ct-' + skia_tool + '-%s' % num for num in range( 189 slave_start_num, slave_start_num + m)]) 190 tasks_to_swarm_hashes.extend(batcharchive_output) 191 # Sort the list to go through tasks in order. 192 tasks_to_swarm_hashes.sort() 193 194 # Trigger all swarming tasks. 195 dimensions={'os': 'Ubuntu-14.04', 'cpu': 'x86-64', 'pool': 'Chrome'} 196 if 'GPU' in buildername: 197 dimensions['gpu'] = '10de:104a' 198 tasks = api.skia_swarming.trigger_swarming_tasks( 199 tasks_to_swarm_hashes, dimensions=dimensions, io_timeout=40*60) 200 201 # Now collect all tasks. 202 env = {'AWS_CREDENTIAL_FILE': None, 'BOTO_CONFIG': None} 203 failed_tasks = [] 204 for task in tasks: 205 try: 206 api.skia_swarming.collect_swarming_task(task) 207 208 if skia_tool == 'nanobench': 209 output_dir = api.skia_swarming.tasks_output_dir.join( 210 task.title).join('0') 211 utc = api.time.utcnow() 212 gs_dest_dir = 'ct/%s/%d/%02d/%02d/%02d/' % ( 213 ct_page_type, utc.year, utc.month, utc.day, utc.hour) 214 for json_output in api.file.listdir( 215 'listdir output dir', output_dir, test_data=['file 1', 'file 2']): 216 with api.context(env=env): 217 api.gsutil.upload( 218 name='upload json output', 219 source=json_output, 220 bucket='skia-perf', 221 dest=gs_dest_dir, 222 args=['-R'] 223 ) 224 225 except api.step.StepFailure as e: 226 # Add SKP links for convenience. 227 api.step.active_result.presentation.links['Webpage rankings'] = ( 228 webpage_rankings_link) 229 api.step.active_result.presentation.links['Download SKPs by rank'] = ( 230 download_skps_link) 231 failed_tasks.append(e) 232 233 if failed_tasks: 234 raise api.step.StepFailure( 235 'Failed steps: %s' % ', '.join([f.name for f in failed_tasks])) 236 237 238def GenTests(api): 239 ct_num_slaves = 5 240 num_per_slave = 10 241 skia_revision = 'abc123' 242 path_config = 'kitchen' 243 244 yield( 245 api.test('CT_DM_10k_SKPs') + 246 api.properties( 247 buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_10k_SKPs', 248 path_config=path_config, 249 swarm_out_dir='[SWARM_OUT_DIR]', 250 ct_num_slaves=ct_num_slaves, 251 num_per_slave=num_per_slave, 252 repository='https://skia.googlesource.com/skia.git', 253 revision=skia_revision, 254 ) 255 ) 256 257 yield( 258 api.test('CT_IMG_DECODE_10k_SKPs') + 259 api.properties( 260 buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_' 261 '10k_SKPs', 262 path_config=path_config, 263 swarm_out_dir='[SWARM_OUT_DIR]', 264 ct_num_slaves=ct_num_slaves, 265 num_per_slave=num_per_slave, 266 repository='https://skia.googlesource.com/skia.git', 267 revision=skia_revision, 268 ) 269 ) 270 271 yield( 272 api.test('CT_DM_100k_SKPs') + 273 api.properties( 274 buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_100k_SKPs', 275 path_config=path_config, 276 swarm_out_dir='[SWARM_OUT_DIR]', 277 ct_num_slaves=ct_num_slaves, 278 num_per_slave=num_per_slave, 279 repository='https://skia.googlesource.com/skia.git', 280 revision=skia_revision, 281 ) 282 ) 283 284 yield( 285 api.test('CT_IMG_DECODE_100k_SKPs') + 286 api.properties( 287 buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_' 288 '100k_SKPs', 289 path_config=path_config, 290 swarm_out_dir='[SWARM_OUT_DIR]', 291 ct_num_slaves=ct_num_slaves, 292 num_per_slave=num_per_slave, 293 repository='https://skia.googlesource.com/skia.git', 294 revision=skia_revision, 295 ) 296 ) 297 298 yield( 299 api.test('CT_GPU_BENCH_1k_SKPs') + 300 api.properties( 301 buildername= 302 'Perf-Ubuntu-GCC-Golo-GPU-GT610-x86_64-Release-CT_BENCH_1k_SKPs', 303 path_config=path_config, 304 swarm_out_dir='[SWARM_OUT_DIR]', 305 ct_num_slaves=ct_num_slaves, 306 num_per_slave=num_per_slave, 307 repository='https://skia.googlesource.com/skia.git', 308 revision=skia_revision, 309 ) + 310 api.path.exists( 311 api.path['start_dir'].join('skia'), 312 api.path['start_dir'].join('src') 313 ) 314 ) 315 316 yield( 317 api.test('CT_CPU_BENCH_10k_SKPs') + 318 api.properties( 319 buildername= 320 'Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-CT_BENCH_10k_SKPs', 321 path_config=path_config, 322 swarm_out_dir='[SWARM_OUT_DIR]', 323 ct_num_slaves=ct_num_slaves, 324 num_per_slave=num_per_slave, 325 repository='https://skia.googlesource.com/skia.git', 326 revision=skia_revision, 327 ) + 328 api.path.exists( 329 api.path['start_dir'].join('skia'), 330 api.path['start_dir'].join('src') 331 ) 332 ) 333 334 yield( 335 api.test('CT_GPU_BENCH_10k_SKPs') + 336 api.properties( 337 buildername= 338 'Perf-Ubuntu-GCC-Golo-GPU-GT610-x86_64-Release-CT_BENCH_10k_SKPs', 339 path_config=path_config, 340 swarm_out_dir='[SWARM_OUT_DIR]', 341 ct_num_slaves=ct_num_slaves, 342 num_per_slave=num_per_slave, 343 repository='https://skia.googlesource.com/skia.git', 344 revision=skia_revision, 345 ) + 346 api.path.exists( 347 api.path['start_dir'].join('skia'), 348 api.path['start_dir'].join('src') 349 ) 350 ) 351 352 yield( 353 api.test('CT_DM_1m_SKPs') + 354 api.properties( 355 buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs', 356 path_config=path_config, 357 swarm_out_dir='[SWARM_OUT_DIR]', 358 ct_num_slaves=ct_num_slaves, 359 num_per_slave=num_per_slave, 360 repository='https://skia.googlesource.com/skia.git', 361 revision=skia_revision, 362 ) 363 ) 364 365 yield ( 366 api.test('CT_DM_SKPs_UnknownBuilder') + 367 api.properties( 368 buildername= 369 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_UnknownRepo_SKPs', 370 path_config=path_config, 371 swarm_out_dir='[SWARM_OUT_DIR]', 372 ct_num_slaves=ct_num_slaves, 373 num_per_slave=num_per_slave, 374 repository='https://skia.googlesource.com/skia.git', 375 revision=skia_revision, 376 ) + 377 api.expect_exception('Exception') 378 ) 379 380 yield ( 381 api.test('CT_10k_SKPs_UnknownBuilder') + 382 api.properties( 383 buildername= 384 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_UnknownTool_10k_SKPs', 385 path_config=path_config, 386 swarm_out_dir='[SWARM_OUT_DIR]', 387 ct_num_slaves=ct_num_slaves, 388 num_per_slave=num_per_slave, 389 repository='https://skia.googlesource.com/skia.git', 390 revision=skia_revision, 391 ) + 392 api.expect_exception('Exception') 393 ) 394 395 yield( 396 api.test('CT_DM_1m_SKPs_slave3_failure') + 397 api.step_data('ct-dm-3', retcode=1) + 398 api.properties( 399 buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs', 400 path_config=path_config, 401 swarm_out_dir='[SWARM_OUT_DIR]', 402 ct_num_slaves=ct_num_slaves, 403 num_per_slave=num_per_slave, 404 repository='https://skia.googlesource.com/skia.git', 405 revision=skia_revision, 406 ) 407 ) 408 409 yield( 410 api.test('CT_DM_1m_SKPs_2slaves_failure') + 411 api.step_data('ct-dm-1', retcode=1) + 412 api.step_data('ct-dm-3', retcode=1) + 413 api.properties( 414 buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs', 415 path_config=path_config, 416 swarm_out_dir='[SWARM_OUT_DIR]', 417 ct_num_slaves=ct_num_slaves, 418 num_per_slave=num_per_slave, 419 repository='https://skia.googlesource.com/skia.git', 420 revision=skia_revision, 421 ) 422 ) 423 424 builder = 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_10k_SKPs' 425 yield( 426 api.test('CT_DM_10k_SKPs_Trybot') + 427 api.properties( 428 buildername=builder, 429 path_config=path_config, 430 swarm_out_dir='[SWARM_OUT_DIR]', 431 ct_num_slaves=ct_num_slaves, 432 num_per_slave=num_per_slave, 433 repository='https://skia.googlesource.com/skia.git', 434 patch_storage='gerrit') + 435 api.properties.tryserver( 436 buildername=builder, 437 gerrit_project='skia', 438 gerrit_url='https://skia-review.googlesource.com/', 439 ) 440 ) 441 442 builder = ('Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_' 443 '10k_SKPs') 444 yield( 445 api.test('CT_IMG_DECODE_10k_SKPs_Trybot') + 446 api.properties( 447 buildername=builder, 448 path_config=path_config, 449 swarm_out_dir='[SWARM_OUT_DIR]', 450 ct_num_slaves=ct_num_slaves, 451 num_per_slave=num_per_slave, 452 repository='https://skia.googlesource.com/skia.git', 453 revision=skia_revision, 454 patch_storage='gerrit') + 455 api.properties.tryserver( 456 buildername=builder, 457 gerrit_project='skia', 458 gerrit_url='https://skia-review.googlesource.com/', 459 ) 460 ) 461