1# Copyright 2016 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5 6# Recipe module for Skia Swarming perf. 7 8 9import calendar 10import os 11 12 13DEPS = [ 14 'env', 15 'flavor', 16 'recipe_engine/file', 17 'recipe_engine/json', 18 'recipe_engine/path', 19 'recipe_engine/platform', 20 'recipe_engine/properties', 21 'recipe_engine/raw_io', 22 'recipe_engine/step', 23 'recipe_engine/time', 24 'run', 25 'vars', 26] 27 28 29def upload_perf_results(buildername): 30 if 'Release' not in buildername: 31 return False 32 skip_upload_bots = [ 33 'ASAN', 34 'Coverage', 35 'MSAN', 36 'TSAN', 37 'UBSAN', 38 'Valgrind', 39 ] 40 for s in skip_upload_bots: 41 if s in buildername: 42 return False 43 return True 44 45 46def nanobench_flags(api, bot): 47 args = ['--pre_log'] 48 49 if 'GPU' in bot: 50 args.append('--images') 51 args.extend(['--gpuStatsDump', 'true']) 52 53 args.extend(['--scales', '1.0', '1.1']) 54 55 if 'iOS' in bot: 56 args.extend(['--skps', 'ignore_skps']) 57 58 configs = [] 59 if api.vars.builder_cfg.get('cpu_or_gpu') == 'CPU': 60 args.append('--nogpu') 61 configs.extend(['8888', 'nonrendering']) 62 63 if 'BonusConfigs' in bot or ('SAN' in bot and 'GCE' in bot): 64 configs += [ 65 'f16', 66 'srgb', 67 'esrgb', 68 'narrow', 69 'enarrow', 70 ] 71 72 elif api.vars.builder_cfg.get('cpu_or_gpu') == 'GPU': 73 args.append('--nocpu') 74 75 gl_prefix = 'gl' 76 sample_count = '8' 77 if 'Android' in bot or 'iOS' in bot: 78 sample_count = '4' 79 # The NVIDIA_Shield has a regular OpenGL implementation. We bench that 80 # instead of ES. 81 if 'NVIDIA_Shield' not in bot: 82 gl_prefix = 'gles' 83 # The NP produces a long error stream when we run with MSAA. 84 # iOS crashes (skia:6399) 85 # Nexus7 (Tegra3) does not support MSAA. 86 if ('NexusPlayer' in bot or 87 'iOS' in bot or 88 'Nexus7' in bot): 89 sample_count = '' 90 elif 'Intel' in bot: 91 sample_count = '' 92 elif 'ChromeOS' in bot: 93 gl_prefix = 'gles' 94 95 configs.extend([gl_prefix, gl_prefix + 'srgb']) 96 if sample_count is not '': 97 configs.append(gl_prefix + 'msaa' + sample_count) 98 if ('TegraX1' in bot or 99 'Quadro' in bot or 100 'GTX' in bot or 101 ('GT610' in bot and 'Ubuntu17' not in bot)): 102 configs.extend([gl_prefix + 'nvpr' + sample_count]) 103 104 # We want to test both the OpenGL config and the GLES config on Linux Intel: 105 # GL is used by Chrome, GLES is used by ChromeOS. 106 if 'Intel' in bot and api.vars.is_linux: 107 configs.extend(['gles', 'glessrgb']) 108 109 if 'CommandBuffer' in bot: 110 configs = ['commandbuffer'] 111 if 'Vulkan' in bot: 112 configs = ['vk'] 113 114 if 'ANGLE' in bot: 115 # Test only ANGLE configs. 116 configs = ['angle_d3d11_es2'] 117 if sample_count is not '': 118 configs.append('angle_d3d11_es2_msaa' + sample_count) 119 if 'QuadroP400' in bot: 120 # See skia:7823 and chromium:693090. 121 configs.append('angle_gl_es2') 122 if sample_count is not '': 123 configs.append('angle_gl_es2_msaa' + sample_count) 124 125 if 'ChromeOS' in bot: 126 # Just run GLES for now - maybe add gles_msaa4 in the future 127 configs = ['gles'] 128 129 args.append('--config') 130 args.extend(configs) 131 132 # By default, we test with GPU threading enabled, unless specifically 133 # disabled. 134 if 'NoGPUThreads' in bot: 135 args.extend(['--gpuThreads', '0']) 136 137 if 'Valgrind' in bot: 138 # Don't care about Valgrind performance. 139 args.extend(['--loops', '1']) 140 args.extend(['--samples', '1']) 141 # Ensure that the bot framework does not think we have timed out. 142 args.extend(['--keepAlive', 'true']) 143 144 if ('QuadroP400' in bot or 145 'Adreno540' in bot or 146 'IntelHD2000' in bot or # gen 6 - sandy bridge 147 'IntelHD4400' in bot or # gen 7 - haswell 148 'IntelHD405' in bot or # gen 8 - cherryview braswell 149 'IntelIris6100' in bot or # gen 8 - broadwell 150 'IntelIris540' in bot or # gen 9 - skylake 151 'IntelIris640' in bot or # gen 9 - kaby lake 152 'IntelIris655' in bot or # gen 9 - coffee lake 153 'MaliT760' in bot or 154 'MaliT860' in bot or 155 'MaliT880' in bot): 156 args.extend(['--reduceOpListSplitting']) 157 158 # Some people don't like verbose output. 159 verbose = False 160 161 match = [] 162 if 'Android' in bot: 163 # Segfaults when run as GPU bench. Very large texture? 164 match.append('~blurroundrect') 165 match.append('~patch_grid') # skia:2847 166 match.append('~desk_carsvg') 167 if 'Nexus5' in bot: 168 match.append('~keymobi_shop_mobileweb_ebay_com.skp') # skia:5178 169 if 'iOS' in bot: 170 match.append('~blurroundrect') 171 match.append('~patch_grid') # skia:2847 172 match.append('~desk_carsvg') 173 match.append('~keymobi') 174 match.append('~path_hairline') 175 match.append('~GLInstancedArraysBench') # skia:4714 176 if 'MoltenVK' in bot: 177 # skbug.com/7962 178 match.append('~^path_text_clipped_uncached$') 179 match.append('~^path_text_uncached$') 180 if ('Intel' in bot and api.vars.is_linux and not 'Vulkan' in bot): 181 # TODO(dogben): Track down what's causing bots to die. 182 verbose = True 183 if 'IntelHD405' in bot and api.vars.is_linux and 'Vulkan' in bot: 184 # skia:7322 185 match.append('~desk_carsvg.skp_1') 186 match.append('~desk_googlehome.skp') 187 match.append('~desk_tiger8svg.skp_1') 188 match.append('~desk_wowwiki.skp') 189 match.append('~desk_ynevsvg.skp_1.1') 190 match.append('~desk_nostroke_tiger8svg.skp') 191 match.append('~keymobi_booking_com.skp_1') 192 match.append('~keymobi_booking_com.skp_1_mpd') 193 match.append('~keymobi_cnn_article.skp_1') 194 match.append('~keymobi_cnn_article.skp_1_mpd') 195 match.append('~keymobi_forecast_io.skp_1') 196 match.append('~keymobi_forecast_io.skp_1_mpd') 197 match.append('~keymobi_sfgate.skp_1') 198 match.append('~keymobi_techcrunch_com.skp_1.1') 199 match.append('~keymobi_techcrunch.skp_1.1') 200 match.append('~keymobi_techcrunch.skp_1.1_mpd') 201 match.append('~svgparse_Seal_of_California.svg_1.1') 202 match.append('~svgparse_NewYork-StateSeal.svg_1.1') 203 match.append('~svgparse_Vermont_state_seal.svg_1') 204 match.append('~tabl_gamedeksiam.skp_1.1') 205 match.append('~tabl_pravda.skp_1') 206 match.append('~top25desk_ebay_com.skp_1.1') 207 match.append('~top25desk_ebay.skp_1.1') 208 match.append('~top25desk_ebay.skp_1.1_mpd') 209 if 'MacBook10.1' in bot and 'CommandBuffer' in bot: 210 match.append('~^desk_micrographygirlsvg.skp_1.1$') 211 if 'IntelIris655' in bot and 'Win10' in bot and 'Vulkan' in bot: 212 # skia:8587 213 match.append('~^GM_varied_text_clipped_lcd$') 214 match.append('~^GM_varied_text_ignorable_clip_lcd$') 215 match.append('~^fontscaler_lcd$') 216 match.append('~^rotated_rects_aa_changing_transparent_src$') 217 match.append('~^rotated_rects_aa_same_transparent_src$') 218 match.append('~^srcmode_rects_1_aa$') 219 match.append('~^desk_skbug6850overlay2.skp_1$') 220 match.append('~^desk_skbug6850overlay2.skp_1.1$') 221 match.append('~^desk_skbug6850overlay2.skp_1.1_mpd$') 222 match.append('~^desk_skbug6850overlay2.skp_1_mpd$') 223 # skia:8659 224 match.append('~^blendmode_mask_DstATop$') 225 match.append('~^blendmode_mask_Src$') 226 match.append('~^blendmode_mask_SrcIn$') 227 match.append('~^blendmode_mask_SrcOut$') 228 match.append('~^desk_carsvg.skp_1$') 229 match.append('~^desk_carsvg.skp_1.1$') 230 match.append('~^desk_carsvg.skp_1.1_mpd$') 231 match.append('~^desk_carsvg.skp_1_mpd$') 232 match.append('~^desk_googlespreadsheet.skp_1$') 233 match.append('~^desk_googlespreadsheet.skp_1.1$') 234 match.append('~^desk_googlespreadsheet.skp_1.1_mpd$') 235 match.append('~^desk_googlespreadsheet.skp_1_mpd$') 236 if 'Release' in bot: 237 match.append('~^rotated_rects_aa_alternating_transparent_and_opaque_src$') 238 match.append('~^shadermask_LCD_FF$') 239 match.append('~^text_16_LCD_88$') 240 match.append('~^text_16_LCD_BK$') 241 match.append('~^text_16_LCD_FF$') 242 match.append('~^text_16_LCD_WT$') 243 if ('ASAN' in bot or 'UBSAN' in bot) and 'CPU' in bot: 244 # floor2int_undef benches undefined behavior, so ASAN correctly complains. 245 match.append('~^floor2int_undef$') 246 if (('Iris655' in bot or 'Iris540' in bot) and 'Release' in bot and 247 'Win10' in bot and 'Vulkan' not in bot and 'ANGLE' not in bot): 248 # skia:8706 249 match.append('~^top25desk_techcrunch.skp_1_mpd$') 250 match.append('~^top25desk_techcrunch.skp_1$') 251 match.append('~^top25desk_techcrunch.skp_1.1_mpd$') 252 match.append('~^top25desk_techcrunch.skp_1.1$') 253 254 # We do not need or want to benchmark the decodes of incomplete images. 255 # In fact, in nanobench we assert that the full image decode succeeds. 256 match.append('~inc0.gif') 257 match.append('~inc1.gif') 258 match.append('~incInterlaced.gif') 259 match.append('~inc0.jpg') 260 match.append('~incGray.jpg') 261 match.append('~inc0.wbmp') 262 match.append('~inc1.wbmp') 263 match.append('~inc0.webp') 264 match.append('~inc1.webp') 265 match.append('~inc0.ico') 266 match.append('~inc1.ico') 267 match.append('~inc0.png') 268 match.append('~inc1.png') 269 match.append('~inc2.png') 270 match.append('~inc12.png') 271 match.append('~inc13.png') 272 match.append('~inc14.png') 273 match.append('~inc0.webp') 274 match.append('~inc1.webp') 275 276 if match: 277 args.append('--match') 278 args.extend(match) 279 280 if verbose: 281 args.append('--verbose') 282 283 return args 284 285 286def perf_steps(api): 287 """Run Skia benchmarks.""" 288 b = api.properties['buildername'] 289 if upload_perf_results(b): 290 api.flavor.create_clean_device_dir( 291 api.flavor.device_dirs.perf_data_dir) 292 293 # Run nanobench. 294 properties = [ 295 '--properties', 296 'gitHash', api.properties['revision'], 297 ] 298 if api.vars.is_trybot: 299 properties.extend([ 300 'issue', api.vars.issue, 301 'patchset', api.vars.patchset, 302 'patch_storage', api.vars.patch_storage, 303 ]) 304 properties.extend(['swarming_bot_id', api.vars.swarming_bot_id]) 305 properties.extend(['swarming_task_id', api.vars.swarming_task_id]) 306 307 target = 'nanobench' 308 args = [ 309 target, 310 '-i', api.flavor.device_dirs.resource_dir, 311 '--skps', api.flavor.device_dirs.skp_dir, 312 '--images', api.flavor.device_path_join( 313 api.flavor.device_dirs.images_dir, 'nanobench'), 314 ] 315 316 # Do not run svgs on Valgrind. 317 if 'Valgrind' not in api.vars.builder_name: 318 args.extend(['--svgs', api.flavor.device_dirs.svg_dir]) 319 320 args.extend(nanobench_flags(api, api.vars.builder_name)) 321 322 if 'Chromecast' in api.vars.builder_cfg.get('os', ''): 323 # Due to limited disk space, run a watered down perf run on Chromecast. 324 args = [target] 325 if api.vars.builder_cfg.get('cpu_or_gpu') == 'CPU': 326 args.extend(['--nogpu', '--config', '8888']) 327 elif api.vars.builder_cfg.get('cpu_or_gpu') == 'GPU': 328 args.extend(['--nocpu', '--config', 'gles']) 329 args.extend([ 330 '-i', api.flavor.device_dirs.resource_dir, 331 '--images', api.flavor.device_path_join( 332 api.flavor.device_dirs.resource_dir, 'images', 'color_wheel.jpg'), 333 '--skps', api.flavor.device_dirs.skp_dir, 334 '--pre_log', 335 '--match', # skia:6687 336 '~matrixconvolution', 337 '~blur_image_filter', 338 '~blur_0.01', 339 '~GM_animated-image-blurs', 340 '~blendmode_mask_', 341 '~desk_carsvg.skp', 342 '~^path_text_clipped', # Bot times out; skia:7190 343 '~shapes_rrect_inner_rrect_50_500x500', # skia:7551 344 '~compositing_images', 345 ]) 346 347 if upload_perf_results(b): 348 now = api.time.utcnow() 349 ts = int(calendar.timegm(now.utctimetuple())) 350 json_path = api.flavor.device_path_join( 351 api.flavor.device_dirs.perf_data_dir, 352 'nanobench_%s_%d.json' % (api.properties['revision'], ts)) 353 args.extend(['--outResultsFile', json_path]) 354 args.extend(properties) 355 356 keys_blacklist = ['configuration', 'role', 'test_filter'] 357 args.append('--key') 358 for k in sorted(api.vars.builder_cfg.keys()): 359 if not k in keys_blacklist: 360 args.extend([k, api.vars.builder_cfg[k]]) 361 362 # See skia:2789. 363 if 'AbandonGpuContext' in api.vars.extra_tokens: 364 args.extend(['--abandonGpuContext']) 365 366 api.run(api.flavor.step, target, cmd=args, 367 abort_on_failure=False) 368 369 # Copy results to swarming out dir. 370 if upload_perf_results(b): 371 api.file.ensure_directory( 372 'makedirs perf_dir', 373 api.flavor.host_dirs.perf_data_dir) 374 api.flavor.copy_directory_contents_to_host( 375 api.flavor.device_dirs.perf_data_dir, 376 api.flavor.host_dirs.perf_data_dir) 377 378 379def RunSteps(api): 380 api.vars.setup() 381 api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir) 382 api.flavor.setup() 383 384 env = {} 385 if 'iOS' in api.vars.builder_name: 386 env['IOS_BUNDLE_ID'] = 'com.google.nanobench' 387 env['IOS_MOUNT_POINT'] = api.vars.slave_dir.join('mnt_iosdevice') 388 with api.env(env): 389 try: 390 if 'Chromecast' in api.vars.builder_name: 391 api.flavor.install(resources=True, skps=True) 392 else: 393 api.flavor.install(skps=True, images=True, svgs=True, resources=True) 394 perf_steps(api) 395 finally: 396 api.flavor.cleanup_steps() 397 api.run.check_failure() 398 399 400TEST_BUILDERS = [ 401 'Perf-Android-Clang-Nexus5-GPU-Adreno330-arm-Debug-All-Android', 402 ('Perf-Android-Clang-Nexus5x-GPU-Adreno418-arm64-Release-All-' 403 'Android_NoGPUThreads'), 404 'Perf-ChromeOS-Clang-ASUSChromebookFlipC100-GPU-MaliT764-arm-Release-All', 405 'Perf-Chromecast-Clang-Chorizo-CPU-Cortex_A7-arm-Debug-All', 406 'Perf-Chromecast-Clang-Chorizo-GPU-Cortex_A7-arm-Release-All', 407 'Perf-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All', 408 'Perf-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-ASAN', 409 'Perf-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-BonusConfigs', 410 'Perf-Debian9-Clang-NUC5PPYH-GPU-IntelHD405-x86_64-Debug-All-Vulkan', 411 'Perf-Debian9-Clang-NUC7i5BNK-GPU-IntelIris640-x86_64-Release-All', 412 ('Perf-Mac10.13-Clang-MacBook10.1-GPU-IntelHD615-x86_64-Release-All-' 413 'CommandBuffer'), 414 ('Perf-Mac10.13-Clang-MacBookPro11.5-GPU-RadeonHD8870M-x86_64-Release-All-' 415 'MoltenVK_Vulkan'), 416 ('Perf-Mac10.13-Clang-MacMini7.1-GPU-IntelIris5100-x86_64-Release-All-' 417 'CommandBuffer'), 418 ('Perf-Ubuntu17-GCC-Golo-GPU-QuadroP400-x86_64-Release-All-' 419 'Valgrind_AbandonGpuContext_SK_CPU_LIMIT_SSE41'), 420 'Perf-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Release-All-ANGLE', 421 'Perf-Win10-Clang-NUC8i5BEK-GPU-IntelIris655-x86_64-Release-All-Vulkan', 422 'Perf-Win10-Clang-NUC8i5BEK-GPU-IntelIris655-x86_64-Release-All', 423 'Perf-iOS-Clang-iPadPro-GPU-PowerVRGT7800-arm64-Release-All', 424] 425 426 427def GenTests(api): 428 for builder in TEST_BUILDERS: 429 test = ( 430 api.test(builder) + 431 api.properties(buildername=builder, 432 revision='abc123', 433 path_config='kitchen', 434 swarm_out_dir='[SWARM_OUT_DIR]') + 435 api.path.exists( 436 api.path['start_dir'].join('skia'), 437 api.path['start_dir'].join('skia', 'infra', 'bots', 'assets', 438 'skimage', 'VERSION'), 439 api.path['start_dir'].join('skia', 'infra', 'bots', 'assets', 440 'skp', 'VERSION'), 441 api.path['start_dir'].join('tmp', 'uninteresting_hashes.txt') 442 ) + 443 api.step_data('get swarming bot id', 444 stdout=api.raw_io.output('skia-bot-123')) + 445 api.step_data('get swarming task id', 446 stdout=api.raw_io.output('123456')) 447 ) 448 if 'Win' in builder: 449 test += api.platform('win', 64) 450 451 if 'Chromecast' in builder: 452 test += api.step_data( 453 'read chromecast ip', 454 stdout=api.raw_io.output('192.168.1.2:5555')) 455 456 if 'ChromeOS' in builder: 457 test += api.step_data( 458 'read chromeos ip', 459 stdout=api.raw_io.output('{"user_ip":"foo@127.0.0.1"}')) 460 461 yield test 462 463 builder = 'Perf-Win10-Clang-NUCD34010WYKH-GPU-IntelHD4400-x86_64-Release-All' 464 yield ( 465 api.test('trybot') + 466 api.properties(buildername=builder, 467 revision='abc123', 468 path_config='kitchen', 469 swarm_out_dir='[SWARM_OUT_DIR]') + 470 api.properties(patch_storage='gerrit') + 471 api.properties.tryserver( 472 buildername=builder, 473 gerrit_project='skia', 474 gerrit_url='https://skia-review.googlesource.com/', 475 )+ 476 api.path.exists( 477 api.path['start_dir'].join('skia'), 478 api.path['start_dir'].join('skia', 'infra', 'bots', 'assets', 479 'skimage', 'VERSION'), 480 api.path['start_dir'].join('skia', 'infra', 'bots', 'assets', 481 'skp', 'VERSION'), 482 api.path['start_dir'].join('skia', 'infra', 'bots', 'assets', 483 'svg', 'VERSION'), 484 api.path['start_dir'].join('tmp', 'uninteresting_hashes.txt') 485 ) 486 ) 487