• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2016 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5
6# Recipe module for Skia Swarming perf.
7
8
9import calendar
10import os
11
12
13DEPS = [
14  'env',
15  'flavor',
16  'recipe_engine/file',
17  'recipe_engine/json',
18  'recipe_engine/path',
19  'recipe_engine/platform',
20  'recipe_engine/properties',
21  'recipe_engine/raw_io',
22  'recipe_engine/step',
23  'recipe_engine/time',
24  'run',
25  'vars',
26]
27
28
29def upload_perf_results(buildername):
30  if 'Release' not in buildername:
31    return False
32  skip_upload_bots = [
33    'ASAN',
34    'Coverage',
35    'MSAN',
36    'TSAN',
37    'UBSAN',
38    'Valgrind',
39  ]
40  for s in skip_upload_bots:
41    if s in buildername:
42      return False
43  return True
44
45
46def nanobench_flags(api, bot):
47  args = ['--pre_log']
48
49  if 'GPU' in bot:
50    args.append('--images')
51    args.extend(['--gpuStatsDump', 'true'])
52
53  args.extend(['--scales', '1.0', '1.1'])
54
55  if 'iOS' in bot:
56    args.extend(['--skps', 'ignore_skps'])
57
58  configs = []
59  if api.vars.builder_cfg.get('cpu_or_gpu') == 'CPU':
60    args.append('--nogpu')
61    configs.extend(['8888', 'nonrendering'])
62
63    if 'BonusConfigs' in bot or ('SAN' in bot and 'GCE' in bot):
64      configs += [
65          'f16',
66          'srgb',
67          'esrgb',
68          'narrow',
69          'enarrow',
70      ]
71
72  elif api.vars.builder_cfg.get('cpu_or_gpu') == 'GPU':
73    args.append('--nocpu')
74
75    gl_prefix = 'gl'
76    sample_count = '8'
77    if 'Android' in bot or 'iOS' in bot:
78      sample_count = '4'
79      # The NVIDIA_Shield has a regular OpenGL implementation. We bench that
80      # instead of ES.
81      if 'NVIDIA_Shield' not in bot:
82        gl_prefix = 'gles'
83      # The NP produces a long error stream when we run with MSAA.
84      # iOS crashes (skia:6399)
85      # Nexus7 (Tegra3) does not support MSAA.
86      if ('NexusPlayer' in bot or
87          'iOS'         in bot or
88          'Nexus7'      in bot):
89        sample_count = ''
90    elif 'Intel' in bot:
91      sample_count = ''
92    elif 'ChromeOS' in bot:
93      gl_prefix = 'gles'
94
95    configs.extend([gl_prefix, gl_prefix + 'srgb'])
96    if sample_count is not '':
97      configs.append(gl_prefix + 'msaa' + sample_count)
98      if ('TegraX1' in bot or
99          'Quadro' in bot or
100          'GTX' in bot or
101          ('GT610' in bot and 'Ubuntu17' not in bot)):
102        configs.extend([gl_prefix + 'nvpr' + sample_count])
103
104    # We want to test both the OpenGL config and the GLES config on Linux Intel:
105    # GL is used by Chrome, GLES is used by ChromeOS.
106    if 'Intel' in bot and api.vars.is_linux:
107      configs.extend(['gles', 'glessrgb'])
108
109    if 'CommandBuffer' in bot:
110      configs = ['commandbuffer']
111    if 'Vulkan' in bot:
112      configs = ['vk']
113
114    if 'ANGLE' in bot:
115      # Test only ANGLE configs.
116      configs = ['angle_d3d11_es2']
117      if sample_count is not '':
118        configs.append('angle_d3d11_es2_msaa' + sample_count)
119      if 'QuadroP400' in bot:
120        # See skia:7823 and chromium:693090.
121        configs.append('angle_gl_es2')
122        if sample_count is not '':
123          configs.append('angle_gl_es2_msaa' + sample_count)
124
125    if 'ChromeOS' in bot:
126      # Just run GLES for now - maybe add gles_msaa4 in the future
127      configs = ['gles']
128
129  args.append('--config')
130  args.extend(configs)
131
132  # By default, we test with GPU threading enabled, unless specifically
133  # disabled.
134  if 'NoGPUThreads' in bot:
135    args.extend(['--gpuThreads', '0'])
136
137  if 'Valgrind' in bot:
138    # Don't care about Valgrind performance.
139    args.extend(['--loops',   '1'])
140    args.extend(['--samples', '1'])
141    # Ensure that the bot framework does not think we have timed out.
142    args.extend(['--keepAlive', 'true'])
143
144  if ('QuadroP400' in bot or
145      'Adreno540' in bot or
146      'IntelHD2000' in bot or   # gen 6 - sandy bridge
147      'IntelHD4400' in bot or   # gen 7 - haswell
148      'IntelHD405' in bot or    # gen 8 - cherryview braswell
149      'IntelIris6100' in bot or # gen 8 - broadwell
150      'IntelIris540' in bot or  # gen 9 - skylake
151      'IntelIris640' in bot or  # gen 9 - kaby lake
152      'IntelIris655' in bot or  # gen 9 - coffee lake
153      'MaliT760' in bot or
154      'MaliT860' in bot or
155      'MaliT880' in bot):
156    args.extend(['--reduceOpListSplitting'])
157
158  # Some people don't like verbose output.
159  verbose = False
160
161  match = []
162  if 'Android' in bot:
163    # Segfaults when run as GPU bench. Very large texture?
164    match.append('~blurroundrect')
165    match.append('~patch_grid')  # skia:2847
166    match.append('~desk_carsvg')
167  if 'Nexus5' in bot:
168    match.append('~keymobi_shop_mobileweb_ebay_com.skp')  # skia:5178
169  if 'iOS' in bot:
170    match.append('~blurroundrect')
171    match.append('~patch_grid')  # skia:2847
172    match.append('~desk_carsvg')
173    match.append('~keymobi')
174    match.append('~path_hairline')
175    match.append('~GLInstancedArraysBench') # skia:4714
176  if 'MoltenVK' in bot:
177    # skbug.com/7962
178    match.append('~^path_text_clipped_uncached$')
179    match.append('~^path_text_uncached$')
180  if ('Intel' in bot and api.vars.is_linux and not 'Vulkan' in bot):
181    # TODO(dogben): Track down what's causing bots to die.
182    verbose = True
183  if 'IntelHD405' in bot and api.vars.is_linux and 'Vulkan' in bot:
184    # skia:7322
185    match.append('~desk_carsvg.skp_1')
186    match.append('~desk_googlehome.skp')
187    match.append('~desk_tiger8svg.skp_1')
188    match.append('~desk_wowwiki.skp')
189    match.append('~desk_ynevsvg.skp_1.1')
190    match.append('~desk_nostroke_tiger8svg.skp')
191    match.append('~keymobi_booking_com.skp_1')
192    match.append('~keymobi_booking_com.skp_1_mpd')
193    match.append('~keymobi_cnn_article.skp_1')
194    match.append('~keymobi_cnn_article.skp_1_mpd')
195    match.append('~keymobi_forecast_io.skp_1')
196    match.append('~keymobi_forecast_io.skp_1_mpd')
197    match.append('~keymobi_sfgate.skp_1')
198    match.append('~keymobi_techcrunch_com.skp_1.1')
199    match.append('~keymobi_techcrunch.skp_1.1')
200    match.append('~keymobi_techcrunch.skp_1.1_mpd')
201    match.append('~svgparse_Seal_of_California.svg_1.1')
202    match.append('~svgparse_NewYork-StateSeal.svg_1.1')
203    match.append('~svgparse_Vermont_state_seal.svg_1')
204    match.append('~tabl_gamedeksiam.skp_1.1')
205    match.append('~tabl_pravda.skp_1')
206    match.append('~top25desk_ebay_com.skp_1.1')
207    match.append('~top25desk_ebay.skp_1.1')
208    match.append('~top25desk_ebay.skp_1.1_mpd')
209  if 'MacBook10.1' in bot and 'CommandBuffer' in bot:
210    match.append('~^desk_micrographygirlsvg.skp_1.1$')
211  if 'IntelIris655' in bot and 'Win10' in bot and 'Vulkan' in bot:
212    # skia:8587
213    match.append('~^GM_varied_text_clipped_lcd$')
214    match.append('~^GM_varied_text_ignorable_clip_lcd$')
215    match.append('~^fontscaler_lcd$')
216    match.append('~^rotated_rects_aa_changing_transparent_src$')
217    match.append('~^rotated_rects_aa_same_transparent_src$')
218    match.append('~^srcmode_rects_1_aa$')
219    match.append('~^desk_skbug6850overlay2.skp_1$')
220    match.append('~^desk_skbug6850overlay2.skp_1.1$')
221    match.append('~^desk_skbug6850overlay2.skp_1.1_mpd$')
222    match.append('~^desk_skbug6850overlay2.skp_1_mpd$')
223    # skia:8659
224    match.append('~^blendmode_mask_DstATop$')
225    match.append('~^blendmode_mask_Src$')
226    match.append('~^blendmode_mask_SrcIn$')
227    match.append('~^blendmode_mask_SrcOut$')
228    match.append('~^desk_carsvg.skp_1$')
229    match.append('~^desk_carsvg.skp_1.1$')
230    match.append('~^desk_carsvg.skp_1.1_mpd$')
231    match.append('~^desk_carsvg.skp_1_mpd$')
232    match.append('~^desk_googlespreadsheet.skp_1$')
233    match.append('~^desk_googlespreadsheet.skp_1.1$')
234    match.append('~^desk_googlespreadsheet.skp_1.1_mpd$')
235    match.append('~^desk_googlespreadsheet.skp_1_mpd$')
236    if 'Release' in bot:
237      match.append('~^rotated_rects_aa_alternating_transparent_and_opaque_src$')
238      match.append('~^shadermask_LCD_FF$')
239      match.append('~^text_16_LCD_88$')
240      match.append('~^text_16_LCD_BK$')
241      match.append('~^text_16_LCD_FF$')
242      match.append('~^text_16_LCD_WT$')
243  if ('ASAN' in bot or 'UBSAN' in bot) and 'CPU' in bot:
244    # floor2int_undef benches undefined behavior, so ASAN correctly complains.
245    match.append('~^floor2int_undef$')
246  if (('Iris655' in bot or 'Iris540' in bot) and 'Release' in bot and
247      'Win10' in bot and 'Vulkan' not in bot and 'ANGLE' not in bot):
248    # skia:8706
249    match.append('~^top25desk_techcrunch.skp_1_mpd$')
250    match.append('~^top25desk_techcrunch.skp_1$')
251    match.append('~^top25desk_techcrunch.skp_1.1_mpd$')
252    match.append('~^top25desk_techcrunch.skp_1.1$')
253
254  # We do not need or want to benchmark the decodes of incomplete images.
255  # In fact, in nanobench we assert that the full image decode succeeds.
256  match.append('~inc0.gif')
257  match.append('~inc1.gif')
258  match.append('~incInterlaced.gif')
259  match.append('~inc0.jpg')
260  match.append('~incGray.jpg')
261  match.append('~inc0.wbmp')
262  match.append('~inc1.wbmp')
263  match.append('~inc0.webp')
264  match.append('~inc1.webp')
265  match.append('~inc0.ico')
266  match.append('~inc1.ico')
267  match.append('~inc0.png')
268  match.append('~inc1.png')
269  match.append('~inc2.png')
270  match.append('~inc12.png')
271  match.append('~inc13.png')
272  match.append('~inc14.png')
273  match.append('~inc0.webp')
274  match.append('~inc1.webp')
275
276  if match:
277    args.append('--match')
278    args.extend(match)
279
280  if verbose:
281    args.append('--verbose')
282
283  return args
284
285
286def perf_steps(api):
287  """Run Skia benchmarks."""
288  b = api.properties['buildername']
289  if upload_perf_results(b):
290    api.flavor.create_clean_device_dir(
291        api.flavor.device_dirs.perf_data_dir)
292
293  # Run nanobench.
294  properties = [
295    '--properties',
296    'gitHash', api.properties['revision'],
297  ]
298  if api.vars.is_trybot:
299    properties.extend([
300      'issue',    api.vars.issue,
301      'patchset', api.vars.patchset,
302      'patch_storage', api.vars.patch_storage,
303    ])
304  properties.extend(['swarming_bot_id', api.vars.swarming_bot_id])
305  properties.extend(['swarming_task_id', api.vars.swarming_task_id])
306
307  target = 'nanobench'
308  args = [
309      target,
310      '-i',       api.flavor.device_dirs.resource_dir,
311      '--skps',   api.flavor.device_dirs.skp_dir,
312      '--images', api.flavor.device_path_join(
313          api.flavor.device_dirs.images_dir, 'nanobench'),
314  ]
315
316  # Do not run svgs on Valgrind.
317  if 'Valgrind' not in api.vars.builder_name:
318    args.extend(['--svgs',  api.flavor.device_dirs.svg_dir])
319
320  args.extend(nanobench_flags(api, api.vars.builder_name))
321
322  if 'Chromecast' in api.vars.builder_cfg.get('os', ''):
323    # Due to limited disk space, run a watered down perf run on Chromecast.
324    args = [target]
325    if api.vars.builder_cfg.get('cpu_or_gpu') == 'CPU':
326      args.extend(['--nogpu', '--config', '8888'])
327    elif api.vars.builder_cfg.get('cpu_or_gpu') == 'GPU':
328      args.extend(['--nocpu', '--config', 'gles'])
329    args.extend([
330      '-i', api.flavor.device_dirs.resource_dir,
331      '--images', api.flavor.device_path_join(
332          api.flavor.device_dirs.resource_dir, 'images', 'color_wheel.jpg'),
333      '--skps',  api.flavor.device_dirs.skp_dir,
334      '--pre_log',
335      '--match', # skia:6687
336      '~matrixconvolution',
337      '~blur_image_filter',
338      '~blur_0.01',
339      '~GM_animated-image-blurs',
340      '~blendmode_mask_',
341      '~desk_carsvg.skp',
342      '~^path_text_clipped', # Bot times out; skia:7190
343      '~shapes_rrect_inner_rrect_50_500x500', # skia:7551
344      '~compositing_images',
345    ])
346
347  if upload_perf_results(b):
348    now = api.time.utcnow()
349    ts = int(calendar.timegm(now.utctimetuple()))
350    json_path = api.flavor.device_path_join(
351        api.flavor.device_dirs.perf_data_dir,
352        'nanobench_%s_%d.json' % (api.properties['revision'], ts))
353    args.extend(['--outResultsFile', json_path])
354    args.extend(properties)
355
356    keys_blacklist = ['configuration', 'role', 'test_filter']
357    args.append('--key')
358    for k in sorted(api.vars.builder_cfg.keys()):
359      if not k in keys_blacklist:
360        args.extend([k, api.vars.builder_cfg[k]])
361
362  # See skia:2789.
363  if 'AbandonGpuContext' in api.vars.extra_tokens:
364    args.extend(['--abandonGpuContext'])
365
366  api.run(api.flavor.step, target, cmd=args,
367          abort_on_failure=False)
368
369  # Copy results to swarming out dir.
370  if upload_perf_results(b):
371    api.file.ensure_directory(
372        'makedirs perf_dir',
373        api.flavor.host_dirs.perf_data_dir)
374    api.flavor.copy_directory_contents_to_host(
375        api.flavor.device_dirs.perf_data_dir,
376        api.flavor.host_dirs.perf_data_dir)
377
378
379def RunSteps(api):
380  api.vars.setup()
381  api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir)
382  api.flavor.setup()
383
384  env = {}
385  if 'iOS' in api.vars.builder_name:
386    env['IOS_BUNDLE_ID'] = 'com.google.nanobench'
387    env['IOS_MOUNT_POINT'] = api.vars.slave_dir.join('mnt_iosdevice')
388  with api.env(env):
389    try:
390      if 'Chromecast' in api.vars.builder_name:
391        api.flavor.install(resources=True, skps=True)
392      else:
393        api.flavor.install(skps=True, images=True, svgs=True, resources=True)
394      perf_steps(api)
395    finally:
396      api.flavor.cleanup_steps()
397    api.run.check_failure()
398
399
400TEST_BUILDERS = [
401  'Perf-Android-Clang-Nexus5-GPU-Adreno330-arm-Debug-All-Android',
402  ('Perf-Android-Clang-Nexus5x-GPU-Adreno418-arm64-Release-All-'
403   'Android_NoGPUThreads'),
404  'Perf-ChromeOS-Clang-ASUSChromebookFlipC100-GPU-MaliT764-arm-Release-All',
405  'Perf-Chromecast-Clang-Chorizo-CPU-Cortex_A7-arm-Debug-All',
406  'Perf-Chromecast-Clang-Chorizo-GPU-Cortex_A7-arm-Release-All',
407  'Perf-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All',
408  'Perf-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-ASAN',
409  'Perf-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-BonusConfigs',
410  'Perf-Debian9-Clang-NUC5PPYH-GPU-IntelHD405-x86_64-Debug-All-Vulkan',
411  'Perf-Debian9-Clang-NUC7i5BNK-GPU-IntelIris640-x86_64-Release-All',
412  ('Perf-Mac10.13-Clang-MacBook10.1-GPU-IntelHD615-x86_64-Release-All-'
413   'CommandBuffer'),
414  ('Perf-Mac10.13-Clang-MacBookPro11.5-GPU-RadeonHD8870M-x86_64-Release-All-'
415   'MoltenVK_Vulkan'),
416  ('Perf-Mac10.13-Clang-MacMini7.1-GPU-IntelIris5100-x86_64-Release-All-'
417   'CommandBuffer'),
418  ('Perf-Ubuntu17-GCC-Golo-GPU-QuadroP400-x86_64-Release-All-'
419    'Valgrind_AbandonGpuContext_SK_CPU_LIMIT_SSE41'),
420  'Perf-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Release-All-ANGLE',
421  'Perf-Win10-Clang-NUC8i5BEK-GPU-IntelIris655-x86_64-Release-All-Vulkan',
422  'Perf-Win10-Clang-NUC8i5BEK-GPU-IntelIris655-x86_64-Release-All',
423  'Perf-iOS-Clang-iPadPro-GPU-PowerVRGT7800-arm64-Release-All',
424]
425
426
427def GenTests(api):
428  for builder in TEST_BUILDERS:
429    test = (
430      api.test(builder) +
431      api.properties(buildername=builder,
432                     revision='abc123',
433                     path_config='kitchen',
434                     swarm_out_dir='[SWARM_OUT_DIR]') +
435      api.path.exists(
436          api.path['start_dir'].join('skia'),
437          api.path['start_dir'].join('skia', 'infra', 'bots', 'assets',
438                                     'skimage', 'VERSION'),
439          api.path['start_dir'].join('skia', 'infra', 'bots', 'assets',
440                                     'skp', 'VERSION'),
441          api.path['start_dir'].join('tmp', 'uninteresting_hashes.txt')
442      ) +
443      api.step_data('get swarming bot id',
444          stdout=api.raw_io.output('skia-bot-123')) +
445      api.step_data('get swarming task id',
446          stdout=api.raw_io.output('123456'))
447    )
448    if 'Win' in builder:
449      test += api.platform('win', 64)
450
451    if 'Chromecast' in builder:
452      test += api.step_data(
453          'read chromecast ip',
454          stdout=api.raw_io.output('192.168.1.2:5555'))
455
456    if 'ChromeOS' in builder:
457      test += api.step_data(
458          'read chromeos ip',
459          stdout=api.raw_io.output('{"user_ip":"foo@127.0.0.1"}'))
460
461    yield test
462
463  builder = 'Perf-Win10-Clang-NUCD34010WYKH-GPU-IntelHD4400-x86_64-Release-All'
464  yield (
465    api.test('trybot') +
466    api.properties(buildername=builder,
467                   revision='abc123',
468                   path_config='kitchen',
469                   swarm_out_dir='[SWARM_OUT_DIR]') +
470    api.properties(patch_storage='gerrit') +
471    api.properties.tryserver(
472          buildername=builder,
473          gerrit_project='skia',
474          gerrit_url='https://skia-review.googlesource.com/',
475      )+
476    api.path.exists(
477        api.path['start_dir'].join('skia'),
478        api.path['start_dir'].join('skia', 'infra', 'bots', 'assets',
479                                     'skimage', 'VERSION'),
480        api.path['start_dir'].join('skia', 'infra', 'bots', 'assets',
481                                     'skp', 'VERSION'),
482        api.path['start_dir'].join('skia', 'infra', 'bots', 'assets',
483                                     'svg', 'VERSION'),
484        api.path['start_dir'].join('tmp', 'uninteresting_hashes.txt')
485    )
486  )
487