• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2016 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5
6# Recipe module for Skia Swarming perf.
7
8
9import calendar
10import os
11
12
13DEPS = [
14  'env',
15  'flavor',
16  'recipe_engine/file',
17  'recipe_engine/json',
18  'recipe_engine/path',
19  'recipe_engine/platform',
20  'recipe_engine/properties',
21  'recipe_engine/raw_io',
22  'recipe_engine/step',
23  'recipe_engine/time',
24  'run',
25  'vars',
26]
27
28
29def upload_perf_results(buildername):
30  if 'Release' not in buildername:
31    return False
32  skip_upload_bots = [
33    'ASAN',
34    'Coverage',
35    'MSAN',
36    'TSAN',
37    'UBSAN',
38    'Valgrind',
39  ]
40  for s in skip_upload_bots:
41    if s in buildername:
42      return False
43  return True
44
45
46def nanobench_flags(api, bot):
47  args = ['--pre_log']
48
49  if 'GPU' in bot:
50    args.append('--images')
51    args.extend(['--gpuStatsDump', 'true'])
52
53  args.extend(['--scales', '1.0', '1.1'])
54
55  if 'iOS' in bot:
56    args.extend(['--skps', 'ignore_skps'])
57
58  configs = []
59  if api.vars.builder_cfg.get('cpu_or_gpu') == 'CPU':
60    args.append('--nogpu')
61    configs.extend(['8888', 'nonrendering'])
62
63    if 'BonusConfigs' in bot or ('SAN' in bot and 'GCE' in bot):
64      configs += [
65          'f16',
66          'srgb',
67          'esrgb',
68          'narrow',
69          'enarrow',
70      ]
71
72    if 'Nexus7' in bot:
73      args.append('--purgeBetweenBenches')  # Debugging skia:8929
74
75  elif api.vars.builder_cfg.get('cpu_or_gpu') == 'GPU':
76    args.append('--nocpu')
77
78    gl_prefix = 'gl'
79    sample_count = '8'
80    if 'Android' in bot or 'iOS' in bot:
81      sample_count = '4'
82      # The NVIDIA_Shield has a regular OpenGL implementation. We bench that
83      # instead of ES.
84      if 'NVIDIA_Shield' not in bot:
85        gl_prefix = 'gles'
86      # iOS crashes with MSAA (skia:6399)
87      # Nexus7 (Tegra3) does not support MSAA.
88      if ('iOS'         in bot or
89          'Nexus7'      in bot):
90        sample_count = ''
91    elif 'Intel' in bot:
92      # MSAA doesn't work well on Intel GPUs chromium:527565, chromium:983926
93      sample_count = ''
94    elif 'ChromeOS' in bot:
95      gl_prefix = 'gles'
96
97    configs.extend([gl_prefix, gl_prefix + 'srgb'])
98    if sample_count:
99      configs.append(gl_prefix + 'msaa' + sample_count)
100
101    # We want to test both the OpenGL config and the GLES config on Linux Intel:
102    # GL is used by Chrome, GLES is used by ChromeOS.
103    if 'Intel' in bot and api.vars.is_linux:
104      configs.extend(['gles', 'glessrgb'])
105
106    if 'CommandBuffer' in bot:
107      configs = ['commandbuffer']
108
109    if 'Vulkan' in bot:
110      configs = ['vk']
111      if 'Android' in bot:
112        # skbug.com/9274
113        if 'Pixel2XL' not in bot:
114          configs.append('vkmsaa4')
115      else:
116        # MSAA doesn't work well on Intel GPUs chromium:527565, chromium:983926, skia:9023
117        if 'Intel' not in bot:
118          configs.append('vkmsaa8')
119
120    if 'Metal' in bot:
121      configs = ['mtl']
122      if 'iOS' in bot:
123        configs.append('mtlmsaa4')
124      else:
125        configs.append('mtlmsaa8')
126
127    if 'ANGLE' in bot:
128      # Test only ANGLE configs.
129      configs = ['angle_d3d11_es2']
130      if sample_count:
131        configs.append('angle_d3d11_es2_msaa' + sample_count)
132      if 'QuadroP400' in bot:
133        # See skia:7823 and chromium:693090.
134        configs.append('angle_gl_es2')
135        if sample_count:
136          configs.append('angle_gl_es2_msaa' + sample_count)
137
138    if 'ChromeOS' in bot:
139      # Just run GLES for now - maybe add gles_msaa4 in the future
140      configs = ['gles']
141
142  args.append('--config')
143  args.extend(configs)
144
145  # By default, we test with GPU threading enabled, unless specifically
146  # disabled.
147  if 'NoGPUThreads' in bot:
148    args.extend(['--gpuThreads', '0'])
149
150  if 'Debug' in bot or 'ASAN' in bot or 'Valgrind' in bot:
151    args.extend(['--loops',   '1'])
152    args.extend(['--samples', '1'])
153    # Ensure that the bot framework does not think we have timed out.
154    args.extend(['--keepAlive', 'true'])
155
156  # skia:9036
157  if 'NVIDIA_Shield' in bot or 'Chorizo' in bot:
158    args.extend(['--dontReduceOpListSplitting'])
159
160  # Some people don't like verbose output.
161  verbose = False
162
163  match = []
164  if 'Android' in bot:
165    # Segfaults when run as GPU bench. Very large texture?
166    match.append('~blurroundrect')
167    match.append('~patch_grid')  # skia:2847
168    match.append('~desk_carsvg')
169  if 'Nexus5' in bot:
170    match.append('~keymobi_shop_mobileweb_ebay_com.skp')  # skia:5178
171  if 'iOS' in bot:
172    match.append('~blurroundrect')
173    match.append('~patch_grid')  # skia:2847
174    match.append('~desk_carsvg')
175    match.append('~keymobi')
176    match.append('~path_hairline')
177    match.append('~GLInstancedArraysBench') # skia:4714
178  if 'MoltenVK' in bot:
179    # skbug.com/7962
180    match.append('~^path_text_clipped_uncached$')
181    match.append('~^path_text_uncached$')
182  if ('Intel' in bot and api.vars.is_linux and not 'Vulkan' in bot):
183    # TODO(dogben): Track down what's causing bots to die.
184    verbose = True
185  if 'IntelHD405' in bot and api.vars.is_linux and 'Vulkan' in bot:
186    # skia:7322
187    match.append('~desk_carsvg.skp_1')
188    match.append('~desk_googlehome.skp')
189    match.append('~desk_tiger8svg.skp_1')
190    match.append('~desk_wowwiki.skp')
191    match.append('~desk_ynevsvg.skp_1.1')
192    match.append('~desk_nostroke_tiger8svg.skp')
193    match.append('~keymobi_booking_com.skp_1')
194    match.append('~keymobi_booking_com.skp_1_mpd')
195    match.append('~keymobi_cnn_article.skp_1')
196    match.append('~keymobi_cnn_article.skp_1_mpd')
197    match.append('~keymobi_forecast_io.skp_1')
198    match.append('~keymobi_forecast_io.skp_1_mpd')
199    match.append('~keymobi_sfgate.skp_1')
200    match.append('~keymobi_techcrunch_com.skp_1.1')
201    match.append('~keymobi_techcrunch.skp_1.1')
202    match.append('~keymobi_techcrunch.skp_1.1_mpd')
203    match.append('~svgparse_Seal_of_California.svg_1.1')
204    match.append('~svgparse_NewYork-StateSeal.svg_1.1')
205    match.append('~svgparse_Vermont_state_seal.svg_1')
206    match.append('~tabl_gamedeksiam.skp_1.1')
207    match.append('~tabl_pravda.skp_1')
208    match.append('~top25desk_ebay_com.skp_1.1')
209    match.append('~top25desk_ebay.skp_1.1')
210    match.append('~top25desk_ebay.skp_1.1_mpd')
211  if 'Vulkan' in bot and ('Nexus5x' in bot or 'GTX660' in bot):
212    # skia:8523 skia:9271
213    match.append('~compositing_images')
214  if 'MacBook10.1' in bot and 'CommandBuffer' in bot:
215    match.append('~^desk_micrographygirlsvg.skp_1.1$')
216  if ('ASAN' in bot or 'UBSAN' in bot) and 'CPU' in bot:
217    # floor2int_undef benches undefined behavior, so ASAN correctly complains.
218    match.append('~^floor2int_undef$')
219  if 'AcerChromebook13_CB5_311-GPU-TegraK1' in bot:
220    # skia:7551
221    match.append('~^shapes_rrect_inner_rrect_50_500x500$')
222
223  # We do not need or want to benchmark the decodes of incomplete images.
224  # In fact, in nanobench we assert that the full image decode succeeds.
225  match.append('~inc0.gif')
226  match.append('~inc1.gif')
227  match.append('~incInterlaced.gif')
228  match.append('~inc0.jpg')
229  match.append('~incGray.jpg')
230  match.append('~inc0.wbmp')
231  match.append('~inc1.wbmp')
232  match.append('~inc0.webp')
233  match.append('~inc1.webp')
234  match.append('~inc0.ico')
235  match.append('~inc1.ico')
236  match.append('~inc0.png')
237  match.append('~inc1.png')
238  match.append('~inc2.png')
239  match.append('~inc12.png')
240  match.append('~inc13.png')
241  match.append('~inc14.png')
242  match.append('~inc0.webp')
243  match.append('~inc1.webp')
244
245  if match:
246    args.append('--match')
247    args.extend(match)
248
249  if verbose:
250    args.append('--verbose')
251
252  return args
253
254
255def perf_steps(api):
256  """Run Skia benchmarks."""
257  b = api.properties['buildername']
258  if upload_perf_results(b):
259    api.flavor.create_clean_device_dir(
260        api.flavor.device_dirs.perf_data_dir)
261
262  # Run nanobench.
263  properties = [
264    '--properties',
265    'gitHash', api.properties['revision'],
266  ]
267  if api.vars.is_trybot:
268    properties.extend([
269      'issue',    api.vars.issue,
270      'patchset', api.vars.patchset,
271      'patch_storage', api.vars.patch_storage,
272    ])
273  properties.extend(['swarming_bot_id', api.vars.swarming_bot_id])
274  properties.extend(['swarming_task_id', api.vars.swarming_task_id])
275
276  target = 'nanobench'
277  args = [
278      target,
279      '-i',       api.flavor.device_dirs.resource_dir,
280      '--skps',   api.flavor.device_dirs.skp_dir,
281      '--images', api.flavor.device_path_join(
282          api.flavor.device_dirs.images_dir, 'nanobench'),
283  ]
284
285  # Do not run svgs on Valgrind.
286  if 'Valgrind' not in api.vars.builder_name:
287    args.extend(['--svgs',  api.flavor.device_dirs.svg_dir])
288
289  args.extend(nanobench_flags(api, api.vars.builder_name))
290
291  if 'Chromecast' in api.vars.builder_cfg.get('os', ''):
292    # Due to limited disk space, run a watered down perf run on Chromecast.
293    args = [target]
294    if api.vars.builder_cfg.get('cpu_or_gpu') == 'CPU':
295      args.extend(['--nogpu', '--config', '8888'])
296    elif api.vars.builder_cfg.get('cpu_or_gpu') == 'GPU':
297      args.extend(['--nocpu', '--config', 'gles'])
298    args.extend([
299      '-i', api.flavor.device_dirs.resource_dir,
300      '--images', api.flavor.device_path_join(
301          api.flavor.device_dirs.resource_dir, 'images', 'color_wheel.jpg'),
302      '--skps',  api.flavor.device_dirs.skp_dir,
303      '--pre_log',
304      '--dontReduceOpListSplitting',
305      '--match', # skia:6687
306      '~matrixconvolution',
307      '~blur_image_filter',
308      '~blur_0.01',
309      '~GM_animated-image-blurs',
310      '~blendmode_mask_',
311      '~desk_carsvg.skp',
312      '~^path_text_clipped', # Bot times out; skia:7190
313      '~shapes_rrect_inner_rrect_50_500x500', # skia:7551
314      '~compositing_images',
315    ])
316    if 'Debug' in api.vars.builder_name:
317      args.extend(['--loops', '1'])
318
319  if upload_perf_results(b):
320    now = api.time.utcnow()
321    ts = int(calendar.timegm(now.utctimetuple()))
322    json_path = api.flavor.device_path_join(
323        api.flavor.device_dirs.perf_data_dir,
324        'nanobench_%s_%d.json' % (api.properties['revision'], ts))
325    args.extend(['--outResultsFile', json_path])
326    args.extend(properties)
327
328    keys_blacklist = ['configuration', 'role', 'test_filter']
329    args.append('--key')
330    for k in sorted(api.vars.builder_cfg.keys()):
331      if not k in keys_blacklist:
332        args.extend([k, api.vars.builder_cfg[k]])
333
334  api.run(api.flavor.step, target, cmd=args,
335          abort_on_failure=False)
336
337  # Copy results to swarming out dir.
338  if upload_perf_results(b):
339    api.file.ensure_directory(
340        'makedirs perf_dir',
341        api.flavor.host_dirs.perf_data_dir)
342    api.flavor.copy_directory_contents_to_host(
343        api.flavor.device_dirs.perf_data_dir,
344        api.flavor.host_dirs.perf_data_dir)
345
346
347def RunSteps(api):
348  api.vars.setup()
349  api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir)
350  api.flavor.setup()
351
352  env = {}
353  if 'iOS' in api.vars.builder_name:
354    env['IOS_BUNDLE_ID'] = 'com.google.nanobench'
355    env['IOS_MOUNT_POINT'] = api.vars.slave_dir.join('mnt_iosdevice')
356  with api.env(env):
357    try:
358      if 'Chromecast' in api.vars.builder_name:
359        api.flavor.install(resources=True, skps=True)
360      else:
361        api.flavor.install(skps=True, images=True, svgs=True, resources=True)
362      perf_steps(api)
363    finally:
364      api.flavor.cleanup_steps()
365    api.run.check_failure()
366
367
368TEST_BUILDERS = [
369  'Perf-Android-Clang-Nexus7-CPU-Tegra3-arm-Debug-All-Android',
370  'Perf-Android-Clang-Nexus5-GPU-Adreno330-arm-Debug-All-Android',
371  ('Perf-Android-Clang-Nexus5x-GPU-Adreno418-arm64-Release-All-'
372   'Android_NoGPUThreads'),
373  'Perf-Android-Clang-Nexus5x-GPU-Adreno418-arm64-Release-All-Android_Vulkan',
374  'Perf-Android-Clang-NVIDIA_Shield-GPU-TegraX1-arm64-Release-All-Android',
375  'Perf-Android-Clang-P30-GPU-MaliG76-arm64-Release-All-Android_Vulkan',
376  'Perf-ChromeOS-Clang-ASUSChromebookFlipC100-GPU-MaliT764-arm-Release-All',
377  'Perf-ChromeOS-Clang-AcerChromebook13_CB5_311-GPU-TegraK1-arm-Release-All',
378  'Perf-Chromecast-Clang-Chorizo-CPU-Cortex_A7-arm-Debug-All',
379  'Perf-Chromecast-Clang-Chorizo-GPU-Cortex_A7-arm-Release-All',
380  'Perf-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All',
381  'Perf-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-ASAN',
382  'Perf-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-BonusConfigs',
383  'Perf-Debian9-Clang-NUC5PPYH-GPU-IntelHD405-x86_64-Debug-All-Vulkan',
384  'Perf-Debian9-Clang-NUC7i5BNK-GPU-IntelIris640-x86_64-Release-All',
385  ('Perf-Mac10.13-Clang-MacBook10.1-GPU-IntelHD615-x86_64-Release-All-'
386   'CommandBuffer'),
387  ('Perf-Mac10.13-Clang-MacBookPro11.5-GPU-RadeonHD8870M-x86_64-Release-All-'
388   'Metal'),
389  ('Perf-Mac10.13-Clang-MacBookPro11.5-GPU-RadeonHD8870M-x86_64-Release-All-'
390   'MoltenVK_Vulkan'),
391  ('Perf-Mac10.13-Clang-MacMini7.1-GPU-IntelIris5100-x86_64-Release-All-'
392   'CommandBuffer'),
393  ('Perf-Ubuntu17-GCC-Golo-GPU-QuadroP400-x86_64-Release-All-'
394    'Valgrind_SK_CPU_LIMIT_SSE41'),
395  'Perf-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Release-All-ANGLE',
396  'Perf-iOS-Clang-iPadPro-GPU-PowerVRGT7800-arm64-Release-All',
397  'Perf-iOS-Clang-iPhone6-GPU-PowerVRGX6450-arm64-Release-All-Metal',
398]
399
400
401def GenTests(api):
402  for builder in TEST_BUILDERS:
403    test = (
404      api.test(builder) +
405      api.properties(buildername=builder,
406                     revision='abc123',
407                     path_config='kitchen',
408                     swarm_out_dir='[SWARM_OUT_DIR]') +
409      api.path.exists(
410          api.path['start_dir'].join('skia'),
411          api.path['start_dir'].join('skia', 'infra', 'bots', 'assets',
412                                     'skimage', 'VERSION'),
413          api.path['start_dir'].join('skia', 'infra', 'bots', 'assets',
414                                     'skp', 'VERSION'),
415          api.path['start_dir'].join('tmp', 'uninteresting_hashes.txt')
416      ) +
417      api.step_data('get swarming bot id',
418          stdout=api.raw_io.output('skia-bot-123')) +
419      api.step_data('get swarming task id',
420          stdout=api.raw_io.output('123456'))
421    )
422    if 'Win' in builder:
423      test += api.platform('win', 64)
424
425    if 'Chromecast' in builder:
426      test += api.step_data(
427          'read chromecast ip',
428          stdout=api.raw_io.output('192.168.1.2:5555'))
429
430    yield test
431
432  builder = 'Perf-Win10-Clang-NUCD34010WYKH-GPU-IntelHD4400-x86_64-Release-All'
433  yield (
434    api.test('trybot') +
435    api.properties(buildername=builder,
436                   revision='abc123',
437                   path_config='kitchen',
438                   swarm_out_dir='[SWARM_OUT_DIR]') +
439    api.properties(patch_storage='gerrit') +
440    api.properties.tryserver(
441          buildername=builder,
442          gerrit_project='skia',
443          gerrit_url='https://skia-review.googlesource.com/',
444      )+
445    api.path.exists(
446        api.path['start_dir'].join('skia'),
447        api.path['start_dir'].join('skia', 'infra', 'bots', 'assets',
448                                     'skimage', 'VERSION'),
449        api.path['start_dir'].join('skia', 'infra', 'bots', 'assets',
450                                     'skp', 'VERSION'),
451        api.path['start_dir'].join('skia', 'infra', 'bots', 'assets',
452                                     'svg', 'VERSION'),
453        api.path['start_dir'].join('tmp', 'uninteresting_hashes.txt')
454    )
455  )
456