• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2
3# Copyright 2016 Google Inc.
4#
5# Use of this source code is governed by a BSD-style license that can be
6# found in the LICENSE file.
7
8from __future__ import print_function
9from _adb import Adb
10from _benchresult import BenchResult
11from _hardware import HardwareException, Hardware
12from argparse import ArgumentParser
13from multiprocessing import Queue
14from threading import Thread, Timer
15import collections
16import glob
17import math
18import re
19import subprocess
20import sys
21import time
22
23__argparse = ArgumentParser(description="""
24
25Executes the skpbench binary with various configs and skps.
26
27Also monitors the output in order to filter out and re-run results that have an
28unacceptable stddev.
29
30""")
31
32__argparse.add_argument('skpbench',
33  help="path to the skpbench binary")
34__argparse.add_argument('--adb',
35  action='store_true', help="execute skpbench over adb")
36__argparse.add_argument('--adb_binary', default='adb',
37  help="The name of the adb binary to use.")
38__argparse.add_argument('-s', '--device-serial',
39  help="if using adb, ID of the specific device to target "
40       "(only required if more than 1 device is attached)")
41__argparse.add_argument('-m', '--max-stddev',
42  type=float, default=4,
43  help="initial max allowable relative standard deviation")
44__argparse.add_argument('-x', '--suffix',
45  help="suffix to append on config (e.g. '_before', '_after')")
46__argparse.add_argument('-w','--write-path',
47  help="directory to save .png proofs to disk.")
48__argparse.add_argument('-v','--verbosity',
49  type=int, default=1, help="level of verbosity (0=none to 5=debug)")
50__argparse.add_argument('-d', '--duration',
51  type=int, help="number of milliseconds to run each benchmark")
52__argparse.add_argument('-l', '--sample-ms',
53  type=int, help="duration of a sample (minimum)")
54__argparse.add_argument('--gpu',
55  action='store_true',
56  help="perform timing on the gpu clock instead of cpu (gpu work only)")
57__argparse.add_argument('--fps',
58  action='store_true', help="use fps instead of ms")
59__argparse.add_argument('--pr',
60  help="comma- or space-separated list of GPU path renderers, including: "
61       "[[~]all [~]default [~]dashline [~]msaa [~]aaconvex "
62       "[~]aalinearizing [~]small [~]tess]")
63__argparse.add_argument('--cc',
64  action='store_true', help="allow coverage counting shortcuts to render paths")
65__argparse.add_argument('--nocache',
66  action='store_true', help="disable caching of path mask textures")
67__argparse.add_argument('--allPathsVolatile',
68  action='store_true',
69  help="Causes all GPU paths to be processed as if 'setIsVolatile' had been called.")
70__argparse.add_argument('-c', '--config',
71  default='gl', help="comma- or space-separated list of GPU configs")
72__argparse.add_argument('-a', '--resultsfile',
73  help="optional file to append results into")
74__argparse.add_argument('--ddl',
75  action='store_true', help="record the skp into DDLs before rendering")
76__argparse.add_argument('--lock-clocks',
77  action='store_true', help="Put device in benchmarking mode (locked clocks, no other processes)")
78__argparse.add_argument('--clock-speed',
79  type=float, default=66.0, help="A number between 0 and 100 indicating how fast to lock the CPU and GPU clock."
80  "Valid speeds are chosen from their respective available frequencies list.")
81__argparse.add_argument('--ddlNumRecordingThreads',
82  type=int, default=0,
83  help="number of DDL recording threads (0=num_cores)")
84__argparse.add_argument('--ddlTilingWidthHeight',
85  type=int, default=0, help="number of tiles along one edge when in DDL mode")
86__argparse.add_argument('--dontReduceOpsTaskSplitting',
87  action='store_true', help="don't reorder GPU tasks to reduce render target swaps")
88__argparse.add_argument('--gpuThreads',
89  type=int, default=-1,
90  help="Create this many extra threads to assist with GPU work, including"
91       " software path rendering. Defaults to two.")
92__argparse.add_argument('--internalSamples',
93  type=int, default=-1,
94  help="Number of samples for internal draws that use MSAA.")
95__argparse.add_argument('srcs',
96  nargs='+',
97  help=".skp files or directories to expand for .skp files, and/or .svg files")
98__argparse.add_argument('--gpuResourceCacheLimit',
99  type=int, default=-1,
100  help="Maximum number of bytes to use for budgeted GPU resources.")
101
102FLAGS = __argparse.parse_args()
103if FLAGS.adb:
104  import _adb_path as _path
105  _path.init(FLAGS.device_serial, FLAGS.adb_binary)
106else:
107  import _os_path as _path
108
109def dump_commandline_if_verbose(commandline):
110  if FLAGS.verbosity >= 5:
111    quoted = ['\'%s\'' % re.sub(r'([\\\'])', r'\\\1', x) for x in commandline]
112    print(' '.join(quoted), file=sys.stderr)
113
114
115class StddevException(Exception):
116  pass
117
118class Message:
119  READLINE = 0,
120  POLL_HARDWARE = 1,
121  EXIT = 2
122  def __init__(self, message, value=None):
123    self.message = message
124    self.value = value
125
126class SubprocessMonitor(Thread):
127  def __init__(self, queue, proc):
128    self._queue = queue
129    self._proc = proc
130    Thread.__init__(self)
131
132  def run(self):
133    """Runs on the background thread."""
134    for line in iter(self._proc.stdout.readline, b''):
135      self._queue.put(Message(Message.READLINE, line.decode('utf-8').rstrip()))
136    self._queue.put(Message(Message.EXIT))
137
138class SKPBench:
139  ARGV = [FLAGS.skpbench, '--verbosity', str(FLAGS.verbosity)]
140  if FLAGS.duration:
141    ARGV.extend(['--duration', str(FLAGS.duration)])
142  if FLAGS.sample_ms:
143    ARGV.extend(['--sampleMs', str(FLAGS.sample_ms)])
144  if FLAGS.gpu:
145    ARGV.extend(['--gpuClock', 'true'])
146  if FLAGS.fps:
147    ARGV.extend(['--fps', 'true'])
148  if FLAGS.pr:
149    ARGV.extend(['--pr'] + re.split(r'[ ,]', FLAGS.pr))
150  if FLAGS.cc:
151    ARGV.extend(['--cc', 'true'])
152  if FLAGS.nocache:
153    ARGV.extend(['--cachePathMasks', 'false'])
154  if FLAGS.allPathsVolatile:
155    ARGV.extend(['--allPathsVolatile', 'true'])
156  if FLAGS.gpuThreads != -1:
157    ARGV.extend(['--gpuThreads', str(FLAGS.gpuThreads)])
158  if FLAGS.internalSamples != -1:
159    ARGV.extend(['--internalSamples', str(FLAGS.internalSamples)])
160
161  # DDL parameters
162  if FLAGS.ddl:
163    ARGV.extend(['--ddl', 'true'])
164  if FLAGS.ddlNumRecordingThreads:
165    ARGV.extend(['--ddlNumRecordingThreads',
166                 str(FLAGS.ddlNumRecordingThreads)])
167  if FLAGS.ddlTilingWidthHeight:
168    ARGV.extend(['--ddlTilingWidthHeight', str(FLAGS.ddlTilingWidthHeight)])
169
170  if FLAGS.dontReduceOpsTaskSplitting:
171    ARGV.extend(['--dontReduceOpsTaskSplitting'])
172
173  if FLAGS.gpuResourceCacheLimit:
174    ARGV.extend(['--gpuResourceCacheLimit', str(FLAGS.gpuResourceCacheLimit)])
175
176  if FLAGS.adb:
177    if FLAGS.device_serial is None:
178      ARGV[:0] = [FLAGS.adb_binary, 'shell']
179    else:
180      ARGV[:0] = [FLAGS.adb_binary, '-s', FLAGS.device_serial, 'shell']
181
182  @classmethod
183  def get_header(cls, outfile=sys.stdout):
184    commandline = cls.ARGV + ['--duration', '0']
185    dump_commandline_if_verbose(commandline)
186    out = subprocess.check_output(commandline, stderr=subprocess.STDOUT)
187    return out.rstrip()
188
189  @classmethod
190  def run_warmup(cls, warmup_time, config):
191    if not warmup_time:
192      return
193    print('running %i second warmup...' % warmup_time, file=sys.stderr)
194    commandline = cls.ARGV + ['--duration', str(warmup_time * 1000),
195                              '--config', config,
196                              '--src', 'warmup']
197    dump_commandline_if_verbose(commandline)
198    output = subprocess.check_output(commandline, stderr=subprocess.STDOUT)
199
200    # validate the warmup run output.
201    for line in output.decode('utf-8').split('\n'):
202      match = BenchResult.match(line.rstrip())
203      if match and match.bench == 'warmup':
204        return
205    raise Exception('Invalid warmup output:\n%s' % output)
206
207  def __init__(self, src, config, max_stddev, best_result=None):
208    self.src = src
209    self.config = config
210    self.max_stddev = max_stddev
211    self.best_result = best_result
212    self._queue = Queue()
213    self._proc = None
214    self._monitor = None
215    self._hw_poll_timer = None
216
217  def __enter__(self):
218    return self
219
220  def __exit__(self, exception_type, exception_value, traceback):
221    if self._proc:
222      self.terminate()
223    if self._hw_poll_timer:
224      self._hw_poll_timer.cancel()
225
226  def execute(self, hardware):
227    hardware.sanity_check()
228    self._schedule_hardware_poll()
229
230    commandline = self.ARGV + ['--config', self.config,
231                               '--src', self.src,
232                               '--suppressHeader', 'true']
233    if FLAGS.write_path:
234      pngfile = _path.join(FLAGS.write_path, self.config,
235                           _path.basename(self.src) + '.png')
236      commandline.extend(['--png', pngfile])
237    dump_commandline_if_verbose(commandline)
238    self._proc = subprocess.Popen(commandline, stdout=subprocess.PIPE,
239                                  stderr=subprocess.STDOUT)
240    self._monitor = SubprocessMonitor(self._queue, self._proc)
241    self._monitor.start()
242
243    while True:
244      message = self._queue.get()
245      if message.message == Message.READLINE:
246        result = BenchResult.match(message.value)
247        if result:
248          hardware.sanity_check()
249          self._process_result(result)
250        elif hardware.filter_line(message.value):
251          print(message.value, file=sys.stderr)
252        continue
253      if message.message == Message.POLL_HARDWARE:
254        hardware.sanity_check()
255        self._schedule_hardware_poll()
256        continue
257      if message.message == Message.EXIT:
258        self._monitor.join()
259        self._proc.wait()
260        if self._proc.returncode != 0:
261          raise Exception("skpbench exited with nonzero exit code %i" %
262                          self._proc.returncode)
263        self._proc = None
264        break
265
266  def _schedule_hardware_poll(self):
267    if self._hw_poll_timer:
268      self._hw_poll_timer.cancel()
269    self._hw_poll_timer = \
270      Timer(1, lambda: self._queue.put(Message(Message.POLL_HARDWARE)))
271    self._hw_poll_timer.start()
272
273  def _process_result(self, result):
274    if not self.best_result or result.stddev <= self.best_result.stddev:
275      self.best_result = result
276    elif FLAGS.verbosity >= 2:
277      print("reusing previous result for %s/%s with lower stddev "
278            "(%s%% instead of %s%%)." %
279            (result.config, result.bench, self.best_result.stddev,
280             result.stddev), file=sys.stderr)
281    if self.max_stddev and self.best_result.stddev > self.max_stddev:
282      raise StddevException()
283
284  def terminate(self):
285    if self._proc:
286      self._proc.terminate()
287      self._monitor.join()
288      self._proc.wait()
289      self._proc = None
290
291def emit_result(line, resultsfile=None):
292  print(line)
293  sys.stdout.flush()
294  if resultsfile:
295    print(line, file=resultsfile)
296    resultsfile.flush()
297
298def run_benchmarks(configs, srcs, hardware, resultsfile=None):
299  hasheader = False
300  benches = collections.deque([(src, config, FLAGS.max_stddev)
301                               for src in srcs
302                               for config in configs])
303  while benches:
304    try:
305      with hardware:
306        SKPBench.run_warmup(hardware.warmup_time, configs[0])
307        if not hasheader:
308          emit_result(SKPBench.get_header(), resultsfile)
309          hasheader = True
310        while benches:
311          benchargs = benches.popleft()
312          with SKPBench(*benchargs) as skpbench:
313            try:
314              skpbench.execute(hardware)
315              if skpbench.best_result:
316                emit_result(skpbench.best_result.format(FLAGS.suffix),
317                            resultsfile)
318              else:
319                print("WARNING: no result for %s with config %s" %
320                      (skpbench.src, skpbench.config), file=sys.stderr)
321
322            except StddevException:
323              retry_max_stddev = skpbench.max_stddev * math.sqrt(2)
324              if FLAGS.verbosity >= 1:
325                print("stddev is too high for %s/%s (%s%%, max=%.2f%%), "
326                      "re-queuing with max=%.2f%%." %
327                      (skpbench.best_result.config, skpbench.best_result.bench,
328                       skpbench.best_result.stddev, skpbench.max_stddev,
329                       retry_max_stddev),
330                      file=sys.stderr)
331              benches.append((skpbench.src, skpbench.config, retry_max_stddev,
332                              skpbench.best_result))
333
334            except HardwareException as exception:
335              skpbench.terminate()
336              if FLAGS.verbosity >= 4:
337                hardware.print_debug_diagnostics()
338              if FLAGS.verbosity >= 1:
339                print("%s; rebooting and taking a %i second nap..." %
340                      (exception.message, exception.sleeptime), file=sys.stderr)
341              benches.appendleft(benchargs) # retry the same bench next time.
342              raise # wake hw up from benchmarking mode before the nap.
343
344    except HardwareException as exception:
345      time.sleep(exception.sleeptime)
346
347def main():
348  # Delimiter is ',' or ' ', skip if nested inside parens (e.g. gpu(a=b,c=d)).
349  DELIMITER = r'[, ](?!(?:[^(]*\([^)]*\))*[^()]*\))'
350  configs = re.split(DELIMITER, FLAGS.config)
351  srcs = _path.find_skps(FLAGS.srcs)
352  assert srcs
353
354
355  if FLAGS.adb:
356    adb = Adb(FLAGS.device_serial, FLAGS.adb_binary,
357              echo=(FLAGS.verbosity >= 5))
358    from _hardware_android import HardwareAndroid
359
360    model = adb.check('getprop ro.product.model').strip()
361    if model == 'Pixel C':
362      from _hardware_pixel_c import HardwarePixelC
363      hardware = HardwarePixelC(adb)
364    elif model == 'Pixel' or model == "Pixel XL":
365      from _hardware_pixel import HardwarePixel
366      hardware = HardwarePixel(adb)
367    elif model == 'Pixel 2':
368      from _hardware_pixel2 import HardwarePixel2
369      hardware = HardwarePixel2(adb)
370    elif model == 'Nexus 6P':
371      from _hardware_nexus_6p import HardwareNexus6P
372      hardware = HardwareNexus6P(adb)
373    else:
374      print("WARNING: %s: don't know how to monitor this hardware; results "
375            "may be unreliable." % model, file=sys.stderr)
376      hardware = HardwareAndroid(adb)
377
378    if FLAGS.lock_clocks:
379      hardware.__enter__()
380      print("Entered benchmarking mode, not running benchmarks. Reboot to restore.");
381      return;
382
383    if FLAGS.clock_speed:
384      hardware.setDesiredClock(FLAGS.clock_speed)
385  else:
386    hardware = Hardware()
387
388  if FLAGS.resultsfile:
389    with open(FLAGS.resultsfile, mode='a+') as resultsfile:
390      run_benchmarks(configs, srcs, hardware, resultsfile=resultsfile)
391  else:
392    run_benchmarks(configs, srcs, hardware)
393
394
395if __name__ == '__main__':
396  main()
397