• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2
3# Copyright 2016 Google Inc.
4#
5# Use of this source code is governed by a BSD-style license that can be
6# found in the LICENSE file.
7
8from __future__ import print_function
9from _adb import Adb
10from _benchresult import BenchResult
11from _hardware import HardwareException, Hardware
12from argparse import ArgumentParser
13from multiprocessing import Queue
14from threading import Thread, Timer
15import collections
16import glob
17import math
18import re
19import subprocess
20import sys
21import time
22
23__argparse = ArgumentParser(description="""
24
25Executes the skpbench binary with various configs and skps.
26
27Also monitors the output in order to filter out and re-run results that have an
28unacceptable stddev.
29
30""")
31
32__argparse.add_argument('skpbench',
33  help="path to the skpbench binary")
34__argparse.add_argument('--adb',
35  action='store_true', help="execute skpbench over adb")
36__argparse.add_argument('--adb_binary', default='adb',
37  help="The name of the adb binary to use.")
38__argparse.add_argument('-s', '--device-serial',
39  help="if using adb, ID of the specific device to target "
40       "(only required if more than 1 device is attached)")
41__argparse.add_argument('-m', '--max-stddev',
42  type=float, default=4,
43  help="initial max allowable relative standard deviation")
44__argparse.add_argument('-x', '--suffix',
45  help="suffix to append on config (e.g. '_before', '_after')")
46__argparse.add_argument('-w','--write-path',
47  help="directory to save .png proofs to disk.")
48__argparse.add_argument('-v','--verbosity',
49  type=int, default=1, help="level of verbosity (0=none to 5=debug)")
50__argparse.add_argument('-d', '--duration',
51  type=int, help="number of milliseconds to run each benchmark")
52__argparse.add_argument('-l', '--sample-ms',
53  type=int, help="duration of a sample (minimum)")
54__argparse.add_argument('--gpu',
55  action='store_true',
56  help="perform timing on the gpu clock instead of cpu (gpu work only)")
57__argparse.add_argument('--fps',
58  action='store_true', help="use fps instead of ms")
59__argparse.add_argument('--pr',
60  help="comma- or space-separated list of GPU path renderers, including: "
61       "[[~]all [~]default [~]dashline [~]nvpr [~]msaa [~]aaconvex "
62       "[~]aalinearizing [~]small [~]tess]")
63__argparse.add_argument('--cc',
64  action='store_true', help="allow coverage counting shortcuts to render paths")
65__argparse.add_argument('--nocache',
66  action='store_true', help="disable caching of path mask textures")
67__argparse.add_argument('-c', '--config',
68  default='gl', help="comma- or space-separated list of GPU configs")
69__argparse.add_argument('-a', '--resultsfile',
70  help="optional file to append results into")
71__argparse.add_argument('--ddl',
72  action='store_true', help="record the skp into DDLs before rendering")
73__argparse.add_argument('--ddlNumAdditionalThreads',
74  type=int, default=0,
75  help="number of DDL recording threads in addition to main one")
76__argparse.add_argument('--ddlTilingWidthHeight',
77  type=int, default=0, help="number of tiles along one edge when in DDL mode")
78__argparse.add_argument('--ddlRecordTime',
79  action='store_true', help="report just the cpu time spent recording DDLs")
80__argparse.add_argument('--gpuThreads',
81  type=int, default=-1,
82  help="Create this many extra threads to assist with GPU work, including"
83       " software path rendering. Defaults to two.")
84__argparse.add_argument('srcs',
85  nargs='+',
86  help=".skp files or directories to expand for .skp files, and/or .svg files")
87
88FLAGS = __argparse.parse_args()
89if FLAGS.adb:
90  import _adb_path as _path
91  _path.init(FLAGS.device_serial, FLAGS.adb_binary)
92else:
93  import _os_path as _path
94
95def dump_commandline_if_verbose(commandline):
96  if FLAGS.verbosity >= 5:
97    quoted = ['\'%s\'' % re.sub(r'([\\\'])', r'\\\1', x) for x in commandline]
98    print(' '.join(quoted), file=sys.stderr)
99
100
101class StddevException(Exception):
102  pass
103
104class Message:
105  READLINE = 0,
106  POLL_HARDWARE = 1,
107  EXIT = 2
108  def __init__(self, message, value=None):
109    self.message = message
110    self.value = value
111
112class SubprocessMonitor(Thread):
113  def __init__(self, queue, proc):
114    self._queue = queue
115    self._proc = proc
116    Thread.__init__(self)
117
118  def run(self):
119    """Runs on the background thread."""
120    for line in iter(self._proc.stdout.readline, b''):
121      self._queue.put(Message(Message.READLINE, line.decode('utf-8').rstrip()))
122    self._queue.put(Message(Message.EXIT))
123
124class SKPBench:
125  ARGV = [FLAGS.skpbench, '--verbosity', str(FLAGS.verbosity)]
126  if FLAGS.duration:
127    ARGV.extend(['--duration', str(FLAGS.duration)])
128  if FLAGS.sample_ms:
129    ARGV.extend(['--sampleMs', str(FLAGS.sample_ms)])
130  if FLAGS.gpu:
131    ARGV.extend(['--gpuClock', 'true'])
132  if FLAGS.fps:
133    ARGV.extend(['--fps', 'true'])
134  if FLAGS.pr:
135    ARGV.extend(['--pr'] + re.split(r'[ ,]', FLAGS.pr))
136  if FLAGS.cc:
137    ARGV.extend(['--cc', 'true'])
138  if FLAGS.nocache:
139    ARGV.extend(['--cachePathMasks', 'false'])
140  if FLAGS.gpuThreads != -1:
141    ARGV.extend(['--gpuThreads', str(FLAGS.gpuThreads)])
142
143  # DDL parameters
144  if FLAGS.ddl:
145    ARGV.extend(['--ddl', 'true'])
146  if FLAGS.ddlNumAdditionalThreads:
147    ARGV.extend(['--ddlNumAdditionalThreads',
148                 str(FLAGS.ddlNumAdditionalThreads)])
149  if FLAGS.ddlTilingWidthHeight:
150    ARGV.extend(['--ddlTilingWidthHeight', str(FLAGS.ddlTilingWidthHeight)])
151  if FLAGS.ddlRecordTime:
152    ARGV.extend(['--ddlRecordTime', 'true'])
153
154  if FLAGS.adb:
155    if FLAGS.device_serial is None:
156      ARGV[:0] = [FLAGS.adb_binary, 'shell']
157    else:
158      ARGV[:0] = [FLAGS.adb_binary, '-s', FLAGS.device_serial, 'shell']
159
160  @classmethod
161  def get_header(cls, outfile=sys.stdout):
162    commandline = cls.ARGV + ['--duration', '0']
163    dump_commandline_if_verbose(commandline)
164    out = subprocess.check_output(commandline, stderr=subprocess.STDOUT)
165    return out.rstrip()
166
167  @classmethod
168  def run_warmup(cls, warmup_time, config):
169    if not warmup_time:
170      return
171    print('running %i second warmup...' % warmup_time, file=sys.stderr)
172    commandline = cls.ARGV + ['--duration', str(warmup_time * 1000),
173                              '--config', config,
174                              '--src', 'warmup']
175    dump_commandline_if_verbose(commandline)
176    output = subprocess.check_output(commandline, stderr=subprocess.STDOUT)
177
178    # validate the warmup run output.
179    for line in output.decode('utf-8').split('\n'):
180      match = BenchResult.match(line.rstrip())
181      if match and match.bench == 'warmup':
182        return
183    raise Exception('Invalid warmup output:\n%s' % output)
184
185  def __init__(self, src, config, max_stddev, best_result=None):
186    self.src = src
187    self.config = config
188    self.max_stddev = max_stddev
189    self.best_result = best_result
190    self._queue = Queue()
191    self._proc = None
192    self._monitor = None
193    self._hw_poll_timer = None
194
195  def __enter__(self):
196    return self
197
198  def __exit__(self, exception_type, exception_value, traceback):
199    if self._proc:
200      self.terminate()
201    if self._hw_poll_timer:
202      self._hw_poll_timer.cancel()
203
204  def execute(self, hardware):
205    hardware.sanity_check()
206    self._schedule_hardware_poll()
207
208    commandline = self.ARGV + ['--config', self.config,
209                               '--src', self.src,
210                               '--suppressHeader', 'true']
211    if FLAGS.write_path:
212      pngfile = _path.join(FLAGS.write_path, self.config,
213                           _path.basename(self.src) + '.png')
214      commandline.extend(['--png', pngfile])
215    dump_commandline_if_verbose(commandline)
216    self._proc = subprocess.Popen(commandline, stdout=subprocess.PIPE,
217                                  stderr=subprocess.STDOUT)
218    self._monitor = SubprocessMonitor(self._queue, self._proc)
219    self._monitor.start()
220
221    while True:
222      message = self._queue.get()
223      if message.message == Message.READLINE:
224        result = BenchResult.match(message.value)
225        if result:
226          hardware.sanity_check()
227          self._process_result(result)
228        elif hardware.filter_line(message.value):
229          print(message.value, file=sys.stderr)
230        continue
231      if message.message == Message.POLL_HARDWARE:
232        hardware.sanity_check()
233        self._schedule_hardware_poll()
234        continue
235      if message.message == Message.EXIT:
236        self._monitor.join()
237        self._proc.wait()
238        if self._proc.returncode != 0:
239          raise Exception("skpbench exited with nonzero exit code %i" %
240                          self._proc.returncode)
241        self._proc = None
242        break
243
244  def _schedule_hardware_poll(self):
245    if self._hw_poll_timer:
246      self._hw_poll_timer.cancel()
247    self._hw_poll_timer = \
248      Timer(1, lambda: self._queue.put(Message(Message.POLL_HARDWARE)))
249    self._hw_poll_timer.start()
250
251  def _process_result(self, result):
252    if not self.best_result or result.stddev <= self.best_result.stddev:
253      self.best_result = result
254    elif FLAGS.verbosity >= 2:
255      print("reusing previous result for %s/%s with lower stddev "
256            "(%s%% instead of %s%%)." %
257            (result.config, result.bench, self.best_result.stddev,
258             result.stddev), file=sys.stderr)
259    if self.max_stddev and self.best_result.stddev > self.max_stddev:
260      raise StddevException()
261
262  def terminate(self):
263    if self._proc:
264      self._proc.terminate()
265      self._monitor.join()
266      self._proc.wait()
267      self._proc = None
268
269def emit_result(line, resultsfile=None):
270  print(line)
271  sys.stdout.flush()
272  if resultsfile:
273    print(line, file=resultsfile)
274    resultsfile.flush()
275
276def run_benchmarks(configs, srcs, hardware, resultsfile=None):
277  hasheader = False
278  benches = collections.deque([(src, config, FLAGS.max_stddev)
279                               for src in srcs
280                               for config in configs])
281  while benches:
282    try:
283      with hardware:
284        SKPBench.run_warmup(hardware.warmup_time, configs[0])
285        if not hasheader:
286          emit_result(SKPBench.get_header(), resultsfile)
287          hasheader = True
288        while benches:
289          benchargs = benches.popleft()
290          with SKPBench(*benchargs) as skpbench:
291            try:
292              skpbench.execute(hardware)
293              if skpbench.best_result:
294                emit_result(skpbench.best_result.format(FLAGS.suffix),
295                            resultsfile)
296              else:
297                print("WARNING: no result for %s with config %s" %
298                      (skpbench.src, skpbench.config), file=sys.stderr)
299
300            except StddevException:
301              retry_max_stddev = skpbench.max_stddev * math.sqrt(2)
302              if FLAGS.verbosity >= 1:
303                print("stddev is too high for %s/%s (%s%%, max=%.2f%%), "
304                      "re-queuing with max=%.2f%%." %
305                      (skpbench.best_result.config, skpbench.best_result.bench,
306                       skpbench.best_result.stddev, skpbench.max_stddev,
307                       retry_max_stddev),
308                      file=sys.stderr)
309              benches.append((skpbench.src, skpbench.config, retry_max_stddev,
310                              skpbench.best_result))
311
312            except HardwareException as exception:
313              skpbench.terminate()
314              if FLAGS.verbosity >= 4:
315                hardware.print_debug_diagnostics()
316              if FLAGS.verbosity >= 1:
317                print("%s; rebooting and taking a %i second nap..." %
318                      (exception.message, exception.sleeptime), file=sys.stderr)
319              benches.appendleft(benchargs) # retry the same bench next time.
320              raise # wake hw up from benchmarking mode before the nap.
321
322    except HardwareException as exception:
323      time.sleep(exception.sleeptime)
324
325def main():
326  # Delimiter is ',' or ' ', skip if nested inside parens (e.g. gpu(a=b,c=d)).
327  DELIMITER = r'[, ](?!(?:[^(]*\([^)]*\))*[^()]*\))'
328  configs = re.split(DELIMITER, FLAGS.config)
329  srcs = _path.find_skps(FLAGS.srcs)
330
331  if FLAGS.adb:
332    adb = Adb(FLAGS.device_serial, FLAGS.adb_binary,
333              echo=(FLAGS.verbosity >= 5))
334    model = adb.check('getprop ro.product.model').strip()
335    if model == 'Pixel C':
336      from _hardware_pixel_c import HardwarePixelC
337      hardware = HardwarePixelC(adb)
338    elif model == 'Pixel':
339      from _hardware_pixel import HardwarePixel
340      hardware = HardwarePixel(adb)
341    elif model == 'Pixel 2':
342      from _hardware_pixel2 import HardwarePixel2
343      hardware = HardwarePixel2(adb)
344    elif model == 'Nexus 6P':
345      from _hardware_nexus_6p import HardwareNexus6P
346      hardware = HardwareNexus6P(adb)
347    else:
348      from _hardware_android import HardwareAndroid
349      print("WARNING: %s: don't know how to monitor this hardware; results "
350            "may be unreliable." % model, file=sys.stderr)
351      hardware = HardwareAndroid(adb)
352  else:
353    hardware = Hardware()
354
355  if FLAGS.resultsfile:
356    with open(FLAGS.resultsfile, mode='a+') as resultsfile:
357      run_benchmarks(configs, srcs, hardware, resultsfile=resultsfile)
358  else:
359    run_benchmarks(configs, srcs, hardware)
360
361
362if __name__ == '__main__':
363  main()
364