• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2
3# Copyright 2016 Google Inc.
4#
5# Use of this source code is governed by a BSD-style license that can be
6# found in the LICENSE file.
7
8from __future__ import print_function
9from _adb import Adb
10from _benchresult import BenchResult
11from _hardware import HardwareException, Hardware
12from argparse import ArgumentParser
13from multiprocessing import Queue
14from threading import Thread, Timer
15import collections
16import glob
17import math
18import re
19import subprocess
20import sys
21import time
22
23__argparse = ArgumentParser(description="""
24
25Executes the skpbench binary with various configs and skps.
26
27Also monitors the output in order to filter out and re-run results that have an
28unacceptable stddev.
29
30""")
31
32__argparse.add_argument('skpbench',
33  help="path to the skpbench binary")
34__argparse.add_argument('--adb',
35  action='store_true', help="execute skpbench over adb")
36__argparse.add_argument('-s', '--device-serial',
37  help="if using adb, ID of the specific device to target "
38       "(only required if more than 1 device is attached)")
39__argparse.add_argument('-m', '--max-stddev',
40  type=float, default=4,
41  help="initial max allowable relative standard deviation")
42__argparse.add_argument('-x', '--suffix',
43  help="suffix to append on config (e.g. '_before', '_after')")
44__argparse.add_argument('-w','--write-path',
45  help="directory to save .png proofs to disk.")
46__argparse.add_argument('-v','--verbosity',
47  type=int, default=1, help="level of verbosity (0=none to 5=debug)")
48__argparse.add_argument('-d', '--duration',
49  type=int, help="number of milliseconds to run each benchmark")
50__argparse.add_argument('-l', '--sample-ms',
51  type=int, help="duration of a sample (minimum)")
52__argparse.add_argument('--gpu',
53  action='store_true',
54  help="perform timing on the gpu clock instead of cpu (gpu work only)")
55__argparse.add_argument('--fps',
56  action='store_true', help="use fps instead of ms")
57__argparse.add_argument('-c', '--config',
58  default='gl', help="comma- or space-separated list of GPU configs")
59__argparse.add_argument('-a', '--resultsfile',
60  help="optional file to append results into")
61__argparse.add_argument('skps',
62  nargs='+',
63  help=".skp files or directories to expand for .skp files")
64
65FLAGS = __argparse.parse_args()
66if FLAGS.adb:
67  import _adb_path as _path
68  _path.init(FLAGS.device_serial)
69else:
70  import _os_path as _path
71
72def dump_commandline_if_verbose(commandline):
73  if FLAGS.verbosity >= 5:
74    quoted = ['\'%s\'' % re.sub(r'([\\\'])', r'\\\1', x) for x in commandline]
75    print(' '.join(quoted), file=sys.stderr)
76
77
78class StddevException(Exception):
79  pass
80
81class Message:
82  READLINE = 0,
83  POLL_HARDWARE = 1,
84  EXIT = 2
85  def __init__(self, message, value=None):
86    self.message = message
87    self.value = value
88
89class SubprocessMonitor(Thread):
90  def __init__(self, queue, proc):
91    self._queue = queue
92    self._proc = proc
93    Thread.__init__(self)
94
95  def run(self):
96    """Runs on the background thread."""
97    for line in iter(self._proc.stdout.readline, b''):
98      self._queue.put(Message(Message.READLINE, line.decode('utf-8').rstrip()))
99    self._queue.put(Message(Message.EXIT))
100
101class SKPBench:
102  ARGV = [FLAGS.skpbench, '--verbosity', str(FLAGS.verbosity)]
103  if FLAGS.duration:
104    ARGV.extend(['--duration', str(FLAGS.duration)])
105  if FLAGS.sample_ms:
106    ARGV.extend(['--sampleMs', str(FLAGS.sample_ms)])
107  if FLAGS.gpu:
108    ARGV.extend(['--gpuClock', 'true'])
109  if FLAGS.fps:
110    ARGV.extend(['--fps', 'true'])
111  if FLAGS.adb:
112    if FLAGS.device_serial is None:
113      ARGV[:0] = ['adb', 'shell']
114    else:
115      ARGV[:0] = ['adb', '-s', FLAGS.device_serial, 'shell']
116
117  @classmethod
118  def get_header(cls, outfile=sys.stdout):
119    commandline = cls.ARGV + ['--duration', '0']
120    dump_commandline_if_verbose(commandline)
121    out = subprocess.check_output(commandline, stderr=subprocess.STDOUT)
122    return out.rstrip()
123
124  @classmethod
125  def run_warmup(cls, warmup_time, config):
126    if not warmup_time:
127      return
128    print('running %i second warmup...' % warmup_time, file=sys.stderr)
129    commandline = cls.ARGV + ['--duration', str(warmup_time * 1000),
130                              '--config', config,
131                              '--skp', 'warmup']
132    dump_commandline_if_verbose(commandline)
133    output = subprocess.check_output(commandline, stderr=subprocess.STDOUT)
134
135    # validate the warmup run output.
136    for line in output.decode('utf-8').split('\n'):
137      match = BenchResult.match(line.rstrip())
138      if match and match.bench == 'warmup':
139        return
140    raise Exception('Invalid warmup output:\n%s' % output)
141
142  def __init__(self, skp, config, max_stddev, best_result=None):
143    self.skp = skp
144    self.config = config
145    self.max_stddev = max_stddev
146    self.best_result = best_result
147    self._queue = Queue()
148    self._proc = None
149    self._monitor = None
150    self._hw_poll_timer = None
151
152  def __enter__(self):
153    return self
154
155  def __exit__(self, exception_type, exception_value, traceback):
156    if self._proc:
157      self.terminate()
158    if self._hw_poll_timer:
159      self._hw_poll_timer.cancel()
160
161  def execute(self, hardware):
162    hardware.sanity_check()
163    self._schedule_hardware_poll()
164
165    commandline = self.ARGV + ['--config', self.config,
166                               '--skp', self.skp,
167                               '--suppressHeader', 'true']
168    if FLAGS.write_path:
169      pngfile = _path.join(FLAGS.write_path, self.config,
170                           _path.basename(self.skp) + '.png')
171      commandline.extend(['--png', pngfile])
172    dump_commandline_if_verbose(commandline)
173    self._proc = subprocess.Popen(commandline, stdout=subprocess.PIPE,
174                                  stderr=subprocess.STDOUT)
175    self._monitor = SubprocessMonitor(self._queue, self._proc)
176    self._monitor.start()
177
178    while True:
179      message = self._queue.get()
180      if message.message == Message.READLINE:
181        result = BenchResult.match(message.value)
182        if result:
183          hardware.sanity_check()
184          self._process_result(result)
185        elif hardware.filter_line(message.value):
186          print(message.value, file=sys.stderr)
187        continue
188      if message.message == Message.POLL_HARDWARE:
189        hardware.sanity_check()
190        self._schedule_hardware_poll()
191        continue
192      if message.message == Message.EXIT:
193        self._monitor.join()
194        self._proc.wait()
195        if self._proc.returncode != 0:
196          raise Exception("skpbench exited with nonzero exit code %i" %
197                          self._proc.returncode)
198        self._proc = None
199        break
200
201  def _schedule_hardware_poll(self):
202    if self._hw_poll_timer:
203      self._hw_poll_timer.cancel()
204    self._hw_poll_timer = \
205      Timer(1, lambda: self._queue.put(Message(Message.POLL_HARDWARE)))
206    self._hw_poll_timer.start()
207
208  def _process_result(self, result):
209    if not self.best_result or result.stddev <= self.best_result.stddev:
210      self.best_result = result
211    elif FLAGS.verbosity >= 2:
212      print("reusing previous result for %s/%s with lower stddev "
213            "(%s%% instead of %s%%)." %
214            (result.config, result.bench, self.best_result.stddev,
215             result.stddev), file=sys.stderr)
216    if self.max_stddev and self.best_result.stddev > self.max_stddev:
217      raise StddevException()
218
219  def terminate(self):
220    if self._proc:
221      self._proc.terminate()
222      self._monitor.join()
223      self._proc.wait()
224      self._proc = None
225
226def emit_result(line, resultsfile=None):
227  print(line)
228  sys.stdout.flush()
229  if resultsfile:
230    print(line, file=resultsfile)
231    resultsfile.flush()
232
233def run_benchmarks(configs, skps, hardware, resultsfile=None):
234  emit_result(SKPBench.get_header(), resultsfile)
235  benches = collections.deque([(skp, config, FLAGS.max_stddev)
236                               for skp in skps
237                               for config in configs])
238  while benches:
239    benchargs = benches.popleft()
240    with SKPBench(*benchargs) as skpbench:
241      try:
242        skpbench.execute(hardware)
243        if skpbench.best_result:
244          emit_result(skpbench.best_result.format(FLAGS.suffix), resultsfile)
245        else:
246          print("WARNING: no result for %s with config %s" %
247                (skpbench.skp, skpbench.config), file=sys.stderr)
248
249      except StddevException:
250        retry_max_stddev = skpbench.max_stddev * math.sqrt(2)
251        if FLAGS.verbosity >= 1:
252          print("stddev is too high for %s/%s (%s%%, max=%.2f%%), "
253                "re-queuing with max=%.2f%%." %
254                (skpbench.best_result.config, skpbench.best_result.bench,
255                 skpbench.best_result.stddev, skpbench.max_stddev,
256                 retry_max_stddev),
257                file=sys.stderr)
258        benches.append((skpbench.skp, skpbench.config, retry_max_stddev,
259                        skpbench.best_result))
260
261      except HardwareException as exception:
262        skpbench.terminate()
263        if FLAGS.verbosity >= 4:
264          hardware.print_debug_diagnostics()
265        if FLAGS.verbosity >= 1:
266          print("%s; taking a %i second nap..." %
267                (exception.message, exception.sleeptime), file=sys.stderr)
268        benches.appendleft(benchargs) # retry the same bench next time.
269        hardware.sleep(exception.sleeptime)
270        if FLAGS.verbosity >= 4:
271          hardware.print_debug_diagnostics()
272        SKPBench.run_warmup(hardware.warmup_time, configs[0])
273
274def main():
275  # Delimiter is ',' or ' ', skip if nested inside parens (e.g. gpu(a=b,c=d)).
276  DELIMITER = r'[, ](?!(?:[^(]*\([^)]*\))*[^()]*\))'
277  configs = re.split(DELIMITER, FLAGS.config)
278  skps = _path.find_skps(FLAGS.skps)
279
280  if FLAGS.adb:
281    adb = Adb(FLAGS.device_serial, echo=(FLAGS.verbosity >= 5))
282    model = adb.check('getprop ro.product.model').strip()
283    if model == 'Pixel C':
284      from _hardware_pixel_c import HardwarePixelC
285      hardware = HardwarePixelC(adb)
286    elif model == 'Nexus 6P':
287      from _hardware_nexus_6p import HardwareNexus6P
288      hardware = HardwareNexus6P(adb)
289    else:
290      from _hardware_android import HardwareAndroid
291      print("WARNING: %s: don't know how to monitor this hardware; results "
292            "may be unreliable." % model, file=sys.stderr)
293      hardware = HardwareAndroid(adb)
294  else:
295    hardware = Hardware()
296
297  with hardware:
298    SKPBench.run_warmup(hardware.warmup_time, configs[0])
299    if FLAGS.resultsfile:
300      with open(FLAGS.resultsfile, mode='a+') as resultsfile:
301        run_benchmarks(configs, skps, hardware, resultsfile=resultsfile)
302    else:
303      run_benchmarks(configs, skps, hardware)
304
305
306if __name__ == '__main__':
307  main()
308