• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2#
3# Copyright (C) 2015 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17"""Simpleperf runtest runner: run simpleperf runtests on host or on device.
18
19For a simpleperf runtest like one_function test, it contains following steps:
201. Run simpleperf record command to record simpleperf_runtest_one_function's
21   running samples, which is generated in perf.data.
222. Run simpleperf report command to parse perf.data, generate perf.report.
234. Parse perf.report and see if it matches expectation.
24
25The information of all runtests is stored in runtest.conf.
26"""
27
28import os
29import os.path
30import re
31import subprocess
32import sys
33import xml.etree.ElementTree as ET
34
35
36class CallTreeNode(object):
37
38  def __init__(self, name):
39    self.name = name
40    self.children = []
41
42  def add_child(self, child):
43    self.children.append(child)
44
45  def __str__(self):
46    return 'CallTreeNode:\n' + '\n'.join(self._dump(1))
47
48  def _dump(self, indent):
49    indent_str = '  ' * indent
50    strs = [indent_str + self.name]
51    for child in self.children:
52      strs.extend(child._dump(indent + 1))
53    return strs
54
55
56class Symbol(object):
57
58  def __init__(self, name, comm, overhead, children_overhead):
59    self.name = name
60    self.comm = comm
61    self.overhead = overhead
62    # children_overhead is the overhead sum of this symbol and functions
63    # called by this symbol.
64    self.children_overhead = children_overhead
65    self.call_tree = None
66
67  def set_call_tree(self, call_tree):
68    self.call_tree = call_tree
69
70  def __str__(self):
71    strs = []
72    strs.append('Symbol name=%s comm=%s overhead=%f children_overhead=%f' % (
73        self.name, self.comm, self.overhead, self.children_overhead))
74    if self.call_tree:
75      strs.append('\t%s' % self.call_tree)
76    return '\n'.join(strs)
77
78
79class SymbolOverheadRequirement(object):
80
81  def __init__(self, symbol_name=None, comm=None, min_overhead=None,
82               max_overhead=None):
83    self.symbol_name = symbol_name
84    self.comm = comm
85    self.min_overhead = min_overhead
86    self.max_overhead = max_overhead
87
88  def __str__(self):
89    strs = []
90    strs.append('SymbolOverheadRequirement')
91    if self.symbol_name is not None:
92      strs.append('symbol_name=%s' % self.symbol_name)
93    if self.comm is not None:
94      strs.append('comm=%s' % self.comm)
95    if self.min_overhead is not None:
96      strs.append('min_overhead=%f' % self.min_overhead)
97    if self.max_overhead is not None:
98      strs.append('max_overhead=%f' % self.max_overhead)
99    return ' '.join(strs)
100
101  def is_match(self, symbol):
102    if self.symbol_name is not None:
103      if self.symbol_name != symbol.name:
104        return False
105    if self.comm is not None:
106      if self.comm != symbol.comm:
107        return False
108    return True
109
110  def check_overhead(self, overhead):
111    if self.min_overhead is not None:
112      if self.min_overhead > overhead:
113        return False
114    if self.max_overhead is not None:
115      if self.max_overhead < overhead:
116        return False
117    return True
118
119
120class SymbolRelationRequirement(object):
121
122  def __init__(self, symbol_name, comm=None):
123    self.symbol_name = symbol_name
124    self.comm = comm
125    self.children = []
126
127  def add_child(self, child):
128    self.children.append(child)
129
130  def __str__(self):
131    return 'SymbolRelationRequirement:\n' + '\n'.join(self._dump(1))
132
133  def _dump(self, indent):
134    indent_str = '  ' * indent
135    strs = [indent_str + self.symbol_name +
136            (' ' + self.comm if self.comm else '')]
137    for child in self.children:
138      strs.extend(child._dump(indent + 1))
139    return strs
140
141  def is_match(self, symbol):
142    if symbol.name != self.symbol_name:
143      return False
144    if self.comm is not None:
145      if symbol.comm != self.comm:
146        return False
147    return True
148
149  def check_relation(self, call_tree):
150    if not call_tree:
151      return False
152    if self.symbol_name != call_tree.name:
153      return False
154    for child in self.children:
155      child_matched = False
156      for node in call_tree.children:
157        if child.check_relation(node):
158          child_matched = True
159          break
160      if not child_matched:
161        return False
162    return True
163
164
165class Test(object):
166
167  def __init__(
168          self,
169          test_name,
170          executable_name,
171          disable_host,
172          record_options,
173          report_options,
174          symbol_overhead_requirements,
175          symbol_children_overhead_requirements,
176          symbol_relation_requirements):
177    self.test_name = test_name
178    self.executable_name = executable_name
179    self.disable_host = disable_host
180    self.record_options = record_options
181    self.report_options = report_options
182    self.symbol_overhead_requirements = symbol_overhead_requirements
183    self.symbol_children_overhead_requirements = (
184        symbol_children_overhead_requirements)
185    self.symbol_relation_requirements = symbol_relation_requirements
186
187  def __str__(self):
188    strs = []
189    strs.append('Test test_name=%s' % self.test_name)
190    strs.append('\texecutable_name=%s' % self.executable_name)
191    strs.append('\tdisable_host=%s' % self.disable_host)
192    strs.append('\trecord_options=%s' % (' '.join(self.record_options)))
193    strs.append('\treport_options=%s' % (' '.join(self.report_options)))
194    strs.append('\tsymbol_overhead_requirements:')
195    for req in self.symbol_overhead_requirements:
196      strs.append('\t\t%s' % req)
197    strs.append('\tsymbol_children_overhead_requirements:')
198    for req in self.symbol_children_overhead_requirements:
199      strs.append('\t\t%s' % req)
200    strs.append('\tsymbol_relation_requirements:')
201    for req in self.symbol_relation_requirements:
202      strs.append('\t\t%s' % req)
203    return '\n'.join(strs)
204
205
206def load_config_file(config_file):
207  tests = []
208  tree = ET.parse(config_file)
209  root = tree.getroot()
210  assert root.tag == 'runtests'
211  for test in root:
212    assert test.tag == 'test'
213    test_name = test.attrib['name']
214    executable_name = None
215    disable_host = False
216    record_options = []
217    report_options = []
218    symbol_overhead_requirements = []
219    symbol_children_overhead_requirements = []
220    symbol_relation_requirements = []
221    for test_item in test:
222      if test_item.tag == 'executable':
223        executable_name = test_item.attrib['name']
224      elif test_item.tag == 'disable_host':
225        disable_host = True
226      elif test_item.tag == 'record':
227        record_options = test_item.attrib['option'].split()
228      elif test_item.tag == 'report':
229        report_options = test_item.attrib['option'].split()
230      elif (test_item.tag == 'symbol_overhead' or
231              test_item.tag == 'symbol_children_overhead'):
232        for symbol_item in test_item:
233          assert symbol_item.tag == 'symbol'
234          symbol_name = None
235          if 'name' in symbol_item.attrib:
236            symbol_name = symbol_item.attrib['name']
237          comm = None
238          if 'comm' in symbol_item.attrib:
239            comm = symbol_item.attrib['comm']
240          overhead_min = None
241          if 'min' in symbol_item.attrib:
242            overhead_min = float(symbol_item.attrib['min'])
243          overhead_max = None
244          if 'max' in symbol_item.attrib:
245            overhead_max = float(symbol_item.attrib['max'])
246
247          if test_item.tag == 'symbol_overhead':
248            symbol_overhead_requirements.append(
249                SymbolOverheadRequirement(
250                    symbol_name,
251                    comm,
252                    overhead_min,
253                    overhead_max)
254            )
255          else:
256            symbol_children_overhead_requirements.append(
257                SymbolOverheadRequirement(
258                    symbol_name,
259                    comm,
260                    overhead_min,
261                    overhead_max))
262      elif test_item.tag == 'symbol_callgraph_relation':
263        for symbol_item in test_item:
264          req = load_symbol_relation_requirement(symbol_item)
265          symbol_relation_requirements.append(req)
266
267    tests.append(
268        Test(
269            test_name,
270            executable_name,
271            disable_host,
272            record_options,
273            report_options,
274            symbol_overhead_requirements,
275            symbol_children_overhead_requirements,
276            symbol_relation_requirements))
277  return tests
278
279
280def load_symbol_relation_requirement(symbol_item):
281  symbol_name = symbol_item.attrib['name']
282  comm = None
283  if 'comm' in symbol_item.attrib:
284    comm = symbol_item.attrib['comm']
285  req = SymbolRelationRequirement(symbol_name, comm)
286  for item in symbol_item:
287    child_req = load_symbol_relation_requirement(item)
288    req.add_child(child_req)
289  return req
290
291
292class Runner(object):
293
294  def __init__(self, target, perf_path):
295    self.target = target
296    self.is32 = target.endswith('32')
297    self.perf_path = perf_path
298    self.use_callgraph = False
299    self.sampler = 'cpu-cycles'
300
301  def record(self, test_executable_name, record_file, additional_options=[]):
302    call_args = [self.perf_path, 'record']
303    call_args += ['--duration', '2']
304    call_args += ['-e', '%s:u' % self.sampler]
305    if self.use_callgraph:
306      call_args += ['-f', '1000', '-g']
307    call_args += ['-o', record_file]
308    call_args += additional_options
309    test_executable_name += '32' if self.is32 else '64'
310    call_args += [test_executable_name]
311    self._call(call_args)
312
313  def report(self, record_file, report_file, additional_options=[]):
314    call_args = [self.perf_path, 'report']
315    call_args += ['-i', record_file]
316    if self.use_callgraph:
317      call_args += ['-g', 'callee']
318    call_args += additional_options
319    self._call(call_args, report_file)
320
321  def _call(self, args, output_file=None):
322    pass
323
324
325class HostRunner(Runner):
326
327  """Run perf test on host."""
328
329  def __init__(self, target):
330    perf_path = 'simpleperf32' if target.endswith('32') else 'simpleperf'
331    super(HostRunner, self).__init__(target, perf_path)
332
333  def _call(self, args, output_file=None):
334    output_fh = None
335    if output_file is not None:
336      output_fh = open(output_file, 'w')
337    subprocess.check_call(args, stdout=output_fh)
338    if output_fh is not None:
339      output_fh.close()
340
341
342class DeviceRunner(Runner):
343
344  """Run perf test on device."""
345
346  def __init__(self, target):
347    self.tmpdir = '/data/local/tmp/'
348    perf_path = 'simpleperf32' if target.endswith('32') else 'simpleperf'
349    super(DeviceRunner, self).__init__(target, self.tmpdir + perf_path)
350    self._download(os.environ['OUT'] + '/system/xbin/' + perf_path, self.tmpdir)
351    lib = 'lib' if self.is32 else 'lib64'
352    self._download(os.environ['OUT'] + '/system/' + lib + '/libsimpleperf_inplace_sampler.so',
353                   self.tmpdir)
354
355  def _call(self, args, output_file=None):
356    output_fh = None
357    if output_file is not None:
358      output_fh = open(output_file, 'w')
359    args_with_adb = ['adb', 'shell']
360    args_with_adb.append('export LD_LIBRARY_PATH=' + self.tmpdir + ' && ' + ' '.join(args))
361    subprocess.check_call(args_with_adb, stdout=output_fh)
362    if output_fh is not None:
363      output_fh.close()
364
365  def _download(self, file, to_dir):
366    args = ['adb', 'push', file, to_dir]
367    subprocess.check_call(args)
368
369  def record(self, test_executable_name, record_file, additional_options=[]):
370    self._download(os.environ['OUT'] + '/system/bin/' + test_executable_name +
371                   ('32' if self.is32 else '64'), self.tmpdir)
372    super(DeviceRunner, self).record(self.tmpdir + test_executable_name,
373                                     self.tmpdir + record_file,
374                                     additional_options)
375
376  def report(self, record_file, report_file, additional_options=[]):
377    super(DeviceRunner, self).report(self.tmpdir + record_file,
378                                     report_file,
379                                     additional_options)
380
381class ReportAnalyzer(object):
382
383  """Check if perf.report matches expectation in Configuration."""
384
385  def _read_report_file(self, report_file, has_callgraph):
386    fh = open(report_file, 'r')
387    lines = fh.readlines()
388    fh.close()
389
390    lines = [x.rstrip() for x in lines]
391    blank_line_index = -1
392    for i in range(len(lines)):
393      if not lines[i]:
394        blank_line_index = i
395    assert blank_line_index != -1
396    assert blank_line_index + 1 < len(lines)
397    title_line = lines[blank_line_index + 1]
398    report_item_lines = lines[blank_line_index + 2:]
399
400    if has_callgraph:
401      assert re.search(r'^Children\s+Self\s+Command.+Symbol$', title_line)
402    else:
403      assert re.search(r'^Overhead\s+Command.+Symbol$', title_line)
404
405    return self._parse_report_items(report_item_lines, has_callgraph)
406
407  def _parse_report_items(self, lines, has_callgraph):
408    symbols = []
409    cur_symbol = None
410    call_tree_stack = {}
411    vertical_columns = []
412    last_node = None
413    last_depth = -1
414
415    for line in lines:
416      if not line:
417        continue
418      if not line[0].isspace():
419        if has_callgraph:
420          items = line.split(None, 6)
421          assert len(items) == 7
422          children_overhead = float(items[0][:-1])
423          overhead = float(items[1][:-1])
424          comm = items[2]
425          symbol_name = items[6]
426          cur_symbol = Symbol(symbol_name, comm, overhead, children_overhead)
427          symbols.append(cur_symbol)
428        else:
429          items = line.split(None, 5)
430          assert len(items) == 6
431          overhead = float(items[0][:-1])
432          comm = items[1]
433          symbol_name = items[5]
434          cur_symbol = Symbol(symbol_name, comm, overhead, 0)
435          symbols.append(cur_symbol)
436        # Each report item can have different column depths.
437        vertical_columns = []
438      else:
439        for i in range(len(line)):
440          if line[i] == '|':
441            if not vertical_columns or vertical_columns[-1] < i:
442              vertical_columns.append(i)
443
444        if not line.strip('| \t'):
445          continue
446        if line.find('-') == -1:
447          function_name = line.strip('| \t')
448          node = CallTreeNode(function_name)
449          last_node.add_child(node)
450          last_node = node
451          call_tree_stack[last_depth] = node
452        else:
453          pos = line.find('-')
454          depth = -1
455          for i in range(len(vertical_columns)):
456            if pos >= vertical_columns[i]:
457              depth = i
458          assert depth != -1
459
460          line = line.strip('|- \t')
461          m = re.search(r'^[\d\.]+%[-\s]+(.+)$', line)
462          if m:
463            function_name = m.group(1)
464          else:
465            function_name = line
466
467          node = CallTreeNode(function_name)
468          if depth == 0:
469            cur_symbol.set_call_tree(node)
470
471          else:
472            call_tree_stack[depth - 1].add_child(node)
473          call_tree_stack[depth] = node
474          last_node = node
475          last_depth = depth
476
477    return symbols
478
479  def check_report_file(self, test, report_file, has_callgraph):
480    symbols = self._read_report_file(report_file, has_callgraph)
481    if not self._check_symbol_overhead_requirements(test, symbols):
482      return False
483    if has_callgraph:
484      if not self._check_symbol_children_overhead_requirements(test, symbols):
485        return False
486      if not self._check_symbol_relation_requirements(test, symbols):
487        return False
488    return True
489
490  def _check_symbol_overhead_requirements(self, test, symbols):
491    result = True
492    matched = [False] * len(test.symbol_overhead_requirements)
493    matched_overhead = [0] * len(test.symbol_overhead_requirements)
494    for symbol in symbols:
495      for i in range(len(test.symbol_overhead_requirements)):
496        req = test.symbol_overhead_requirements[i]
497        if req.is_match(symbol):
498          matched[i] = True
499          matched_overhead[i] += symbol.overhead
500    for i in range(len(matched)):
501      if not matched[i]:
502        print 'requirement (%s) has no matched symbol in test %s' % (
503            test.symbol_overhead_requirements[i], test)
504        result = False
505      else:
506        fulfilled = req.check_overhead(matched_overhead[i])
507        if not fulfilled:
508          print "Symbol (%s) doesn't match requirement (%s) in test %s" % (
509              symbol, req, test)
510          result = False
511    return result
512
513  def _check_symbol_children_overhead_requirements(self, test, symbols):
514    result = True
515    matched = [False] * len(test.symbol_children_overhead_requirements)
516    for symbol in symbols:
517      for i in range(len(test.symbol_children_overhead_requirements)):
518        req = test.symbol_children_overhead_requirements[i]
519        if req.is_match(symbol):
520          matched[i] = True
521          fulfilled = req.check_overhead(symbol.children_overhead)
522          if not fulfilled:
523            print "Symbol (%s) doesn't match requirement (%s) in test %s" % (
524                symbol, req, test)
525            result = False
526    for i in range(len(matched)):
527      if not matched[i]:
528        print 'requirement (%s) has no matched symbol in test %s' % (
529            test.symbol_children_overhead_requirements[i], test)
530        result = False
531    return result
532
533  def _check_symbol_relation_requirements(self, test, symbols):
534    result = True
535    matched = [False] * len(test.symbol_relation_requirements)
536    for symbol in symbols:
537      for i in range(len(test.symbol_relation_requirements)):
538        req = test.symbol_relation_requirements[i]
539        if req.is_match(symbol):
540          matched[i] = True
541          fulfilled = req.check_relation(symbol.call_tree)
542          if not fulfilled:
543            print "Symbol (%s) doesn't match requirement (%s) in test %s" % (
544                symbol, req, test)
545            result = False
546    for i in range(len(matched)):
547      if not matched[i]:
548        print 'requirement (%s) has no matched symbol in test %s' % (
549            test.symbol_relation_requirements[i], test)
550        result = False
551    return result
552
553
554def build_runner(target, use_callgraph, sampler):
555  if target == 'host32' and use_callgraph:
556    print "Current 64bit linux host doesn't support `simpleperf32 record -g`"
557    return None
558  if target.startswith('host'):
559    runner = HostRunner(target)
560  else:
561    runner = DeviceRunner(target)
562  runner.use_callgraph = use_callgraph
563  runner.sampler = sampler
564  return runner
565
566
567def test_with_runner(runner, tests):
568  report_analyzer = ReportAnalyzer()
569  for test in tests:
570    if test.disable_host and runner.target.startswith('host'):
571      print('Skip test %s on %s' % (test.test_name, runner.target))
572      continue
573    runner.record(test.executable_name, 'perf.data', additional_options = test.record_options)
574    if runner.sampler == 'inplace-sampler':
575      # TODO: fix this when inplace-sampler actually works.
576      runner.report('perf.data', 'perf.report')
577      symbols = report_analyzer._read_report_file('perf.report', runner.use_callgraph)
578      result = False
579      if len(symbols) == 1 and symbols[0].name.find('FakeFunction()') != -1:
580        result = True
581    else:
582      runner.report('perf.data', 'perf.report', additional_options = test.report_options)
583      result = report_analyzer.check_report_file(test, 'perf.report', runner.use_callgraph)
584    str = 'test %s on %s ' % (test.test_name, runner.target)
585    if runner.use_callgraph:
586      str += 'with call graph '
587    str += 'using %s ' % runner.sampler
588    str += ' Succeeded' if result else 'Failed'
589    print str
590    if not result:
591      exit(1)
592
593
594def runtest(target_options, use_callgraph_options, sampler_options, selected_tests):
595  tests = load_config_file(os.path.dirname(os.path.realpath(__file__)) + \
596                           '/runtest.conf')
597  if selected_tests is not None:
598    new_tests = []
599    for test in tests:
600      if test.test_name in selected_tests:
601        new_tests.append(test)
602    tests = new_tests
603  for target in target_options:
604    for use_callgraph in use_callgraph_options:
605      for sampler in sampler_options:
606        runner = build_runner(target, use_callgraph, sampler)
607        if runner is not None:
608          test_with_runner(runner, tests)
609
610
611def main():
612  target_options = ['host64', 'host32', 'device64', 'device32']
613  use_callgraph_options = [False, True]
614  sampler_options = ['cpu-cycles', 'inplace-sampler']
615  selected_tests = None
616  i = 1
617  while i < len(sys.argv):
618    if sys.argv[i] == '--host':
619      target_options = ['host64', 'host32']
620    elif sys.argv[i] == '--device':
621      target_options = ['device64', 'device32']
622    elif sys.argv[i] == '--normal':
623      use_callgraph_options = [False]
624    elif sys.argv[i] == '--callgraph':
625      use_callgraph_options = [True]
626    elif sys.argv[i] == '--no-inplace-sampler':
627      sampler_options = ['cpu-cycles']
628    elif sys.argv[i] == '--inplace-sampler':
629      sampler_options = ['inplace-sampler']
630    elif sys.argv[i] == '--test':
631      if i < len(sys.argv):
632        i += 1
633        for test in sys.argv[i].split(','):
634          if selected_tests is None:
635            selected_tests = {}
636          selected_tests[test] = True
637    i += 1
638  runtest(target_options, use_callgraph_options, sampler_options, selected_tests)
639
640if __name__ == '__main__':
641  main()
642