1#!/usr/bin/env python 2# 3# Copyright (C) 2015 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17"""Simpleperf runtest runner: run simpleperf runtests on host or on device. 18 19For a simpleperf runtest like one_function test, it contains following steps: 201. Run simpleperf record command to record simpleperf_runtest_one_function's 21 running samples, which is generated in perf.data. 222. Run simpleperf report command to parse perf.data, generate perf.report. 234. Parse perf.report and see if it matches expectation. 24 25The information of all runtests is stored in runtest.conf. 26""" 27 28import os 29import os.path 30import re 31import subprocess 32import sys 33import xml.etree.ElementTree as ET 34 35 36class CallTreeNode(object): 37 38 def __init__(self, name): 39 self.name = name 40 self.children = [] 41 42 def add_child(self, child): 43 self.children.append(child) 44 45 def __str__(self): 46 return 'CallTreeNode:\n' + '\n'.join(self._dump(1)) 47 48 def _dump(self, indent): 49 indent_str = ' ' * indent 50 strs = [indent_str + self.name] 51 for child in self.children: 52 strs.extend(child._dump(indent + 1)) 53 return strs 54 55 56class Symbol(object): 57 58 def __init__(self, name, comm, overhead, children_overhead): 59 self.name = name 60 self.comm = comm 61 self.overhead = overhead 62 # children_overhead is the overhead sum of this symbol and functions 63 # called by this symbol. 64 self.children_overhead = children_overhead 65 self.call_tree = None 66 67 def set_call_tree(self, call_tree): 68 self.call_tree = call_tree 69 70 def __str__(self): 71 strs = [] 72 strs.append('Symbol name=%s comm=%s overhead=%f children_overhead=%f' % ( 73 self.name, self.comm, self.overhead, self.children_overhead)) 74 if self.call_tree: 75 strs.append('\t%s' % self.call_tree) 76 return '\n'.join(strs) 77 78 79class SymbolOverheadRequirement(object): 80 81 def __init__(self, symbol_name=None, comm=None, min_overhead=None, 82 max_overhead=None): 83 self.symbol_name = symbol_name 84 self.comm = comm 85 self.min_overhead = min_overhead 86 self.max_overhead = max_overhead 87 88 def __str__(self): 89 strs = [] 90 strs.append('SymbolOverheadRequirement') 91 if self.symbol_name is not None: 92 strs.append('symbol_name=%s' % self.symbol_name) 93 if self.comm is not None: 94 strs.append('comm=%s' % self.comm) 95 if self.min_overhead is not None: 96 strs.append('min_overhead=%f' % self.min_overhead) 97 if self.max_overhead is not None: 98 strs.append('max_overhead=%f' % self.max_overhead) 99 return ' '.join(strs) 100 101 def is_match(self, symbol): 102 if self.symbol_name is not None: 103 if self.symbol_name != symbol.name: 104 return False 105 if self.comm is not None: 106 if self.comm != symbol.comm: 107 return False 108 return True 109 110 def check_overhead(self, overhead): 111 if self.min_overhead is not None: 112 if self.min_overhead > overhead: 113 return False 114 if self.max_overhead is not None: 115 if self.max_overhead < overhead: 116 return False 117 return True 118 119 120class SymbolRelationRequirement(object): 121 122 def __init__(self, symbol_name, comm=None): 123 self.symbol_name = symbol_name 124 self.comm = comm 125 self.children = [] 126 127 def add_child(self, child): 128 self.children.append(child) 129 130 def __str__(self): 131 return 'SymbolRelationRequirement:\n' + '\n'.join(self._dump(1)) 132 133 def _dump(self, indent): 134 indent_str = ' ' * indent 135 strs = [indent_str + self.symbol_name + 136 (' ' + self.comm if self.comm else '')] 137 for child in self.children: 138 strs.extend(child._dump(indent + 1)) 139 return strs 140 141 def is_match(self, symbol): 142 if symbol.name != self.symbol_name: 143 return False 144 if self.comm is not None: 145 if symbol.comm != self.comm: 146 return False 147 return True 148 149 def check_relation(self, call_tree): 150 if not call_tree: 151 return False 152 if self.symbol_name != call_tree.name: 153 return False 154 for child in self.children: 155 child_matched = False 156 for node in call_tree.children: 157 if child.check_relation(node): 158 child_matched = True 159 break 160 if not child_matched: 161 return False 162 return True 163 164 165class Test(object): 166 167 def __init__( 168 self, 169 test_name, 170 executable_name, 171 disable_host, 172 record_options, 173 report_options, 174 symbol_overhead_requirements, 175 symbol_children_overhead_requirements, 176 symbol_relation_requirements): 177 self.test_name = test_name 178 self.executable_name = executable_name 179 self.disable_host = disable_host 180 self.record_options = record_options 181 self.report_options = report_options 182 self.symbol_overhead_requirements = symbol_overhead_requirements 183 self.symbol_children_overhead_requirements = ( 184 symbol_children_overhead_requirements) 185 self.symbol_relation_requirements = symbol_relation_requirements 186 187 def __str__(self): 188 strs = [] 189 strs.append('Test test_name=%s' % self.test_name) 190 strs.append('\texecutable_name=%s' % self.executable_name) 191 strs.append('\tdisable_host=%s' % self.disable_host) 192 strs.append('\trecord_options=%s' % (' '.join(self.record_options))) 193 strs.append('\treport_options=%s' % (' '.join(self.report_options))) 194 strs.append('\tsymbol_overhead_requirements:') 195 for req in self.symbol_overhead_requirements: 196 strs.append('\t\t%s' % req) 197 strs.append('\tsymbol_children_overhead_requirements:') 198 for req in self.symbol_children_overhead_requirements: 199 strs.append('\t\t%s' % req) 200 strs.append('\tsymbol_relation_requirements:') 201 for req in self.symbol_relation_requirements: 202 strs.append('\t\t%s' % req) 203 return '\n'.join(strs) 204 205 206def load_config_file(config_file): 207 tests = [] 208 tree = ET.parse(config_file) 209 root = tree.getroot() 210 assert root.tag == 'runtests' 211 for test in root: 212 assert test.tag == 'test' 213 test_name = test.attrib['name'] 214 executable_name = None 215 disable_host = False 216 record_options = [] 217 report_options = [] 218 symbol_overhead_requirements = [] 219 symbol_children_overhead_requirements = [] 220 symbol_relation_requirements = [] 221 for test_item in test: 222 if test_item.tag == 'executable': 223 executable_name = test_item.attrib['name'] 224 elif test_item.tag == 'disable_host': 225 disable_host = True 226 elif test_item.tag == 'record': 227 record_options = test_item.attrib['option'].split() 228 elif test_item.tag == 'report': 229 report_options = test_item.attrib['option'].split() 230 elif (test_item.tag == 'symbol_overhead' or 231 test_item.tag == 'symbol_children_overhead'): 232 for symbol_item in test_item: 233 assert symbol_item.tag == 'symbol' 234 symbol_name = None 235 if 'name' in symbol_item.attrib: 236 symbol_name = symbol_item.attrib['name'] 237 comm = None 238 if 'comm' in symbol_item.attrib: 239 comm = symbol_item.attrib['comm'] 240 overhead_min = None 241 if 'min' in symbol_item.attrib: 242 overhead_min = float(symbol_item.attrib['min']) 243 overhead_max = None 244 if 'max' in symbol_item.attrib: 245 overhead_max = float(symbol_item.attrib['max']) 246 247 if test_item.tag == 'symbol_overhead': 248 symbol_overhead_requirements.append( 249 SymbolOverheadRequirement( 250 symbol_name, 251 comm, 252 overhead_min, 253 overhead_max) 254 ) 255 else: 256 symbol_children_overhead_requirements.append( 257 SymbolOverheadRequirement( 258 symbol_name, 259 comm, 260 overhead_min, 261 overhead_max)) 262 elif test_item.tag == 'symbol_callgraph_relation': 263 for symbol_item in test_item: 264 req = load_symbol_relation_requirement(symbol_item) 265 symbol_relation_requirements.append(req) 266 267 tests.append( 268 Test( 269 test_name, 270 executable_name, 271 disable_host, 272 record_options, 273 report_options, 274 symbol_overhead_requirements, 275 symbol_children_overhead_requirements, 276 symbol_relation_requirements)) 277 return tests 278 279 280def load_symbol_relation_requirement(symbol_item): 281 symbol_name = symbol_item.attrib['name'] 282 comm = None 283 if 'comm' in symbol_item.attrib: 284 comm = symbol_item.attrib['comm'] 285 req = SymbolRelationRequirement(symbol_name, comm) 286 for item in symbol_item: 287 child_req = load_symbol_relation_requirement(item) 288 req.add_child(child_req) 289 return req 290 291 292class Runner(object): 293 294 def __init__(self, target, perf_path): 295 self.target = target 296 self.is32 = target.endswith('32') 297 self.perf_path = perf_path 298 self.use_callgraph = False 299 self.sampler = 'cpu-cycles' 300 301 def record(self, test_executable_name, record_file, additional_options=[]): 302 call_args = [self.perf_path, 'record'] 303 call_args += ['--duration', '2'] 304 call_args += ['-e', '%s:u' % self.sampler] 305 if self.use_callgraph: 306 call_args += ['-f', '1000', '-g'] 307 call_args += ['-o', record_file] 308 call_args += additional_options 309 test_executable_name += '32' if self.is32 else '64' 310 call_args += [test_executable_name] 311 self._call(call_args) 312 313 def report(self, record_file, report_file, additional_options=[]): 314 call_args = [self.perf_path, 'report'] 315 call_args += ['-i', record_file] 316 if self.use_callgraph: 317 call_args += ['-g', 'callee'] 318 call_args += additional_options 319 self._call(call_args, report_file) 320 321 def _call(self, args, output_file=None): 322 pass 323 324 325class HostRunner(Runner): 326 327 """Run perf test on host.""" 328 329 def __init__(self, target): 330 perf_path = 'simpleperf32' if target.endswith('32') else 'simpleperf' 331 super(HostRunner, self).__init__(target, perf_path) 332 333 def _call(self, args, output_file=None): 334 output_fh = None 335 if output_file is not None: 336 output_fh = open(output_file, 'w') 337 subprocess.check_call(args, stdout=output_fh) 338 if output_fh is not None: 339 output_fh.close() 340 341 342class DeviceRunner(Runner): 343 344 """Run perf test on device.""" 345 346 def __init__(self, target): 347 self.tmpdir = '/data/local/tmp/' 348 perf_path = 'simpleperf32' if target.endswith('32') else 'simpleperf' 349 super(DeviceRunner, self).__init__(target, self.tmpdir + perf_path) 350 self._download(os.environ['OUT'] + '/system/xbin/' + perf_path, self.tmpdir) 351 lib = 'lib' if self.is32 else 'lib64' 352 self._download(os.environ['OUT'] + '/system/' + lib + '/libsimpleperf_inplace_sampler.so', 353 self.tmpdir) 354 355 def _call(self, args, output_file=None): 356 output_fh = None 357 if output_file is not None: 358 output_fh = open(output_file, 'w') 359 args_with_adb = ['adb', 'shell'] 360 args_with_adb.append('export LD_LIBRARY_PATH=' + self.tmpdir + ' && ' + ' '.join(args)) 361 subprocess.check_call(args_with_adb, stdout=output_fh) 362 if output_fh is not None: 363 output_fh.close() 364 365 def _download(self, file, to_dir): 366 args = ['adb', 'push', file, to_dir] 367 subprocess.check_call(args) 368 369 def record(self, test_executable_name, record_file, additional_options=[]): 370 self._download(os.environ['OUT'] + '/system/bin/' + test_executable_name + 371 ('32' if self.is32 else '64'), self.tmpdir) 372 super(DeviceRunner, self).record(self.tmpdir + test_executable_name, 373 self.tmpdir + record_file, 374 additional_options) 375 376 def report(self, record_file, report_file, additional_options=[]): 377 super(DeviceRunner, self).report(self.tmpdir + record_file, 378 report_file, 379 additional_options) 380 381class ReportAnalyzer(object): 382 383 """Check if perf.report matches expectation in Configuration.""" 384 385 def _read_report_file(self, report_file, has_callgraph): 386 fh = open(report_file, 'r') 387 lines = fh.readlines() 388 fh.close() 389 390 lines = [x.rstrip() for x in lines] 391 blank_line_index = -1 392 for i in range(len(lines)): 393 if not lines[i]: 394 blank_line_index = i 395 assert blank_line_index != -1 396 assert blank_line_index + 1 < len(lines) 397 title_line = lines[blank_line_index + 1] 398 report_item_lines = lines[blank_line_index + 2:] 399 400 if has_callgraph: 401 assert re.search(r'^Children\s+Self\s+Command.+Symbol$', title_line) 402 else: 403 assert re.search(r'^Overhead\s+Command.+Symbol$', title_line) 404 405 return self._parse_report_items(report_item_lines, has_callgraph) 406 407 def _parse_report_items(self, lines, has_callgraph): 408 symbols = [] 409 cur_symbol = None 410 call_tree_stack = {} 411 vertical_columns = [] 412 last_node = None 413 last_depth = -1 414 415 for line in lines: 416 if not line: 417 continue 418 if not line[0].isspace(): 419 if has_callgraph: 420 items = line.split(None, 6) 421 assert len(items) == 7 422 children_overhead = float(items[0][:-1]) 423 overhead = float(items[1][:-1]) 424 comm = items[2] 425 symbol_name = items[6] 426 cur_symbol = Symbol(symbol_name, comm, overhead, children_overhead) 427 symbols.append(cur_symbol) 428 else: 429 items = line.split(None, 5) 430 assert len(items) == 6 431 overhead = float(items[0][:-1]) 432 comm = items[1] 433 symbol_name = items[5] 434 cur_symbol = Symbol(symbol_name, comm, overhead, 0) 435 symbols.append(cur_symbol) 436 # Each report item can have different column depths. 437 vertical_columns = [] 438 else: 439 for i in range(len(line)): 440 if line[i] == '|': 441 if not vertical_columns or vertical_columns[-1] < i: 442 vertical_columns.append(i) 443 444 if not line.strip('| \t'): 445 continue 446 if line.find('-') == -1: 447 function_name = line.strip('| \t') 448 node = CallTreeNode(function_name) 449 last_node.add_child(node) 450 last_node = node 451 call_tree_stack[last_depth] = node 452 else: 453 pos = line.find('-') 454 depth = -1 455 for i in range(len(vertical_columns)): 456 if pos >= vertical_columns[i]: 457 depth = i 458 assert depth != -1 459 460 line = line.strip('|- \t') 461 m = re.search(r'^[\d\.]+%[-\s]+(.+)$', line) 462 if m: 463 function_name = m.group(1) 464 else: 465 function_name = line 466 467 node = CallTreeNode(function_name) 468 if depth == 0: 469 cur_symbol.set_call_tree(node) 470 471 else: 472 call_tree_stack[depth - 1].add_child(node) 473 call_tree_stack[depth] = node 474 last_node = node 475 last_depth = depth 476 477 return symbols 478 479 def check_report_file(self, test, report_file, has_callgraph): 480 symbols = self._read_report_file(report_file, has_callgraph) 481 if not self._check_symbol_overhead_requirements(test, symbols): 482 return False 483 if has_callgraph: 484 if not self._check_symbol_children_overhead_requirements(test, symbols): 485 return False 486 if not self._check_symbol_relation_requirements(test, symbols): 487 return False 488 return True 489 490 def _check_symbol_overhead_requirements(self, test, symbols): 491 result = True 492 matched = [False] * len(test.symbol_overhead_requirements) 493 matched_overhead = [0] * len(test.symbol_overhead_requirements) 494 for symbol in symbols: 495 for i in range(len(test.symbol_overhead_requirements)): 496 req = test.symbol_overhead_requirements[i] 497 if req.is_match(symbol): 498 matched[i] = True 499 matched_overhead[i] += symbol.overhead 500 for i in range(len(matched)): 501 if not matched[i]: 502 print 'requirement (%s) has no matched symbol in test %s' % ( 503 test.symbol_overhead_requirements[i], test) 504 result = False 505 else: 506 fulfilled = req.check_overhead(matched_overhead[i]) 507 if not fulfilled: 508 print "Symbol (%s) doesn't match requirement (%s) in test %s" % ( 509 symbol, req, test) 510 result = False 511 return result 512 513 def _check_symbol_children_overhead_requirements(self, test, symbols): 514 result = True 515 matched = [False] * len(test.symbol_children_overhead_requirements) 516 for symbol in symbols: 517 for i in range(len(test.symbol_children_overhead_requirements)): 518 req = test.symbol_children_overhead_requirements[i] 519 if req.is_match(symbol): 520 matched[i] = True 521 fulfilled = req.check_overhead(symbol.children_overhead) 522 if not fulfilled: 523 print "Symbol (%s) doesn't match requirement (%s) in test %s" % ( 524 symbol, req, test) 525 result = False 526 for i in range(len(matched)): 527 if not matched[i]: 528 print 'requirement (%s) has no matched symbol in test %s' % ( 529 test.symbol_children_overhead_requirements[i], test) 530 result = False 531 return result 532 533 def _check_symbol_relation_requirements(self, test, symbols): 534 result = True 535 matched = [False] * len(test.symbol_relation_requirements) 536 for symbol in symbols: 537 for i in range(len(test.symbol_relation_requirements)): 538 req = test.symbol_relation_requirements[i] 539 if req.is_match(symbol): 540 matched[i] = True 541 fulfilled = req.check_relation(symbol.call_tree) 542 if not fulfilled: 543 print "Symbol (%s) doesn't match requirement (%s) in test %s" % ( 544 symbol, req, test) 545 result = False 546 for i in range(len(matched)): 547 if not matched[i]: 548 print 'requirement (%s) has no matched symbol in test %s' % ( 549 test.symbol_relation_requirements[i], test) 550 result = False 551 return result 552 553 554def build_runner(target, use_callgraph, sampler): 555 if target == 'host32' and use_callgraph: 556 print "Current 64bit linux host doesn't support `simpleperf32 record -g`" 557 return None 558 if target.startswith('host'): 559 runner = HostRunner(target) 560 else: 561 runner = DeviceRunner(target) 562 runner.use_callgraph = use_callgraph 563 runner.sampler = sampler 564 return runner 565 566 567def test_with_runner(runner, tests): 568 report_analyzer = ReportAnalyzer() 569 for test in tests: 570 if test.disable_host and runner.target.startswith('host'): 571 print('Skip test %s on %s' % (test.test_name, runner.target)) 572 continue 573 runner.record(test.executable_name, 'perf.data', additional_options = test.record_options) 574 if runner.sampler == 'inplace-sampler': 575 # TODO: fix this when inplace-sampler actually works. 576 runner.report('perf.data', 'perf.report') 577 symbols = report_analyzer._read_report_file('perf.report', runner.use_callgraph) 578 result = False 579 if len(symbols) == 1 and symbols[0].name.find('FakeFunction()') != -1: 580 result = True 581 else: 582 runner.report('perf.data', 'perf.report', additional_options = test.report_options) 583 result = report_analyzer.check_report_file(test, 'perf.report', runner.use_callgraph) 584 str = 'test %s on %s ' % (test.test_name, runner.target) 585 if runner.use_callgraph: 586 str += 'with call graph ' 587 str += 'using %s ' % runner.sampler 588 str += ' Succeeded' if result else 'Failed' 589 print str 590 if not result: 591 exit(1) 592 593 594def runtest(target_options, use_callgraph_options, sampler_options, selected_tests): 595 tests = load_config_file(os.path.dirname(os.path.realpath(__file__)) + \ 596 '/runtest.conf') 597 if selected_tests is not None: 598 new_tests = [] 599 for test in tests: 600 if test.test_name in selected_tests: 601 new_tests.append(test) 602 tests = new_tests 603 for target in target_options: 604 for use_callgraph in use_callgraph_options: 605 for sampler in sampler_options: 606 runner = build_runner(target, use_callgraph, sampler) 607 if runner is not None: 608 test_with_runner(runner, tests) 609 610 611def main(): 612 target_options = ['host64', 'host32', 'device64', 'device32'] 613 use_callgraph_options = [False, True] 614 sampler_options = ['cpu-cycles', 'inplace-sampler'] 615 selected_tests = None 616 i = 1 617 while i < len(sys.argv): 618 if sys.argv[i] == '--host': 619 target_options = ['host64', 'host32'] 620 elif sys.argv[i] == '--device': 621 target_options = ['device64', 'device32'] 622 elif sys.argv[i] == '--normal': 623 use_callgraph_options = [False] 624 elif sys.argv[i] == '--callgraph': 625 use_callgraph_options = [True] 626 elif sys.argv[i] == '--no-inplace-sampler': 627 sampler_options = ['cpu-cycles'] 628 elif sys.argv[i] == '--inplace-sampler': 629 sampler_options = ['inplace-sampler'] 630 elif sys.argv[i] == '--test': 631 if i < len(sys.argv): 632 i += 1 633 for test in sys.argv[i].split(','): 634 if selected_tests is None: 635 selected_tests = {} 636 selected_tests[test] = True 637 i += 1 638 runtest(target_options, use_callgraph_options, sampler_options, selected_tests) 639 640if __name__ == '__main__': 641 main() 642