1#!/usr/bin/env python 2# 3# Copyright (C) 2015 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17"""Simpleperf runtest runner: run simpleperf runtests on host or on device. 18 19For a simpleperf runtest like one_function test, it contains following steps: 201. Run simpleperf record command to record simpleperf_runtest_one_function's 21 running samples, which is generated in perf.data. 222. Run simpleperf report command to parse perf.data, generate perf.report. 234. Parse perf.report and see if it matches expectation. 24 25The information of all runtests is stored in runtest.conf. 26""" 27 28import re 29import subprocess 30import sys 31import xml.etree.ElementTree as ET 32 33 34class CallTreeNode(object): 35 36 def __init__(self, name): 37 self.name = name 38 self.children = [] 39 40 def add_child(self, child): 41 self.children.append(child) 42 43 def __str__(self): 44 return 'CallTreeNode:\n' + '\n'.join(self._dump(1)) 45 46 def _dump(self, indent): 47 indent_str = ' ' * indent 48 strs = [indent_str + self.name] 49 for child in self.children: 50 strs.extend(child._dump(indent + 1)) 51 return strs 52 53 54class Symbol(object): 55 56 def __init__(self, name, comm, overhead, children_overhead): 57 self.name = name 58 self.comm = comm 59 self.overhead = overhead 60 # children_overhead is the overhead sum of this symbol and functions 61 # called by this symbol. 62 self.children_overhead = children_overhead 63 self.call_tree = None 64 65 def set_call_tree(self, call_tree): 66 self.call_tree = call_tree 67 68 def __str__(self): 69 strs = [] 70 strs.append('Symbol name=%s comm=%s overhead=%f children_overhead=%f' % ( 71 self.name, self.comm, self.overhead, self.children_overhead)) 72 if self.call_tree: 73 strs.append('\t%s' % self.call_tree) 74 return '\n'.join(strs) 75 76 77class SymbolOverheadRequirement(object): 78 79 def __init__(self, symbol_name=None, comm=None, min_overhead=None, 80 max_overhead=None): 81 self.symbol_name = symbol_name 82 self.comm = comm 83 self.min_overhead = min_overhead 84 self.max_overhead = max_overhead 85 86 def __str__(self): 87 strs = [] 88 strs.append('SymbolOverheadRequirement') 89 if self.symbol_name is not None: 90 strs.append('symbol_name=%s' % self.symbol_name) 91 if self.comm is not None: 92 strs.append('comm=%s' % self.comm) 93 if self.min_overhead is not None: 94 strs.append('min_overhead=%f' % self.min_overhead) 95 if self.max_overhead is not None: 96 strs.append('max_overhead=%f' % self.max_overhead) 97 return ' '.join(strs) 98 99 def is_match(self, symbol): 100 if self.symbol_name is not None: 101 if self.symbol_name != symbol.name: 102 return False 103 if self.comm is not None: 104 if self.comm != symbol.comm: 105 return False 106 return True 107 108 def check_overhead(self, overhead): 109 if self.min_overhead is not None: 110 if self.min_overhead > overhead: 111 return False 112 if self.max_overhead is not None: 113 if self.max_overhead < overhead: 114 return False 115 return True 116 117 118class SymbolRelationRequirement(object): 119 120 def __init__(self, symbol_name, comm=None): 121 self.symbol_name = symbol_name 122 self.comm = comm 123 self.children = [] 124 125 def add_child(self, child): 126 self.children.append(child) 127 128 def __str__(self): 129 return 'SymbolRelationRequirement:\n' + '\n'.join(self._dump(1)) 130 131 def _dump(self, indent): 132 indent_str = ' ' * indent 133 strs = [indent_str + self.symbol_name + 134 (' ' + self.comm if self.comm else '')] 135 for child in self.children: 136 strs.extend(child._dump(indent + 1)) 137 return strs 138 139 def is_match(self, symbol): 140 if symbol.name != self.symbol_name: 141 return False 142 if self.comm is not None: 143 if symbol.comm != self.comm: 144 return False 145 return True 146 147 def check_relation(self, call_tree): 148 if not call_tree: 149 return False 150 if self.symbol_name != call_tree.name: 151 return False 152 for child in self.children: 153 child_matched = False 154 for node in call_tree.children: 155 if child.check_relation(node): 156 child_matched = True 157 break 158 if not child_matched: 159 return False 160 return True 161 162 163class Test(object): 164 165 def __init__( 166 self, 167 test_name, 168 executable_name, 169 report_options, 170 symbol_overhead_requirements, 171 symbol_children_overhead_requirements, 172 symbol_relation_requirements): 173 self.test_name = test_name 174 self.executable_name = executable_name 175 self.report_options = report_options 176 self.symbol_overhead_requirements = symbol_overhead_requirements 177 self.symbol_children_overhead_requirements = ( 178 symbol_children_overhead_requirements) 179 self.symbol_relation_requirements = symbol_relation_requirements 180 181 def __str__(self): 182 strs = [] 183 strs.append('Test test_name=%s' % self.test_name) 184 strs.append('\texecutable_name=%s' % self.executable_name) 185 strs.append('\treport_options=%s' % (' '.join(self.report_options))) 186 strs.append('\tsymbol_overhead_requirements:') 187 for req in self.symbol_overhead_requirements: 188 strs.append('\t\t%s' % req) 189 strs.append('\tsymbol_children_overhead_requirements:') 190 for req in self.symbol_children_overhead_requirements: 191 strs.append('\t\t%s' % req) 192 strs.append('\tsymbol_relation_requirements:') 193 for req in self.symbol_relation_requirements: 194 strs.append('\t\t%s' % req) 195 return '\n'.join(strs) 196 197 198def load_config_file(config_file): 199 tests = [] 200 tree = ET.parse(config_file) 201 root = tree.getroot() 202 assert root.tag == 'runtests' 203 for test in root: 204 assert test.tag == 'test' 205 test_name = test.attrib['name'] 206 executable_name = None 207 report_options = [] 208 symbol_overhead_requirements = [] 209 symbol_children_overhead_requirements = [] 210 symbol_relation_requirements = [] 211 for test_item in test: 212 if test_item.tag == 'executable': 213 executable_name = test_item.attrib['name'] 214 elif test_item.tag == 'report': 215 report_options = test_item.attrib['option'].split() 216 elif (test_item.tag == 'symbol_overhead' or 217 test_item.tag == 'symbol_children_overhead'): 218 for symbol_item in test_item: 219 assert symbol_item.tag == 'symbol' 220 symbol_name = None 221 if 'name' in symbol_item.attrib: 222 symbol_name = symbol_item.attrib['name'] 223 comm = None 224 if 'comm' in symbol_item.attrib: 225 comm = symbol_item.attrib['comm'] 226 overhead_min = None 227 if 'min' in symbol_item.attrib: 228 overhead_min = float(symbol_item.attrib['min']) 229 overhead_max = None 230 if 'max' in symbol_item.attrib: 231 overhead_max = float(symbol_item.attrib['max']) 232 233 if test_item.tag == 'symbol_overhead': 234 symbol_overhead_requirements.append( 235 SymbolOverheadRequirement( 236 symbol_name, 237 comm, 238 overhead_min, 239 overhead_max) 240 ) 241 else: 242 symbol_children_overhead_requirements.append( 243 SymbolOverheadRequirement( 244 symbol_name, 245 comm, 246 overhead_min, 247 overhead_max)) 248 elif test_item.tag == 'symbol_callgraph_relation': 249 for symbol_item in test_item: 250 req = load_symbol_relation_requirement(symbol_item) 251 symbol_relation_requirements.append(req) 252 253 tests.append( 254 Test( 255 test_name, 256 executable_name, 257 report_options, 258 symbol_overhead_requirements, 259 symbol_children_overhead_requirements, 260 symbol_relation_requirements)) 261 return tests 262 263 264def load_symbol_relation_requirement(symbol_item): 265 symbol_name = symbol_item.attrib['name'] 266 comm = None 267 if 'comm' in symbol_item.attrib: 268 comm = symbol_item.attrib['comm'] 269 req = SymbolRelationRequirement(symbol_name, comm) 270 for item in symbol_item: 271 child_req = load_symbol_relation_requirement(item) 272 req.add_child(child_req) 273 return req 274 275 276class Runner(object): 277 278 def __init__(self, perf_path): 279 self.perf_path = perf_path 280 281 def record(self, test_executable_name, record_file, additional_options=[]): 282 call_args = [self.perf_path, 283 'record'] + additional_options + ['-e', 284 'cpu-cycles:u', 285 '-o', 286 record_file, 287 test_executable_name] 288 self._call(call_args) 289 290 def report(self, record_file, report_file, additional_options=[]): 291 call_args = [self.perf_path, 292 'report'] + additional_options + ['-i', 293 record_file] 294 self._call(call_args, report_file) 295 296 def _call(self, args, output_file=None): 297 pass 298 299 300class HostRunner(Runner): 301 302 """Run perf test on host.""" 303 304 def _call(self, args, output_file=None): 305 output_fh = None 306 if output_file is not None: 307 output_fh = open(output_file, 'w') 308 subprocess.check_call(args, stdout=output_fh) 309 if output_fh is not None: 310 output_fh.close() 311 312 313class DeviceRunner(Runner): 314 315 """Run perf test on device.""" 316 317 def _call(self, args, output_file=None): 318 output_fh = None 319 if output_file is not None: 320 output_fh = open(output_file, 'w') 321 args_with_adb = ['adb', 'shell'] 322 args_with_adb.extend(args) 323 subprocess.check_call(args_with_adb, stdout=output_fh) 324 if output_fh is not None: 325 output_fh.close() 326 327 328class ReportAnalyzer(object): 329 330 """Check if perf.report matches expectation in Configuration.""" 331 332 def _read_report_file(self, report_file, has_callgraph): 333 fh = open(report_file, 'r') 334 lines = fh.readlines() 335 fh.close() 336 337 lines = [x.rstrip() for x in lines] 338 blank_line_index = -1 339 for i in range(len(lines)): 340 if not lines[i]: 341 blank_line_index = i 342 assert blank_line_index != -1 343 assert blank_line_index + 1 < len(lines) 344 title_line = lines[blank_line_index + 1] 345 report_item_lines = lines[blank_line_index + 2:] 346 347 if has_callgraph: 348 assert re.search(r'^Children\s+Self\s+Command.+Symbol$', title_line) 349 else: 350 assert re.search(r'^Overhead\s+Command.+Symbol$', title_line) 351 352 return self._parse_report_items(report_item_lines, has_callgraph) 353 354 def _parse_report_items(self, lines, has_callgraph): 355 symbols = [] 356 cur_symbol = None 357 call_tree_stack = {} 358 vertical_columns = [] 359 last_node = None 360 last_depth = -1 361 362 for line in lines: 363 if not line: 364 continue 365 if not line[0].isspace(): 366 if has_callgraph: 367 m = re.search(r'^([\d\.]+)%\s+([\d\.]+)%\s+(\S+).*\s+(\S+)$', line) 368 children_overhead = float(m.group(1)) 369 overhead = float(m.group(2)) 370 comm = m.group(3) 371 symbol_name = m.group(4) 372 cur_symbol = Symbol(symbol_name, comm, overhead, children_overhead) 373 symbols.append(cur_symbol) 374 else: 375 m = re.search(r'^([\d\.]+)%\s+(\S+).*\s+(\S+)$', line) 376 overhead = float(m.group(1)) 377 comm = m.group(2) 378 symbol_name = m.group(3) 379 cur_symbol = Symbol(symbol_name, comm, overhead, 0) 380 symbols.append(cur_symbol) 381 # Each report item can have different column depths. 382 vertical_columns = [] 383 else: 384 for i in range(len(line)): 385 if line[i] == '|': 386 if not vertical_columns or vertical_columns[-1] < i: 387 vertical_columns.append(i) 388 389 if not line.strip('| \t'): 390 continue 391 if line.find('-') == -1: 392 function_name = line.strip('| \t') 393 node = CallTreeNode(function_name) 394 last_node.add_child(node) 395 last_node = node 396 call_tree_stack[last_depth] = node 397 else: 398 pos = line.find('-') 399 depth = -1 400 for i in range(len(vertical_columns)): 401 if pos >= vertical_columns[i]: 402 depth = i 403 assert depth != -1 404 405 line = line.strip('|- \t') 406 m = re.search(r'^[\d\.]+%[-\s]+(.+)$', line) 407 if m: 408 function_name = m.group(1) 409 else: 410 function_name = line 411 412 node = CallTreeNode(function_name) 413 if depth == 0: 414 cur_symbol.set_call_tree(node) 415 416 else: 417 call_tree_stack[depth - 1].add_child(node) 418 call_tree_stack[depth] = node 419 last_node = node 420 last_depth = depth 421 422 return symbols 423 424 def check_report_file(self, test, report_file, has_callgraph): 425 symbols = self._read_report_file(report_file, has_callgraph) 426 if not self._check_symbol_overhead_requirements(test, symbols): 427 return False 428 if has_callgraph: 429 if not self._check_symbol_children_overhead_requirements(test, symbols): 430 return False 431 if not self._check_symbol_relation_requirements(test, symbols): 432 return False 433 return True 434 435 def _check_symbol_overhead_requirements(self, test, symbols): 436 result = True 437 matched = [False] * len(test.symbol_overhead_requirements) 438 matched_overhead = [0] * len(test.symbol_overhead_requirements) 439 for symbol in symbols: 440 for i in range(len(test.symbol_overhead_requirements)): 441 req = test.symbol_overhead_requirements[i] 442 if req.is_match(symbol): 443 matched[i] = True 444 matched_overhead[i] += symbol.overhead 445 for i in range(len(matched)): 446 if not matched[i]: 447 print 'requirement (%s) has no matched symbol in test %s' % ( 448 test.symbol_overhead_requirements[i], test) 449 result = False 450 else: 451 fulfilled = req.check_overhead(matched_overhead[i]) 452 if not fulfilled: 453 print "Symbol (%s) doesn't match requirement (%s) in test %s" % ( 454 symbol, req, test) 455 result = False 456 return result 457 458 def _check_symbol_children_overhead_requirements(self, test, symbols): 459 result = True 460 matched = [False] * len(test.symbol_children_overhead_requirements) 461 for symbol in symbols: 462 for i in range(len(test.symbol_children_overhead_requirements)): 463 req = test.symbol_children_overhead_requirements[i] 464 if req.is_match(symbol): 465 matched[i] = True 466 fulfilled = req.check_overhead(symbol.children_overhead) 467 if not fulfilled: 468 print "Symbol (%s) doesn't match requirement (%s) in test %s" % ( 469 symbol, req, test) 470 result = False 471 for i in range(len(matched)): 472 if not matched[i]: 473 print 'requirement (%s) has no matched symbol in test %s' % ( 474 test.symbol_children_overhead_requirements[i], test) 475 result = False 476 return result 477 478 def _check_symbol_relation_requirements(self, test, symbols): 479 result = True 480 matched = [False] * len(test.symbol_relation_requirements) 481 for symbol in symbols: 482 for i in range(len(test.symbol_relation_requirements)): 483 req = test.symbol_relation_requirements[i] 484 if req.is_match(symbol): 485 matched[i] = True 486 fulfilled = req.check_relation(symbol.call_tree) 487 if not fulfilled: 488 print "Symbol (%s) doesn't match requirement (%s) in test %s" % ( 489 symbol, req, test) 490 result = False 491 for i in range(len(matched)): 492 if not matched[i]: 493 print 'requirement (%s) has no matched symbol in test %s' % ( 494 test.symbol_relation_requirements[i], test) 495 result = False 496 return result 497 498 499def runtest(host, device, normal, callgraph, selected_tests): 500 tests = load_config_file('runtest.conf') 501 host_runner = HostRunner('simpleperf') 502 device_runner = DeviceRunner('simpleperf') 503 report_analyzer = ReportAnalyzer() 504 for test in tests: 505 if selected_tests is not None: 506 if test.test_name not in selected_tests: 507 continue 508 if host and normal: 509 host_runner.record(test.executable_name, 'perf.data') 510 host_runner.report('perf.data', 'perf.report', 511 additional_options = test.report_options) 512 result = report_analyzer.check_report_file( 513 test, 'perf.report', False) 514 print 'test %s on host %s' % ( 515 test.test_name, 'Succeeded' if result else 'Failed') 516 if not result: 517 exit(1) 518 519 if device and normal: 520 device_runner.record(test.executable_name, '/data/perf.data') 521 device_runner.report('/data/perf.data', 'perf.report', 522 additional_options = test.report_options) 523 result = report_analyzer.check_report_file(test, 'perf.report', False) 524 print 'test %s on device %s' % ( 525 test.test_name, 'Succeeded' if result else 'Failed') 526 if not result: 527 exit(1) 528 529 if host and callgraph: 530 host_runner.record( 531 test.executable_name, 532 'perf_g.data', 533 additional_options=['-g']) 534 host_runner.report( 535 'perf_g.data', 536 'perf_g.report', 537 additional_options=['-g'] + test.report_options) 538 result = report_analyzer.check_report_file(test, 'perf_g.report', True) 539 print 'call-graph test %s on host %s' % ( 540 test.test_name, 'Succeeded' if result else 'Failed') 541 if not result: 542 exit(1) 543 544 if device and callgraph: 545 device_runner.record( 546 test.executable_name, 547 '/data/perf_g.data', 548 additional_options=['-g']) 549 device_runner.report( 550 '/data/perf_g.data', 551 'perf_g.report', 552 additional_options=['-g'] + test.report_options) 553 result = report_analyzer.check_report_file(test, 'perf_g.report', True) 554 print 'call-graph test %s on device %s' % ( 555 test.test_name, 'Succeeded' if result else 'Failed') 556 if not result: 557 exit(1) 558 559def main(): 560 host = True 561 device = True 562 normal = True 563 callgraph = True 564 selected_tests = None 565 i = 1 566 while i < len(sys.argv): 567 if sys.argv[i] == '--host': 568 host = True 569 device = False 570 elif sys.argv[i] == '--device': 571 host = False 572 device = True 573 elif sys.argv[i] == '--normal': 574 normal = True 575 callgraph = False 576 elif sys.argv[i] == '--callgraph': 577 normal = False 578 callgraph = True 579 elif sys.argv[i] == '--test': 580 if i < len(sys.argv): 581 i += 1 582 for test in sys.argv[i].split(','): 583 if selected_tests is None: 584 selected_tests = {} 585 selected_tests[test] = True 586 i += 1 587 runtest(host, device, normal, callgraph, selected_tests) 588 589if __name__ == '__main__': 590 main() 591