1#!/usr/bin/env python 2# Copyright 2014 the V8 project authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6""" 7Performance runner for d8. 8 9Call e.g. with tools/run-perf.py --arch ia32 some_suite.json 10 11The suite json format is expected to be: 12{ 13 "path": <relative path chunks to perf resources and main file>, 14 "name": <optional suite name, file name is default>, 15 "archs": [<architecture name for which this suite is run>, ...], 16 "binary": <name of binary to run, default "d8">, 17 "flags": [<flag to d8>, ...], 18 "test_flags": [<flag to the test file>, ...], 19 "run_count": <how often will this suite run (optional)>, 20 "run_count_XXX": <how often will this suite run for arch XXX (optional)>, 21 "resources": [<js file to be moved to android device>, ...] 22 "main": <main js perf runner file>, 23 "results_regexp": <optional regexp>, 24 "results_processor": <optional python results processor script>, 25 "units": <the unit specification for the performance dashboard>, 26 "tests": [ 27 { 28 "name": <name of the trace>, 29 "results_regexp": <optional more specific regexp>, 30 "results_processor": <optional python results processor script>, 31 "units": <the unit specification for the performance dashboard>, 32 }, ... 33 ] 34} 35 36The tests field can also nest other suites in arbitrary depth. A suite 37with a "main" file is a leaf suite that can contain one more level of 38tests. 39 40A suite's results_regexp is expected to have one string place holder 41"%s" for the trace name. A trace's results_regexp overwrites suite 42defaults. 43 44A suite's results_processor may point to an optional python script. If 45specified, it is called after running the tests like this (with a path 46relatve to the suite level's path): 47<results_processor file> <same flags as for d8> <suite level name> <output> 48 49The <output> is a temporary file containing d8 output. The results_regexp will 50be applied to the output of this script. 51 52A suite without "tests" is considered a performance test itself. 53 54Full example (suite with one runner): 55{ 56 "path": ["."], 57 "flags": ["--expose-gc"], 58 "test_flags": ["5"], 59 "archs": ["ia32", "x64"], 60 "run_count": 5, 61 "run_count_ia32": 3, 62 "main": "run.js", 63 "results_regexp": "^%s: (.+)$", 64 "units": "score", 65 "tests": [ 66 {"name": "Richards"}, 67 {"name": "DeltaBlue"}, 68 {"name": "NavierStokes", 69 "results_regexp": "^NavierStokes: (.+)$"} 70 ] 71} 72 73Full example (suite with several runners): 74{ 75 "path": ["."], 76 "flags": ["--expose-gc"], 77 "archs": ["ia32", "x64"], 78 "run_count": 5, 79 "units": "score", 80 "tests": [ 81 {"name": "Richards", 82 "path": ["richards"], 83 "main": "run.js", 84 "run_count": 3, 85 "results_regexp": "^Richards: (.+)$"}, 86 {"name": "NavierStokes", 87 "path": ["navier_stokes"], 88 "main": "run.js", 89 "results_regexp": "^NavierStokes: (.+)$"} 90 ] 91} 92 93Path pieces are concatenated. D8 is always run with the suite's path as cwd. 94 95The test flags are passed to the js test file after '--'. 96""" 97 98from collections import OrderedDict 99import json 100import logging 101import math 102import optparse 103import os 104import re 105import subprocess 106import sys 107 108from testrunner.local import commands 109from testrunner.local import utils 110 111ARCH_GUESS = utils.DefaultArch() 112SUPPORTED_ARCHS = ["arm", 113 "ia32", 114 "mips", 115 "mipsel", 116 "nacl_ia32", 117 "nacl_x64", 118 "x64", 119 "arm64"] 120 121GENERIC_RESULTS_RE = re.compile(r"^RESULT ([^:]+): ([^=]+)= ([^ ]+) ([^ ]*)$") 122RESULT_STDDEV_RE = re.compile(r"^\{([^\}]+)\}$") 123RESULT_LIST_RE = re.compile(r"^\[([^\]]+)\]$") 124TOOLS_BASE = os.path.abspath(os.path.dirname(__file__)) 125 126 127def LoadAndroidBuildTools(path): # pragma: no cover 128 assert os.path.exists(path) 129 sys.path.insert(0, path) 130 131 from pylib.device import adb_wrapper # pylint: disable=F0401 132 from pylib.device import device_errors # pylint: disable=F0401 133 from pylib.device import device_utils # pylint: disable=F0401 134 from pylib.perf import cache_control # pylint: disable=F0401 135 from pylib.perf import perf_control # pylint: disable=F0401 136 global adb_wrapper 137 global cache_control 138 global device_errors 139 global device_utils 140 global perf_control 141 142 143def GeometricMean(values): 144 """Returns the geometric mean of a list of values. 145 146 The mean is calculated using log to avoid overflow. 147 """ 148 values = map(float, values) 149 return str(math.exp(sum(map(math.log, values)) / len(values))) 150 151 152class Results(object): 153 """Place holder for result traces.""" 154 def __init__(self, traces=None, errors=None): 155 self.traces = traces or [] 156 self.errors = errors or [] 157 158 def ToDict(self): 159 return {"traces": self.traces, "errors": self.errors} 160 161 def WriteToFile(self, file_name): 162 with open(file_name, "w") as f: 163 f.write(json.dumps(self.ToDict())) 164 165 def __add__(self, other): 166 self.traces += other.traces 167 self.errors += other.errors 168 return self 169 170 def __str__(self): # pragma: no cover 171 return str(self.ToDict()) 172 173 174class Measurement(object): 175 """Represents a series of results of one trace. 176 177 The results are from repetitive runs of the same executable. They are 178 gathered by repeated calls to ConsumeOutput. 179 """ 180 def __init__(self, graphs, units, results_regexp, stddev_regexp): 181 self.name = graphs[-1] 182 self.graphs = graphs 183 self.units = units 184 self.results_regexp = results_regexp 185 self.stddev_regexp = stddev_regexp 186 self.results = [] 187 self.errors = [] 188 self.stddev = "" 189 190 def ConsumeOutput(self, stdout): 191 try: 192 result = re.search(self.results_regexp, stdout, re.M).group(1) 193 self.results.append(str(float(result))) 194 except ValueError: 195 self.errors.append("Regexp \"%s\" returned a non-numeric for test %s." 196 % (self.results_regexp, self.name)) 197 except: 198 self.errors.append("Regexp \"%s\" didn't match for test %s." 199 % (self.results_regexp, self.name)) 200 201 try: 202 if self.stddev_regexp and self.stddev: 203 self.errors.append("Test %s should only run once since a stddev " 204 "is provided by the test." % self.name) 205 if self.stddev_regexp: 206 self.stddev = re.search(self.stddev_regexp, stdout, re.M).group(1) 207 except: 208 self.errors.append("Regexp \"%s\" didn't match for test %s." 209 % (self.stddev_regexp, self.name)) 210 211 def GetResults(self): 212 return Results([{ 213 "graphs": self.graphs, 214 "units": self.units, 215 "results": self.results, 216 "stddev": self.stddev, 217 }], self.errors) 218 219 220class NullMeasurement(object): 221 """Null object to avoid having extra logic for configurations that didn't 222 run like running without patch on trybots. 223 """ 224 def ConsumeOutput(self, stdout): 225 pass 226 227 def GetResults(self): 228 return Results() 229 230 231def Unzip(iterable): 232 left = [] 233 right = [] 234 for l, r in iterable: 235 left.append(l) 236 right.append(r) 237 return lambda: iter(left), lambda: iter(right) 238 239 240def AccumulateResults( 241 graph_names, trace_configs, iter_output, trybot, no_patch, calc_total): 242 """Iterates over the output of multiple benchmark reruns and accumulates 243 results for a configured list of traces. 244 245 Args: 246 graph_names: List of names that configure the base path of the traces. E.g. 247 ['v8', 'Octane']. 248 trace_configs: List of "TraceConfig" instances. Each trace config defines 249 how to perform a measurement. 250 iter_output: Iterator over the standard output of each test run. 251 trybot: Indicates that this is run in trybot mode, i.e. run twice, once 252 with once without patch. 253 no_patch: Indicates weather this is a trybot run without patch. 254 calc_total: Boolean flag to speficy the calculation of a summary trace. 255 Returns: A "Results" object. 256 """ 257 measurements = [ 258 trace.CreateMeasurement(trybot, no_patch) for trace in trace_configs] 259 for stdout in iter_output(): 260 for measurement in measurements: 261 measurement.ConsumeOutput(stdout) 262 263 res = reduce(lambda r, m: r + m.GetResults(), measurements, Results()) 264 265 if not res.traces or not calc_total: 266 return res 267 268 # Assume all traces have the same structure. 269 if len(set(map(lambda t: len(t["results"]), res.traces))) != 1: 270 res.errors.append("Not all traces have the same number of results.") 271 return res 272 273 # Calculate the geometric means for all traces. Above we made sure that 274 # there is at least one trace and that the number of results is the same 275 # for each trace. 276 n_results = len(res.traces[0]["results"]) 277 total_results = [GeometricMean(t["results"][i] for t in res.traces) 278 for i in range(0, n_results)] 279 res.traces.append({ 280 "graphs": graph_names + ["Total"], 281 "units": res.traces[0]["units"], 282 "results": total_results, 283 "stddev": "", 284 }) 285 return res 286 287 288def AccumulateGenericResults(graph_names, suite_units, iter_output): 289 """Iterates over the output of multiple benchmark reruns and accumulates 290 generic results. 291 292 Args: 293 graph_names: List of names that configure the base path of the traces. E.g. 294 ['v8', 'Octane']. 295 suite_units: Measurement default units as defined by the benchmark suite. 296 iter_output: Iterator over the standard output of each test run. 297 Returns: A "Results" object. 298 """ 299 traces = OrderedDict() 300 for stdout in iter_output(): 301 if stdout is None: 302 # The None value is used as a null object to simplify logic. 303 continue 304 for line in stdout.strip().splitlines(): 305 match = GENERIC_RESULTS_RE.match(line) 306 if match: 307 stddev = "" 308 graph = match.group(1) 309 trace = match.group(2) 310 body = match.group(3) 311 units = match.group(4) 312 match_stddev = RESULT_STDDEV_RE.match(body) 313 match_list = RESULT_LIST_RE.match(body) 314 errors = [] 315 if match_stddev: 316 result, stddev = map(str.strip, match_stddev.group(1).split(",")) 317 results = [result] 318 elif match_list: 319 results = map(str.strip, match_list.group(1).split(",")) 320 else: 321 results = [body.strip()] 322 323 try: 324 results = map(lambda r: str(float(r)), results) 325 except ValueError: 326 results = [] 327 errors = ["Found non-numeric in %s" % 328 "/".join(graph_names + [graph, trace])] 329 330 trace_result = traces.setdefault(trace, Results([{ 331 "graphs": graph_names + [graph, trace], 332 "units": (units or suite_units).strip(), 333 "results": [], 334 "stddev": "", 335 }], errors)) 336 trace_result.traces[0]["results"].extend(results) 337 trace_result.traces[0]["stddev"] = stddev 338 339 return reduce(lambda r, t: r + t, traces.itervalues(), Results()) 340 341 342class Node(object): 343 """Represents a node in the suite tree structure.""" 344 def __init__(self, *args): 345 self._children = [] 346 347 def AppendChild(self, child): 348 self._children.append(child) 349 350 351class DefaultSentinel(Node): 352 """Fake parent node with all default values.""" 353 def __init__(self, binary = "d8"): 354 super(DefaultSentinel, self).__init__() 355 self.binary = binary 356 self.run_count = 10 357 self.timeout = 60 358 self.path = [] 359 self.graphs = [] 360 self.flags = [] 361 self.test_flags = [] 362 self.resources = [] 363 self.results_regexp = None 364 self.stddev_regexp = None 365 self.units = "score" 366 self.total = False 367 368 369class GraphConfig(Node): 370 """Represents a suite definition. 371 372 Can either be a leaf or an inner node that provides default values. 373 """ 374 def __init__(self, suite, parent, arch): 375 super(GraphConfig, self).__init__() 376 self._suite = suite 377 378 assert isinstance(suite.get("path", []), list) 379 assert isinstance(suite["name"], basestring) 380 assert isinstance(suite.get("flags", []), list) 381 assert isinstance(suite.get("test_flags", []), list) 382 assert isinstance(suite.get("resources", []), list) 383 384 # Accumulated values. 385 self.path = parent.path[:] + suite.get("path", []) 386 self.graphs = parent.graphs[:] + [suite["name"]] 387 self.flags = parent.flags[:] + suite.get("flags", []) 388 self.test_flags = parent.test_flags[:] + suite.get("test_flags", []) 389 390 # Values independent of parent node. 391 self.resources = suite.get("resources", []) 392 393 # Descrete values (with parent defaults). 394 self.binary = suite.get("binary", parent.binary) 395 self.run_count = suite.get("run_count", parent.run_count) 396 self.run_count = suite.get("run_count_%s" % arch, self.run_count) 397 self.timeout = suite.get("timeout", parent.timeout) 398 self.timeout = suite.get("timeout_%s" % arch, self.timeout) 399 self.units = suite.get("units", parent.units) 400 self.total = suite.get("total", parent.total) 401 402 # A regular expression for results. If the parent graph provides a 403 # regexp and the current suite has none, a string place holder for the 404 # suite name is expected. 405 # TODO(machenbach): Currently that makes only sense for the leaf level. 406 # Multiple place holders for multiple levels are not supported. 407 if parent.results_regexp: 408 regexp_default = parent.results_regexp % re.escape(suite["name"]) 409 else: 410 regexp_default = None 411 self.results_regexp = suite.get("results_regexp", regexp_default) 412 413 # A similar regular expression for the standard deviation (optional). 414 if parent.stddev_regexp: 415 stddev_default = parent.stddev_regexp % re.escape(suite["name"]) 416 else: 417 stddev_default = None 418 self.stddev_regexp = suite.get("stddev_regexp", stddev_default) 419 420 421class TraceConfig(GraphConfig): 422 """Represents a leaf in the suite tree structure.""" 423 def __init__(self, suite, parent, arch): 424 super(TraceConfig, self).__init__(suite, parent, arch) 425 assert self.results_regexp 426 427 def CreateMeasurement(self, trybot, no_patch): 428 if not trybot and no_patch: 429 # Use null object for no-patch logic if this is not a trybot run. 430 return NullMeasurement() 431 432 return Measurement( 433 self.graphs, 434 self.units, 435 self.results_regexp, 436 self.stddev_regexp, 437 ) 438 439 440class RunnableConfig(GraphConfig): 441 """Represents a runnable suite definition (i.e. has a main file). 442 """ 443 @property 444 def main(self): 445 return self._suite.get("main", "") 446 447 def ChangeCWD(self, suite_path): 448 """Changes the cwd to to path defined in the current graph. 449 450 The tests are supposed to be relative to the suite configuration. 451 """ 452 suite_dir = os.path.abspath(os.path.dirname(suite_path)) 453 bench_dir = os.path.normpath(os.path.join(*self.path)) 454 os.chdir(os.path.join(suite_dir, bench_dir)) 455 456 def GetCommandFlags(self, extra_flags=None): 457 suffix = ["--"] + self.test_flags if self.test_flags else [] 458 return self.flags + (extra_flags or []) + [self.main] + suffix 459 460 def GetCommand(self, shell_dir, extra_flags=None): 461 # TODO(machenbach): This requires +.exe if run on windows. 462 extra_flags = extra_flags or [] 463 cmd = [os.path.join(shell_dir, self.binary)] 464 if self.binary != 'd8' and '--prof' in extra_flags: 465 print "Profiler supported only on a benchmark run with d8" 466 return cmd + self.GetCommandFlags(extra_flags=extra_flags) 467 468 def Run(self, runner, trybot): 469 """Iterates over several runs and handles the output for all traces.""" 470 stdout_with_patch, stdout_no_patch = Unzip(runner()) 471 return ( 472 AccumulateResults( 473 self.graphs, 474 self._children, 475 iter_output=stdout_with_patch, 476 trybot=trybot, 477 no_patch=False, 478 calc_total=self.total, 479 ), 480 AccumulateResults( 481 self.graphs, 482 self._children, 483 iter_output=stdout_no_patch, 484 trybot=trybot, 485 no_patch=True, 486 calc_total=self.total, 487 ), 488 ) 489 490 491class RunnableTraceConfig(TraceConfig, RunnableConfig): 492 """Represents a runnable suite definition that is a leaf.""" 493 def __init__(self, suite, parent, arch): 494 super(RunnableTraceConfig, self).__init__(suite, parent, arch) 495 496 def Run(self, runner, trybot): 497 """Iterates over several runs and handles the output.""" 498 measurement_with_patch = self.CreateMeasurement(trybot, False) 499 measurement_no_patch = self.CreateMeasurement(trybot, True) 500 for stdout_with_patch, stdout_no_patch in runner(): 501 measurement_with_patch.ConsumeOutput(stdout_with_patch) 502 measurement_no_patch.ConsumeOutput(stdout_no_patch) 503 return ( 504 measurement_with_patch.GetResults(), 505 measurement_no_patch.GetResults(), 506 ) 507 508 509class RunnableGenericConfig(RunnableConfig): 510 """Represents a runnable suite definition with generic traces.""" 511 def __init__(self, suite, parent, arch): 512 super(RunnableGenericConfig, self).__init__(suite, parent, arch) 513 514 def Run(self, runner, trybot): 515 stdout_with_patch, stdout_no_patch = Unzip(runner()) 516 return ( 517 AccumulateGenericResults(self.graphs, self.units, stdout_with_patch), 518 AccumulateGenericResults(self.graphs, self.units, stdout_no_patch), 519 ) 520 521 522def MakeGraphConfig(suite, arch, parent): 523 """Factory method for making graph configuration objects.""" 524 if isinstance(parent, RunnableConfig): 525 # Below a runnable can only be traces. 526 return TraceConfig(suite, parent, arch) 527 elif suite.get("main") is not None: 528 # A main file makes this graph runnable. Empty strings are accepted. 529 if suite.get("tests"): 530 # This graph has subgraphs (traces). 531 return RunnableConfig(suite, parent, arch) 532 else: 533 # This graph has no subgraphs, it's a leaf. 534 return RunnableTraceConfig(suite, parent, arch) 535 elif suite.get("generic"): 536 # This is a generic suite definition. It is either a runnable executable 537 # or has a main js file. 538 return RunnableGenericConfig(suite, parent, arch) 539 elif suite.get("tests"): 540 # This is neither a leaf nor a runnable. 541 return GraphConfig(suite, parent, arch) 542 else: # pragma: no cover 543 raise Exception("Invalid suite configuration.") 544 545 546def BuildGraphConfigs(suite, arch, parent): 547 """Builds a tree structure of graph objects that corresponds to the suite 548 configuration. 549 """ 550 551 # TODO(machenbach): Implement notion of cpu type? 552 if arch not in suite.get("archs", SUPPORTED_ARCHS): 553 return None 554 555 graph = MakeGraphConfig(suite, arch, parent) 556 for subsuite in suite.get("tests", []): 557 BuildGraphConfigs(subsuite, arch, graph) 558 parent.AppendChild(graph) 559 return graph 560 561 562def FlattenRunnables(node, node_cb): 563 """Generator that traverses the tree structure and iterates over all 564 runnables. 565 """ 566 node_cb(node) 567 if isinstance(node, RunnableConfig): 568 yield node 569 elif isinstance(node, Node): 570 for child in node._children: 571 for result in FlattenRunnables(child, node_cb): 572 yield result 573 else: # pragma: no cover 574 raise Exception("Invalid suite configuration.") 575 576 577class Platform(object): 578 def __init__(self, options): 579 self.shell_dir = options.shell_dir 580 self.shell_dir_no_patch = options.shell_dir_no_patch 581 self.extra_flags = options.extra_flags.split() 582 583 @staticmethod 584 def GetPlatform(options): 585 if options.android_build_tools: 586 return AndroidPlatform(options) 587 else: 588 return DesktopPlatform(options) 589 590 def _Run(self, runnable, count, no_patch=False): 591 raise NotImplementedError() # pragma: no cover 592 593 def Run(self, runnable, count): 594 """Execute the benchmark's main file. 595 596 If options.shell_dir_no_patch is specified, the benchmark is run once with 597 and once without patch. 598 Args: 599 runnable: A Runnable benchmark instance. 600 count: The number of this (repeated) run. 601 Returns: A tuple with the benchmark outputs with and without patch. The 602 latter will be None if options.shell_dir_no_patch was not 603 specified. 604 """ 605 stdout = self._Run(runnable, count, no_patch=False) 606 if self.shell_dir_no_patch: 607 return stdout, self._Run(runnable, count, no_patch=True) 608 else: 609 return stdout, None 610 611 612class DesktopPlatform(Platform): 613 def __init__(self, options): 614 super(DesktopPlatform, self).__init__(options) 615 self.command_prefix = [] 616 617 if options.prioritize or options.affinitize != None: 618 self.command_prefix = ["schedtool"] 619 if options.prioritize: 620 self.command_prefix += ["-n", "-20"] 621 if options.affinitize != None: 622 # schedtool expects a bit pattern when setting affinity, where each 623 # bit set to '1' corresponds to a core where the process may run on. 624 # First bit corresponds to CPU 0. Since the 'affinitize' parameter is 625 # a core number, we need to map to said bit pattern. 626 cpu = int(options.affinitize) 627 core = 1 << cpu 628 self.command_prefix += ["-a", ("0x%x" % core)] 629 self.command_prefix += ["-e"] 630 631 def PreExecution(self): 632 pass 633 634 def PostExecution(self): 635 pass 636 637 def PreTests(self, node, path): 638 if isinstance(node, RunnableConfig): 639 node.ChangeCWD(path) 640 641 def _Run(self, runnable, count, no_patch=False): 642 suffix = ' - without patch' if no_patch else '' 643 shell_dir = self.shell_dir_no_patch if no_patch else self.shell_dir 644 title = ">>> %%s (#%d)%s:" % ((count + 1), suffix) 645 command = self.command_prefix + runnable.GetCommand(shell_dir, 646 self.extra_flags) 647 try: 648 output = commands.Execute( 649 command, 650 timeout=runnable.timeout, 651 ) 652 except OSError as e: # pragma: no cover 653 print title % "OSError" 654 print e 655 return "" 656 657 print title % "Stdout" 658 print output.stdout 659 if output.stderr: # pragma: no cover 660 # Print stderr for debugging. 661 print title % "Stderr" 662 print output.stderr 663 if output.timed_out: 664 print ">>> Test timed out after %ss." % runnable.timeout 665 if '--prof' in self.extra_flags: 666 os_prefix = {"linux": "linux", "macos": "mac"}.get(utils.GuessOS()) 667 if os_prefix: 668 tick_tools = os.path.join(TOOLS_BASE, "%s-tick-processor" % os_prefix) 669 subprocess.check_call(tick_tools + " --only-summary", shell=True) 670 else: # pragma: no cover 671 print "Profiler option currently supported on Linux and Mac OS." 672 return output.stdout 673 674 675class AndroidPlatform(Platform): # pragma: no cover 676 DEVICE_DIR = "/data/local/tmp/v8/" 677 678 def __init__(self, options): 679 super(AndroidPlatform, self).__init__(options) 680 LoadAndroidBuildTools(options.android_build_tools) 681 682 if not options.device: 683 # Detect attached device if not specified. 684 devices = adb_wrapper.AdbWrapper.Devices() 685 assert devices and len(devices) == 1, ( 686 "None or multiple devices detected. Please specify the device on " 687 "the command-line with --device") 688 options.device = str(devices[0]) 689 self.adb_wrapper = adb_wrapper.AdbWrapper(options.device) 690 self.device = device_utils.DeviceUtils(self.adb_wrapper) 691 692 def PreExecution(self): 693 perf = perf_control.PerfControl(self.device) 694 perf.SetHighPerfMode() 695 696 # Remember what we have already pushed to the device. 697 self.pushed = set() 698 699 def PostExecution(self): 700 perf = perf_control.PerfControl(self.device) 701 perf.SetDefaultPerfMode() 702 self.device.RunShellCommand(["rm", "-rf", AndroidPlatform.DEVICE_DIR]) 703 704 def _PushFile(self, host_dir, file_name, target_rel=".", 705 skip_if_missing=False): 706 file_on_host = os.path.join(host_dir, file_name) 707 file_on_device_tmp = os.path.join( 708 AndroidPlatform.DEVICE_DIR, "_tmp_", file_name) 709 file_on_device = os.path.join( 710 AndroidPlatform.DEVICE_DIR, target_rel, file_name) 711 folder_on_device = os.path.dirname(file_on_device) 712 713 # Only attempt to push files that exist. 714 if not os.path.exists(file_on_host): 715 if not skip_if_missing: 716 logging.critical('Missing file on host: %s' % file_on_host) 717 return 718 719 # Only push files not yet pushed in one execution. 720 if file_on_host in self.pushed: 721 return 722 else: 723 self.pushed.add(file_on_host) 724 725 # Work-around for "text file busy" errors. Push the files to a temporary 726 # location and then copy them with a shell command. 727 output = self.adb_wrapper.Push(file_on_host, file_on_device_tmp) 728 # Success looks like this: "3035 KB/s (12512056 bytes in 4.025s)". 729 # Errors look like this: "failed to copy ... ". 730 if output and not re.search('^[0-9]', output.splitlines()[-1]): 731 logging.critical('PUSH FAILED: ' + output) 732 self.adb_wrapper.Shell("mkdir -p %s" % folder_on_device) 733 self.adb_wrapper.Shell("cp %s %s" % (file_on_device_tmp, file_on_device)) 734 735 def _PushExecutable(self, shell_dir, target_dir, binary): 736 self._PushFile(shell_dir, binary, target_dir) 737 738 # Push external startup data. Backwards compatible for revisions where 739 # these files didn't exist. 740 self._PushFile( 741 shell_dir, 742 "natives_blob.bin", 743 target_dir, 744 skip_if_missing=True, 745 ) 746 self._PushFile( 747 shell_dir, 748 "snapshot_blob.bin", 749 target_dir, 750 skip_if_missing=True, 751 ) 752 self._PushFile( 753 shell_dir, 754 "snapshot_blob_ignition.bin", 755 target_dir, 756 skip_if_missing=True, 757 ) 758 759 def PreTests(self, node, path): 760 suite_dir = os.path.abspath(os.path.dirname(path)) 761 if node.path: 762 bench_rel = os.path.normpath(os.path.join(*node.path)) 763 bench_abs = os.path.join(suite_dir, bench_rel) 764 else: 765 bench_rel = "." 766 bench_abs = suite_dir 767 768 self._PushExecutable(self.shell_dir, "bin", node.binary) 769 if self.shell_dir_no_patch: 770 self._PushExecutable( 771 self.shell_dir_no_patch, "bin_no_patch", node.binary) 772 773 if isinstance(node, RunnableConfig): 774 self._PushFile(bench_abs, node.main, bench_rel) 775 for resource in node.resources: 776 self._PushFile(bench_abs, resource, bench_rel) 777 778 def _Run(self, runnable, count, no_patch=False): 779 suffix = ' - without patch' if no_patch else '' 780 target_dir = "bin_no_patch" if no_patch else "bin" 781 title = ">>> %%s (#%d)%s:" % ((count + 1), suffix) 782 cache = cache_control.CacheControl(self.device) 783 cache.DropRamCaches() 784 binary_on_device = os.path.join( 785 AndroidPlatform.DEVICE_DIR, target_dir, runnable.binary) 786 cmd = [binary_on_device] + runnable.GetCommandFlags(self.extra_flags) 787 788 # Relative path to benchmark directory. 789 if runnable.path: 790 bench_rel = os.path.normpath(os.path.join(*runnable.path)) 791 else: 792 bench_rel = "." 793 794 try: 795 output = self.device.RunShellCommand( 796 cmd, 797 cwd=os.path.join(AndroidPlatform.DEVICE_DIR, bench_rel), 798 timeout=runnable.timeout, 799 retries=0, 800 ) 801 stdout = "\n".join(output) 802 print title % "Stdout" 803 print stdout 804 except device_errors.CommandTimeoutError: 805 print ">>> Test timed out after %ss." % runnable.timeout 806 stdout = "" 807 return stdout 808 809class CustomMachineConfiguration: 810 def __init__(self, disable_aslr = False, governor = None): 811 self.aslr_backup = None 812 self.governor_backup = None 813 self.disable_aslr = disable_aslr 814 self.governor = governor 815 816 def __enter__(self): 817 if self.disable_aslr: 818 self.aslr_backup = CustomMachineConfiguration.GetASLR() 819 CustomMachineConfiguration.SetASLR(0) 820 if self.governor != None: 821 self.governor_backup = CustomMachineConfiguration.GetCPUGovernor() 822 CustomMachineConfiguration.SetCPUGovernor(self.governor) 823 return self 824 825 def __exit__(self, type, value, traceback): 826 if self.aslr_backup != None: 827 CustomMachineConfiguration.SetASLR(self.aslr_backup) 828 if self.governor_backup != None: 829 CustomMachineConfiguration.SetCPUGovernor(self.governor_backup) 830 831 @staticmethod 832 def GetASLR(): 833 try: 834 with open("/proc/sys/kernel/randomize_va_space", "r") as f: 835 return int(f.readline().strip()) 836 except Exception as e: 837 print "Failed to get current ASLR settings." 838 raise e 839 840 @staticmethod 841 def SetASLR(value): 842 try: 843 with open("/proc/sys/kernel/randomize_va_space", "w") as f: 844 f.write(str(value)) 845 except Exception as e: 846 print "Failed to update ASLR to %s." % value 847 print "Are we running under sudo?" 848 raise e 849 850 new_value = CustomMachineConfiguration.GetASLR() 851 if value != new_value: 852 raise Exception("Present value is %s" % new_value) 853 854 @staticmethod 855 def GetCPUCoresRange(): 856 try: 857 with open("/sys/devices/system/cpu/present", "r") as f: 858 indexes = f.readline() 859 r = map(int, indexes.split("-")) 860 if len(r) == 1: 861 return range(r[0], r[0] + 1) 862 return range(r[0], r[1] + 1) 863 except Exception as e: 864 print "Failed to retrieve number of CPUs." 865 raise e 866 867 @staticmethod 868 def GetCPUPathForId(cpu_index): 869 ret = "/sys/devices/system/cpu/cpu" 870 ret += str(cpu_index) 871 ret += "/cpufreq/scaling_governor" 872 return ret 873 874 @staticmethod 875 def GetCPUGovernor(): 876 try: 877 cpu_indices = CustomMachineConfiguration.GetCPUCoresRange() 878 ret = None 879 for cpu_index in cpu_indices: 880 cpu_device = CustomMachineConfiguration.GetCPUPathForId(cpu_index) 881 with open(cpu_device, "r") as f: 882 # We assume the governors of all CPUs are set to the same value 883 val = f.readline().strip() 884 if ret == None: 885 ret = val 886 elif ret != val: 887 raise Exception("CPU cores have differing governor settings") 888 return ret 889 except Exception as e: 890 print "Failed to get the current CPU governor." 891 print "Is the CPU governor disabled? Check BIOS." 892 raise e 893 894 @staticmethod 895 def SetCPUGovernor(value): 896 try: 897 cpu_indices = CustomMachineConfiguration.GetCPUCoresRange() 898 for cpu_index in cpu_indices: 899 cpu_device = CustomMachineConfiguration.GetCPUPathForId(cpu_index) 900 with open(cpu_device, "w") as f: 901 f.write(value) 902 903 except Exception as e: 904 print "Failed to change CPU governor to %s." % value 905 print "Are we running under sudo?" 906 raise e 907 908 cur_value = CustomMachineConfiguration.GetCPUGovernor() 909 if cur_value != value: 910 raise Exception("Could not set CPU governor. Present value is %s" 911 % cur_value ) 912 913# TODO: Implement results_processor. 914def Main(args): 915 logging.getLogger().setLevel(logging.INFO) 916 parser = optparse.OptionParser() 917 parser.add_option("--android-build-tools", 918 help="Path to chromium's build/android. Specifying this " 919 "option will run tests using android platform.") 920 parser.add_option("--arch", 921 help=("The architecture to run tests for, " 922 "'auto' or 'native' for auto-detect"), 923 default="x64") 924 parser.add_option("--buildbot", 925 help="Adapt to path structure used on buildbots", 926 default=False, action="store_true") 927 parser.add_option("--device", 928 help="The device ID to run Android tests on. If not given " 929 "it will be autodetected.") 930 parser.add_option("--extra-flags", 931 help="Additional flags to pass to the test executable", 932 default="") 933 parser.add_option("--json-test-results", 934 help="Path to a file for storing json results.") 935 parser.add_option("--json-test-results-no-patch", 936 help="Path to a file for storing json results from run " 937 "without patch.") 938 parser.add_option("--outdir", help="Base directory with compile output", 939 default="out") 940 parser.add_option("--outdir-no-patch", 941 help="Base directory with compile output without patch") 942 parser.add_option("--binary-override-path", 943 help="JavaScript engine binary. By default, d8 under " 944 "architecture-specific build dir. " 945 "Not supported in conjunction with outdir-no-patch.") 946 parser.add_option("--prioritize", 947 help="Raise the priority to nice -20 for the benchmarking " 948 "process.Requires Linux, schedtool, and sudo privileges.", 949 default=False, action="store_true") 950 parser.add_option("--affinitize", 951 help="Run benchmarking process on the specified core. " 952 "For example: " 953 "--affinitize=0 will run the benchmark process on core 0. " 954 "--affinitize=3 will run the benchmark process on core 3. " 955 "Requires Linux, schedtool, and sudo privileges.", 956 default=None) 957 parser.add_option("--noaslr", 958 help="Disable ASLR for the duration of the benchmarked " 959 "process. Requires Linux and sudo privileges.", 960 default=False, action="store_true") 961 parser.add_option("--cpu-governor", 962 help="Set cpu governor to specified policy for the " 963 "duration of the benchmarked process. Typical options: " 964 "'powersave' for more stable results, or 'performance' " 965 "for shorter completion time of suite, with potentially " 966 "more noise in results.") 967 968 (options, args) = parser.parse_args(args) 969 970 if len(args) == 0: # pragma: no cover 971 parser.print_help() 972 return 1 973 974 if options.arch in ["auto", "native"]: # pragma: no cover 975 options.arch = ARCH_GUESS 976 977 if not options.arch in SUPPORTED_ARCHS: # pragma: no cover 978 print "Unknown architecture %s" % options.arch 979 return 1 980 981 if options.device and not options.android_build_tools: # pragma: no cover 982 print "Specifying a device requires Android build tools." 983 return 1 984 985 if (options.json_test_results_no_patch and 986 not options.outdir_no_patch): # pragma: no cover 987 print("For writing json test results without patch, an outdir without " 988 "patch must be specified.") 989 return 1 990 991 workspace = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) 992 993 if options.buildbot: 994 build_config = "Release" 995 else: 996 build_config = "%s.release" % options.arch 997 998 if options.binary_override_path == None: 999 options.shell_dir = os.path.join(workspace, options.outdir, build_config) 1000 default_binary_name = "d8" 1001 else: 1002 if not os.path.isfile(options.binary_override_path): 1003 print "binary-override-path must be a file name" 1004 return 1 1005 if options.outdir_no_patch: 1006 print "specify either binary-override-path or outdir-no-patch" 1007 return 1 1008 options.shell_dir = os.path.dirname(options.binary_override_path) 1009 default_binary_name = os.path.basename(options.binary_override_path) 1010 1011 if options.outdir_no_patch: 1012 options.shell_dir_no_patch = os.path.join( 1013 workspace, options.outdir_no_patch, build_config) 1014 else: 1015 options.shell_dir_no_patch = None 1016 1017 prev_aslr = None 1018 prev_cpu_gov = None 1019 platform = Platform.GetPlatform(options) 1020 1021 results = Results() 1022 results_no_patch = Results() 1023 with CustomMachineConfiguration(governor = options.cpu_governor, 1024 disable_aslr = options.noaslr) as conf: 1025 for path in args: 1026 path = os.path.abspath(path) 1027 1028 if not os.path.exists(path): # pragma: no cover 1029 results.errors.append("Configuration file %s does not exist." % path) 1030 continue 1031 1032 with open(path) as f: 1033 suite = json.loads(f.read()) 1034 1035 # If no name is given, default to the file name without .json. 1036 suite.setdefault("name", os.path.splitext(os.path.basename(path))[0]) 1037 1038 # Setup things common to one test suite. 1039 platform.PreExecution() 1040 1041 # Build the graph/trace tree structure. 1042 default_parent = DefaultSentinel(default_binary_name) 1043 root = BuildGraphConfigs(suite, options.arch, default_parent) 1044 1045 # Callback to be called on each node on traversal. 1046 def NodeCB(node): 1047 platform.PreTests(node, path) 1048 1049 # Traverse graph/trace tree and interate over all runnables. 1050 for runnable in FlattenRunnables(root, NodeCB): 1051 print ">>> Running suite: %s" % "/".join(runnable.graphs) 1052 1053 def Runner(): 1054 """Output generator that reruns several times.""" 1055 for i in xrange(0, max(1, runnable.run_count)): 1056 # TODO(machenbach): Allow timeout per arch like with run_count per 1057 # arch. 1058 yield platform.Run(runnable, i) 1059 1060 # Let runnable iterate over all runs and handle output. 1061 result, result_no_patch = runnable.Run( 1062 Runner, trybot=options.shell_dir_no_patch) 1063 results += result 1064 results_no_patch += result_no_patch 1065 platform.PostExecution() 1066 1067 if options.json_test_results: 1068 results.WriteToFile(options.json_test_results) 1069 else: # pragma: no cover 1070 print results 1071 1072 if options.json_test_results_no_patch: 1073 results_no_patch.WriteToFile(options.json_test_results_no_patch) 1074 else: # pragma: no cover 1075 print results_no_patch 1076 1077 return min(1, len(results.errors)) 1078 1079if __name__ == "__main__": # pragma: no cover 1080 sys.exit(Main(sys.argv[1:])) 1081