1#!/usr/bin/env python 2# Copyright 2014 the V8 project authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6from collections import namedtuple 7import coverage 8import json 9from mock import DEFAULT 10from mock import MagicMock 11import os 12from os import path, sys 13import platform 14import shutil 15import subprocess 16import tempfile 17import unittest 18 19# Requires python-coverage and python-mock. Native python coverage 20# version >= 3.7.1 should be installed to get the best speed. 21 22BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 23RUN_PERF = os.path.join(BASE_DIR, 'run_perf.py') 24TEST_DATA = os.path.join(BASE_DIR, 'unittests', 'testdata') 25 26TEST_WORKSPACE = path.join(tempfile.gettempdir(), "test-v8-run-perf") 27 28V8_JSON = { 29 "path": ["."], 30 "binary": "d7", 31 "flags": ["--flag"], 32 "main": "run.js", 33 "run_count": 1, 34 "results_regexp": "^%s: (.+)$", 35 "tests": [ 36 {"name": "Richards"}, 37 {"name": "DeltaBlue"}, 38 ] 39} 40 41V8_NESTED_SUITES_JSON = { 42 "path": ["."], 43 "flags": ["--flag"], 44 "run_count": 1, 45 "units": "score", 46 "tests": [ 47 {"name": "Richards", 48 "path": ["richards"], 49 "binary": "d7", 50 "main": "run.js", 51 "resources": ["file1.js", "file2.js"], 52 "run_count": 2, 53 "results_regexp": "^Richards: (.+)$"}, 54 {"name": "Sub", 55 "path": ["sub"], 56 "tests": [ 57 {"name": "Leaf", 58 "path": ["leaf"], 59 "run_count_x64": 3, 60 "units": "ms", 61 "main": "run.js", 62 "results_regexp": "^Simple: (.+) ms.$"}, 63 ] 64 }, 65 {"name": "DeltaBlue", 66 "path": ["delta_blue"], 67 "main": "run.js", 68 "flags": ["--flag2"], 69 "results_regexp": "^DeltaBlue: (.+)$"}, 70 {"name": "ShouldntRun", 71 "path": ["."], 72 "archs": ["arm"], 73 "main": "run.js"}, 74 ] 75} 76 77V8_GENERIC_JSON = { 78 "path": ["."], 79 "binary": "cc", 80 "flags": ["--flag"], 81 "generic": True, 82 "run_count": 1, 83 "units": "ms", 84} 85 86Output = namedtuple("Output", "stdout, stderr, timed_out") 87 88class PerfTest(unittest.TestCase): 89 @classmethod 90 def setUpClass(cls): 91 cls.base = path.dirname(path.dirname(path.abspath(__file__))) 92 sys.path.append(cls.base) 93 cls._cov = coverage.coverage( 94 include=([os.path.join(cls.base, "run_perf.py")])) 95 cls._cov.start() 96 import run_perf 97 from testrunner.local import commands 98 global commands 99 global run_perf 100 101 @classmethod 102 def tearDownClass(cls): 103 cls._cov.stop() 104 print "" 105 print cls._cov.report() 106 107 def setUp(self): 108 self.maxDiff = None 109 if path.exists(TEST_WORKSPACE): 110 shutil.rmtree(TEST_WORKSPACE) 111 os.makedirs(TEST_WORKSPACE) 112 113 def tearDown(self): 114 if path.exists(TEST_WORKSPACE): 115 shutil.rmtree(TEST_WORKSPACE) 116 117 def _WriteTestInput(self, json_content): 118 self._test_input = path.join(TEST_WORKSPACE, "test.json") 119 with open(self._test_input, "w") as f: 120 f.write(json.dumps(json_content)) 121 122 def _MockCommand(self, *args, **kwargs): 123 # Fake output for each test run. 124 test_outputs = [Output(stdout=arg, 125 stderr=None, 126 timed_out=kwargs.get("timed_out", False)) 127 for arg in args[1]] 128 def execute(*args, **kwargs): 129 return test_outputs.pop() 130 commands.Execute = MagicMock(side_effect=execute) 131 132 # Check that d8 is called from the correct cwd for each test run. 133 dirs = [path.join(TEST_WORKSPACE, arg) for arg in args[0]] 134 def chdir(*args, **kwargs): 135 self.assertEquals(dirs.pop(), args[0]) 136 os.chdir = MagicMock(side_effect=chdir) 137 138 subprocess.check_call = MagicMock() 139 platform.system = MagicMock(return_value='Linux') 140 141 def _CallMain(self, *args): 142 self._test_output = path.join(TEST_WORKSPACE, "results.json") 143 all_args=[ 144 "--json-test-results", 145 self._test_output, 146 self._test_input, 147 ] 148 all_args += args 149 return run_perf.Main(all_args) 150 151 def _LoadResults(self, file_name=None): 152 with open(file_name or self._test_output) as f: 153 return json.load(f) 154 155 def _VerifyResults(self, suite, units, traces, file_name=None): 156 self.assertEquals([ 157 {"units": units, 158 "graphs": [suite, trace["name"]], 159 "results": trace["results"], 160 "stddev": trace["stddev"]} for trace in traces], 161 self._LoadResults(file_name)["traces"]) 162 163 def _VerifyErrors(self, errors): 164 self.assertEquals(errors, self._LoadResults()["errors"]) 165 166 def _VerifyMock(self, binary, *args, **kwargs): 167 arg = [path.join(path.dirname(self.base), binary)] 168 arg += args 169 commands.Execute.assert_called_with( 170 arg, timeout=kwargs.get("timeout", 60)) 171 172 def _VerifyMockMultiple(self, *args, **kwargs): 173 expected = [] 174 for arg in args: 175 a = [path.join(path.dirname(self.base), arg[0])] 176 a += arg[1:] 177 expected.append(((a,), {"timeout": kwargs.get("timeout", 60)})) 178 self.assertEquals(expected, commands.Execute.call_args_list) 179 180 def testOneRun(self): 181 self._WriteTestInput(V8_JSON) 182 self._MockCommand(["."], ["x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n"]) 183 self.assertEquals(0, self._CallMain()) 184 self._VerifyResults("test", "score", [ 185 {"name": "Richards", "results": ["1.234"], "stddev": ""}, 186 {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""}, 187 ]) 188 self._VerifyErrors([]) 189 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") 190 191 def testOneRunWithTestFlags(self): 192 test_input = dict(V8_JSON) 193 test_input["test_flags"] = ["2", "test_name"] 194 self._WriteTestInput(test_input) 195 self._MockCommand(["."], ["Richards: 1.234\nDeltaBlue: 10657567"]) 196 self.assertEquals(0, self._CallMain()) 197 self._VerifyResults("test", "score", [ 198 {"name": "Richards", "results": ["1.234"], "stddev": ""}, 199 {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""}, 200 ]) 201 self._VerifyErrors([]) 202 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js", 203 "--", "2", "test_name") 204 205 def testTwoRuns_Units_SuiteName(self): 206 test_input = dict(V8_JSON) 207 test_input["run_count"] = 2 208 test_input["name"] = "v8" 209 test_input["units"] = "ms" 210 self._WriteTestInput(test_input) 211 self._MockCommand([".", "."], 212 ["Richards: 100\nDeltaBlue: 200\n", 213 "Richards: 50\nDeltaBlue: 300\n"]) 214 self.assertEquals(0, self._CallMain()) 215 self._VerifyResults("v8", "ms", [ 216 {"name": "Richards", "results": ["50.0", "100.0"], "stddev": ""}, 217 {"name": "DeltaBlue", "results": ["300.0", "200.0"], "stddev": ""}, 218 ]) 219 self._VerifyErrors([]) 220 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") 221 222 def testTwoRuns_SubRegexp(self): 223 test_input = dict(V8_JSON) 224 test_input["run_count"] = 2 225 del test_input["results_regexp"] 226 test_input["tests"][0]["results_regexp"] = "^Richards: (.+)$" 227 test_input["tests"][1]["results_regexp"] = "^DeltaBlue: (.+)$" 228 self._WriteTestInput(test_input) 229 self._MockCommand([".", "."], 230 ["Richards: 100\nDeltaBlue: 200\n", 231 "Richards: 50\nDeltaBlue: 300\n"]) 232 self.assertEquals(0, self._CallMain()) 233 self._VerifyResults("test", "score", [ 234 {"name": "Richards", "results": ["50.0", "100.0"], "stddev": ""}, 235 {"name": "DeltaBlue", "results": ["300.0", "200.0"], "stddev": ""}, 236 ]) 237 self._VerifyErrors([]) 238 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") 239 240 def testNestedSuite(self): 241 self._WriteTestInput(V8_NESTED_SUITES_JSON) 242 self._MockCommand(["delta_blue", "sub/leaf", "richards"], 243 ["DeltaBlue: 200\n", 244 "Simple: 1 ms.\n", 245 "Simple: 2 ms.\n", 246 "Simple: 3 ms.\n", 247 "Richards: 100\n", 248 "Richards: 50\n"]) 249 self.assertEquals(0, self._CallMain()) 250 self.assertEquals([ 251 {"units": "score", 252 "graphs": ["test", "Richards"], 253 "results": ["50.0", "100.0"], 254 "stddev": ""}, 255 {"units": "ms", 256 "graphs": ["test", "Sub", "Leaf"], 257 "results": ["3.0", "2.0", "1.0"], 258 "stddev": ""}, 259 {"units": "score", 260 "graphs": ["test", "DeltaBlue"], 261 "results": ["200.0"], 262 "stddev": ""}, 263 ], self._LoadResults()["traces"]) 264 self._VerifyErrors([]) 265 self._VerifyMockMultiple( 266 (path.join("out", "x64.release", "d7"), "--flag", "run.js"), 267 (path.join("out", "x64.release", "d7"), "--flag", "run.js"), 268 (path.join("out", "x64.release", "d8"), "--flag", "run.js"), 269 (path.join("out", "x64.release", "d8"), "--flag", "run.js"), 270 (path.join("out", "x64.release", "d8"), "--flag", "run.js"), 271 (path.join("out", "x64.release", "d8"), "--flag", "--flag2", "run.js")) 272 273 def testOneRunStdDevRegExp(self): 274 test_input = dict(V8_JSON) 275 test_input["stddev_regexp"] = "^%s\-stddev: (.+)$" 276 self._WriteTestInput(test_input) 277 self._MockCommand(["."], ["Richards: 1.234\nRichards-stddev: 0.23\n" 278 "DeltaBlue: 10657567\nDeltaBlue-stddev: 106\n"]) 279 self.assertEquals(0, self._CallMain()) 280 self._VerifyResults("test", "score", [ 281 {"name": "Richards", "results": ["1.234"], "stddev": "0.23"}, 282 {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": "106"}, 283 ]) 284 self._VerifyErrors([]) 285 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") 286 287 def testTwoRunsStdDevRegExp(self): 288 test_input = dict(V8_JSON) 289 test_input["stddev_regexp"] = "^%s\-stddev: (.+)$" 290 test_input["run_count"] = 2 291 self._WriteTestInput(test_input) 292 self._MockCommand(["."], ["Richards: 3\nRichards-stddev: 0.7\n" 293 "DeltaBlue: 6\nDeltaBlue-boom: 0.9\n", 294 "Richards: 2\nRichards-stddev: 0.5\n" 295 "DeltaBlue: 5\nDeltaBlue-stddev: 0.8\n"]) 296 self.assertEquals(1, self._CallMain()) 297 self._VerifyResults("test", "score", [ 298 {"name": "Richards", "results": ["2.0", "3.0"], "stddev": "0.7"}, 299 {"name": "DeltaBlue", "results": ["5.0", "6.0"], "stddev": "0.8"}, 300 ]) 301 self._VerifyErrors( 302 ["Test test/Richards should only run once since a stddev is provided " 303 "by the test.", 304 "Test test/DeltaBlue should only run once since a stddev is provided " 305 "by the test.", 306 "Regexp \"^DeltaBlue\-stddev: (.+)$\" didn't match for test " 307 "test/DeltaBlue."]) 308 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") 309 310 def testBuildbot(self): 311 self._WriteTestInput(V8_JSON) 312 self._MockCommand(["."], ["Richards: 1.234\nDeltaBlue: 10657567\n"]) 313 self.assertEquals(0, self._CallMain("--buildbot")) 314 self._VerifyResults("test", "score", [ 315 {"name": "Richards", "results": ["1.234"], "stddev": ""}, 316 {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""}, 317 ]) 318 self._VerifyErrors([]) 319 self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js") 320 321 def testBuildbotWithTotal(self): 322 test_input = dict(V8_JSON) 323 test_input["total"] = True 324 self._WriteTestInput(test_input) 325 self._MockCommand(["."], ["Richards: 1.234\nDeltaBlue: 10657567\n"]) 326 self.assertEquals(0, self._CallMain("--buildbot")) 327 self._VerifyResults("test", "score", [ 328 {"name": "Richards", "results": ["1.234"], "stddev": ""}, 329 {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""}, 330 {"name": "Total", "results": ["3626.49109719"], "stddev": ""}, 331 ]) 332 self._VerifyErrors([]) 333 self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js") 334 335 def testBuildbotWithTotalAndErrors(self): 336 test_input = dict(V8_JSON) 337 test_input["total"] = True 338 self._WriteTestInput(test_input) 339 self._MockCommand(["."], ["x\nRichards: bla\nDeltaBlue: 10657567\ny\n"]) 340 self.assertEquals(1, self._CallMain("--buildbot")) 341 self._VerifyResults("test", "score", [ 342 {"name": "Richards", "results": [], "stddev": ""}, 343 {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""}, 344 ]) 345 self._VerifyErrors( 346 ["Regexp \"^Richards: (.+)$\" " 347 "returned a non-numeric for test test/Richards.", 348 "Not all traces have the same number of results."]) 349 self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js") 350 351 def testRegexpNoMatch(self): 352 self._WriteTestInput(V8_JSON) 353 self._MockCommand(["."], ["x\nRichaards: 1.234\nDeltaBlue: 10657567\ny\n"]) 354 self.assertEquals(1, self._CallMain()) 355 self._VerifyResults("test", "score", [ 356 {"name": "Richards", "results": [], "stddev": ""}, 357 {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""}, 358 ]) 359 self._VerifyErrors( 360 ["Regexp \"^Richards: (.+)$\" didn't match for test test/Richards."]) 361 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") 362 363 def testOneRunGeneric(self): 364 test_input = dict(V8_GENERIC_JSON) 365 self._WriteTestInput(test_input) 366 self._MockCommand(["."], [ 367 "RESULT Infra: Constant1= 11 count\n" 368 "RESULT Infra: Constant2= [10,5,10,15] count\n" 369 "RESULT Infra: Constant3= {12,1.2} count\n" 370 "RESULT Infra: Constant4= [10,5,error,15] count\n"]) 371 self.assertEquals(1, self._CallMain()) 372 self.assertEquals([ 373 {"units": "count", 374 "graphs": ["test", "Infra", "Constant1"], 375 "results": ["11.0"], 376 "stddev": ""}, 377 {"units": "count", 378 "graphs": ["test", "Infra", "Constant2"], 379 "results": ["10.0", "5.0", "10.0", "15.0"], 380 "stddev": ""}, 381 {"units": "count", 382 "graphs": ["test", "Infra", "Constant3"], 383 "results": ["12.0"], 384 "stddev": "1.2"}, 385 {"units": "count", 386 "graphs": ["test", "Infra", "Constant4"], 387 "results": [], 388 "stddev": ""}, 389 ], self._LoadResults()["traces"]) 390 self._VerifyErrors(["Found non-numeric in test/Infra/Constant4"]) 391 self._VerifyMock(path.join("out", "x64.release", "cc"), "--flag", "") 392 393 def testOneRunTimingOut(self): 394 test_input = dict(V8_JSON) 395 test_input["timeout"] = 70 396 self._WriteTestInput(test_input) 397 self._MockCommand(["."], [""], timed_out=True) 398 self.assertEquals(1, self._CallMain()) 399 self._VerifyResults("test", "score", [ 400 {"name": "Richards", "results": [], "stddev": ""}, 401 {"name": "DeltaBlue", "results": [], "stddev": ""}, 402 ]) 403 self._VerifyErrors([ 404 "Regexp \"^Richards: (.+)$\" didn't match for test test/Richards.", 405 "Regexp \"^DeltaBlue: (.+)$\" didn't match for test test/DeltaBlue.", 406 ]) 407 self._VerifyMock( 408 path.join("out", "x64.release", "d7"), "--flag", "run.js", timeout=70) 409 410 # Simple test that mocks out the android platform. Testing the platform would 411 # require lots of complicated mocks for the android tools. 412 def testAndroid(self): 413 self._WriteTestInput(V8_JSON) 414 # FIXME(machenbach): This is not test-local! 415 platform = run_perf.AndroidPlatform 416 platform.PreExecution = MagicMock(return_value=None) 417 platform.PostExecution = MagicMock(return_value=None) 418 platform.PreTests = MagicMock(return_value=None) 419 platform.Run = MagicMock( 420 return_value=("Richards: 1.234\nDeltaBlue: 10657567\n", None)) 421 run_perf.AndroidPlatform = MagicMock(return_value=platform) 422 self.assertEquals( 423 0, self._CallMain("--android-build-tools", "/some/dir", 424 "--arch", "arm")) 425 self._VerifyResults("test", "score", [ 426 {"name": "Richards", "results": ["1.234"], "stddev": ""}, 427 {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""}, 428 ]) 429 430 def testTwoRuns_Trybot(self): 431 test_input = dict(V8_JSON) 432 test_input["run_count"] = 2 433 self._WriteTestInput(test_input) 434 self._MockCommand([".", ".", ".", "."], 435 ["Richards: 100\nDeltaBlue: 200\n", 436 "Richards: 200\nDeltaBlue: 20\n", 437 "Richards: 50\nDeltaBlue: 200\n", 438 "Richards: 100\nDeltaBlue: 20\n"]) 439 test_output_no_patch = path.join(TEST_WORKSPACE, "results_no_patch.json") 440 self.assertEquals(0, self._CallMain( 441 "--outdir-no-patch", "out-no-patch", 442 "--json-test-results-no-patch", test_output_no_patch, 443 )) 444 self._VerifyResults("test", "score", [ 445 {"name": "Richards", "results": ["100.0", "200.0"], "stddev": ""}, 446 {"name": "DeltaBlue", "results": ["20.0", "20.0"], "stddev": ""}, 447 ]) 448 self._VerifyResults("test", "score", [ 449 {"name": "Richards", "results": ["50.0", "100.0"], "stddev": ""}, 450 {"name": "DeltaBlue", "results": ["200.0", "200.0"], "stddev": ""}, 451 ], test_output_no_patch) 452 self._VerifyErrors([]) 453 self._VerifyMockMultiple( 454 (path.join("out", "x64.release", "d7"), "--flag", "run.js"), 455 (path.join("out-no-patch", "x64.release", "d7"), "--flag", "run.js"), 456 (path.join("out", "x64.release", "d7"), "--flag", "run.js"), 457 (path.join("out-no-patch", "x64.release", "d7"), "--flag", "run.js"), 458 ) 459 460 def testWrongBinaryWithProf(self): 461 test_input = dict(V8_JSON) 462 self._WriteTestInput(test_input) 463 self._MockCommand(["."], ["x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n"]) 464 self.assertEquals(0, self._CallMain("--extra-flags=--prof")) 465 self._VerifyResults("test", "score", [ 466 {"name": "Richards", "results": ["1.234"], "stddev": ""}, 467 {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""}, 468 ]) 469 self._VerifyErrors([]) 470 self._VerifyMock(path.join("out", "x64.release", "d7"), 471 "--flag", "--prof", "run.js") 472 473 def testUnzip(self): 474 def Gen(): 475 for i in [1, 2, 3]: 476 yield i, i + 1 477 l, r = run_perf.Unzip(Gen()) 478 self.assertEquals([1, 2, 3], list(l())) 479 self.assertEquals([2, 3, 4], list(r())) 480 481 ############################################################################# 482 ### System tests 483 484 def _RunPerf(self, mocked_d8, test_json): 485 output_json = path.join(TEST_WORKSPACE, "output.json") 486 args = [ 487 sys.executable, RUN_PERF, 488 "--binary-override-path", os.path.join(TEST_DATA, mocked_d8), 489 "--json-test-results", output_json, 490 os.path.join(TEST_DATA, test_json), 491 ] 492 subprocess.check_output(args) 493 return self._LoadResults(output_json) 494 495 def testNormal(self): 496 results = self._RunPerf("d8_mocked1.py", "test1.json") 497 self.assertEquals([], results['errors']) 498 self.assertEquals([ 499 { 500 'units': 'score', 501 'graphs': ['test1', 'Richards'], 502 'results': [u'1.2', u'1.2'], 503 'stddev': '', 504 }, 505 { 506 'units': 'score', 507 'graphs': ['test1', 'DeltaBlue'], 508 'results': [u'2.1', u'2.1'], 509 'stddev': '', 510 }, 511 ], results['traces']) 512 513 def testResultsProcessor(self): 514 results = self._RunPerf("d8_mocked2.py", "test2.json") 515 self.assertEquals([], results['errors']) 516 self.assertEquals([ 517 { 518 'units': 'score', 519 'graphs': ['test2', 'Richards'], 520 'results': [u'1.2', u'1.2'], 521 'stddev': '', 522 }, 523 { 524 'units': 'score', 525 'graphs': ['test2', 'DeltaBlue'], 526 'results': [u'2.1', u'2.1'], 527 'stddev': '', 528 }, 529 ], results['traces']) 530 531 def testResultsProcessorNested(self): 532 results = self._RunPerf("d8_mocked2.py", "test3.json") 533 self.assertEquals([], results['errors']) 534 self.assertEquals([ 535 { 536 'units': 'score', 537 'graphs': ['test3', 'Octane', 'Richards'], 538 'results': [u'1.2'], 539 'stddev': '', 540 }, 541 { 542 'units': 'score', 543 'graphs': ['test3', 'Octane', 'DeltaBlue'], 544 'results': [u'2.1'], 545 'stddev': '', 546 }, 547 ], results['traces']) 548