1// Copyright 2012 the V8 project authors. All rights reserved. 2// Redistribution and use in source and binary forms, with or without 3// modification, are permitted provided that the following conditions are 4// met: 5// 6// * Redistributions of source code must retain the above copyright 7// notice, this list of conditions and the following disclaimer. 8// * Redistributions in binary form must reproduce the above 9// copyright notice, this list of conditions and the following 10// disclaimer in the documentation and/or other materials provided 11// with the distribution. 12// * Neither the name of Google Inc. nor the names of its 13// contributors may be used to endorse or promote products derived 14// from this software without specific prior written permission. 15// 16// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 28 29// Simple framework for running the benchmark suites and 30// computing a score based on the timing measurements. 31 32 33// A benchmark has a name (string) and a function that will be run to 34// do the performance measurement. The optional setup and tearDown 35// arguments are functions that will be invoked before and after 36// running the benchmark, but the running time of these functions will 37// not be accounted for in the benchmark score. 38function Benchmark(name, run, setup, tearDown) { 39 this.name = name; 40 this.run = run; 41 this.Setup = setup ? setup : function() { }; 42 this.TearDown = tearDown ? tearDown : function() { }; 43} 44 45 46// Benchmark results hold the benchmark and the measured time used to 47// run the benchmark. The benchmark score is computed later once a 48// full benchmark suite has run to completion. 49function BenchmarkResult(benchmark, time) { 50 this.benchmark = benchmark; 51 this.time = time; 52} 53 54 55// Automatically convert results to numbers. Used by the geometric 56// mean computation. 57BenchmarkResult.prototype.valueOf = function() { 58 return this.time; 59} 60 61 62// Suites of benchmarks consist of a name and the set of benchmarks in 63// addition to the reference timing that the final score will be based 64// on. This way, all scores are relative to a reference run and higher 65// scores implies better performance. 66function BenchmarkSuite(name, reference, benchmarks) { 67 this.name = name; 68 this.reference = reference; 69 this.benchmarks = benchmarks; 70 BenchmarkSuite.suites.push(this); 71} 72 73 74// Keep track of all declared benchmark suites. 75BenchmarkSuite.suites = []; 76 77 78// Scores are not comparable across versions. Bump the version if 79// you're making changes that will affect that scores, e.g. if you add 80// a new benchmark or change an existing one. 81BenchmarkSuite.version = '7'; 82 83 84// To make the benchmark results predictable, we replace Math.random 85// with a 100% deterministic alternative. 86Math.random = (function() { 87 var seed = 49734321; 88 return function() { 89 // Robert Jenkins' 32 bit integer hash function. 90 seed = ((seed + 0x7ed55d16) + (seed << 12)) & 0xffffffff; 91 seed = ((seed ^ 0xc761c23c) ^ (seed >>> 19)) & 0xffffffff; 92 seed = ((seed + 0x165667b1) + (seed << 5)) & 0xffffffff; 93 seed = ((seed + 0xd3a2646c) ^ (seed << 9)) & 0xffffffff; 94 seed = ((seed + 0xfd7046c5) + (seed << 3)) & 0xffffffff; 95 seed = ((seed ^ 0xb55a4f09) ^ (seed >>> 16)) & 0xffffffff; 96 return (seed & 0xfffffff) / 0x10000000; 97 }; 98})(); 99 100 101// Runs all registered benchmark suites and optionally yields between 102// each individual benchmark to avoid running for too long in the 103// context of browsers. Once done, the final score is reported to the 104// runner. 105BenchmarkSuite.RunSuites = function(runner) { 106 var continuation = null; 107 var suites = BenchmarkSuite.suites; 108 var length = suites.length; 109 BenchmarkSuite.scores = []; 110 var index = 0; 111 function RunStep() { 112 while (continuation || index < length) { 113 if (continuation) { 114 continuation = continuation(); 115 } else { 116 var suite = suites[index++]; 117 if (runner.NotifyStart) runner.NotifyStart(suite.name); 118 continuation = suite.RunStep(runner); 119 } 120 if (continuation && typeof window != 'undefined' && window.setTimeout) { 121 window.setTimeout(RunStep, 25); 122 return; 123 } 124 } 125 if (runner.NotifyScore) { 126 var score = BenchmarkSuite.GeometricMean(BenchmarkSuite.scores); 127 var formatted = BenchmarkSuite.FormatScore(100 * score); 128 runner.NotifyScore(formatted); 129 } 130 } 131 RunStep(); 132} 133 134 135// Counts the total number of registered benchmarks. Useful for 136// showing progress as a percentage. 137BenchmarkSuite.CountBenchmarks = function() { 138 var result = 0; 139 var suites = BenchmarkSuite.suites; 140 for (var i = 0; i < suites.length; i++) { 141 result += suites[i].benchmarks.length; 142 } 143 return result; 144} 145 146 147// Computes the geometric mean of a set of numbers. 148BenchmarkSuite.GeometricMean = function(numbers) { 149 var log = 0; 150 for (var i = 0; i < numbers.length; i++) { 151 log += Math.log(numbers[i]); 152 } 153 return Math.pow(Math.E, log / numbers.length); 154} 155 156 157// Converts a score value to a string with at least three significant 158// digits. 159BenchmarkSuite.FormatScore = function(value) { 160 if (value > 100) { 161 return value.toFixed(0); 162 } else { 163 return value.toPrecision(3); 164 } 165} 166 167// Notifies the runner that we're done running a single benchmark in 168// the benchmark suite. This can be useful to report progress. 169BenchmarkSuite.prototype.NotifyStep = function(result) { 170 this.results.push(result); 171 if (this.runner.NotifyStep) this.runner.NotifyStep(result.benchmark.name); 172} 173 174 175// Notifies the runner that we're done with running a suite and that 176// we have a result which can be reported to the user if needed. 177BenchmarkSuite.prototype.NotifyResult = function() { 178 var mean = BenchmarkSuite.GeometricMean(this.results); 179 var score = this.reference / mean; 180 BenchmarkSuite.scores.push(score); 181 if (this.runner.NotifyResult) { 182 var formatted = BenchmarkSuite.FormatScore(100 * score); 183 this.runner.NotifyResult(this.name, formatted); 184 } 185} 186 187 188// Notifies the runner that running a benchmark resulted in an error. 189BenchmarkSuite.prototype.NotifyError = function(error) { 190 if (this.runner.NotifyError) { 191 this.runner.NotifyError(this.name, error); 192 } 193 if (this.runner.NotifyStep) { 194 this.runner.NotifyStep(this.name); 195 } 196} 197 198 199// Runs a single benchmark for at least a second and computes the 200// average time it takes to run a single iteration. 201BenchmarkSuite.prototype.RunSingleBenchmark = function(benchmark, data) { 202 function Measure(data) { 203 var elapsed = 0; 204 var start = new Date(); 205 for (var n = 0; elapsed < 1000; n++) { 206 benchmark.run(); 207 elapsed = new Date() - start; 208 } 209 if (data != null) { 210 data.runs += n; 211 data.elapsed += elapsed; 212 } 213 } 214 215 if (data == null) { 216 // Measure the benchmark once for warm up and throw the result 217 // away. Return a fresh data object. 218 Measure(null); 219 return { runs: 0, elapsed: 0 }; 220 } else { 221 Measure(data); 222 // If we've run too few iterations, we continue for another second. 223 if (data.runs < 32) return data; 224 var usec = (data.elapsed * 1000) / data.runs; 225 this.NotifyStep(new BenchmarkResult(benchmark, usec)); 226 return null; 227 } 228} 229 230 231// This function starts running a suite, but stops between each 232// individual benchmark in the suite and returns a continuation 233// function which can be invoked to run the next benchmark. Once the 234// last benchmark has been executed, null is returned. 235BenchmarkSuite.prototype.RunStep = function(runner) { 236 this.results = []; 237 this.runner = runner; 238 var length = this.benchmarks.length; 239 var index = 0; 240 var suite = this; 241 var data; 242 243 // Run the setup, the actual benchmark, and the tear down in three 244 // separate steps to allow the framework to yield between any of the 245 // steps. 246 247 function RunNextSetup() { 248 if (index < length) { 249 try { 250 suite.benchmarks[index].Setup(); 251 } catch (e) { 252 suite.NotifyError(e); 253 return null; 254 } 255 return RunNextBenchmark; 256 } 257 suite.NotifyResult(); 258 return null; 259 } 260 261 function RunNextBenchmark() { 262 try { 263 data = suite.RunSingleBenchmark(suite.benchmarks[index], data); 264 } catch (e) { 265 suite.NotifyError(e); 266 return null; 267 } 268 // If data is null, we're done with this benchmark. 269 return (data == null) ? RunNextTearDown : RunNextBenchmark(); 270 } 271 272 function RunNextTearDown() { 273 try { 274 suite.benchmarks[index++].TearDown(); 275 } catch (e) { 276 suite.NotifyError(e); 277 return null; 278 } 279 return RunNextSetup; 280 } 281 282 // Start out running the setup. 283 return RunNextSetup(); 284} 285