• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# -*- coding: utf-8 -*-
2# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Define a type that wraps a Benchmark instance."""
7
8from __future__ import division
9from __future__ import print_function
10
11import math
12# FIXME(denik): Fix the import in chroot.
13# pylint: disable=import-error
14from scipy import stats
15
16# See crbug.com/673558 for how these are estimated.
17_estimated_stddev = {
18    'octane': 0.015,
19    'kraken': 0.019,
20    'speedometer': 0.007,
21    'speedometer2': 0.006,
22    'dromaeo.domcoreattr': 0.023,
23    'dromaeo.domcoremodify': 0.011,
24    'graphics_WebGLAquarium': 0.008,
25    'page_cycler_v2.typical_25': 0.021,
26    'loading.desktop': 0.021,  # Copied from page_cycler initially
27}
28
29
30# Get #samples needed to guarantee a given confidence interval, assuming the
31# samples follow normal distribution.
32def _samples(b):
33  # TODO: Make this an option
34  # CI = (0.9, 0.02), i.e., 90% chance that |sample mean - true mean| < 2%.
35  p = 0.9
36  e = 0.02
37  if b not in _estimated_stddev:
38    return 1
39  d = _estimated_stddev[b]
40  # Get at least 2 samples so as to calculate standard deviation, which is
41  # needed in T-test for p-value.
42  n = int(math.ceil((stats.norm.isf((1 - p) / 2) * d / e)**2))
43  return n if n > 1 else 2
44
45
46class Benchmark(object):
47  """Class representing a benchmark to be run.
48
49  Contains details of the benchmark suite, arguments to pass to the suite,
50  iterations to run the benchmark suite and so on. Note that the benchmark name
51  can be different to the test suite name. For example, you may want to have
52  two different benchmarks which run the same test_name with different
53  arguments.
54  """
55
56  def __init__(self,
57               name,
58               test_name,
59               test_args,
60               iterations,
61               rm_chroot_tmp,
62               perf_args,
63               suite='',
64               show_all_results=False,
65               retries=0,
66               run_local=False,
67               cwp_dso='',
68               weight=0):
69    self.name = name
70    # For telemetry, this is the benchmark name.
71    self.test_name = test_name
72    # For telemetry, this is the data.
73    self.test_args = test_args
74    self.iterations = iterations if iterations > 0 else _samples(name)
75    self.perf_args = perf_args
76    self.rm_chroot_tmp = rm_chroot_tmp
77    self.iteration_adjusted = False
78    self.suite = suite
79    self.show_all_results = show_all_results
80    self.retries = retries
81    if self.suite == 'telemetry':
82      self.show_all_results = True
83    if run_local and self.suite != 'telemetry_Crosperf':
84      raise RuntimeError('run_local is only supported by telemetry_Crosperf.')
85    self.run_local = run_local
86    self.cwp_dso = cwp_dso
87    self.weight = weight
88