• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import optparse
6
7from telemetry import decorators
8from telemetry.internal import story_runner
9from telemetry.internal.util import command_line
10from telemetry.page import legacy_page_test
11from telemetry.web_perf import timeline_based_measurement
12
13Disabled = decorators.Disabled
14Enabled = decorators.Enabled
15Owner = decorators.Owner
16
17class InvalidOptionsError(Exception):
18  """Raised for invalid benchmark options."""
19  pass
20
21
22class BenchmarkMetadata(object):
23  def __init__(self, name, description='', rerun_options=None):
24    self._name = name
25    self._description = description
26    self._rerun_options = rerun_options
27
28  @property
29  def name(self):
30    return self._name
31
32  @property
33  def description(self):
34    return self._description
35
36  @property
37  def rerun_options(self):
38    return self._rerun_options
39
40  def AsDict(self):
41    return {
42      'type': 'telemetry_benchmark',
43      'name': self._name,
44      'description': self._description,
45      'rerun_options': self._rerun_options,
46    }
47
48
49class Benchmark(command_line.Command):
50  """Base class for a Telemetry benchmark.
51
52  A benchmark packages a measurement and a PageSet together.
53  Benchmarks default to using TBM unless you override the value of
54  Benchmark.test, or override the CreatePageTest method.
55
56  New benchmarks should override CreateStorySet.
57  """
58  options = {}
59  page_set = None
60  test = timeline_based_measurement.TimelineBasedMeasurement
61
62  def __init__(self, max_failures=None):
63    """Creates a new Benchmark.
64
65    Args:
66      max_failures: The number of story run's failures before bailing
67          from executing subsequent page runs. If None, we never bail.
68    """
69    self._max_failures = max_failures
70    self._has_original_tbm_options = (
71        self.CreateTimelineBasedMeasurementOptions.__func__ ==
72        Benchmark.CreateTimelineBasedMeasurementOptions.__func__)
73    has_original_create_page_test = (
74        self.CreatePageTest.__func__ == Benchmark.CreatePageTest.__func__)
75    assert self._has_original_tbm_options or has_original_create_page_test, (
76        'Cannot override both CreatePageTest and '
77        'CreateTimelineBasedMeasurementOptions.')
78
79  # pylint: disable=unused-argument
80  @classmethod
81  def ShouldDisable(cls, possible_browser):
82    """Override this method to disable a benchmark under specific conditions.
83
84     Supports logic too complex for simple Enabled and Disabled decorators.
85     Decorators are still respected in cases where this function returns False.
86     """
87    return False
88
89  def Run(self, finder_options):
90    """Do not override this method."""
91    return story_runner.RunBenchmark(self, finder_options)
92
93  @property
94  def max_failures(self):
95    return self._max_failures
96
97  @classmethod
98  def Name(cls):
99    return '%s.%s' % (cls.__module__.split('.')[-1], cls.__name__)
100
101  @classmethod
102  def ShouldTearDownStateAfterEachStoryRun(cls):
103    """Override to specify whether to tear down state after each story run.
104
105    Tearing down all states after each story run, e.g., clearing profiles,
106    stopping the browser, stopping local server, etc. So the browser will not be
107    reused among multiple stories. This is particularly useful to get the
108    startup part of launching the browser in each story.
109
110    This should only be used by TimelineBasedMeasurement (TBM) benchmarks, but
111    not by PageTest based benchmarks.
112    """
113    return True
114
115  # NOTE: this is a temporary workaround for crbug.com/645329, do not rely on
116  # this as a stable public API as we may remove this without public notice.
117  @classmethod
118  def IsShouldTearDownStateAfterEachStoryRunOverriden(cls):
119    return (cls.ShouldTearDownStateAfterEachStoryRun.__func__ !=
120            Benchmark.ShouldTearDownStateAfterEachStoryRun.__func__)
121
122  @classmethod
123  def ShouldTearDownStateAfterEachStorySetRun(cls):
124    """Override to specify whether to tear down state after each story set run.
125
126    Defaults to True in order to reset the state and make individual story set
127    repeats more independent of each other. The intended effect is to average
128    out noise in measurements between repeats.
129
130    Long running benchmarks willing to stess test the browser and have it run
131    for long periods of time may switch this value to False.
132
133    This should only be used by TimelineBasedMeasurement (TBM) benchmarks, but
134    not by PageTest based benchmarks.
135    """
136    return True
137
138  @classmethod
139  def AddCommandLineArgs(cls, parser):
140    group = optparse.OptionGroup(parser, '%s test options' % cls.Name())
141    cls.AddBenchmarkCommandLineArgs(group)
142
143    if cls.HasTraceRerunDebugOption():
144      group.add_option(
145          '--rerun-with-debug-trace',
146          action='store_true',
147          help='Rerun option that enables more extensive tracing.')
148
149    if group.option_list:
150      parser.add_option_group(group)
151
152  @classmethod
153  def AddBenchmarkCommandLineArgs(cls, group):
154    del group  # unused
155
156  @classmethod
157  def HasTraceRerunDebugOption(cls):
158    return False
159
160  def GetTraceRerunCommands(self):
161    if self.HasTraceRerunDebugOption():
162      return [['Debug Trace', '--rerun-with-debug-trace']]
163    return []
164
165  def SetupTraceRerunOptions(self, browser_options, tbm_options):
166    if self.HasTraceRerunDebugOption():
167      if browser_options.rerun_with_debug_trace:
168        self.SetupBenchmarkDebugTraceRerunOptions(tbm_options)
169      else:
170        self.SetupBenchmarkDefaultTraceRerunOptions(tbm_options)
171
172  def SetupBenchmarkDefaultTraceRerunOptions(self, tbm_options):
173    """Setup tracing categories associated with default trace option."""
174
175  def SetupBenchmarkDebugTraceRerunOptions(self, tbm_options):
176    """Setup tracing categories associated with debug trace option."""
177
178  @classmethod
179  def SetArgumentDefaults(cls, parser):
180    default_values = parser.get_default_values()
181    invalid_options = [
182        o for o in cls.options if not hasattr(default_values, o)]
183    if invalid_options:
184      raise InvalidOptionsError('Invalid benchmark options: %s',
185                                ', '.join(invalid_options))
186    parser.set_defaults(**cls.options)
187
188  @classmethod
189  def ProcessCommandLineArgs(cls, parser, args):
190    pass
191
192  # pylint: disable=unused-argument
193  @classmethod
194  def ValueCanBeAddedPredicate(cls, value, is_first_result):
195    """Returns whether |value| can be added to the test results.
196
197    Override this method to customize the logic of adding values to test
198    results.
199
200    Args:
201      value: a value.Value instance (except failure.FailureValue,
202        skip.SkipValue or trace.TraceValue which will always be added).
203      is_first_result: True if |value| is the first result for its
204          corresponding story.
205
206    Returns:
207      True if |value| should be added to the test results.
208      Otherwise, it returns False.
209    """
210    return True
211
212  def CustomizeBrowserOptions(self, options):
213    """Add browser options that are required by this benchmark."""
214
215  def GetMetadata(self):
216    return BenchmarkMetadata(
217        self.Name(), self.__doc__, self.GetTraceRerunCommands())
218
219  def CreateTimelineBasedMeasurementOptions(self):
220    """Return the TimelineBasedMeasurementOptions for this Benchmark.
221
222    Override this method to configure a TimelineBasedMeasurement benchmark.
223    Otherwise, override CreatePageTest for PageTest tests. Do not override
224    both methods.
225    """
226    return timeline_based_measurement.Options()
227
228  def CreatePageTest(self, options):  # pylint: disable=unused-argument
229    """Return the PageTest for this Benchmark.
230
231    Override this method for PageTest tests.
232    Override, override CreateTimelineBasedMeasurementOptions to configure
233    TimelineBasedMeasurement tests. Do not override both methods.
234
235    Args:
236      options: a browser_options.BrowserFinderOptions instance
237    Returns:
238      |test()| if |test| is a PageTest class.
239      Otherwise, a TimelineBasedMeasurement instance.
240    """
241    is_page_test = issubclass(self.test, legacy_page_test.LegacyPageTest)
242    is_tbm = self.test == timeline_based_measurement.TimelineBasedMeasurement
243    if not is_page_test and not is_tbm:
244      raise TypeError('"%s" is not a PageTest or a TimelineBasedMeasurement.' %
245                      self.test.__name__)
246    if is_page_test:
247      assert self._has_original_tbm_options, (
248          'Cannot override CreateTimelineBasedMeasurementOptions '
249          'with a PageTest.')
250      return self.test()  # pylint: disable=no-value-for-parameter
251
252    opts = self.CreateTimelineBasedMeasurementOptions()
253    self.SetupTraceRerunOptions(options, opts)
254    return timeline_based_measurement.TimelineBasedMeasurement(opts)
255
256  def CreateStorySet(self, options):
257    """Creates the instance of StorySet used to run the benchmark.
258
259    Can be overridden by subclasses.
260    """
261    del options  # unused
262    # TODO(aiolos, nednguyen, eakufner): replace class attribute page_set with
263    # story_set.
264    if not self.page_set:
265      raise NotImplementedError('This test has no "page_set" attribute.')
266    return self.page_set()  # pylint: disable=not-callable
267
268
269def AddCommandLineArgs(parser):
270  story_runner.AddCommandLineArgs(parser)
271
272
273def ProcessCommandLineArgs(parser, args):
274  story_runner.ProcessCommandLineArgs(parser, args)
275