• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import collections
6import copy
7import datetime
8import logging
9import random
10import sys
11import traceback
12
13from py_utils import cloud_storage  # pylint: disable=import-error
14
15from telemetry.internal.results import json_output_formatter
16from telemetry.internal.results import progress_reporter as reporter_module
17from telemetry.internal.results import story_run
18from telemetry import value as value_module
19from telemetry.value import failure
20from telemetry.value import skip
21from telemetry.value import trace
22
23
24class IterationInfo(object):
25  def __init__(self):
26    self._benchmark_name = None
27    self._benchmark_start_ms = None
28    self._label = None
29    self._story_display_name = ''
30    self._story_grouping_keys = {}
31    self._story_repeat_counter = 0
32    self._story_url = ''
33    self._storyset_repeat_counter = 0
34
35  @property
36  def benchmark_name(self):
37    return self._benchmark_name
38
39  @benchmark_name.setter
40  def benchmark_name(self, benchmark_name):
41    assert self.benchmark_name is None, (
42      'benchmark_name must be set exactly once')
43    self._benchmark_name = benchmark_name
44
45  @property
46  def benchmark_start_ms(self):
47    return self._benchmark_start_ms
48
49  @benchmark_start_ms.setter
50  def benchmark_start_ms(self, benchmark_start_ms):
51    assert self.benchmark_start_ms is None, (
52      'benchmark_start_ms must be set exactly once')
53    self._benchmark_start_ms = benchmark_start_ms
54
55  @property
56  def label(self):
57    return self._label
58
59  @label.setter
60  def label(self, label):
61    assert self.label is None, 'label cannot be set more than once'
62    self._label = label
63
64  @property
65  def story_display_name(self):
66    return self._story_display_name
67
68  @property
69  def story_url(self):
70    return self._story_url
71
72  @property
73  def story_grouping_keys(self):
74    return self._story_grouping_keys
75
76  @property
77  def storyset_repeat_counter(self):
78    return self._storyset_repeat_counter
79
80  @property
81  def story_repeat_counter(self):
82    return self._story_repeat_counter
83
84  def WillRunStory(self, story, storyset_repeat_counter, story_repeat_counter):
85    self._story_display_name = story.display_name
86    self._story_url = story.url
87    if story.grouping_keys:
88      self._story_grouping_keys = story.grouping_keys
89    self._storyset_repeat_counter = storyset_repeat_counter
90    self._story_repeat_counter = story_repeat_counter
91
92  def AsDict(self):
93    assert self.benchmark_name is not None, (
94        'benchmark_name must be set exactly once')
95    assert self.benchmark_start_ms is not None, (
96        'benchmark_start_ms must be set exactly once')
97    d = {}
98    d['benchmarkName'] = self.benchmark_name
99    d['benchmarkStartMs'] = self.benchmark_start_ms
100    if self.label:
101      d['label'] = self.label
102    d['storyDisplayName'] = self.story_display_name
103    d['storyGroupingKeys'] = self.story_grouping_keys
104    d['storyRepeatCounter'] = self.story_repeat_counter
105    d['storyUrl'] = self.story_url
106    d['storysetRepeatCounter'] = self.storyset_repeat_counter
107    return d
108
109
110class PageTestResults(object):
111  def __init__(self, output_formatters=None,
112               progress_reporter=None, trace_tag='', output_dir=None,
113               value_can_be_added_predicate=lambda v, is_first: True):
114    """
115    Args:
116      output_formatters: A list of output formatters. The output
117          formatters are typically used to format the test results, such
118          as CsvPivotTableOutputFormatter, which output the test results as CSV.
119      progress_reporter: An instance of progress_reporter.ProgressReporter,
120          to be used to output test status/results progressively.
121      trace_tag: A string to append to the buildbot trace name. Currently only
122          used for buildbot.
123      output_dir: A string specified the directory where to store the test
124          artifacts, e.g: trace, videos,...
125      value_can_be_added_predicate: A function that takes two arguments:
126          a value.Value instance (except failure.FailureValue, skip.SkipValue
127          or trace.TraceValue) and a boolean (True when the value is part of
128          the first result for the story). It returns True if the value
129          can be added to the test results and False otherwise.
130    """
131    # TODO(chrishenry): Figure out if trace_tag is still necessary.
132
133    super(PageTestResults, self).__init__()
134    self._progress_reporter = (
135        progress_reporter if progress_reporter is not None
136        else reporter_module.ProgressReporter())
137    self._output_formatters = (
138        output_formatters if output_formatters is not None else [])
139    self._trace_tag = trace_tag
140    self._output_dir = output_dir
141    self._value_can_be_added_predicate = value_can_be_added_predicate
142
143    self._current_page_run = None
144    self._all_page_runs = []
145    self._all_stories = set()
146    self._representative_value_for_each_value_name = {}
147    self._all_summary_values = []
148    self._serialized_trace_file_ids_to_paths = {}
149    self._pages_to_profiling_files = collections.defaultdict(list)
150    self._pages_to_profiling_files_cloud_url = collections.defaultdict(list)
151
152    # You'd expect this to be a set(), but Values are dictionaries, which are
153    # unhashable. We could wrap Values with custom __eq/hash__, but we don't
154    # actually need set-ness in python.
155    self._value_set = []
156
157    self._iteration_info = IterationInfo()
158
159  @property
160  def iteration_info(self):
161    return self._iteration_info
162
163  @property
164  def value_set(self):
165    return self._value_set
166
167  def __copy__(self):
168    cls = self.__class__
169    result = cls.__new__(cls)
170    for k, v in self.__dict__.items():
171      if isinstance(v, collections.Container):
172        v = copy.copy(v)
173      setattr(result, k, v)
174    return result
175
176  @property
177  def pages_to_profiling_files(self):
178    return self._pages_to_profiling_files
179
180  @property
181  def serialized_trace_file_ids_to_paths(self):
182    return self._serialized_trace_file_ids_to_paths
183
184  @property
185  def pages_to_profiling_files_cloud_url(self):
186    return self._pages_to_profiling_files_cloud_url
187
188  @property
189  def all_page_specific_values(self):
190    values = []
191    for run in self._all_page_runs:
192      values += run.values
193    if self._current_page_run:
194      values += self._current_page_run.values
195    return values
196
197  @property
198  def all_summary_values(self):
199    return self._all_summary_values
200
201  @property
202  def current_page(self):
203    assert self._current_page_run, 'Not currently running test.'
204    return self._current_page_run.story
205
206  @property
207  def current_page_run(self):
208    assert self._current_page_run, 'Not currently running test.'
209    return self._current_page_run
210
211  @property
212  def all_page_runs(self):
213    return self._all_page_runs
214
215  @property
216  def pages_that_succeeded(self):
217    """Returns the set of pages that succeeded."""
218    pages = set(run.story for run in self.all_page_runs)
219    pages.difference_update(self.pages_that_failed)
220    return pages
221
222  @property
223  def pages_that_failed(self):
224    """Returns the set of failed pages."""
225    failed_pages = set()
226    for run in self.all_page_runs:
227      if run.failed:
228        failed_pages.add(run.story)
229    return failed_pages
230
231  @property
232  def failures(self):
233    values = self.all_page_specific_values
234    return [v for v in values if isinstance(v, failure.FailureValue)]
235
236  @property
237  def skipped_values(self):
238    values = self.all_page_specific_values
239    return [v for v in values if isinstance(v, skip.SkipValue)]
240
241  def _GetStringFromExcInfo(self, err):
242    return ''.join(traceback.format_exception(*err))
243
244  def CleanUp(self):
245    """Clean up any TraceValues contained within this results object."""
246    for run in self._all_page_runs:
247      for v in run.values:
248        if isinstance(v, trace.TraceValue):
249          v.CleanUp()
250          run.values.remove(v)
251
252  def __enter__(self):
253    return self
254
255  def __exit__(self, _, __, ___):
256    self.CleanUp()
257
258  def WillRunPage(self, page, storyset_repeat_counter=0,
259                  story_repeat_counter=0):
260    assert not self._current_page_run, 'Did not call DidRunPage.'
261    self._current_page_run = story_run.StoryRun(page)
262    self._progress_reporter.WillRunPage(self)
263    self.iteration_info.WillRunStory(
264        page, storyset_repeat_counter, story_repeat_counter)
265
266  def DidRunPage(self, page):  # pylint: disable=unused-argument
267    """
268    Args:
269      page: The current page under test.
270    """
271    assert self._current_page_run, 'Did not call WillRunPage.'
272    self._progress_reporter.DidRunPage(self)
273    self._all_page_runs.append(self._current_page_run)
274    self._all_stories.add(self._current_page_run.story)
275    self._current_page_run = None
276
277  def AddValue(self, value):
278    assert self._current_page_run, 'Not currently running test.'
279    self._ValidateValue(value)
280    is_first_result = (
281      self._current_page_run.story not in self._all_stories)
282
283    story_keys = self._current_page_run.story.grouping_keys
284
285    if story_keys:
286      for k, v in story_keys.iteritems():
287        assert k not in value.grouping_keys, (
288            'Tried to add story grouping key ' + k + ' already defined by ' +
289            'value')
290        value.grouping_keys[k] = v
291
292      # We sort by key name to make building the tir_label deterministic.
293      story_keys_label = '_'.join(v for _, v in sorted(story_keys.iteritems()))
294      if value.tir_label:
295        assert value.tir_label == story_keys_label, (
296            'Value has an explicit tir_label (%s) that does not match the '
297            'one computed from story_keys (%s)' % (value.tir_label, story_keys))
298      else:
299        value.tir_label = story_keys_label
300
301    if not (isinstance(value, skip.SkipValue) or
302            isinstance(value, failure.FailureValue) or
303            isinstance(value, trace.TraceValue) or
304            self._value_can_be_added_predicate(value, is_first_result)):
305      return
306    # TODO(eakuefner/chrishenry): Add only one skip per pagerun assert here
307    self._current_page_run.AddValue(value)
308    self._progress_reporter.DidAddValue(value)
309
310  def AddProfilingFile(self, page, file_handle):
311    self._pages_to_profiling_files[page].append(file_handle)
312
313  def AddSummaryValue(self, value):
314    assert value.page is None
315    self._ValidateValue(value)
316    self._all_summary_values.append(value)
317
318  def _ValidateValue(self, value):
319    assert isinstance(value, value_module.Value)
320    if value.name not in self._representative_value_for_each_value_name:
321      self._representative_value_for_each_value_name[value.name] = value
322    representative_value = self._representative_value_for_each_value_name[
323        value.name]
324    assert value.IsMergableWith(representative_value)
325
326  def PrintSummary(self):
327    self._progress_reporter.DidFinishAllTests(self)
328
329    # Only serialize the trace if output_format is json.
330    if (self._output_dir and
331        any(isinstance(o, json_output_formatter.JsonOutputFormatter)
332            for o in self._output_formatters)):
333      self._SerializeTracesToDirPath(self._output_dir)
334    for output_formatter in self._output_formatters:
335      output_formatter.Format(self)
336
337  def FindValues(self, predicate):
338    """Finds all values matching the specified predicate.
339
340    Args:
341      predicate: A function that takes a Value and returns a bool.
342    Returns:
343      A list of values matching |predicate|.
344    """
345    values = []
346    for value in self.all_page_specific_values:
347      if predicate(value):
348        values.append(value)
349    return values
350
351  def FindPageSpecificValuesForPage(self, page, value_name):
352    return self.FindValues(lambda v: v.page == page and v.name == value_name)
353
354  def FindAllPageSpecificValuesNamed(self, value_name):
355    return self.FindValues(lambda v: v.name == value_name)
356
357  def FindAllPageSpecificValuesFromIRNamed(self, tir_label, value_name):
358    return self.FindValues(lambda v: v.name == value_name
359                           and v.tir_label == tir_label)
360
361  def FindAllTraceValues(self):
362    return self.FindValues(lambda v: isinstance(v, trace.TraceValue))
363
364  def _SerializeTracesToDirPath(self, dir_path):
365    """ Serialize all trace values to files in dir_path and return a list of
366    file handles to those files. """
367    for value in self.FindAllTraceValues():
368      fh = value.Serialize(dir_path)
369      self._serialized_trace_file_ids_to_paths[fh.id] = fh.GetAbsPath()
370
371  def UploadTraceFilesToCloud(self, bucket):
372    for value in self.FindAllTraceValues():
373      value.UploadToCloud(bucket)
374
375  def UploadProfilingFilesToCloud(self, bucket):
376    for page, file_handle_list in self._pages_to_profiling_files.iteritems():
377      for file_handle in file_handle_list:
378        remote_path = ('profiler-file-id_%s-%s%-d%s' % (
379            file_handle.id,
380            datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'),
381            random.randint(1, 100000),
382            file_handle.extension))
383        try:
384          cloud_url = cloud_storage.Insert(
385              bucket, remote_path, file_handle.GetAbsPath())
386          sys.stderr.write(
387              'View generated profiler files online at %s for page %s\n' %
388              (cloud_url, page.display_name))
389          self._pages_to_profiling_files_cloud_url[page].append(cloud_url)
390        except cloud_storage.PermissionError as e:
391          logging.error('Cannot upload profiling files to cloud storage due to '
392                        ' permission error: %s' % e.message)
393