• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4"""Parse data from benchmark_runs for tabulator."""
5
6from __future__ import print_function
7
8import errno
9import json
10import os
11import re
12import sys
13
14from cros_utils import misc
15
16_TELEMETRY_RESULT_DEFAULTS_FILE = 'default-telemetry-results.json'
17_DUP_KEY_REGEX = re.compile(r'(\w+)\{(\d+)\}')
18
19
20def _AdjustIteration(benchmarks, max_dup, bench):
21  """Adjust the interation numbers if they have keys like ABCD{i}."""
22  for benchmark in benchmarks:
23    if benchmark.name != bench or benchmark.iteration_adjusted:
24      continue
25    benchmark.iteration_adjusted = True
26    benchmark.iterations *= (max_dup + 1)
27
28
29def _GetMaxDup(data):
30  """Find the maximum i inside ABCD{i}.
31
32  data should be a [[[Key]]], where Key is a string that may look like
33  ABCD{i}.
34  """
35  max_dup = 0
36  for label in data:
37    for run in label:
38      for key in run:
39        match = _DUP_KEY_REGEX.match(key)
40        if match:
41          max_dup = max(max_dup, int(match.group(2)))
42  return max_dup
43
44
45def _Repeat(func, times):
46  """Returns the result of running func() n times."""
47  return [func() for _ in xrange(times)]
48
49
50def _GetNonDupLabel(max_dup, runs):
51  """Create new list for the runs of the same label.
52
53  Specifically, this will split out keys like foo{0}, foo{1} from one run into
54  their own runs. For example, given a run like:
55    {"foo": 1, "bar{0}": 2, "baz": 3, "qux{1}": 4, "pirate{0}": 5}
56
57  You'll get:
58    [{"foo": 1, "baz": 3}, {"bar": 2, "pirate": 5}, {"qux": 4}]
59
60  Hands back the lists of transformed runs, all concatenated together.
61  """
62  new_runs = []
63  for run in runs:
64    new_run = {}
65    added_runs = _Repeat(dict, max_dup)
66    for key, value in run.iteritems():
67      match = _DUP_KEY_REGEX.match(key)
68      if not match:
69        new_run[key] = value
70      else:
71        new_key, index_str = match.groups()
72        added_runs[int(index_str)-1][new_key] = str(value)
73    new_runs.append(new_run)
74    new_runs += added_runs
75  return new_runs
76
77
78def _DuplicatePass(result, benchmarks):
79  """Properly expands keys like `foo{1}` in `result`."""
80  for bench, data in result.iteritems():
81    max_dup = _GetMaxDup(data)
82    # If there's nothing to expand, there's nothing to do.
83    if not max_dup:
84      continue
85    for i, runs in enumerate(data):
86      data[i] = _GetNonDupLabel(max_dup, runs)
87    _AdjustIteration(benchmarks, max_dup, bench)
88
89
90def _ReadSummaryFile(filename):
91  """Reads the summary file at filename."""
92  dirname, _ = misc.GetRoot(filename)
93  fullname = os.path.join(dirname, _TELEMETRY_RESULT_DEFAULTS_FILE)
94  try:
95    # Slurp the summary file into a dictionary. The keys in the dictionary are
96    # the benchmark names. The value for a key is a list containing the names
97    # of all the result fields that should be returned in a 'default' report.
98    with open(fullname) as in_file:
99      return json.load(in_file)
100  except IOError as e:
101    # ENOENT means "no such file or directory"
102    if e.errno == errno.ENOENT:
103      return {}
104    raise
105
106
107def _MakeOrganizeResultOutline(benchmark_runs, labels):
108  """Creates the "outline" of the OrganizeResults result for a set of runs.
109
110  Report generation returns lists of different sizes, depending on the input
111  data. Depending on the order in which we iterate through said input data, we
112  may populate the Nth index of a list, then the N-1st, then the N+Mth, ...
113
114  It's cleaner to figure out the "skeleton"/"outline" ahead of time, so we don't
115  have to worry about resizing while computing results.
116  """
117  # Count how many iterations exist for each benchmark run.
118  # We can't simply count up, since we may be given an incomplete set of
119  # iterations (e.g. [r.iteration for r in benchmark_runs] == [1, 3])
120  iteration_count = {}
121  for run in benchmark_runs:
122    name = run.benchmark.name
123    old_iterations = iteration_count.get(name, -1)
124    # N.B. run.iteration starts at 1, not 0.
125    iteration_count[name] = max(old_iterations, run.iteration)
126
127  # Result structure: {benchmark_name: [[{key: val}]]}
128  result = {}
129  for run in benchmark_runs:
130    name = run.benchmark.name
131    num_iterations = iteration_count[name]
132    # default param makes cros lint be quiet about defining num_iterations in a
133    # loop.
134    make_dicts = lambda n=num_iterations: _Repeat(dict, n)
135    result[name] = _Repeat(make_dicts, len(labels))
136  return result
137
138def OrganizeResults(benchmark_runs, labels, benchmarks=None, json_report=False):
139  """Create a dict from benchmark_runs.
140
141  The structure of the output dict is as follows:
142  {"benchmark_1":[
143    [{"key1":"v1", "key2":"v2"},{"key1":"v1", "key2","v2"}]
144    #one label
145    []
146    #the other label
147    ]
148   "benchmark_2":
149    [
150    ]}.
151  """
152  result = _MakeOrganizeResultOutline(benchmark_runs, labels)
153  label_names = [label.name for label in labels]
154  label_indices = {name: i for i, name in enumerate(label_names)}
155  summary_file = _ReadSummaryFile(sys.argv[0])
156  if benchmarks is None:
157    benchmarks = []
158
159  for benchmark_run in benchmark_runs:
160    if not benchmark_run.result:
161      continue
162    benchmark = benchmark_run.benchmark
163    label_index = label_indices[benchmark_run.label.name]
164    cur_label_list = result[benchmark.name][label_index]
165    cur_dict = cur_label_list[benchmark_run.iteration - 1]
166
167    show_all_results = json_report or benchmark.show_all_results
168    if not show_all_results:
169      summary_list = summary_file.get(benchmark.test_name)
170      if summary_list:
171        summary_list.append('retval')
172      else:
173        # Did not find test_name in json file; show everything.
174        show_all_results = True
175    for test_key in benchmark_run.result.keyvals:
176      if show_all_results or test_key in summary_list:
177        cur_dict[test_key] = benchmark_run.result.keyvals[test_key]
178    # Occasionally Telemetry tests will not fail but they will not return a
179    # result, either.  Look for those cases, and force them to be a fail.
180    # (This can happen if, for example, the test has been disabled.)
181    if len(cur_dict) == 1 and cur_dict['retval'] == 0:
182      cur_dict['retval'] = 1
183      # TODO: This output should be sent via logger.
184      print("WARNING: Test '%s' appears to have succeeded but returned"
185            ' no results.' % benchmark.name,
186            file=sys.stderr)
187    if json_report and benchmark_run.machine:
188      cur_dict['machine'] = benchmark_run.machine.name
189      cur_dict['machine_checksum'] = benchmark_run.machine.checksum
190      cur_dict['machine_string'] = benchmark_run.machine.checksum_string
191  _DuplicatePass(result, benchmarks)
192  return result
193