• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# -*- coding: utf-8 -*-
2# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Parse data from benchmark_runs for tabulator."""
7
8from __future__ import print_function
9
10import errno
11import json
12import os
13import re
14import sys
15
16from cros_utils import misc
17
18_TELEMETRY_RESULT_DEFAULTS_FILE = 'default-telemetry-results.json'
19_DUP_KEY_REGEX = re.compile(r'(\w+)\{(\d+)\}')
20
21
22def _AdjustIteration(benchmarks, max_dup, bench):
23  """Adjust the interation numbers if they have keys like ABCD{i}."""
24  for benchmark in benchmarks:
25    if benchmark.name != bench or benchmark.iteration_adjusted:
26      continue
27    benchmark.iteration_adjusted = True
28    benchmark.iterations *= (max_dup + 1)
29
30
31def _GetMaxDup(data):
32  """Find the maximum i inside ABCD{i}.
33
34  data should be a [[[Key]]], where Key is a string that may look like
35  ABCD{i}.
36  """
37  max_dup = 0
38  for label in data:
39    for run in label:
40      for key in run:
41        match = _DUP_KEY_REGEX.match(key)
42        if match:
43          max_dup = max(max_dup, int(match.group(2)))
44  return max_dup
45
46
47def _Repeat(func, times):
48  """Returns the result of running func() n times."""
49  return [func() for _ in range(times)]
50
51
52def _DictWithReturnValues(retval, pass_fail):
53  """Create a new dictionary pre-populated with success/fail values."""
54  new_dict = {}
55  # Note: 0 is a valid retval; test to make sure it's not None.
56  if retval is not None:
57    new_dict['retval'] = retval
58  if pass_fail:
59    new_dict[''] = pass_fail
60  return new_dict
61
62
63def _GetNonDupLabel(max_dup, runs):
64  """Create new list for the runs of the same label.
65
66  Specifically, this will split out keys like foo{0}, foo{1} from one run into
67  their own runs. For example, given a run like:
68    {"foo": 1, "bar{0}": 2, "baz": 3, "qux{1}": 4, "pirate{0}": 5}
69
70  You'll get:
71    [{"foo": 1, "baz": 3}, {"bar": 2, "pirate": 5}, {"qux": 4}]
72
73  Hands back the lists of transformed runs, all concatenated together.
74  """
75  new_runs = []
76  for run in runs:
77    run_retval = run.get('retval', None)
78    run_pass_fail = run.get('', None)
79    new_run = {}
80    # pylint: disable=cell-var-from-loop
81    added_runs = _Repeat(
82        lambda: _DictWithReturnValues(run_retval, run_pass_fail), max_dup)
83    for key, value in run.items():
84      match = _DUP_KEY_REGEX.match(key)
85      if not match:
86        new_run[key] = value
87      else:
88        new_key, index_str = match.groups()
89        added_runs[int(index_str) - 1][new_key] = str(value)
90    new_runs.append(new_run)
91    new_runs += added_runs
92  return new_runs
93
94
95def _DuplicatePass(result, benchmarks):
96  """Properly expands keys like `foo{1}` in `result`."""
97  for bench, data in result.items():
98    max_dup = _GetMaxDup(data)
99    # If there's nothing to expand, there's nothing to do.
100    if not max_dup:
101      continue
102    for i, runs in enumerate(data):
103      data[i] = _GetNonDupLabel(max_dup, runs)
104    _AdjustIteration(benchmarks, max_dup, bench)
105
106
107def _ReadSummaryFile(filename):
108  """Reads the summary file at filename."""
109  dirname, _ = misc.GetRoot(filename)
110  fullname = os.path.join(dirname, _TELEMETRY_RESULT_DEFAULTS_FILE)
111  try:
112    # Slurp the summary file into a dictionary. The keys in the dictionary are
113    # the benchmark names. The value for a key is a list containing the names
114    # of all the result fields that should be returned in a 'default' report.
115    with open(fullname) as in_file:
116      return json.load(in_file)
117  except IOError as e:
118    # ENOENT means "no such file or directory"
119    if e.errno == errno.ENOENT:
120      return {}
121    raise
122
123
124def _MakeOrganizeResultOutline(benchmark_runs, labels):
125  """Creates the "outline" of the OrganizeResults result for a set of runs.
126
127  Report generation returns lists of different sizes, depending on the input
128  data. Depending on the order in which we iterate through said input data, we
129  may populate the Nth index of a list, then the N-1st, then the N+Mth, ...
130
131  It's cleaner to figure out the "skeleton"/"outline" ahead of time, so we don't
132  have to worry about resizing while computing results.
133  """
134  # Count how many iterations exist for each benchmark run.
135  # We can't simply count up, since we may be given an incomplete set of
136  # iterations (e.g. [r.iteration for r in benchmark_runs] == [1, 3])
137  iteration_count = {}
138  for run in benchmark_runs:
139    name = run.benchmark.name
140    old_iterations = iteration_count.get(name, -1)
141    # N.B. run.iteration starts at 1, not 0.
142    iteration_count[name] = max(old_iterations, run.iteration)
143
144  # Result structure: {benchmark_name: [[{key: val}]]}
145  result = {}
146  for run in benchmark_runs:
147    name = run.benchmark.name
148    num_iterations = iteration_count[name]
149    # default param makes cros lint be quiet about defining num_iterations in a
150    # loop.
151    make_dicts = lambda n=num_iterations: _Repeat(dict, n)
152    result[name] = _Repeat(make_dicts, len(labels))
153  return result
154
155
156def OrganizeResults(benchmark_runs, labels, benchmarks=None, json_report=False):
157  """Create a dict from benchmark_runs.
158
159  The structure of the output dict is as follows:
160  {"benchmark_1":[
161    [{"key1":"v1", "key2":"v2"},{"key1":"v1", "key2","v2"}]
162    #one label
163    []
164    #the other label
165    ]
166   "benchmark_2":
167    [
168    ]}.
169  """
170  result = _MakeOrganizeResultOutline(benchmark_runs, labels)
171  label_names = [label.name for label in labels]
172  label_indices = {name: i for i, name in enumerate(label_names)}
173  summary_file = _ReadSummaryFile(sys.argv[0])
174
175  if benchmarks is None:
176    benchmarks = []
177
178  for benchmark_run in benchmark_runs:
179    if not benchmark_run.result:
180      continue
181    benchmark = benchmark_run.benchmark
182    label_index = label_indices[benchmark_run.label.name]
183    cur_label_list = result[benchmark.name][label_index]
184    cur_dict = cur_label_list[benchmark_run.iteration - 1]
185
186    show_all_results = json_report or benchmark.show_all_results
187    if not show_all_results:
188      summary_list = summary_file.get(benchmark.name)
189      if summary_list:
190        for key in benchmark_run.result.keyvals.keys():
191          if any(
192              key.startswith(added_key)
193              for added_key in ['retval', 'cpufreq', 'cputemp']):
194            summary_list.append(key)
195      else:
196        # Did not find test_name in json file; show everything.
197        show_all_results = True
198    if benchmark_run.result.cwp_dso:
199      # If we are in cwp approximation mode, we only care about samples
200      if 'samples' in benchmark_run.result.keyvals:
201        cur_dict['samples'] = benchmark_run.result.keyvals['samples']
202      cur_dict['retval'] = benchmark_run.result.keyvals['retval']
203      for key, value in benchmark_run.result.keyvals.items():
204        if any(
205            key.startswith(cpustat_keyword)
206            for cpustat_keyword in ['cpufreq', 'cputemp']):
207          cur_dict[key] = value
208    else:
209      for test_key in benchmark_run.result.keyvals:
210        if show_all_results or test_key in summary_list:
211          cur_dict[test_key] = benchmark_run.result.keyvals[test_key]
212    # Occasionally Telemetry tests will not fail but they will not return a
213    # result, either.  Look for those cases, and force them to be a fail.
214    # (This can happen if, for example, the test has been disabled.)
215    if len(cur_dict) == 1 and cur_dict['retval'] == 0:
216      cur_dict['retval'] = 1
217      benchmark_run.result.keyvals['retval'] = 1
218      # TODO: This output should be sent via logger.
219      print(
220          "WARNING: Test '%s' appears to have succeeded but returned"
221          ' no results.' % benchmark.name,
222          file=sys.stderr)
223    if json_report and benchmark_run.machine:
224      cur_dict['machine'] = benchmark_run.machine.name
225      cur_dict['machine_checksum'] = benchmark_run.machine.checksum
226      cur_dict['machine_string'] = benchmark_run.machine.checksum_string
227  _DuplicatePass(result, benchmarks)
228  return result
229