• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python2
2#
3# Copyright 2017 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6#
7# pylint: disable=cros-logging-import
8"""Transforms skia benchmark results to ones that crosperf can understand."""
9
10from __future__ import print_function
11
12import itertools
13import logging
14import json
15import sys
16
17# Turn the logging level to INFO before importing other autotest
18# code, to avoid having failed import logging messages confuse the
19# test_droid user.
20logging.basicConfig(level=logging.INFO)
21
22# All of the results we care about, by name.
23# Each of these *must* end in _ns, _us, _ms, or _s, since all the metrics we
24# collect (so far) are related to time, and we alter the results based on the
25# suffix of these strings (so we don't have 0.000421ms per sample, for example)
26_RESULT_RENAMES = {
27    'memset32_100000_640_480_nonrendering': 'memset_time_ms',
28    'path_equality_50%_640_480_nonrendering': 'path_equality_ns',
29    'sort_qsort_backward_640_480_nonrendering': 'qsort_us'
30}
31
32
33def _GetFamiliarName(name):
34    r = _RESULT_RENAMES[name]
35    return r if r else name
36
37
38def _IsResultInteresting(name):
39    return name in _RESULT_RENAMES
40
41
42def _GetTimeMultiplier(label_name):
43    """Given a time (in milliseconds), normalize it to what label_name expects.
44
45    "What label_name expects" meaning "we pattern match against the last few
46    non-space chars in label_name."
47
48    This expects the time unit to be separated from anything else by '_'.
49    """
50    ms_mul = 1000 * 1000.
51    endings = [('_ns', 1), ('_us', 1000),
52               ('_ms', ms_mul), ('_s', ms_mul * 1000)]
53    for end, mul in endings:
54        if label_name.endswith(end):
55            return ms_mul / mul
56    raise ValueError('Unknown ending in "%s"; expecting one of %s' %
57                     (label_name, [end for end, _ in endings]))
58
59
60def _GetTimeDenom(ms):
61    """Given a list of times (in milliseconds), find a sane time unit for them.
62
63    Returns the unit name, and `ms` normalized to that time unit.
64
65    >>> _GetTimeDenom([1, 2, 3])
66    ('ms', [1.0, 2.0, 3.0])
67    >>> _GetTimeDenom([.1, .2, .3])
68    ('us', [100.0, 200.0, 300.0])
69    """
70
71    ms_mul = 1000 * 1000
72    units = [('us', 1000), ('ms', ms_mul), ('s', ms_mul * 1000)]
73    for name, mul in reversed(units):
74        normalized = [float(t) * ms_mul / mul for t in ms]
75        average = sum(normalized) / len(normalized)
76        if all(n > 0.1 for n in normalized) and average >= 1:
77            return name, normalized
78
79    normalized = [float(t) * ms_mul for t in ms]
80    return 'ns', normalized
81
82
83def _TransformBenchmarks(raw_benchmarks):
84    # We get {"results": {"bench_name": Results}}
85    # where
86    #   Results = {"config_name": {"samples": [float], etc.}}
87    #
88    # We want {"data": {"skia": [[BenchmarkData]]},
89    #          "platforms": ["platform1, ..."]}
90    # where
91    #   BenchmarkData = {"bench_name": bench_samples[N], ..., "retval": 0}
92    #
93    # Note that retval is awkward -- crosperf's JSON reporter reports the result
94    # as a failure if it's not there. Everything else treats it like a
95    # statistic...
96    benchmarks = raw_benchmarks['results']
97    results = []
98    for bench_name, bench_result in benchmarks.iteritems():
99        try:
100            for cfg_name, keyvals in bench_result.iteritems():
101                # Some benchmarks won't have timing data (either it won't exist
102                # at all, or it'll be empty); skip them.
103                samples = keyvals.get('samples')
104                if not samples:
105                    continue
106
107                bench_name = '%s_%s' % (bench_name, cfg_name)
108                if not _IsResultInteresting(bench_name):
109                    continue
110
111                friendly_name = _GetFamiliarName(bench_name)
112                if len(results) < len(samples):
113                    results.extend({
114                        'retval': 0
115                    } for _ in xrange(len(samples) - len(results)))
116
117                time_mul = _GetTimeMultiplier(friendly_name)
118                for sample, app in itertools.izip(samples, results):
119                    assert friendly_name not in app
120                    app[friendly_name] = sample * time_mul
121        except (KeyError, ValueError) as e:
122            logging.error('While converting "%s" (key: %s): %s',
123                          bench_result, bench_name, e.message)
124            raise
125
126    # Realistically, [results] should be multiple results, where each entry in
127    # the list is the result for a different label. Because we only deal with
128    # one label at the moment, we need to wrap it in its own list.
129    return results
130
131
132if __name__ == '__main__':
133
134    def _GetUserFile(argv):
135        if not argv or argv[0] == '-':
136            return sys.stdin
137        return open(argv[0])
138
139    def _Main():
140        with _GetUserFile(sys.argv[1:]) as in_file:
141            obj = json.load(in_file)
142        output = _TransformBenchmarks(obj)
143        json.dump(output, sys.stdout)
144
145    _Main()
146