• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python2.7
2#
3# Copyright 2017 gRPC authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16""" Computes the diff between two qps runs and outputs significant results """
17
18import argparse
19import json
20import multiprocessing
21import os
22import qps_scenarios
23import shutil
24import subprocess
25import sys
26import tabulate
27
28sys.path.append(
29    os.path.join(os.path.dirname(sys.argv[0]), '..', 'microbenchmarks',
30                 'bm_diff'))
31import bm_speedup
32
33sys.path.append(
34    os.path.join(os.path.dirname(sys.argv[0]), '..', '..', 'run_tests',
35                 'python_utils'))
36import check_on_pr
37
38
39def _args():
40    argp = argparse.ArgumentParser(description='Perform diff on QPS Driver')
41    argp.add_argument('-d',
42                      '--diff_base',
43                      type=str,
44                      help='Commit or branch to compare the current one to')
45    argp.add_argument(
46        '-l',
47        '--loops',
48        type=int,
49        default=4,
50        help='Number of loops for each benchmark. More loops cuts down on noise'
51    )
52    argp.add_argument('-j',
53                      '--jobs',
54                      type=int,
55                      default=multiprocessing.cpu_count(),
56                      help='Number of CPUs to use')
57    args = argp.parse_args()
58    assert args.diff_base, "diff_base must be set"
59    return args
60
61
62def _make_cmd(jobs):
63    return ['make', '-j', '%d' % jobs, 'qps_json_driver', 'qps_worker']
64
65
66def build(name, jobs):
67    shutil.rmtree('qps_diff_%s' % name, ignore_errors=True)
68    subprocess.check_call(['git', 'submodule', 'update'])
69    try:
70        subprocess.check_call(_make_cmd(jobs))
71    except subprocess.CalledProcessError, e:
72        subprocess.check_call(['make', 'clean'])
73        subprocess.check_call(_make_cmd(jobs))
74    os.rename('bins', 'qps_diff_%s' % name)
75
76
77def _run_cmd(name, scenario, fname):
78    return [
79        'qps_diff_%s/opt/qps_json_driver' % name, '--scenarios_json', scenario,
80        '--json_file_out', fname
81    ]
82
83
84def run(name, scenarios, loops):
85    for sn in scenarios:
86        for i in range(0, loops):
87            fname = "%s.%s.%d.json" % (sn, name, i)
88            subprocess.check_call(_run_cmd(name, scenarios[sn], fname))
89
90
91def _load_qps(fname):
92    try:
93        with open(fname) as f:
94            return json.loads(f.read())['qps']
95    except IOError, e:
96        print("IOError occurred reading file: %s" % fname)
97        return None
98    except ValueError, e:
99        print("ValueError occurred reading file: %s" % fname)
100        return None
101
102
103def _median(ary):
104    assert (len(ary))
105    ary = sorted(ary)
106    n = len(ary)
107    if n % 2 == 0:
108        return (ary[(n - 1) / 2] + ary[(n - 1) / 2 + 1]) / 2.0
109    else:
110        return ary[n / 2]
111
112
113def diff(scenarios, loops, old, new):
114    old_data = {}
115    new_data = {}
116
117    # collect data
118    for sn in scenarios:
119        old_data[sn] = []
120        new_data[sn] = []
121        for i in range(loops):
122            old_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, old, i)))
123            new_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, new, i)))
124
125    # crunch data
126    headers = ['Benchmark', 'qps']
127    rows = []
128    for sn in scenarios:
129        mdn_diff = abs(_median(new_data[sn]) - _median(old_data[sn]))
130        print('%s: %s=%r %s=%r mdn_diff=%r' %
131              (sn, new, new_data[sn], old, old_data[sn], mdn_diff))
132        s = bm_speedup.speedup(new_data[sn], old_data[sn], 10e-5)
133        if abs(s) > 3 and mdn_diff > 0.5:
134            rows.append([sn, '%+d%%' % s])
135
136    if rows:
137        return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f')
138    else:
139        return None
140
141
142def main(args):
143    build('new', args.jobs)
144
145    if args.diff_base:
146        where_am_i = subprocess.check_output(
147            ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
148        subprocess.check_call(['git', 'checkout', args.diff_base])
149        try:
150            build('old', args.jobs)
151        finally:
152            subprocess.check_call(['git', 'checkout', where_am_i])
153            subprocess.check_call(['git', 'submodule', 'update'])
154
155    run('new', qps_scenarios._SCENARIOS, args.loops)
156    run('old', qps_scenarios._SCENARIOS, args.loops)
157
158    diff_output = diff(qps_scenarios._SCENARIOS, args.loops, 'old', 'new')
159
160    if diff_output:
161        text = '[qps] Performance differences noted:\n%s' % diff_output
162    else:
163        text = '[qps] No significant performance differences'
164    print('%s' % text)
165    check_on_pr.check_on_pr('QPS', '```\n%s\n```' % text)
166
167
168if __name__ == '__main__':
169    args = _args()
170    main(args)
171