• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python2.7
2#
3# Copyright 2017 gRPC authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16""" Computes the diff between two qps runs and outputs significant results """
17
18import argparse
19import json
20import multiprocessing
21import os
22import qps_scenarios
23import shutil
24import subprocess
25import sys
26import tabulate
27
28sys.path.append(
29    os.path.join(
30        os.path.dirname(sys.argv[0]), '..', 'microbenchmarks', 'bm_diff'))
31import bm_speedup
32
33sys.path.append(
34    os.path.join(
35        os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils'))
36import comment_on_pr
37
38
39def _args():
40    argp = argparse.ArgumentParser(description='Perform diff on QPS Driver')
41    argp.add_argument(
42        '-d',
43        '--diff_base',
44        type=str,
45        help='Commit or branch to compare the current one to')
46    argp.add_argument(
47        '-l',
48        '--loops',
49        type=int,
50        default=4,
51        help='Number of loops for each benchmark. More loops cuts down on noise'
52    )
53    argp.add_argument(
54        '-j',
55        '--jobs',
56        type=int,
57        default=multiprocessing.cpu_count(),
58        help='Number of CPUs to use')
59    args = argp.parse_args()
60    assert args.diff_base, "diff_base must be set"
61    return args
62
63
64def _make_cmd(jobs):
65    return ['make', '-j', '%d' % jobs, 'qps_json_driver', 'qps_worker']
66
67
68def build(name, jobs):
69    shutil.rmtree('qps_diff_%s' % name, ignore_errors=True)
70    subprocess.check_call(['git', 'submodule', 'update'])
71    try:
72        subprocess.check_call(_make_cmd(jobs))
73    except subprocess.CalledProcessError, e:
74        subprocess.check_call(['make', 'clean'])
75        subprocess.check_call(_make_cmd(jobs))
76    os.rename('bins', 'qps_diff_%s' % name)
77
78
79def _run_cmd(name, scenario, fname):
80    return [
81        'qps_diff_%s/opt/qps_json_driver' % name, '--scenarios_json', scenario,
82        '--json_file_out', fname
83    ]
84
85
86def run(name, scenarios, loops):
87    for sn in scenarios:
88        for i in range(0, loops):
89            fname = "%s.%s.%d.json" % (sn, name, i)
90            subprocess.check_call(_run_cmd(name, scenarios[sn], fname))
91
92
93def _load_qps(fname):
94    try:
95        with open(fname) as f:
96            return json.loads(f.read())['qps']
97    except IOError, e:
98        print("IOError occurred reading file: %s" % fname)
99        return None
100    except ValueError, e:
101        print("ValueError occurred reading file: %s" % fname)
102        return None
103
104
105def _median(ary):
106    assert (len(ary))
107    ary = sorted(ary)
108    n = len(ary)
109    if n % 2 == 0:
110        return (ary[(n - 1) / 2] + ary[(n - 1) / 2 + 1]) / 2.0
111    else:
112        return ary[n / 2]
113
114
115def diff(scenarios, loops, old, new):
116    old_data = {}
117    new_data = {}
118
119    # collect data
120    for sn in scenarios:
121        old_data[sn] = []
122        new_data[sn] = []
123        for i in range(loops):
124            old_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, old, i)))
125            new_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, new, i)))
126
127    # crunch data
128    headers = ['Benchmark', 'qps']
129    rows = []
130    for sn in scenarios:
131        mdn_diff = abs(_median(new_data[sn]) - _median(old_data[sn]))
132        print('%s: %s=%r %s=%r mdn_diff=%r' % (sn, new, new_data[sn], old,
133                                               old_data[sn], mdn_diff))
134        s = bm_speedup.speedup(new_data[sn], old_data[sn], 10e-5)
135        if abs(s) > 3 and mdn_diff > 0.5:
136            rows.append([sn, '%+d%%' % s])
137
138    if rows:
139        return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f')
140    else:
141        return None
142
143
144def main(args):
145    build('new', args.jobs)
146
147    if args.diff_base:
148        where_am_i = subprocess.check_output(
149            ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
150        subprocess.check_call(['git', 'checkout', args.diff_base])
151        try:
152            build('old', args.jobs)
153        finally:
154            subprocess.check_call(['git', 'checkout', where_am_i])
155            subprocess.check_call(['git', 'submodule', 'update'])
156
157    run('new', qps_scenarios._SCENARIOS, args.loops)
158    run('old', qps_scenarios._SCENARIOS, args.loops)
159
160    diff_output = diff(qps_scenarios._SCENARIOS, args.loops, 'old', 'new')
161
162    if diff_output:
163        text = '[qps] Performance differences noted:\n%s' % diff_output
164    else:
165        text = '[qps] No significant performance differences'
166    print('%s' % text)
167    comment_on_pr.comment_on_pr('```\n%s\n```' % text)
168
169
170if __name__ == '__main__':
171    args = _args()
172    main(args)
173