• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python2.7
2#
3# Copyright 2017 gRPC authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16""" Runs the entire bm_*.py pipeline, and possible comments on the PR """
17
18import bm_constants
19import bm_build
20import bm_run
21import bm_diff
22
23import sys
24import os
25import random
26import argparse
27import multiprocessing
28import subprocess
29
30sys.path.append(
31    os.path.join(os.path.dirname(sys.argv[0]), '..', '..', 'run_tests',
32                 'python_utils'))
33import check_on_pr
34
35sys.path.append(
36    os.path.join(os.path.dirname(sys.argv[0]), '..', '..', '..', 'run_tests',
37                 'python_utils'))
38import jobset
39
40
41def _args():
42    argp = argparse.ArgumentParser(
43        description='Perform diff on microbenchmarks')
44    argp.add_argument('-t',
45                      '--track',
46                      choices=sorted(bm_constants._INTERESTING),
47                      nargs='+',
48                      default=sorted(bm_constants._INTERESTING),
49                      help='Which metrics to track')
50    argp.add_argument('-b',
51                      '--benchmarks',
52                      nargs='+',
53                      choices=bm_constants._AVAILABLE_BENCHMARK_TESTS,
54                      default=bm_constants._AVAILABLE_BENCHMARK_TESTS,
55                      help='Which benchmarks to run')
56    argp.add_argument('-d',
57                      '--diff_base',
58                      type=str,
59                      help='Commit or branch to compare the current one to')
60    argp.add_argument(
61        '-o',
62        '--old',
63        default='old',
64        type=str,
65        help='Name of baseline run to compare to. Usually just called "old"')
66    argp.add_argument('-r',
67                      '--regex',
68                      type=str,
69                      default="",
70                      help='Regex to filter benchmarks run')
71    argp.add_argument(
72        '-l',
73        '--loops',
74        type=int,
75        default=10,
76        help=
77        'Number of times to loops the benchmarks. More loops cuts down on noise'
78    )
79    argp.add_argument('-j',
80                      '--jobs',
81                      type=int,
82                      default=multiprocessing.cpu_count(),
83                      help='Number of CPUs to use')
84    argp.add_argument('--pr_comment_name',
85                      type=str,
86                      default="microbenchmarks",
87                      help='Name that Jenkins will use to comment on the PR')
88    argp.add_argument('--counters', dest='counters', action='store_true')
89    argp.add_argument('--no-counters', dest='counters', action='store_false')
90    argp.set_defaults(counters=True)
91    args = argp.parse_args()
92    assert args.diff_base or args.old, "One of diff_base or old must be set!"
93    if args.loops < 3:
94        print("WARNING: This run will likely be noisy. Increase loops.")
95    return args
96
97
98def eintr_be_gone(fn):
99    """Run fn until it doesn't stop because of EINTR"""
100
101    def inner(*args):
102        while True:
103            try:
104                return fn(*args)
105            except IOError as e:
106                if e.errno != errno.EINTR:
107                    raise
108
109    return inner
110
111
112def main(args):
113
114    bm_build.build('new', args.benchmarks, args.jobs, args.counters)
115
116    old = args.old
117    if args.diff_base:
118        old = 'old'
119        where_am_i = subprocess.check_output(
120            ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
121        subprocess.check_call(['git', 'checkout', args.diff_base])
122        try:
123            bm_build.build(old, args.benchmarks, args.jobs, args.counters)
124        finally:
125            subprocess.check_call(['git', 'checkout', where_am_i])
126            subprocess.check_call(['git', 'submodule', 'update'])
127
128    jobs_list = []
129    jobs_list += bm_run.create_jobs('new', args.benchmarks, args.loops,
130                                    args.regex, args.counters)
131    jobs_list += bm_run.create_jobs(old, args.benchmarks, args.loops,
132                                    args.regex, args.counters)
133
134    # shuffle all jobs to eliminate noise from GCE CPU drift
135    random.shuffle(jobs_list, random.SystemRandom().random)
136    jobset.run(jobs_list, maxjobs=args.jobs)
137
138    diff, note = bm_diff.diff(args.benchmarks, args.loops, args.regex,
139                              args.track, old, 'new', args.counters)
140    if diff:
141        text = '[%s] Performance differences noted:\n%s' % (
142            args.pr_comment_name, diff)
143    else:
144        text = '[%s] No significant performance differences' % args.pr_comment_name
145    if note:
146        text = note + '\n\n' + text
147    print('%s' % text)
148    check_on_pr.check_on_pr('Benchmark', '```\n%s\n```' % text)
149
150
151if __name__ == '__main__':
152    args = _args()
153    main(args)
154