• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright 2017 gRPC authors.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import cgi
17import multiprocessing
18import os
19import subprocess
20import sys
21import argparse
22
23import python_utils.jobset as jobset
24import python_utils.start_port_server as start_port_server
25
26sys.path.append(
27    os.path.join(
28        os.path.dirname(sys.argv[0]), '..', 'profiling', 'microbenchmarks',
29        'bm_diff'))
30import bm_constants
31
32flamegraph_dir = os.path.join(os.path.expanduser('~'), 'FlameGraph')
33
34os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
35if not os.path.exists('reports'):
36    os.makedirs('reports')
37
38start_port_server.start_port_server()
39
40
41def fnize(s):
42    out = ''
43    for c in s:
44        if c in '<>, /':
45            if len(out) and out[-1] == '_': continue
46            out += '_'
47        else:
48            out += c
49    return out
50
51
52# index html
53index_html = """
54<html>
55<head>
56<title>Microbenchmark Results</title>
57</head>
58<body>
59"""
60
61
62def heading(name):
63    global index_html
64    index_html += "<h1>%s</h1>\n" % name
65
66
67def link(txt, tgt):
68    global index_html
69    index_html += "<p><a href=\"%s\">%s</a></p>\n" % (
70        cgi.escape(tgt, quote=True), cgi.escape(txt))
71
72
73def text(txt):
74    global index_html
75    index_html += "<p><pre>%s</pre></p>\n" % cgi.escape(txt)
76
77
78def collect_latency(bm_name, args):
79    """generate latency profiles"""
80    benchmarks = []
81    profile_analysis = []
82    cleanup = []
83
84    heading('Latency Profiles: %s' % bm_name)
85    subprocess.check_call([
86        'make', bm_name, 'CONFIG=basicprof', '-j',
87        '%d' % multiprocessing.cpu_count()
88    ])
89    for line in subprocess.check_output(
90        ['bins/basicprof/%s' % bm_name, '--benchmark_list_tests']).splitlines():
91        link(line, '%s.txt' % fnize(line))
92        benchmarks.append(
93            jobset.JobSpec(
94                [
95                    'bins/basicprof/%s' % bm_name,
96                    '--benchmark_filter=^%s$' % line,
97                    '--benchmark_min_time=0.05'
98                ],
99                environ={'LATENCY_TRACE': '%s.trace' % fnize(line)},
100                shortname='profile-%s' % fnize(line)))
101        profile_analysis.append(
102            jobset.JobSpec(
103                [
104                    sys.executable,
105                    'tools/profiling/latency_profile/profile_analyzer.py',
106                    '--source',
107                    '%s.trace' % fnize(line), '--fmt', 'simple', '--out',
108                    'reports/%s.txt' % fnize(line)
109                ],
110                timeout_seconds=20 * 60,
111                shortname='analyze-%s' % fnize(line)))
112        cleanup.append(jobset.JobSpec(['rm', '%s.trace' % fnize(line)]))
113        # periodically flush out the list of jobs: profile_analysis jobs at least
114        # consume upwards of five gigabytes of ram in some cases, and so analysing
115        # hundreds of them at once is impractical -- but we want at least some
116        # concurrency or the work takes too long
117        if len(benchmarks) >= min(16, multiprocessing.cpu_count()):
118            # run up to half the cpu count: each benchmark can use up to two cores
119            # (one for the microbenchmark, one for the data flush)
120            jobset.run(
121                benchmarks, maxjobs=max(1,
122                                        multiprocessing.cpu_count() / 2))
123            jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
124            jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
125            benchmarks = []
126            profile_analysis = []
127            cleanup = []
128    # run the remaining benchmarks that weren't flushed
129    if len(benchmarks):
130        jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count() / 2))
131        jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
132        jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
133
134
135def collect_perf(bm_name, args):
136    """generate flamegraphs"""
137    heading('Flamegraphs: %s' % bm_name)
138    subprocess.check_call([
139        'make', bm_name, 'CONFIG=mutrace', '-j',
140        '%d' % multiprocessing.cpu_count()
141    ])
142    benchmarks = []
143    profile_analysis = []
144    cleanup = []
145    for line in subprocess.check_output(
146        ['bins/mutrace/%s' % bm_name, '--benchmark_list_tests']).splitlines():
147        link(line, '%s.svg' % fnize(line))
148        benchmarks.append(
149            jobset.JobSpec(
150                [
151                    'perf', 'record', '-o',
152                    '%s-perf.data' % fnize(line), '-g', '-F', '997',
153                    'bins/mutrace/%s' % bm_name,
154                    '--benchmark_filter=^%s$' % line, '--benchmark_min_time=10'
155                ],
156                shortname='perf-%s' % fnize(line)))
157        profile_analysis.append(
158            jobset.JobSpec(
159                [
160                    'tools/run_tests/performance/process_local_perf_flamegraphs.sh'
161                ],
162                environ={
163                    'PERF_BASE_NAME': fnize(line),
164                    'OUTPUT_DIR': 'reports',
165                    'OUTPUT_FILENAME': fnize(line),
166                },
167                shortname='flame-%s' % fnize(line)))
168        cleanup.append(jobset.JobSpec(['rm', '%s-perf.data' % fnize(line)]))
169        cleanup.append(jobset.JobSpec(['rm', '%s-out.perf' % fnize(line)]))
170        # periodically flush out the list of jobs: temporary space required for this
171        # processing is large
172        if len(benchmarks) >= 20:
173            # run up to half the cpu count: each benchmark can use up to two cores
174            # (one for the microbenchmark, one for the data flush)
175            jobset.run(benchmarks, maxjobs=1)
176            jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
177            jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
178            benchmarks = []
179            profile_analysis = []
180            cleanup = []
181    # run the remaining benchmarks that weren't flushed
182    if len(benchmarks):
183        jobset.run(benchmarks, maxjobs=1)
184        jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
185        jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
186
187
188def run_summary(bm_name, cfg, base_json_name):
189    subprocess.check_call([
190        'make', bm_name,
191        'CONFIG=%s' % cfg, '-j',
192        '%d' % multiprocessing.cpu_count()
193    ])
194    cmd = [
195        'bins/%s/%s' % (cfg, bm_name),
196        '--benchmark_out=%s.%s.json' % (base_json_name, cfg),
197        '--benchmark_out_format=json'
198    ]
199    if args.summary_time is not None:
200        cmd += ['--benchmark_min_time=%d' % args.summary_time]
201    return subprocess.check_output(cmd)
202
203
204def collect_summary(bm_name, args):
205    heading('Summary: %s [no counters]' % bm_name)
206    text(run_summary(bm_name, 'opt', bm_name))
207    heading('Summary: %s [with counters]' % bm_name)
208    text(run_summary(bm_name, 'counters', bm_name))
209    if args.bigquery_upload:
210        with open('%s.csv' % bm_name, 'w') as f:
211            f.write(
212                subprocess.check_output([
213                    'tools/profiling/microbenchmarks/bm2bq.py',
214                    '%s.counters.json' % bm_name,
215                    '%s.opt.json' % bm_name
216                ]))
217        subprocess.check_call([
218            'bq', 'load', 'microbenchmarks.microbenchmarks',
219            '%s.csv' % bm_name
220        ])
221
222
223collectors = {
224    'latency': collect_latency,
225    'perf': collect_perf,
226    'summary': collect_summary,
227}
228
229argp = argparse.ArgumentParser(description='Collect data from microbenchmarks')
230argp.add_argument(
231    '-c',
232    '--collect',
233    choices=sorted(collectors.keys()),
234    nargs='*',
235    default=sorted(collectors.keys()),
236    help='Which collectors should be run against each benchmark')
237argp.add_argument(
238    '-b',
239    '--benchmarks',
240    choices=bm_constants._AVAILABLE_BENCHMARK_TESTS,
241    default=bm_constants._AVAILABLE_BENCHMARK_TESTS,
242    nargs='+',
243    type=str,
244    help='Which microbenchmarks should be run')
245argp.add_argument(
246    '--bigquery_upload',
247    default=False,
248    action='store_const',
249    const=True,
250    help='Upload results from summary collection to bigquery')
251argp.add_argument(
252    '--summary_time',
253    default=None,
254    type=int,
255    help='Minimum time to run benchmarks for the summary collection')
256args = argp.parse_args()
257
258try:
259    for collect in args.collect:
260        for bm_name in args.benchmarks:
261            collectors[collect](bm_name, args)
262finally:
263    if not os.path.exists('reports'):
264        os.makedirs('reports')
265    index_html += "</body>\n</html>\n"
266    with open('reports/index.html', 'w') as f:
267        f.write(index_html)
268