• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1import subprocess
2import argparse
3import os
4import random
5from collections import OrderedDict
6from parse import parse
7from bokeh.io import export_png
8from bokeh.plotting import figure, output_file, show, save
9from bokeh.models import ColumnDataSource, FactorRange
10from bokeh.transform import factor_cmap
11from bokeh.layouts import gridplot
12from bokeh.embed import components
13
14parser = argparse.ArgumentParser(description='IWKV Benchmarks')
15parser.add_argument(
16    '-b', '--basedir', help='Base directory with benchmark executables', default='.', nargs='?')
17args = parser.parse_args()
18
19basedir = os.path.abspath(args.basedir)
20print('Base directory:', basedir)
21
22benchmarks = [
23    'iwkv',
24    'lmdb',
25    'bdb',
26    'wiredtiger',
27    'kyc',
28    'tc'
29    #'leveldb'
30]
31
32runs = []
33
34runs += [{'b': 'fillrandom2', 'n': n, 'vz': vz, 'rs': 2853624176, 'sizestats': True}
35         for n in (int(1e6),)
36         for vz in (1000,)]
37
38runs += [{'b': 'fillrandom2,readrandom,deleterandom', 'n': n, 'vz': vz, 'kz': kz, 'rs': 2105940112}
39         for n in (int(2e6),)
40         for vz in (40, 400,)
41         for kz in (16, 1024,)]
42
43runs += [{'b': 'fillseq,overwrite,deleteseq', 'n': n, 'kz': kz, 'rs': 570078848}
44         for n in (int(2e6),)
45         for vz in (400,)
46         for kz in (16, 1024,)]
47
48runs += [{'b': 'fillrandom2,readrandom,readseq,readreverse', 'n': n, 'vz': vz, 'rs': 1513135152}
49         for n in (int(10e6),)
50         for vz in (200,)]
51
52runs += [{'b': 'fillrandom2', 'n': n, 'vz': vz, 'rs': 3434783568}
53         for n in (int(10e3),)
54         for vz in ((200 * 1024),)]
55
56results = OrderedDict()
57
58
59def fill_result(bm, run, sizestats, line):
60    key = ' '.join(['-{} {}'.format(a, v) for a, v in run.items()])
61    if key not in results:
62        results[key] = OrderedDict()
63    if bm not in results[key]:
64        results[key][bm] = OrderedDict()
65    res = results[key][bm]
66
67    pval = parse('done: {} in {}', line)
68    if sizestats:
69        pval = parse('db size: {} ({})', line)
70        if pval and 'db size' not in res:
71            print(line, flush=True)
72            res['db size'] = int(pval[0]) / (1024 * 1024)
73    elif pval:
74        print(line, flush=True)
75        res[pval[0]] = int(pval[1])
76
77
78def run_benchmark_run(bm, run):
79    args = ['{}/{}_benchmark'.format(basedir, bm)]
80    sizestats = False
81    for a, v in run.items():
82        if a in ('sizestats',):
83            sizestats = True
84            continue
85        args.append('-{}'.format(a))
86        args.append(str(v))
87    print('Run {}'.format(' '.join(args)), flush=True)
88    with subprocess.Popen(args,
89                          stderr=subprocess.STDOUT,
90                          stdout=subprocess.PIPE,
91                          universal_newlines=True,
92                          cwd=basedir,
93                          bufsize=1) as output:
94        for line in output.stdout:
95            fill_result(bm, run, sizestats, line.strip())
96        output.wait()
97
98
99def run_benchmark(bm):
100    for run in runs:
101        run_benchmark_run(bm, run)
102
103
104def run():
105    for b in benchmarks:
106        run_benchmark(b)
107
108
109def main():
110    run()
111    plots = []
112    palette = ["#00B377", "#e84d60", "#0054AE", "#c9d9d3",
113               "#BFF500", "#555555", "#DFBFFF", "#B1D28F",
114               "#FFAA00", "#A18353", "#888888", "#718dbf"]
115    for bn, rmap in results.items():
116        pfactors = None
117        x = [(bm, brun) for bm in iter(rmap) for brun in iter(rmap[bm])]
118        if len([v for v in x if v[1] == 'db size']):
119            sizestats = True
120        else:
121            sizestats = False
122        if pfactors is None:
123            pfactors = [f[1] for f in x]
124        counts = [rmap[bm][brun]
125                  for bm in iter(rmap) for brun in iter(rmap[bm])]
126        source = ColumnDataSource(data=dict(x=x, counts=counts))
127        p = figure(x_range=FactorRange(*x), plot_height=350, plot_width=750,
128                   title=bn)  # y_axis_type="log"
129        p.vbar(x='x', top='counts', width=0.9, source=source, line_color='white',
130               fill_color=factor_cmap('x', palette=palette, factors=pfactors, start=1, end=2))
131        p.y_range.start = 0
132        p.yaxis.axis_label = 'Time ms' if not sizestats else 'Database file size (MB)'
133        p.x_range.range_padding = 0.1
134        p.xaxis.major_label_orientation = 1
135        p.xgrid.grid_line_color = None
136        p.toolbar_location = None
137        plots.append(p)
138        os.makedirs("charts", exist_ok=True)
139        export_png(p, filename="charts/{}.png".format(bn))
140        p.toolbar_location = "right"
141
142    grid = gridplot(plots, ncols=1, merge_tools=False)
143    output_file('benchmark_results_raw.html')
144    save(grid)
145    show(grid)
146
147
148if __name__ == '__main__':
149    main()
150