1#! /usr/bin/env python 2# 3# btt_plot.py: Generate matplotlib plots for BTT generate data files 4# 5# (C) Copyright 2009 Hewlett-Packard Development Company, L.P. 6# 7# This program is free software; you can redistribute it and/or modify 8# it under the terms of the GNU General Public License as published by 9# the Free Software Foundation; either version 2 of the License, or 10# (at your option) any later version. 11# 12# This program is distributed in the hope that it will be useful, 13# but WITHOUT ANY WARRANTY; without even the implied warranty of 14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15# GNU General Public License for more details. 16# 17# You should have received a copy of the GNU General Public License 18# along with this program; if not, write to the Free Software 19# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 20# 21 22""" 23btt_plot.py: Generate matplotlib plots for BTT generated data files 24 25Files handled: 26 AQD - Average Queue Depth Running average of queue depths 27 28 BNOS - Block numbers accessed Markers for each block 29 30 Q2D - Queue to Issue latencies Running averages 31 D2C - Issue to Complete latencies Running averages 32 Q2C - Queue to Complete latencies Running averages 33 34Usage: 35 btt_plot_aqd.py equivalent to: btt_plot.py -t aqd <type>=aqd 36 btt_plot_bnos.py equivalent to: btt_plot.py -t bnos <type>=bnos 37 btt_plot_q2d.py equivalent to: btt_plot.py -t q2d <type>=q2d 38 btt_plot_d2c.py equivalent to: btt_plot.py -t d2c <type>=d2c 39 btt_plot_q2c.py equivalent to: btt_plot.py -t q2c <type>=q2c 40 41Arguments: 42 [ -A | --generate-all ] Default: False 43 [ -L | --no-legend ] Default: Legend table produced 44 [ -o <file> | --output=<file> ] Default: <type>.png 45 [ -T <string> | --title=<string> ] Default: Based upon <type> 46 [ -v | --verbose ] Default: False 47 <data-files...> 48 49 The -A (--generate-all) argument is different: when this is specified, 50 an attempt is made to generate default plots for all 5 types (aqd, bnos, 51 q2d, d2c and q2c). It will find files with the appropriate suffix for 52 each type ('aqd.dat' for example). If such files are found, a plot for 53 that type will be made. The output file name will be the default for 54 each type. The -L (--no-legend) option will be obeyed for all plots, 55 but the -o (--output) and -T (--title) options will be ignored. 56""" 57 58__author__ = 'Alan D. Brunelle <alan.brunelle@hp.com>' 59 60#------------------------------------------------------------------------------ 61 62import matplotlib 63matplotlib.use('Agg') 64import getopt, glob, os, sys 65import matplotlib.pyplot as plt 66 67plot_size = [10.9, 8.4] # inches... 68 69add_legend = True 70generate_all = False 71output_file = None 72title_str = None 73type = None 74verbose = False 75 76types = [ 'aqd', 'q2d', 'd2c', 'q2c', 'live', 'bnos' ] 77progs = [ 'btt_plot_%s.py' % t for t in types ] 78 79get_base = lambda file: file[file.find('_')+1:file.rfind('_')] 80 81#------------------------------------------------------------------------------ 82def fatal(msg): 83 """Generate fatal error message and exit""" 84 85 print >>sys.stderr, 'FATAL: %s' % msg 86 sys.exit(1) 87 88#------------------------------------------------------------------------------ 89def gen_legends(ax, legends): 90 leg = ax.legend(legends, 'best', shadow=True) 91 frame = leg.get_frame() 92 frame.set_facecolor('0.80') 93 for t in leg.get_texts(): 94 t.set_fontsize('xx-small') 95 96#---------------------------------------------------------------------- 97def get_data(files): 98 """Retrieve data from files provided. 99 100 Returns a database containing: 101 'min_x', 'max_x' - Minimum and maximum X values found 102 'min_y', 'max_y' - Minimum and maximum Y values found 103 'x', 'y' - X & Y value arrays 104 'ax', 'ay' - Running average over X & Y -- 105 if > 10 values provided... 106 """ 107 #-------------------------------------------------------------- 108 def check(mn, mx, v): 109 """Returns new min, max, and float value for those passed in""" 110 111 v = float(v) 112 if mn == None or v < mn: mn = v 113 if mx == None or v > mx: mx = v 114 return mn, mx, v 115 116 #-------------------------------------------------------------- 117 def avg(xs, ys): 118 """Computes running average for Xs and Ys""" 119 120 #------------------------------------------------------ 121 def _avg(vals): 122 """Computes average for array of values passed""" 123 124 total = 0.0 125 for val in vals: 126 total += val 127 return total / len(vals) 128 129 #------------------------------------------------------ 130 if len(xs) < 1000: 131 return xs, ys 132 133 axs = [xs[0]] 134 ays = [ys[0]] 135 _xs = [xs[0]] 136 _ys = [ys[0]] 137 138 x_range = (xs[-1] - xs[0]) / 100 139 for idx in range(1, len(ys)): 140 if (xs[idx] - _xs[0]) > x_range: 141 axs.append(_avg(_xs)) 142 ays.append(_avg(_ys)) 143 del _xs, _ys 144 145 _xs = [xs[idx]] 146 _ys = [ys[idx]] 147 else: 148 _xs.append(xs[idx]) 149 _ys.append(ys[idx]) 150 151 if len(_xs) > 1: 152 axs.append(_avg(_xs)) 153 ays.append(_avg(_ys)) 154 155 return axs, ays 156 157 #-------------------------------------------------------------- 158 global verbose 159 160 db = {} 161 min_x = max_x = min_y = max_y = None 162 for file in files: 163 if not os.path.exists(file): 164 fatal('%s not found' % file) 165 elif verbose: 166 print 'Processing %s' % file 167 168 xs = [] 169 ys = [] 170 for line in open(file, 'r'): 171 f = line.rstrip().split(None) 172 if line.find('#') == 0 or len(f) < 2: 173 continue 174 (min_x, max_x, x) = check(min_x, max_x, f[0]) 175 (min_y, max_y, y) = check(min_y, max_y, f[1]) 176 xs.append(x) 177 ys.append(y) 178 179 db[file] = {'x':xs, 'y':ys} 180 if len(xs) > 10: 181 db[file]['ax'], db[file]['ay'] = avg(xs, ys) 182 else: 183 db[file]['ax'] = db[file]['ay'] = None 184 185 db['min_x'] = min_x 186 db['max_x'] = max_x 187 db['min_y'] = min_y 188 db['max_y'] = max_y 189 return db 190 191#---------------------------------------------------------------------- 192def parse_args(args): 193 """Parse command line arguments. 194 195 Returns list of (data) files that need to be processed -- /unless/ 196 the -A (--generate-all) option is passed, in which case superfluous 197 data files are ignored... 198 """ 199 200 global add_legend, output_file, title_str, type, verbose 201 global generate_all 202 203 prog = args[0][args[0].rfind('/')+1:] 204 if prog == 'btt_plot.py': 205 pass 206 elif not prog in progs: 207 fatal('%s not a valid command name' % prog) 208 else: 209 type = prog[prog.rfind('_')+1:prog.rfind('.py')] 210 211 s_opts = 'ALo:t:T:v' 212 l_opts = [ 'generate-all', 'type', 'no-legend', 'output', 'title', 213 'verbose' ] 214 215 try: 216 (opts, args) = getopt.getopt(args[1:], s_opts, l_opts) 217 except getopt.error, msg: 218 print >>sys.stderr, msg 219 fatal(__doc__) 220 221 for (o, a) in opts: 222 if o in ('-A', '--generate-all'): 223 generate_all = True 224 elif o in ('-L', '--no-legend'): 225 add_legend = False 226 elif o in ('-o', '--output'): 227 output_file = a 228 elif o in ('-t', '--type'): 229 if not a in types: 230 fatal('Type %s not supported' % a) 231 type = a 232 elif o in ('-T', '--title'): 233 title_str = a 234 elif o in ('-v', '--verbose'): 235 verbose = True 236 237 if type == None and not generate_all: 238 fatal('Need type of data files to process - (-t <type>)') 239 240 return args 241 242#------------------------------------------------------------------------------ 243def gen_title(fig, type, title_str): 244 """Sets the title for the figure based upon the type /or/ user title""" 245 246 if title_str != None: 247 pass 248 elif type == 'aqd': 249 title_str = 'Average Queue Depth' 250 elif type == 'bnos': 251 title_str = 'Block Numbers Accessed' 252 elif type == 'q2d': 253 title_str = 'Queue (Q) To Issue (D) Average Latencies' 254 elif type == 'd2c': 255 title_str = 'Issue (D) To Complete (C) Average Latencies' 256 elif type == 'q2c': 257 title_str = 'Queue (Q) To Complete (C) Average Latencies' 258 259 title = fig.text(.5, .95, title_str, horizontalalignment='center') 260 title.set_fontsize('large') 261 262#------------------------------------------------------------------------------ 263def gen_labels(db, ax, type): 264 """Generate X & Y 'axis'""" 265 266 #---------------------------------------------------------------------- 267 def gen_ylabel(ax, type): 268 """Set the Y axis label based upon the type""" 269 270 if type == 'aqd': 271 str = 'Number of Requests Queued' 272 elif type == 'bnos': 273 str = 'Block Number' 274 else: 275 str = 'Seconds' 276 ax.set_ylabel(str) 277 278 #---------------------------------------------------------------------- 279 xdelta = 0.1 * (db['max_x'] - db['min_x']) 280 ydelta = 0.1 * (db['max_y'] - db['min_y']) 281 282 ax.set_xlim(db['min_x'] - xdelta, db['max_x'] + xdelta) 283 ax.set_ylim(db['min_y'] - ydelta, db['max_y'] + ydelta) 284 ax.set_xlabel('Runtime (seconds)') 285 ax.grid(True) 286 gen_ylabel(ax, type) 287 288#------------------------------------------------------------------------------ 289def generate_output(type, db): 290 """Generate the output plot based upon the type and database""" 291 292 #---------------------------------------------------------------------- 293 def color(idx, style): 294 """Returns a color/symbol type based upon the index passed.""" 295 296 colors = [ 'b', 'g', 'r', 'c', 'm', 'y', 'k' ] 297 l_styles = [ '-', ':', '--', '-.' ] 298 m_styles = [ 'o', '+', '.', ',', 's', 'v', 'x', '<', '>' ] 299 300 color = colors[idx % len(colors)] 301 if style == 'line': 302 style = l_styles[(idx / len(l_styles)) % len(l_styles)] 303 elif style == 'marker': 304 style = m_styles[(idx / len(m_styles)) % len(m_styles)] 305 306 return '%s%s' % (color, style) 307 308 #---------------------------------------------------------------------- 309 global add_legend, output_file, title_str, verbose 310 311 if output_file != None: 312 ofile = output_file 313 else: 314 ofile = '%s.png' % type 315 316 if verbose: 317 print 'Generating plot into %s' % ofile 318 319 fig = plt.figure(figsize=plot_size) 320 ax = fig.add_subplot(111) 321 322 gen_title(fig, type, title_str) 323 gen_labels(db, ax, type) 324 325 idx = 0 326 if add_legend: 327 legends = [] 328 else: 329 legends = None 330 331 keys = [] 332 for file in db.iterkeys(): 333 if not file in ['min_x', 'max_x', 'min_y', 'max_y']: 334 keys.append(file) 335 336 keys.sort() 337 for file in keys: 338 dat = db[file] 339 if type == 'bnos': 340 ax.plot(dat['x'], dat['y'], color(idx, 'marker'), 341 markersize=1) 342 elif dat['ax'] == None: 343 continue # Don't add legend 344 else: 345 ax.plot(dat['ax'], dat['ay'], color(idx, 'line'), 346 linewidth=1.0) 347 if add_legend: 348 legends.append(get_base(file)) 349 idx += 1 350 351 if add_legend and len(legends) > 0: 352 gen_legends(ax, legends) 353 plt.savefig(ofile) 354 355#------------------------------------------------------------------------------ 356def get_files(type): 357 """Returns the list of files for the -A option based upon type""" 358 359 if type == 'bnos': 360 files = [] 361 for fn in glob.glob('*c.dat'): 362 for t in [ 'q2q', 'd2d', 'q2c', 'd2c' ]: 363 if fn.find(t) >= 0: 364 break 365 else: 366 files.append(fn) 367 else: 368 files = glob.glob('*%s.dat' % type) 369 return files 370 371#------------------------------------------------------------------------------ 372def do_bnos(files): 373 for file in files: 374 base = get_base(file) 375 title_str = 'Block Numbers Accessed: %s' % base 376 output_file = 'bnos_%s.png' % base 377 generate_output(t, get_data([file])) 378 379#------------------------------------------------------------------------------ 380def do_live(files): 381 global plot_size 382 383 #---------------------------------------------------------------------- 384 def get_live_data(fn): 385 xs = [] 386 ys = [] 387 for line in open(fn, 'r'): 388 f = line.rstrip().split() 389 if f[0] != '#' and len(f) == 2: 390 xs.append(float(f[0])) 391 ys.append(float(f[1])) 392 return xs, ys 393 394 #---------------------------------------------------------------------- 395 def live_sort(a, b): 396 if a[0] == 'sys' and b[0] == 'sys': 397 return 0 398 elif a[0] == 'sys' or a[2][0] < b[2][0]: 399 return -1 400 elif b[0] == 'sys' or a[2][0] > b[2][0]: 401 return 1 402 else: 403 return 0 404 405 #---------------------------------------------------------------------- 406 def turn_off_ticks(ax): 407 for tick in ax.xaxis.get_major_ticks(): 408 tick.tick1On = tick.tick2On = False 409 for tick in ax.yaxis.get_major_ticks(): 410 tick.tick1On = tick.tick2On = False 411 for tick in ax.xaxis.get_minor_ticks(): 412 tick.tick1On = tick.tick2On = False 413 for tick in ax.yaxis.get_minor_ticks(): 414 tick.tick1On = tick.tick2On = False 415 416 #---------------------------------------------------------------------- 417 fig = plt.figure(figsize=plot_size) 418 ax = fig.add_subplot(111) 419 420 db = [] 421 for fn in files: 422 if not os.path.exists(fn): 423 continue 424 (xs, ys) = get_live_data(fn) 425 db.append([fn[:fn.find('_live.dat')], xs, ys]) 426 db.sort(live_sort) 427 428 for rec in db: 429 ax.plot(rec[1], rec[2]) 430 431 gen_title(fig, 'live', 'Active I/O Per Device') 432 ax.set_xlabel('Runtime (seconds)') 433 ax.set_ylabel('Device') 434 ax.grid(False) 435 436 ax.set_xlim(-0.1, db[0][1][-1]+1) 437 ax.set_yticks([idx for idx in range(0, len(db))]) 438 ax.yaxis.set_ticklabels([rec[0] for rec in db]) 439 turn_off_ticks(ax) 440 441 plt.savefig('live.png') 442 plt.savefig('live.eps') 443 444#------------------------------------------------------------------------------ 445if __name__ == '__main__': 446 files = parse_args(sys.argv) 447 448 if generate_all: 449 output_file = title_str = type = None 450 for t in types: 451 files = get_files(t) 452 if len(files) == 0: 453 continue 454 elif t == 'bnos': 455 do_bnos(files) 456 elif t == 'live': 457 do_live(files) 458 else: 459 generate_output(t, get_data(files)) 460 continue 461 462 elif len(files) < 1: 463 fatal('Need data files to process') 464 else: 465 generate_output(type, get_data(files)) 466 sys.exit(0) 467