1# -*- coding: utf-8 -*- 2# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Table generating, analyzing and printing functions. 7 8This defines several classes that are used to generate, analyze and print 9tables. 10 11Example usage: 12 13 from cros_utils import tabulator 14 15 data = [["benchmark1", "33", "44"],["benchmark2", "44", "33"]] 16 tabulator.GetSimpleTable(data) 17 18You could also use it to generate more complex tables with analysis such as 19p-values, custom colors, etc. Tables are generated by TableGenerator and 20analyzed/formatted by TableFormatter. TableFormatter can take in a list of 21columns with custom result computation and coloring, and will compare values in 22each row according to taht scheme. Here is a complex example on printing a 23table: 24 25 from cros_utils import tabulator 26 27 runs = [[{"k1": "10", "k2": "12", "k5": "40", "k6": "40", 28 "ms_1": "20", "k7": "FAIL", "k8": "PASS", "k9": "PASS", 29 "k10": "0"}, 30 {"k1": "13", "k2": "14", "k3": "15", "ms_1": "10", "k8": "PASS", 31 "k9": "FAIL", "k10": "0"}], 32 [{"k1": "50", "k2": "51", "k3": "52", "k4": "53", "k5": "35", "k6": 33 "45", "ms_1": "200", "ms_2": "20", "k7": "FAIL", "k8": "PASS", "k9": 34 "PASS"}]] 35 labels = ["vanilla", "modified"] 36 tg = TableGenerator(runs, labels, TableGenerator.SORT_BY_VALUES_DESC) 37 table = tg.GetTable() 38 columns = [Column(LiteralResult(), 39 Format(), 40 "Literal"), 41 Column(AmeanResult(), 42 Format()), 43 Column(StdResult(), 44 Format()), 45 Column(CoeffVarResult(), 46 CoeffVarFormat()), 47 Column(NonEmptyCountResult(), 48 Format()), 49 Column(AmeanRatioResult(), 50 PercentFormat()), 51 Column(AmeanRatioResult(), 52 RatioFormat()), 53 Column(GmeanRatioResult(), 54 RatioFormat()), 55 Column(PValueResult(), 56 PValueFormat()), 57 ] 58 tf = TableFormatter(table, columns) 59 cell_table = tf.GetCellTable() 60 tp = TablePrinter(cell_table, out_to) 61 print tp.Print() 62""" 63 64from __future__ import division 65from __future__ import print_function 66 67import getpass 68import math 69import sys 70# TODO(zhizhouy): Drop numpy in the future 71# pylint: disable=import-error 72import numpy 73import scipy 74 75from cros_utils.email_sender import EmailSender 76from cros_utils import misc 77 78 79def _AllFloat(values): 80 return all([misc.IsFloat(v) for v in values]) 81 82 83def _GetFloats(values): 84 return [float(v) for v in values] 85 86 87def _StripNone(results): 88 res = [] 89 for result in results: 90 if result is not None: 91 res.append(result) 92 return res 93 94 95def _RemoveMinMax(cell, values): 96 if len(values) < 3: 97 print('WARNING: Values count is less than 3, not ignoring min/max values') 98 print('WARNING: Cell name:', cell.name, 'Values:', values) 99 return values 100 101 values.remove(min(values)) 102 values.remove(max(values)) 103 return values 104 105 106class TableGenerator(object): 107 """Creates a table from a list of list of dicts. 108 109 The main public function is called GetTable(). 110 """ 111 SORT_BY_KEYS = 0 112 SORT_BY_KEYS_DESC = 1 113 SORT_BY_VALUES = 2 114 SORT_BY_VALUES_DESC = 3 115 116 MISSING_VALUE = 'x' 117 118 def __init__(self, d, l, sort=SORT_BY_KEYS, key_name='keys'): 119 self._runs = d 120 self._labels = l 121 self._sort = sort 122 self._key_name = key_name 123 124 def _AggregateKeys(self): 125 keys = set([]) 126 for run_list in self._runs: 127 for run in run_list: 128 keys = keys.union(run.keys()) 129 return keys 130 131 def _GetHighestValue(self, key): 132 values = [] 133 for run_list in self._runs: 134 for run in run_list: 135 if key in run: 136 values.append(run[key]) 137 values = _StripNone(values) 138 if _AllFloat(values): 139 values = _GetFloats(values) 140 return max(values) 141 142 def _GetLowestValue(self, key): 143 values = [] 144 for run_list in self._runs: 145 for run in run_list: 146 if key in run: 147 values.append(run[key]) 148 values = _StripNone(values) 149 if _AllFloat(values): 150 values = _GetFloats(values) 151 return min(values) 152 153 def _SortKeys(self, keys): 154 if self._sort == self.SORT_BY_KEYS: 155 return sorted(keys) 156 elif self._sort == self.SORT_BY_VALUES: 157 # pylint: disable=unnecessary-lambda 158 return sorted(keys, key=lambda x: self._GetLowestValue(x)) 159 elif self._sort == self.SORT_BY_VALUES_DESC: 160 # pylint: disable=unnecessary-lambda 161 return sorted(keys, key=lambda x: self._GetHighestValue(x), reverse=True) 162 else: 163 assert 0, 'Unimplemented sort %s' % self._sort 164 165 def _GetKeys(self): 166 keys = self._AggregateKeys() 167 return self._SortKeys(keys) 168 169 def GetTable(self, number_of_rows=sys.maxsize): 170 """Returns a table from a list of list of dicts. 171 172 Examples: 173 We have the following runs: 174 [[{"k1": "v1", "k2": "v2"}, {"k1": "v3"}], 175 [{"k1": "v4", "k4": "v5"}]] 176 and the following labels: 177 ["vanilla", "modified"] 178 it will return: 179 [["Key", "vanilla", "modified"] 180 ["k1", ["v1", "v3"], ["v4"]] 181 ["k2", ["v2"], []] 182 ["k4", [], ["v5"]]] 183 The returned table can then be processed further by other classes in this 184 module. 185 186 The list of list of dicts is passed into the constructor of TableGenerator. 187 This method converts that into a canonical list of lists which represents a 188 table of values. 189 190 Args: 191 number_of_rows: Maximum number of rows to return from the table. 192 193 Returns: 194 A list of lists which is the table. 195 """ 196 keys = self._GetKeys() 197 header = [self._key_name] + self._labels 198 table = [header] 199 rows = 0 200 for k in keys: 201 row = [k] 202 unit = None 203 for run_list in self._runs: 204 v = [] 205 for run in run_list: 206 if k in run: 207 if isinstance(run[k], list): 208 val = run[k][0] 209 unit = run[k][1] 210 else: 211 val = run[k] 212 v.append(val) 213 else: 214 v.append(None) 215 row.append(v) 216 # If we got a 'unit' value, append the units name to the key name. 217 if unit: 218 keyname = row[0] + ' (%s) ' % unit 219 row[0] = keyname 220 table.append(row) 221 rows += 1 222 if rows == number_of_rows: 223 break 224 return table 225 226 227class SamplesTableGenerator(TableGenerator): 228 """Creates a table with only samples from the results 229 230 The main public function is called GetTable(). 231 232 Different than TableGenerator, self._runs is now a dict of {benchmark: runs} 233 We are expecting there is 'samples' in `runs`. 234 """ 235 236 def __init__(self, run_keyvals, label_list, iter_counts, weights): 237 TableGenerator.__init__( 238 self, run_keyvals, label_list, key_name='Benchmarks') 239 self._iter_counts = iter_counts 240 self._weights = weights 241 242 def _GetKeys(self): 243 keys = self._runs.keys() 244 return self._SortKeys(keys) 245 246 def GetTable(self, number_of_rows=sys.maxsize): 247 """Returns a tuple, which contains three args: 248 249 1) a table from a list of list of dicts. 250 2) updated benchmark_results run_keyvals with composite benchmark 251 3) updated benchmark_results iter_count with composite benchmark 252 253 The dict of list of list of dicts is passed into the constructor of 254 SamplesTableGenerator. 255 This method converts that into a canonical list of lists which 256 represents a table of values. 257 258 Examples: 259 We have the following runs: 260 {bench1: [[{"samples": "v1"}, {"samples": "v2"}], 261 [{"samples": "v3"}, {"samples": "v4"}]] 262 bench2: [[{"samples": "v21"}, None], 263 [{"samples": "v22"}, {"samples": "v23"}]]} 264 and weights of benchmarks: 265 {bench1: w1, bench2: w2} 266 and the following labels: 267 ["vanilla", "modified"] 268 it will return: 269 [["Benchmark", "Weights", "vanilla", "modified"] 270 ["bench1", w1, 271 ((2, 0), ["v1*w1", "v2*w1"]), ((2, 0), ["v3*w1", "v4*w1"])] 272 ["bench2", w2, 273 ((1, 1), ["v21*w2", None]), ((2, 0), ["v22*w2", "v23*w2"])] 274 ["Composite Benchmark", N/A, 275 ((1, 1), ["v1*w1+v21*w2", None]), 276 ((2, 0), ["v3*w1+v22*w2", "v4*w1+ v23*w2"])]] 277 The returned table can then be processed further by other classes in this 278 module. 279 280 Args: 281 number_of_rows: Maximum number of rows to return from the table. 282 283 Returns: 284 A list of lists which is the table. 285 """ 286 keys = self._GetKeys() 287 header = [self._key_name, 'Weights'] + self._labels 288 table = [header] 289 rows = 0 290 iterations = 0 291 292 for k in keys: 293 bench_runs = self._runs[k] 294 unit = None 295 all_runs_empty = all(not dict for label in bench_runs for dict in label) 296 if all_runs_empty: 297 cell = Cell() 298 cell.string_value = 'Benchmark %s contains no result.' + \ 299 ' Is the benchmark name valid?' % k 300 table.append([cell]) 301 else: 302 row = [k] 303 row.append(self._weights[k]) 304 for run_list in bench_runs: 305 run_pass = 0 306 run_fail = 0 307 v = [] 308 for run in run_list: 309 if 'samples' in run: 310 if isinstance(run['samples'], list): 311 val = run['samples'][0] * self._weights[k] 312 unit = run['samples'][1] 313 else: 314 val = run['samples'] * self._weights[k] 315 v.append(val) 316 run_pass += 1 317 else: 318 v.append(None) 319 run_fail += 1 320 one_tuple = ((run_pass, run_fail), v) 321 if iterations != 0 and iterations != run_pass + run_fail: 322 raise ValueError('Iterations of each benchmark run ' \ 323 'are not the same') 324 iterations = run_pass + run_fail 325 row.append(one_tuple) 326 if unit: 327 keyname = row[0] + ' (%s) ' % unit 328 row[0] = keyname 329 table.append(row) 330 rows += 1 331 if rows == number_of_rows: 332 break 333 334 k = 'Composite Benchmark' 335 if k in keys: 336 raise RuntimeError('Composite benchmark already exists in results') 337 338 # Create a new composite benchmark row at the bottom of the summary table 339 # The new row will be like the format in example: 340 # ["Composite Benchmark", N/A, 341 # ((1, 1), ["v1*w1+v21*w2", None]), 342 # ((2, 0), ["v3*w1+v22*w2", "v4*w1+ v23*w2"])]] 343 # First we will create a row of [key, weight, [[0] * iterations] * labels] 344 row = [None] * len(header) 345 row[0] = '%s (samples)' % k 346 row[1] = 'N/A' 347 for label_index in range(2, len(row)): 348 row[label_index] = [0] * iterations 349 350 for cur_row in table[1:]: 351 # Iterate through each benchmark 352 if len(cur_row) > 1: 353 for label_index in range(2, len(cur_row)): 354 # Iterate through each run in a single benchmark 355 # each result should look like ((pass, fail), [values_list]) 356 bench_runs = cur_row[label_index][1] 357 for index in range(iterations): 358 # Accumulate each run result to composite benchmark run 359 # If any run fails, then we set this run for composite benchmark 360 # to None so that we know it fails. 361 if bench_runs[index] and row[label_index][index] != None: 362 row[label_index][index] += bench_runs[index] 363 else: 364 row[label_index][index] = None 365 else: 366 # One benchmark totally fails, no valid data will be in final result 367 for label_index in range(2, len(row)): 368 row[label_index] = [None] * iterations 369 break 370 # Calculate pass and fail count for composite benchmark 371 for label_index in range(2, len(row)): 372 run_pass = 0 373 run_fail = 0 374 for run in row[label_index]: 375 if run: 376 run_pass += 1 377 else: 378 run_fail += 1 379 row[label_index] = ((run_pass, run_fail), row[label_index]) 380 table.append(row) 381 382 # Now that we have the table genearted, we want to store this new composite 383 # benchmark into the benchmark_result in ResultReport object. 384 # This will be used to generate a full table which contains our composite 385 # benchmark. 386 # We need to create composite benchmark result and add it to keyvals in 387 # benchmark_results. 388 v = [] 389 for label in row[2:]: 390 # each label's result looks like ((pass, fail), [values]) 391 benchmark_runs = label[1] 392 # List of values of each label 393 single_run_list = [] 394 for run in benchmark_runs: 395 # Result of each run under the same label is a dict of keys. 396 # Here the only key we will add for composite benchmark is the 397 # weighted_samples we added up. 398 one_dict = {} 399 if run: 400 one_dict[u'weighted_samples'] = [run, u'samples'] 401 one_dict['retval'] = 0 402 else: 403 one_dict['retval'] = 1 404 single_run_list.append(one_dict) 405 v.append(single_run_list) 406 407 self._runs[k] = v 408 self._iter_counts[k] = iterations 409 410 return (table, self._runs, self._iter_counts) 411 412 413class Result(object): 414 """A class that respresents a single result. 415 416 This single result is obtained by condensing the information from a list of 417 runs and a list of baseline runs. 418 """ 419 420 def __init__(self): 421 pass 422 423 def _AllStringsSame(self, values): 424 values_set = set(values) 425 return len(values_set) == 1 426 427 def NeedsBaseline(self): 428 return False 429 430 # pylint: disable=unused-argument 431 def _Literal(self, cell, values, baseline_values): 432 cell.value = ' '.join([str(v) for v in values]) 433 434 def _ComputeFloat(self, cell, values, baseline_values): 435 self._Literal(cell, values, baseline_values) 436 437 def _ComputeString(self, cell, values, baseline_values): 438 self._Literal(cell, values, baseline_values) 439 440 def _InvertIfLowerIsBetter(self, cell): 441 pass 442 443 def _GetGmean(self, values): 444 if not values: 445 return float('nan') 446 if any([v < 0 for v in values]): 447 return float('nan') 448 if any([v == 0 for v in values]): 449 return 0.0 450 log_list = [math.log(v) for v in values] 451 gmean_log = sum(log_list) / len(log_list) 452 return math.exp(gmean_log) 453 454 def Compute(self, cell, values, baseline_values): 455 """Compute the result given a list of values and baseline values. 456 457 Args: 458 cell: A cell data structure to populate. 459 values: List of values. 460 baseline_values: List of baseline values. Can be none if this is the 461 baseline itself. 462 """ 463 all_floats = True 464 values = _StripNone(values) 465 if not values: 466 cell.value = '' 467 return 468 if _AllFloat(values): 469 float_values = _GetFloats(values) 470 else: 471 all_floats = False 472 if baseline_values: 473 baseline_values = _StripNone(baseline_values) 474 if baseline_values: 475 if _AllFloat(baseline_values): 476 float_baseline_values = _GetFloats(baseline_values) 477 else: 478 all_floats = False 479 else: 480 if self.NeedsBaseline(): 481 cell.value = '' 482 return 483 float_baseline_values = None 484 if all_floats: 485 self._ComputeFloat(cell, float_values, float_baseline_values) 486 self._InvertIfLowerIsBetter(cell) 487 else: 488 self._ComputeString(cell, values, baseline_values) 489 490 491class LiteralResult(Result): 492 """A literal result.""" 493 494 def __init__(self, iteration=0): 495 super(LiteralResult, self).__init__() 496 self.iteration = iteration 497 498 def Compute(self, cell, values, baseline_values): 499 try: 500 cell.value = values[self.iteration] 501 except IndexError: 502 cell.value = '-' 503 504 505class NonEmptyCountResult(Result): 506 """A class that counts the number of non-empty results. 507 508 The number of non-empty values will be stored in the cell. 509 """ 510 511 def Compute(self, cell, values, baseline_values): 512 """Put the number of non-empty values in the cell result. 513 514 Args: 515 cell: Put the result in cell.value. 516 values: A list of values for the row. 517 baseline_values: A list of baseline values for the row. 518 """ 519 cell.value = len(_StripNone(values)) 520 if not baseline_values: 521 return 522 base_value = len(_StripNone(baseline_values)) 523 if cell.value == base_value: 524 return 525 f = ColorBoxFormat() 526 len_values = len(values) 527 len_baseline_values = len(baseline_values) 528 tmp_cell = Cell() 529 tmp_cell.value = 1.0 + ( 530 float(cell.value - base_value) / (max(len_values, len_baseline_values))) 531 f.Compute(tmp_cell) 532 cell.bgcolor = tmp_cell.bgcolor 533 534 535class StringMeanResult(Result): 536 """Mean of string values.""" 537 538 def _ComputeString(self, cell, values, baseline_values): 539 if self._AllStringsSame(values): 540 cell.value = str(values[0]) 541 else: 542 cell.value = '?' 543 544 545class AmeanResult(StringMeanResult): 546 """Arithmetic mean.""" 547 548 def __init__(self, ignore_min_max=False): 549 super(AmeanResult, self).__init__() 550 self.ignore_min_max = ignore_min_max 551 552 def _ComputeFloat(self, cell, values, baseline_values): 553 if self.ignore_min_max: 554 values = _RemoveMinMax(cell, values) 555 cell.value = numpy.mean(values) 556 557 558class RawResult(Result): 559 """Raw result.""" 560 pass 561 562 563class IterationResult(Result): 564 """Iteration result.""" 565 pass 566 567 568class MinResult(Result): 569 """Minimum.""" 570 571 def _ComputeFloat(self, cell, values, baseline_values): 572 cell.value = min(values) 573 574 def _ComputeString(self, cell, values, baseline_values): 575 if values: 576 cell.value = min(values) 577 else: 578 cell.value = '' 579 580 581class MaxResult(Result): 582 """Maximum.""" 583 584 def _ComputeFloat(self, cell, values, baseline_values): 585 cell.value = max(values) 586 587 def _ComputeString(self, cell, values, baseline_values): 588 if values: 589 cell.value = max(values) 590 else: 591 cell.value = '' 592 593 594class NumericalResult(Result): 595 """Numerical result.""" 596 597 def _ComputeString(self, cell, values, baseline_values): 598 cell.value = '?' 599 600 601class StdResult(NumericalResult): 602 """Standard deviation.""" 603 604 def __init__(self, ignore_min_max=False): 605 super(StdResult, self).__init__() 606 self.ignore_min_max = ignore_min_max 607 608 def _ComputeFloat(self, cell, values, baseline_values): 609 if self.ignore_min_max: 610 values = _RemoveMinMax(cell, values) 611 cell.value = numpy.std(values) 612 613 614class CoeffVarResult(NumericalResult): 615 """Standard deviation / Mean""" 616 617 def __init__(self, ignore_min_max=False): 618 super(CoeffVarResult, self).__init__() 619 self.ignore_min_max = ignore_min_max 620 621 def _ComputeFloat(self, cell, values, baseline_values): 622 if self.ignore_min_max: 623 values = _RemoveMinMax(cell, values) 624 if numpy.mean(values) != 0.0: 625 noise = numpy.abs(numpy.std(values) / numpy.mean(values)) 626 else: 627 noise = 0.0 628 cell.value = noise 629 630 631class ComparisonResult(Result): 632 """Same or Different.""" 633 634 def NeedsBaseline(self): 635 return True 636 637 def _ComputeString(self, cell, values, baseline_values): 638 value = None 639 baseline_value = None 640 if self._AllStringsSame(values): 641 value = values[0] 642 if self._AllStringsSame(baseline_values): 643 baseline_value = baseline_values[0] 644 if value is not None and baseline_value is not None: 645 if value == baseline_value: 646 cell.value = 'SAME' 647 else: 648 cell.value = 'DIFFERENT' 649 else: 650 cell.value = '?' 651 652 653class PValueResult(ComparisonResult): 654 """P-value.""" 655 656 def __init__(self, ignore_min_max=False): 657 super(PValueResult, self).__init__() 658 self.ignore_min_max = ignore_min_max 659 660 def _ComputeFloat(self, cell, values, baseline_values): 661 if self.ignore_min_max: 662 values = _RemoveMinMax(cell, values) 663 baseline_values = _RemoveMinMax(cell, baseline_values) 664 if len(values) < 2 or len(baseline_values) < 2: 665 cell.value = float('nan') 666 return 667 _, cell.value = scipy.stats.ttest_ind(values, baseline_values) 668 669 def _ComputeString(self, cell, values, baseline_values): 670 return float('nan') 671 672 673class KeyAwareComparisonResult(ComparisonResult): 674 """Automatic key aware comparison.""" 675 676 def _IsLowerBetter(self, key): 677 # Units in histograms should include directions 678 if 'smallerIsBetter' in key: 679 return True 680 if 'biggerIsBetter' in key: 681 return False 682 683 # For units in chartjson: 684 # TODO(llozano): Trying to guess direction by looking at the name of the 685 # test does not seem like a good idea. Test frameworks should provide this 686 # info explicitly. I believe Telemetry has this info. Need to find it out. 687 # 688 # Below are some test names for which we are not sure what the 689 # direction is. 690 # 691 # For these we dont know what the direction is. But, since we dont 692 # specify anything, crosperf will assume higher is better: 693 # --percent_impl_scrolled--percent_impl_scrolled--percent 694 # --solid_color_tiles_analyzed--solid_color_tiles_analyzed--count 695 # --total_image_cache_hit_count--total_image_cache_hit_count--count 696 # --total_texture_upload_time_by_url 697 # 698 # About these we are doubtful but we made a guess: 699 # --average_num_missing_tiles_by_url--*--units (low is good) 700 # --experimental_mean_frame_time_by_url--*--units (low is good) 701 # --experimental_median_frame_time_by_url--*--units (low is good) 702 # --texture_upload_count--texture_upload_count--count (high is good) 703 # --total_deferred_image_decode_count--count (low is good) 704 # --total_tiles_analyzed--total_tiles_analyzed--count (high is good) 705 lower_is_better_keys = [ 706 'milliseconds', 'ms_', 'seconds_', 'KB', 'rdbytes', 'wrbytes', 707 'dropped_percent', '(ms)', '(seconds)', '--ms', 708 '--average_num_missing_tiles', '--experimental_jank', 709 '--experimental_mean_frame', '--experimental_median_frame_time', 710 '--total_deferred_image_decode_count', '--seconds', 'samples' 711 ] 712 713 return any([l in key for l in lower_is_better_keys]) 714 715 def _InvertIfLowerIsBetter(self, cell): 716 if self._IsLowerBetter(cell.name): 717 if cell.value: 718 cell.value = 1.0 / cell.value 719 720 721class AmeanRatioResult(KeyAwareComparisonResult): 722 """Ratio of arithmetic means of values vs. baseline values.""" 723 724 def __init__(self, ignore_min_max=False): 725 super(AmeanRatioResult, self).__init__() 726 self.ignore_min_max = ignore_min_max 727 728 def _ComputeFloat(self, cell, values, baseline_values): 729 if self.ignore_min_max: 730 values = _RemoveMinMax(cell, values) 731 baseline_values = _RemoveMinMax(cell, baseline_values) 732 if numpy.mean(baseline_values) != 0: 733 cell.value = numpy.mean(values) / numpy.mean(baseline_values) 734 elif numpy.mean(values) != 0: 735 cell.value = 0.00 736 # cell.value = 0 means the values and baseline_values have big difference 737 else: 738 cell.value = 1.00 739 # no difference if both values and baseline_values are 0 740 741 742class GmeanRatioResult(KeyAwareComparisonResult): 743 """Ratio of geometric means of values vs. baseline values.""" 744 745 def __init__(self, ignore_min_max=False): 746 super(GmeanRatioResult, self).__init__() 747 self.ignore_min_max = ignore_min_max 748 749 def _ComputeFloat(self, cell, values, baseline_values): 750 if self.ignore_min_max: 751 values = _RemoveMinMax(cell, values) 752 baseline_values = _RemoveMinMax(cell, baseline_values) 753 if self._GetGmean(baseline_values) != 0: 754 cell.value = self._GetGmean(values) / self._GetGmean(baseline_values) 755 elif self._GetGmean(values) != 0: 756 cell.value = 0.00 757 else: 758 cell.value = 1.00 759 760 761class Color(object): 762 """Class that represents color in RGBA format.""" 763 764 def __init__(self, r=0, g=0, b=0, a=0): 765 self.r = r 766 self.g = g 767 self.b = b 768 self.a = a 769 770 def __str__(self): 771 return 'r: %s g: %s: b: %s: a: %s' % (self.r, self.g, self.b, self.a) 772 773 def Round(self): 774 """Round RGBA values to the nearest integer.""" 775 self.r = int(self.r) 776 self.g = int(self.g) 777 self.b = int(self.b) 778 self.a = int(self.a) 779 780 def GetRGB(self): 781 """Get a hex representation of the color.""" 782 return '%02x%02x%02x' % (self.r, self.g, self.b) 783 784 @classmethod 785 def Lerp(cls, ratio, a, b): 786 """Perform linear interpolation between two colors. 787 788 Args: 789 ratio: The ratio to use for linear polation. 790 a: The first color object (used when ratio is 0). 791 b: The second color object (used when ratio is 1). 792 793 Returns: 794 Linearly interpolated color. 795 """ 796 ret = cls() 797 ret.r = (b.r - a.r) * ratio + a.r 798 ret.g = (b.g - a.g) * ratio + a.g 799 ret.b = (b.b - a.b) * ratio + a.b 800 ret.a = (b.a - a.a) * ratio + a.a 801 return ret 802 803 804class Format(object): 805 """A class that represents the format of a column.""" 806 807 def __init__(self): 808 pass 809 810 def Compute(self, cell): 811 """Computes the attributes of a cell based on its value. 812 813 Attributes typically are color, width, etc. 814 815 Args: 816 cell: The cell whose attributes are to be populated. 817 """ 818 if cell.value is None: 819 cell.string_value = '' 820 if isinstance(cell.value, float): 821 self._ComputeFloat(cell) 822 else: 823 self._ComputeString(cell) 824 825 def _ComputeFloat(self, cell): 826 cell.string_value = '{0:.2f}'.format(cell.value) 827 828 def _ComputeString(self, cell): 829 cell.string_value = str(cell.value) 830 831 def _GetColor(self, value, low, mid, high, power=6, mid_value=1.0): 832 min_value = 0.0 833 max_value = 2.0 834 if math.isnan(value): 835 return mid 836 if value > mid_value: 837 value = max_value - mid_value / value 838 839 return self._GetColorBetweenRange(value, min_value, mid_value, max_value, 840 low, mid, high, power) 841 842 def _GetColorBetweenRange(self, value, min_value, mid_value, max_value, 843 low_color, mid_color, high_color, power): 844 assert value <= max_value 845 assert value >= min_value 846 if value > mid_value: 847 value = (max_value - value) / (max_value - mid_value) 848 value **= power 849 ret = Color.Lerp(value, high_color, mid_color) 850 else: 851 value = (value - min_value) / (mid_value - min_value) 852 value **= power 853 ret = Color.Lerp(value, low_color, mid_color) 854 ret.Round() 855 return ret 856 857 858class PValueFormat(Format): 859 """Formatting for p-value.""" 860 861 def _ComputeFloat(self, cell): 862 cell.string_value = '%0.2f' % float(cell.value) 863 if float(cell.value) < 0.05: 864 cell.bgcolor = self._GetColor( 865 cell.value, 866 Color(255, 255, 0, 0), 867 Color(255, 255, 255, 0), 868 Color(255, 255, 255, 0), 869 mid_value=0.05, 870 power=1) 871 872 873class WeightFormat(Format): 874 """Formatting for weight in cwp mode.""" 875 876 def _ComputeFloat(self, cell): 877 cell.string_value = '%0.4f' % float(cell.value) 878 879 880class StorageFormat(Format): 881 """Format the cell as a storage number. 882 883 Examples: 884 If the cell contains a value of 1024, the string_value will be 1.0K. 885 """ 886 887 def _ComputeFloat(self, cell): 888 base = 1024 889 suffices = ['K', 'M', 'G'] 890 v = float(cell.value) 891 current = 0 892 while v >= base**(current + 1) and current < len(suffices): 893 current += 1 894 895 if current: 896 divisor = base**current 897 cell.string_value = '%1.1f%s' % ((v / divisor), suffices[current - 1]) 898 else: 899 cell.string_value = str(cell.value) 900 901 902class CoeffVarFormat(Format): 903 """Format the cell as a percent. 904 905 Examples: 906 If the cell contains a value of 1.5, the string_value will be +150%. 907 """ 908 909 def _ComputeFloat(self, cell): 910 cell.string_value = '%1.1f%%' % (float(cell.value) * 100) 911 cell.color = self._GetColor( 912 cell.value, 913 Color(0, 255, 0, 0), 914 Color(0, 0, 0, 0), 915 Color(255, 0, 0, 0), 916 mid_value=0.02, 917 power=1) 918 919 920class PercentFormat(Format): 921 """Format the cell as a percent. 922 923 Examples: 924 If the cell contains a value of 1.5, the string_value will be +50%. 925 """ 926 927 def _ComputeFloat(self, cell): 928 cell.string_value = '%+1.1f%%' % ((float(cell.value) - 1) * 100) 929 cell.color = self._GetColor(cell.value, Color(255, 0, 0, 0), 930 Color(0, 0, 0, 0), Color(0, 255, 0, 0)) 931 932 933class RatioFormat(Format): 934 """Format the cell as a ratio. 935 936 Examples: 937 If the cell contains a value of 1.5642, the string_value will be 1.56. 938 """ 939 940 def _ComputeFloat(self, cell): 941 cell.string_value = '%+1.1f%%' % ((cell.value - 1) * 100) 942 cell.color = self._GetColor(cell.value, Color(255, 0, 0, 0), 943 Color(0, 0, 0, 0), Color(0, 255, 0, 0)) 944 945 946class ColorBoxFormat(Format): 947 """Format the cell as a color box. 948 949 Examples: 950 If the cell contains a value of 1.5, it will get a green color. 951 If the cell contains a value of 0.5, it will get a red color. 952 The intensity of the green/red will be determined by how much above or below 953 1.0 the value is. 954 """ 955 956 def _ComputeFloat(self, cell): 957 cell.string_value = '--' 958 bgcolor = self._GetColor(cell.value, Color(255, 0, 0, 0), 959 Color(255, 255, 255, 0), Color(0, 255, 0, 0)) 960 cell.bgcolor = bgcolor 961 cell.color = bgcolor 962 963 964class Cell(object): 965 """A class to represent a cell in a table. 966 967 Attributes: 968 value: The raw value of the cell. 969 color: The color of the cell. 970 bgcolor: The background color of the cell. 971 string_value: The string value of the cell. 972 suffix: A string suffix to be attached to the value when displaying. 973 prefix: A string prefix to be attached to the value when displaying. 974 color_row: Indicates whether the whole row is to inherit this cell's color. 975 bgcolor_row: Indicates whether the whole row is to inherit this cell's 976 bgcolor. 977 width: Optional specifier to make a column narrower than the usual width. 978 The usual width of a column is the max of all its cells widths. 979 colspan: Set the colspan of the cell in the HTML table, this is used for 980 table headers. Default value is 1. 981 name: the test name of the cell. 982 header: Whether this is a header in html. 983 """ 984 985 def __init__(self): 986 self.value = None 987 self.color = None 988 self.bgcolor = None 989 self.string_value = None 990 self.suffix = None 991 self.prefix = None 992 # Entire row inherits this color. 993 self.color_row = False 994 self.bgcolor_row = False 995 self.width = 0 996 self.colspan = 1 997 self.name = None 998 self.header = False 999 1000 def __str__(self): 1001 l = [] 1002 l.append('value: %s' % self.value) 1003 l.append('string_value: %s' % self.string_value) 1004 return ' '.join(l) 1005 1006 1007class Column(object): 1008 """Class representing a column in a table. 1009 1010 Attributes: 1011 result: an object of the Result class. 1012 fmt: an object of the Format class. 1013 """ 1014 1015 def __init__(self, result, fmt, name=''): 1016 self.result = result 1017 self.fmt = fmt 1018 self.name = name 1019 1020 1021# Takes in: 1022# ["Key", "Label1", "Label2"] 1023# ["k", ["v", "v2"], [v3]] 1024# etc. 1025# Also takes in a format string. 1026# Returns a table like: 1027# ["Key", "Label1", "Label2"] 1028# ["k", avg("v", "v2"), stddev("v", "v2"), etc.]] 1029# according to format string 1030class TableFormatter(object): 1031 """Class to convert a plain table into a cell-table. 1032 1033 This class takes in a table generated by TableGenerator and a list of column 1034 formats to apply to the table and returns a table of cells. 1035 """ 1036 1037 def __init__(self, table, columns, samples_table=False): 1038 """The constructor takes in a table and a list of columns. 1039 1040 Args: 1041 table: A list of lists of values. 1042 columns: A list of column containing what to produce and how to format 1043 it. 1044 samples_table: A flag to check whether we are generating a table of 1045 samples in CWP apporximation mode. 1046 """ 1047 self._table = table 1048 self._columns = columns 1049 self._samples_table = samples_table 1050 self._table_columns = [] 1051 self._out_table = [] 1052 1053 def GenerateCellTable(self, table_type): 1054 row_index = 0 1055 all_failed = False 1056 1057 for row in self._table[1:]: 1058 # If we are generating samples_table, the second value will be weight 1059 # rather than values. 1060 start_col = 2 if self._samples_table else 1 1061 # It does not make sense to put retval in the summary table. 1062 if str(row[0]) == 'retval' and table_type == 'summary': 1063 # Check to see if any runs passed, and update all_failed. 1064 all_failed = True 1065 for values in row[start_col:]: 1066 if 0 in values: 1067 all_failed = False 1068 continue 1069 key = Cell() 1070 key.string_value = str(row[0]) 1071 out_row = [key] 1072 if self._samples_table: 1073 # Add one column for weight if in samples_table mode 1074 weight = Cell() 1075 weight.value = row[1] 1076 f = WeightFormat() 1077 f.Compute(weight) 1078 out_row.append(weight) 1079 baseline = None 1080 for results in row[start_col:]: 1081 column_start = 0 1082 values = None 1083 # If generating sample table, we will split a tuple of iterations info 1084 # from the results 1085 if isinstance(results, tuple): 1086 it, values = results 1087 column_start = 1 1088 cell = Cell() 1089 cell.string_value = '[%d: %d]' % (it[0], it[1]) 1090 out_row.append(cell) 1091 if not row_index: 1092 self._table_columns.append(self._columns[0]) 1093 else: 1094 values = results 1095 # Parse each column 1096 for column in self._columns[column_start:]: 1097 cell = Cell() 1098 cell.name = key.string_value 1099 if not column.result.NeedsBaseline() or baseline is not None: 1100 column.result.Compute(cell, values, baseline) 1101 column.fmt.Compute(cell) 1102 out_row.append(cell) 1103 if not row_index: 1104 self._table_columns.append(column) 1105 1106 if baseline is None: 1107 baseline = values 1108 self._out_table.append(out_row) 1109 row_index += 1 1110 1111 # If this is a summary table, and the only row in it is 'retval', and 1112 # all the test runs failed, we need to a 'Results' row to the output 1113 # table. 1114 if table_type == 'summary' and all_failed and len(self._table) == 2: 1115 labels_row = self._table[0] 1116 key = Cell() 1117 key.string_value = 'Results' 1118 out_row = [key] 1119 baseline = None 1120 for _ in labels_row[1:]: 1121 for column in self._columns: 1122 cell = Cell() 1123 cell.name = key.string_value 1124 column.result.Compute(cell, ['Fail'], baseline) 1125 column.fmt.Compute(cell) 1126 out_row.append(cell) 1127 if not row_index: 1128 self._table_columns.append(column) 1129 self._out_table.append(out_row) 1130 1131 def AddColumnName(self): 1132 """Generate Column name at the top of table.""" 1133 key = Cell() 1134 key.header = True 1135 key.string_value = 'Keys' if not self._samples_table else 'Benchmarks' 1136 header = [key] 1137 if self._samples_table: 1138 weight = Cell() 1139 weight.header = True 1140 weight.string_value = 'Weights' 1141 header.append(weight) 1142 for column in self._table_columns: 1143 cell = Cell() 1144 cell.header = True 1145 if column.name: 1146 cell.string_value = column.name 1147 else: 1148 result_name = column.result.__class__.__name__ 1149 format_name = column.fmt.__class__.__name__ 1150 1151 cell.string_value = '%s %s' % (result_name.replace('Result', ''), 1152 format_name.replace('Format', '')) 1153 1154 header.append(cell) 1155 1156 self._out_table = [header] + self._out_table 1157 1158 def AddHeader(self, s): 1159 """Put additional string on the top of the table.""" 1160 cell = Cell() 1161 cell.header = True 1162 cell.string_value = str(s) 1163 header = [cell] 1164 colspan = max(1, max(len(row) for row in self._table)) 1165 cell.colspan = colspan 1166 self._out_table = [header] + self._out_table 1167 1168 def GetPassesAndFails(self, values): 1169 passes = 0 1170 fails = 0 1171 for val in values: 1172 if val == 0: 1173 passes = passes + 1 1174 else: 1175 fails = fails + 1 1176 return passes, fails 1177 1178 def AddLabelName(self): 1179 """Put label on the top of the table.""" 1180 top_header = [] 1181 base_colspan = len( 1182 [c for c in self._columns if not c.result.NeedsBaseline()]) 1183 compare_colspan = len(self._columns) 1184 # Find the row with the key 'retval', if it exists. This 1185 # will be used to calculate the number of iterations that passed and 1186 # failed for each image label. 1187 retval_row = None 1188 for row in self._table: 1189 if row[0] == 'retval': 1190 retval_row = row 1191 # The label is organized as follows 1192 # "keys" label_base, label_comparison1, label_comparison2 1193 # The first cell has colspan 1, the second is base_colspan 1194 # The others are compare_colspan 1195 column_position = 0 1196 for label in self._table[0]: 1197 cell = Cell() 1198 cell.header = True 1199 # Put the number of pass/fail iterations in the image label header. 1200 if column_position > 0 and retval_row: 1201 retval_values = retval_row[column_position] 1202 if isinstance(retval_values, list): 1203 passes, fails = self.GetPassesAndFails(retval_values) 1204 cell.string_value = str(label) + ' (pass:%d fail:%d)' % (passes, 1205 fails) 1206 else: 1207 cell.string_value = str(label) 1208 else: 1209 cell.string_value = str(label) 1210 if top_header: 1211 if not self._samples_table or (self._samples_table and 1212 len(top_header) == 2): 1213 cell.colspan = base_colspan 1214 if len(top_header) > 1: 1215 if not self._samples_table or (self._samples_table and 1216 len(top_header) > 2): 1217 cell.colspan = compare_colspan 1218 top_header.append(cell) 1219 column_position = column_position + 1 1220 self._out_table = [top_header] + self._out_table 1221 1222 def _PrintOutTable(self): 1223 o = '' 1224 for row in self._out_table: 1225 for cell in row: 1226 o += str(cell) + ' ' 1227 o += '\n' 1228 print(o) 1229 1230 def GetCellTable(self, table_type='full', headers=True): 1231 """Function to return a table of cells. 1232 1233 The table (list of lists) is converted into a table of cells by this 1234 function. 1235 1236 Args: 1237 table_type: Can be 'full' or 'summary' 1238 headers: A boolean saying whether we want default headers 1239 1240 Returns: 1241 A table of cells with each cell having the properties and string values as 1242 requiested by the columns passed in the constructor. 1243 """ 1244 # Generate the cell table, creating a list of dynamic columns on the fly. 1245 if not self._out_table: 1246 self.GenerateCellTable(table_type) 1247 if headers: 1248 self.AddColumnName() 1249 self.AddLabelName() 1250 return self._out_table 1251 1252 1253class TablePrinter(object): 1254 """Class to print a cell table to the console, file or html.""" 1255 PLAIN = 0 1256 CONSOLE = 1 1257 HTML = 2 1258 TSV = 3 1259 EMAIL = 4 1260 1261 def __init__(self, table, output_type): 1262 """Constructor that stores the cell table and output type.""" 1263 self._table = table 1264 self._output_type = output_type 1265 self._row_styles = [] 1266 self._column_styles = [] 1267 1268 # Compute whole-table properties like max-size, etc. 1269 def _ComputeStyle(self): 1270 self._row_styles = [] 1271 for row in self._table: 1272 row_style = Cell() 1273 for cell in row: 1274 if cell.color_row: 1275 assert cell.color, 'Cell color not set but color_row set!' 1276 assert not row_style.color, 'Multiple row_style.colors found!' 1277 row_style.color = cell.color 1278 if cell.bgcolor_row: 1279 assert cell.bgcolor, 'Cell bgcolor not set but bgcolor_row set!' 1280 assert not row_style.bgcolor, 'Multiple row_style.bgcolors found!' 1281 row_style.bgcolor = cell.bgcolor 1282 self._row_styles.append(row_style) 1283 1284 self._column_styles = [] 1285 if len(self._table) < 2: 1286 return 1287 1288 for i in range(max(len(row) for row in self._table)): 1289 column_style = Cell() 1290 for row in self._table: 1291 if not any([cell.colspan != 1 for cell in row]): 1292 column_style.width = max(column_style.width, len(row[i].string_value)) 1293 self._column_styles.append(column_style) 1294 1295 def _GetBGColorFix(self, color): 1296 if self._output_type == self.CONSOLE: 1297 prefix = misc.rgb2short(color.r, color.g, color.b) 1298 # pylint: disable=anomalous-backslash-in-string 1299 prefix = '\033[48;5;%sm' % prefix 1300 suffix = '\033[0m' 1301 elif self._output_type in [self.EMAIL, self.HTML]: 1302 rgb = color.GetRGB() 1303 prefix = ('<FONT style="BACKGROUND-COLOR:#{0}">'.format(rgb)) 1304 suffix = '</FONT>' 1305 elif self._output_type in [self.PLAIN, self.TSV]: 1306 prefix = '' 1307 suffix = '' 1308 return prefix, suffix 1309 1310 def _GetColorFix(self, color): 1311 if self._output_type == self.CONSOLE: 1312 prefix = misc.rgb2short(color.r, color.g, color.b) 1313 # pylint: disable=anomalous-backslash-in-string 1314 prefix = '\033[38;5;%sm' % prefix 1315 suffix = '\033[0m' 1316 elif self._output_type in [self.EMAIL, self.HTML]: 1317 rgb = color.GetRGB() 1318 prefix = '<FONT COLOR=#{0}>'.format(rgb) 1319 suffix = '</FONT>' 1320 elif self._output_type in [self.PLAIN, self.TSV]: 1321 prefix = '' 1322 suffix = '' 1323 return prefix, suffix 1324 1325 def Print(self): 1326 """Print the table to a console, html, etc. 1327 1328 Returns: 1329 A string that contains the desired representation of the table. 1330 """ 1331 self._ComputeStyle() 1332 return self._GetStringValue() 1333 1334 def _GetCellValue(self, i, j): 1335 cell = self._table[i][j] 1336 out = cell.string_value 1337 raw_width = len(out) 1338 1339 if cell.color: 1340 p, s = self._GetColorFix(cell.color) 1341 out = '%s%s%s' % (p, out, s) 1342 1343 if cell.bgcolor: 1344 p, s = self._GetBGColorFix(cell.bgcolor) 1345 out = '%s%s%s' % (p, out, s) 1346 1347 if self._output_type in [self.PLAIN, self.CONSOLE, self.EMAIL]: 1348 if cell.width: 1349 width = cell.width 1350 else: 1351 if self._column_styles: 1352 width = self._column_styles[j].width 1353 else: 1354 width = len(cell.string_value) 1355 if cell.colspan > 1: 1356 width = 0 1357 start = 0 1358 for k in range(j): 1359 start += self._table[i][k].colspan 1360 for k in range(cell.colspan): 1361 width += self._column_styles[start + k].width 1362 if width > raw_width: 1363 padding = ('%' + str(width - raw_width) + 's') % '' 1364 out = padding + out 1365 1366 if self._output_type == self.HTML: 1367 if cell.header: 1368 tag = 'th' 1369 else: 1370 tag = 'td' 1371 out = '<{0} colspan = "{2}"> {1} </{0}>'.format(tag, out, cell.colspan) 1372 1373 return out 1374 1375 def _GetHorizontalSeparator(self): 1376 if self._output_type in [self.CONSOLE, self.PLAIN, self.EMAIL]: 1377 return ' ' 1378 if self._output_type == self.HTML: 1379 return '' 1380 if self._output_type == self.TSV: 1381 return '\t' 1382 1383 def _GetVerticalSeparator(self): 1384 if self._output_type in [self.PLAIN, self.CONSOLE, self.TSV, self.EMAIL]: 1385 return '\n' 1386 if self._output_type == self.HTML: 1387 return '</tr>\n<tr>' 1388 1389 def _GetPrefix(self): 1390 if self._output_type in [self.PLAIN, self.CONSOLE, self.TSV, self.EMAIL]: 1391 return '' 1392 if self._output_type == self.HTML: 1393 return '<p></p><table id="box-table-a">\n<tr>' 1394 1395 def _GetSuffix(self): 1396 if self._output_type in [self.PLAIN, self.CONSOLE, self.TSV, self.EMAIL]: 1397 return '' 1398 if self._output_type == self.HTML: 1399 return '</tr>\n</table>' 1400 1401 def _GetStringValue(self): 1402 o = '' 1403 o += self._GetPrefix() 1404 for i in range(len(self._table)): 1405 row = self._table[i] 1406 # Apply row color and bgcolor. 1407 p = s = bgp = bgs = '' 1408 if self._row_styles[i].bgcolor: 1409 bgp, bgs = self._GetBGColorFix(self._row_styles[i].bgcolor) 1410 if self._row_styles[i].color: 1411 p, s = self._GetColorFix(self._row_styles[i].color) 1412 o += p + bgp 1413 for j in range(len(row)): 1414 out = self._GetCellValue(i, j) 1415 o += out + self._GetHorizontalSeparator() 1416 o += s + bgs 1417 o += self._GetVerticalSeparator() 1418 o += self._GetSuffix() 1419 return o 1420 1421 1422# Some common drivers 1423def GetSimpleTable(table, out_to=TablePrinter.CONSOLE): 1424 """Prints a simple table. 1425 1426 This is used by code that has a very simple list-of-lists and wants to 1427 produce a table with ameans, a percentage ratio of ameans and a colorbox. 1428 1429 Examples: 1430 GetSimpleConsoleTable([["binary", "b1", "b2"],["size", "300", "400"]]) 1431 will produce a colored table that can be printed to the console. 1432 1433 Args: 1434 table: a list of lists. 1435 out_to: specify the fomat of output. Currently it supports HTML and CONSOLE. 1436 1437 Returns: 1438 A string version of the table that can be printed to the console. 1439 """ 1440 columns = [ 1441 Column(AmeanResult(), Format()), 1442 Column(AmeanRatioResult(), PercentFormat()), 1443 Column(AmeanRatioResult(), ColorBoxFormat()), 1444 ] 1445 our_table = [table[0]] 1446 for row in table[1:]: 1447 our_row = [row[0]] 1448 for v in row[1:]: 1449 our_row.append([v]) 1450 our_table.append(our_row) 1451 1452 tf = TableFormatter(our_table, columns) 1453 cell_table = tf.GetCellTable() 1454 tp = TablePrinter(cell_table, out_to) 1455 return tp.Print() 1456 1457 1458# pylint: disable=redefined-outer-name 1459def GetComplexTable(runs, labels, out_to=TablePrinter.CONSOLE): 1460 """Prints a complex table. 1461 1462 This can be used to generate a table with arithmetic mean, standard deviation, 1463 coefficient of variation, p-values, etc. 1464 1465 Args: 1466 runs: A list of lists with data to tabulate. 1467 labels: A list of labels that correspond to the runs. 1468 out_to: specifies the format of the table (example CONSOLE or HTML). 1469 1470 Returns: 1471 A string table that can be printed to the console or put in an HTML file. 1472 """ 1473 tg = TableGenerator(runs, labels, TableGenerator.SORT_BY_VALUES_DESC) 1474 table = tg.GetTable() 1475 columns = [ 1476 Column(LiteralResult(), Format(), 'Literal'), 1477 Column(AmeanResult(), Format()), 1478 Column(StdResult(), Format()), 1479 Column(CoeffVarResult(), CoeffVarFormat()), 1480 Column(NonEmptyCountResult(), Format()), 1481 Column(AmeanRatioResult(), PercentFormat()), 1482 Column(AmeanRatioResult(), RatioFormat()), 1483 Column(GmeanRatioResult(), RatioFormat()), 1484 Column(PValueResult(), PValueFormat()) 1485 ] 1486 tf = TableFormatter(table, columns) 1487 cell_table = tf.GetCellTable() 1488 tp = TablePrinter(cell_table, out_to) 1489 return tp.Print() 1490 1491 1492if __name__ == '__main__': 1493 # Run a few small tests here. 1494 runs = [[{ 1495 'k1': '10', 1496 'k2': '12', 1497 'k5': '40', 1498 'k6': '40', 1499 'ms_1': '20', 1500 'k7': 'FAIL', 1501 'k8': 'PASS', 1502 'k9': 'PASS', 1503 'k10': '0' 1504 }, 1505 { 1506 'k1': '13', 1507 'k2': '14', 1508 'k3': '15', 1509 'ms_1': '10', 1510 'k8': 'PASS', 1511 'k9': 'FAIL', 1512 'k10': '0' 1513 }], 1514 [{ 1515 'k1': '50', 1516 'k2': '51', 1517 'k3': '52', 1518 'k4': '53', 1519 'k5': '35', 1520 'k6': '45', 1521 'ms_1': '200', 1522 'ms_2': '20', 1523 'k7': 'FAIL', 1524 'k8': 'PASS', 1525 'k9': 'PASS' 1526 }]] 1527 labels = ['vanilla', 'modified'] 1528 t = GetComplexTable(runs, labels, TablePrinter.CONSOLE) 1529 print(t) 1530 email = GetComplexTable(runs, labels, TablePrinter.EMAIL) 1531 1532 runs = [[{ 1533 'k1': '1' 1534 }, { 1535 'k1': '1.1' 1536 }, { 1537 'k1': '1.2' 1538 }], [{ 1539 'k1': '5' 1540 }, { 1541 'k1': '5.1' 1542 }, { 1543 'k1': '5.2' 1544 }]] 1545 t = GetComplexTable(runs, labels, TablePrinter.CONSOLE) 1546 print(t) 1547 1548 simple_table = [ 1549 ['binary', 'b1', 'b2', 'b3'], 1550 ['size', 100, 105, 108], 1551 ['rodata', 100, 80, 70], 1552 ['data', 100, 100, 100], 1553 ['debug', 100, 140, 60], 1554 ] 1555 t = GetSimpleTable(simple_table) 1556 print(t) 1557 email += GetSimpleTable(simple_table, TablePrinter.HTML) 1558 email_to = [getpass.getuser()] 1559 email = "<pre style='font-size: 13px'>%s</pre>" % email 1560 EmailSender().SendEmail(email_to, 'SimpleTableTest', email, msg_type='html') 1561