1# -*- coding: utf-8 -*- 2# Copyright 2013 The ChromiumOS Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Table generating, analyzing and printing functions. 7 8This defines several classes that are used to generate, analyze and print 9tables. 10 11Example usage: 12 13 from cros_utils import tabulator 14 15 data = [["benchmark1", "33", "44"],["benchmark2", "44", "33"]] 16 tabulator.GetSimpleTable(data) 17 18You could also use it to generate more complex tables with analysis such as 19p-values, custom colors, etc. Tables are generated by TableGenerator and 20analyzed/formatted by TableFormatter. TableFormatter can take in a list of 21columns with custom result computation and coloring, and will compare values in 22each row according to taht scheme. Here is a complex example on printing a 23table: 24 25 from cros_utils import tabulator 26 27 runs = [[{"k1": "10", "k2": "12", "k5": "40", "k6": "40", 28 "ms_1": "20", "k7": "FAIL", "k8": "PASS", "k9": "PASS", 29 "k10": "0"}, 30 {"k1": "13", "k2": "14", "k3": "15", "ms_1": "10", "k8": "PASS", 31 "k9": "FAIL", "k10": "0"}], 32 [{"k1": "50", "k2": "51", "k3": "52", "k4": "53", "k5": "35", "k6": 33 "45", "ms_1": "200", "ms_2": "20", "k7": "FAIL", "k8": "PASS", "k9": 34 "PASS"}]] 35 labels = ["vanilla", "modified"] 36 tg = TableGenerator(runs, labels, TableGenerator.SORT_BY_VALUES_DESC) 37 table = tg.GetTable() 38 columns = [Column(LiteralResult(), 39 Format(), 40 "Literal"), 41 Column(AmeanResult(), 42 Format()), 43 Column(StdResult(), 44 Format()), 45 Column(CoeffVarResult(), 46 CoeffVarFormat()), 47 Column(NonEmptyCountResult(), 48 Format()), 49 Column(AmeanRatioResult(), 50 PercentFormat()), 51 Column(AmeanRatioResult(), 52 RatioFormat()), 53 Column(GmeanRatioResult(), 54 RatioFormat()), 55 Column(PValueResult(), 56 PValueFormat()), 57 ] 58 tf = TableFormatter(table, columns) 59 cell_table = tf.GetCellTable() 60 tp = TablePrinter(cell_table, out_to) 61 print tp.Print() 62""" 63 64 65import collections 66import getpass 67import math 68import statistics 69import sys 70 71from cros_utils import misc 72from cros_utils.email_sender import EmailSender 73 74# TODO(crbug.com/980719): Drop scipy in the future. 75# pylint: disable=import-error 76import scipy 77 78 79def _AllFloat(values): 80 return all([misc.IsFloat(v) for v in values]) 81 82 83def _GetFloats(values): 84 return [float(v) for v in values] 85 86 87def _StripNone(results): 88 res = [] 89 for result in results: 90 if result is not None: 91 res.append(result) 92 return res 93 94 95def _RemoveMinMax(cell, values): 96 if len(values) < 3: 97 print( 98 "WARNING: Values count is less than 3, not ignoring min/max values" 99 ) 100 print("WARNING: Cell name:", cell.name, "Values:", values) 101 return values 102 103 values.remove(min(values)) 104 values.remove(max(values)) 105 return values 106 107 108class TableGenerator(object): 109 """Creates a table from a list of list of dicts. 110 111 The main public function is called GetTable(). 112 """ 113 114 SORT_BY_KEYS = 0 115 SORT_BY_KEYS_DESC = 1 116 SORT_BY_VALUES = 2 117 SORT_BY_VALUES_DESC = 3 118 NO_SORT = 4 119 120 MISSING_VALUE = "x" 121 122 def __init__(self, d, l, sort=NO_SORT, key_name="keys"): 123 self._runs = d 124 self._labels = l 125 self._sort = sort 126 self._key_name = key_name 127 128 def _AggregateKeys(self): 129 keys = collections.OrderedDict() 130 for run_list in self._runs: 131 for run in run_list: 132 keys.update(dict.fromkeys(run.keys())) 133 return list(keys.keys()) 134 135 def _GetHighestValue(self, key): 136 values = [] 137 for run_list in self._runs: 138 for run in run_list: 139 if key in run: 140 values.append(run[key]) 141 values = _StripNone(values) 142 if _AllFloat(values): 143 values = _GetFloats(values) 144 return max(values) 145 146 def _GetLowestValue(self, key): 147 values = [] 148 for run_list in self._runs: 149 for run in run_list: 150 if key in run: 151 values.append(run[key]) 152 values = _StripNone(values) 153 if _AllFloat(values): 154 values = _GetFloats(values) 155 return min(values) 156 157 def _SortKeys(self, keys): 158 if self._sort == self.SORT_BY_KEYS: 159 return sorted(keys) 160 elif self._sort == self.SORT_BY_VALUES: 161 # pylint: disable=unnecessary-lambda 162 return sorted(keys, key=lambda x: self._GetLowestValue(x)) 163 elif self._sort == self.SORT_BY_VALUES_DESC: 164 # pylint: disable=unnecessary-lambda 165 return sorted( 166 keys, key=lambda x: self._GetHighestValue(x), reverse=True 167 ) 168 elif self._sort == self.NO_SORT: 169 return keys 170 else: 171 assert 0, "Unimplemented sort %s" % self._sort 172 173 def _GetKeys(self): 174 keys = self._AggregateKeys() 175 return self._SortKeys(keys) 176 177 def GetTable(self, number_of_rows=sys.maxsize): 178 """Returns a table from a list of list of dicts. 179 180 Examples: 181 We have the following runs: 182 [[{"k1": "v1", "k2": "v2"}, {"k1": "v3"}], 183 [{"k1": "v4", "k4": "v5"}]] 184 and the following labels: 185 ["vanilla", "modified"] 186 it will return: 187 [["Key", "vanilla", "modified"] 188 ["k1", ["v1", "v3"], ["v4"]] 189 ["k2", ["v2"], []] 190 ["k4", [], ["v5"]]] 191 The returned table can then be processed further by other classes in this 192 module. 193 194 The list of list of dicts is passed into the constructor of TableGenerator. 195 This method converts that into a canonical list of lists which represents a 196 table of values. 197 198 Args: 199 number_of_rows: Maximum number of rows to return from the table. 200 201 Returns: 202 A list of lists which is the table. 203 """ 204 keys = self._GetKeys() 205 header = [self._key_name] + self._labels 206 table = [header] 207 rows = 0 208 for k in keys: 209 row = [k] 210 unit = None 211 for run_list in self._runs: 212 v = [] 213 for run in run_list: 214 if k in run: 215 if isinstance(run[k], list): 216 val = run[k][0] 217 unit = run[k][1] 218 else: 219 val = run[k] 220 v.append(val) 221 else: 222 v.append(None) 223 row.append(v) 224 # If we got a 'unit' value, append the units name to the key name. 225 if unit: 226 keyname = row[0] + " (%s) " % unit 227 row[0] = keyname 228 table.append(row) 229 rows += 1 230 if rows == number_of_rows: 231 break 232 return table 233 234 235class SamplesTableGenerator(TableGenerator): 236 """Creates a table with only samples from the results 237 238 The main public function is called GetTable(). 239 240 Different than TableGenerator, self._runs is now a dict of {benchmark: runs} 241 We are expecting there is 'samples' in `runs`. 242 """ 243 244 def __init__(self, run_keyvals, label_list, iter_counts, weights): 245 TableGenerator.__init__( 246 self, run_keyvals, label_list, key_name="Benchmarks" 247 ) 248 self._iter_counts = iter_counts 249 self._weights = weights 250 251 def _GetKeys(self): 252 keys = self._runs.keys() 253 return self._SortKeys(keys) 254 255 def GetTable(self, number_of_rows=sys.maxsize): 256 """Returns a tuple, which contains three args: 257 258 1) a table from a list of list of dicts. 259 2) updated benchmark_results run_keyvals with composite benchmark 260 3) updated benchmark_results iter_count with composite benchmark 261 262 The dict of list of list of dicts is passed into the constructor of 263 SamplesTableGenerator. 264 This method converts that into a canonical list of lists which 265 represents a table of values. 266 267 Examples: 268 We have the following runs: 269 {bench1: [[{"samples": "v1"}, {"samples": "v2"}], 270 [{"samples": "v3"}, {"samples": "v4"}]] 271 bench2: [[{"samples": "v21"}, None], 272 [{"samples": "v22"}, {"samples": "v23"}]]} 273 and weights of benchmarks: 274 {bench1: w1, bench2: w2} 275 and the following labels: 276 ["vanilla", "modified"] 277 it will return: 278 [["Benchmark", "Weights", "vanilla", "modified"] 279 ["bench1", w1, 280 ((2, 0), ["v1*w1", "v2*w1"]), ((2, 0), ["v3*w1", "v4*w1"])] 281 ["bench2", w2, 282 ((1, 1), ["v21*w2", None]), ((2, 0), ["v22*w2", "v23*w2"])] 283 ["Composite Benchmark", N/A, 284 ((1, 1), ["v1*w1+v21*w2", None]), 285 ((2, 0), ["v3*w1+v22*w2", "v4*w1+ v23*w2"])]] 286 The returned table can then be processed further by other classes in this 287 module. 288 289 Args: 290 number_of_rows: Maximum number of rows to return from the table. 291 292 Returns: 293 A list of lists which is the table. 294 """ 295 keys = self._GetKeys() 296 header = [self._key_name, "Weights"] + self._labels 297 table = [header] 298 rows = 0 299 iterations = 0 300 301 for k in keys: 302 bench_runs = self._runs[k] 303 unit = None 304 all_runs_empty = all( 305 not dict for label in bench_runs for dict in label 306 ) 307 if all_runs_empty: 308 cell = Cell() 309 cell.string_value = ( 310 "Benchmark %s contains no result." 311 " Is the benchmark name valid?" % k 312 ) 313 table.append([cell]) 314 else: 315 row = [k] 316 row.append(self._weights[k]) 317 for run_list in bench_runs: 318 run_pass = 0 319 run_fail = 0 320 v = [] 321 for run in run_list: 322 if "samples" in run: 323 if isinstance(run["samples"], list): 324 val = run["samples"][0] * self._weights[k] 325 unit = run["samples"][1] 326 else: 327 val = run["samples"] * self._weights[k] 328 v.append(val) 329 run_pass += 1 330 else: 331 v.append(None) 332 run_fail += 1 333 one_tuple = ((run_pass, run_fail), v) 334 if iterations not in (0, run_pass + run_fail): 335 raise ValueError( 336 "Iterations of each benchmark run " 337 "are not the same" 338 ) 339 iterations = run_pass + run_fail 340 row.append(one_tuple) 341 if unit: 342 keyname = row[0] + " (%s) " % unit 343 row[0] = keyname 344 table.append(row) 345 rows += 1 346 if rows == number_of_rows: 347 break 348 349 k = "Composite Benchmark" 350 if k in keys: 351 raise RuntimeError("Composite benchmark already exists in results") 352 353 # Create a new composite benchmark row at the bottom of the summary table 354 # The new row will be like the format in example: 355 # ["Composite Benchmark", N/A, 356 # ((1, 1), ["v1*w1+v21*w2", None]), 357 # ((2, 0), ["v3*w1+v22*w2", "v4*w1+ v23*w2"])]] 358 # First we will create a row of [key, weight, [[0] * iterations] * labels] 359 row = [None] * len(header) 360 row[0] = "%s (samples)" % k 361 row[1] = "N/A" 362 for label_index in range(2, len(row)): 363 row[label_index] = [0] * iterations 364 365 for cur_row in table[1:]: 366 # Iterate through each benchmark 367 if len(cur_row) > 1: 368 for label_index in range(2, len(cur_row)): 369 # Iterate through each run in a single benchmark 370 # each result should look like ((pass, fail), [values_list]) 371 bench_runs = cur_row[label_index][1] 372 for index in range(iterations): 373 # Accumulate each run result to composite benchmark run 374 # If any run fails, then we set this run for composite benchmark 375 # to None so that we know it fails. 376 if ( 377 bench_runs[index] 378 and row[label_index][index] is not None 379 ): 380 row[label_index][index] += bench_runs[index] 381 else: 382 row[label_index][index] = None 383 else: 384 # One benchmark totally fails, no valid data will be in final result 385 for label_index in range(2, len(row)): 386 row[label_index] = [None] * iterations 387 break 388 # Calculate pass and fail count for composite benchmark 389 for label_index in range(2, len(row)): 390 run_pass = 0 391 run_fail = 0 392 for run in row[label_index]: 393 if run: 394 run_pass += 1 395 else: 396 run_fail += 1 397 row[label_index] = ((run_pass, run_fail), row[label_index]) 398 table.append(row) 399 400 # Now that we have the table genearted, we want to store this new composite 401 # benchmark into the benchmark_result in ResultReport object. 402 # This will be used to generate a full table which contains our composite 403 # benchmark. 404 # We need to create composite benchmark result and add it to keyvals in 405 # benchmark_results. 406 v = [] 407 for label in row[2:]: 408 # each label's result looks like ((pass, fail), [values]) 409 benchmark_runs = label[1] 410 # List of values of each label 411 single_run_list = [] 412 for run in benchmark_runs: 413 # Result of each run under the same label is a dict of keys. 414 # Here the only key we will add for composite benchmark is the 415 # weighted_samples we added up. 416 one_dict = {} 417 if run: 418 one_dict[u"weighted_samples"] = [run, u"samples"] 419 one_dict["retval"] = 0 420 else: 421 one_dict["retval"] = 1 422 single_run_list.append(one_dict) 423 v.append(single_run_list) 424 425 self._runs[k] = v 426 self._iter_counts[k] = iterations 427 428 return (table, self._runs, self._iter_counts) 429 430 431class Result(object): 432 """A class that respresents a single result. 433 434 This single result is obtained by condensing the information from a list of 435 runs and a list of baseline runs. 436 """ 437 438 def __init__(self): 439 pass 440 441 def _AllStringsSame(self, values): 442 values_set = set(values) 443 return len(values_set) == 1 444 445 def NeedsBaseline(self): 446 return False 447 448 # pylint: disable=unused-argument 449 def _Literal(self, cell, values, baseline_values): 450 cell.value = " ".join([str(v) for v in values]) 451 452 def _ComputeFloat(self, cell, values, baseline_values): 453 self._Literal(cell, values, baseline_values) 454 455 def _ComputeString(self, cell, values, baseline_values): 456 self._Literal(cell, values, baseline_values) 457 458 def _InvertIfLowerIsBetter(self, cell): 459 pass 460 461 def _GetGmean(self, values): 462 if not values: 463 return float("nan") 464 if any([v < 0 for v in values]): 465 return float("nan") 466 if any([v == 0 for v in values]): 467 return 0.0 468 log_list = [math.log(v) for v in values] 469 gmean_log = sum(log_list) / len(log_list) 470 return math.exp(gmean_log) 471 472 def Compute(self, cell, values, baseline_values): 473 """Compute the result given a list of values and baseline values. 474 475 Args: 476 cell: A cell data structure to populate. 477 values: List of values. 478 baseline_values: List of baseline values. Can be none if this is the 479 baseline itself. 480 """ 481 all_floats = True 482 values = _StripNone(values) 483 if not values: 484 cell.value = "" 485 return 486 if _AllFloat(values): 487 float_values = _GetFloats(values) 488 else: 489 all_floats = False 490 if baseline_values: 491 baseline_values = _StripNone(baseline_values) 492 if baseline_values: 493 if _AllFloat(baseline_values): 494 float_baseline_values = _GetFloats(baseline_values) 495 else: 496 all_floats = False 497 else: 498 if self.NeedsBaseline(): 499 cell.value = "" 500 return 501 float_baseline_values = None 502 if all_floats: 503 self._ComputeFloat(cell, float_values, float_baseline_values) 504 self._InvertIfLowerIsBetter(cell) 505 else: 506 self._ComputeString(cell, values, baseline_values) 507 508 509class LiteralResult(Result): 510 """A literal result.""" 511 512 def __init__(self, iteration=0): 513 super(LiteralResult, self).__init__() 514 self.iteration = iteration 515 516 def Compute(self, cell, values, baseline_values): 517 try: 518 cell.value = values[self.iteration] 519 except IndexError: 520 cell.value = "-" 521 522 523class NonEmptyCountResult(Result): 524 """A class that counts the number of non-empty results. 525 526 The number of non-empty values will be stored in the cell. 527 """ 528 529 def Compute(self, cell, values, baseline_values): 530 """Put the number of non-empty values in the cell result. 531 532 Args: 533 cell: Put the result in cell.value. 534 values: A list of values for the row. 535 baseline_values: A list of baseline values for the row. 536 """ 537 cell.value = len(_StripNone(values)) 538 if not baseline_values: 539 return 540 base_value = len(_StripNone(baseline_values)) 541 if cell.value == base_value: 542 return 543 f = ColorBoxFormat() 544 len_values = len(values) 545 len_baseline_values = len(baseline_values) 546 tmp_cell = Cell() 547 tmp_cell.value = 1.0 + ( 548 float(cell.value - base_value) 549 / (max(len_values, len_baseline_values)) 550 ) 551 f.Compute(tmp_cell) 552 cell.bgcolor = tmp_cell.bgcolor 553 554 555class StringMeanResult(Result): 556 """Mean of string values.""" 557 558 def _ComputeString(self, cell, values, baseline_values): 559 if self._AllStringsSame(values): 560 cell.value = str(values[0]) 561 else: 562 cell.value = "?" 563 564 565class AmeanResult(StringMeanResult): 566 """Arithmetic mean.""" 567 568 def __init__(self, ignore_min_max=False): 569 super(AmeanResult, self).__init__() 570 self.ignore_min_max = ignore_min_max 571 572 def _ComputeFloat(self, cell, values, baseline_values): 573 if self.ignore_min_max: 574 values = _RemoveMinMax(cell, values) 575 cell.value = statistics.mean(values) 576 577 578class RawResult(Result): 579 """Raw result.""" 580 581 582class IterationResult(Result): 583 """Iteration result.""" 584 585 586class MinResult(Result): 587 """Minimum.""" 588 589 def _ComputeFloat(self, cell, values, baseline_values): 590 cell.value = min(values) 591 592 def _ComputeString(self, cell, values, baseline_values): 593 if values: 594 cell.value = min(values) 595 else: 596 cell.value = "" 597 598 599class MaxResult(Result): 600 """Maximum.""" 601 602 def _ComputeFloat(self, cell, values, baseline_values): 603 cell.value = max(values) 604 605 def _ComputeString(self, cell, values, baseline_values): 606 if values: 607 cell.value = max(values) 608 else: 609 cell.value = "" 610 611 612class NumericalResult(Result): 613 """Numerical result.""" 614 615 def _ComputeString(self, cell, values, baseline_values): 616 cell.value = "?" 617 618 619class StdResult(NumericalResult): 620 """Standard deviation.""" 621 622 def __init__(self, ignore_min_max=False): 623 super(StdResult, self).__init__() 624 self.ignore_min_max = ignore_min_max 625 626 def _ComputeFloat(self, cell, values, baseline_values): 627 if self.ignore_min_max: 628 values = _RemoveMinMax(cell, values) 629 cell.value = statistics.pstdev(values) 630 631 632class CoeffVarResult(NumericalResult): 633 """Standard deviation / Mean""" 634 635 def __init__(self, ignore_min_max=False): 636 super(CoeffVarResult, self).__init__() 637 self.ignore_min_max = ignore_min_max 638 639 def _ComputeFloat(self, cell, values, baseline_values): 640 if self.ignore_min_max: 641 values = _RemoveMinMax(cell, values) 642 if statistics.mean(values) != 0.0: 643 noise = abs(statistics.pstdev(values) / statistics.mean(values)) 644 else: 645 noise = 0.0 646 cell.value = noise 647 648 649class ComparisonResult(Result): 650 """Same or Different.""" 651 652 def NeedsBaseline(self): 653 return True 654 655 def _ComputeString(self, cell, values, baseline_values): 656 value = None 657 baseline_value = None 658 if self._AllStringsSame(values): 659 value = values[0] 660 if self._AllStringsSame(baseline_values): 661 baseline_value = baseline_values[0] 662 if value is not None and baseline_value is not None: 663 if value == baseline_value: 664 cell.value = "SAME" 665 else: 666 cell.value = "DIFFERENT" 667 else: 668 cell.value = "?" 669 670 671class PValueResult(ComparisonResult): 672 """P-value.""" 673 674 def __init__(self, ignore_min_max=False): 675 super(PValueResult, self).__init__() 676 self.ignore_min_max = ignore_min_max 677 678 def _ComputeFloat(self, cell, values, baseline_values): 679 if self.ignore_min_max: 680 values = _RemoveMinMax(cell, values) 681 baseline_values = _RemoveMinMax(cell, baseline_values) 682 if len(values) < 2 or len(baseline_values) < 2: 683 cell.value = float("nan") 684 return 685 _, cell.value = scipy.stats.ttest_ind(values, baseline_values) 686 687 def _ComputeString(self, cell, values, baseline_values): 688 return float("nan") 689 690 691class KeyAwareComparisonResult(ComparisonResult): 692 """Automatic key aware comparison.""" 693 694 def _IsLowerBetter(self, key): 695 # Units in histograms should include directions 696 if "smallerIsBetter" in key: 697 return True 698 if "biggerIsBetter" in key: 699 return False 700 701 # For units in chartjson: 702 # TODO(llozano): Trying to guess direction by looking at the name of the 703 # test does not seem like a good idea. Test frameworks should provide this 704 # info explicitly. I believe Telemetry has this info. Need to find it out. 705 # 706 # Below are some test names for which we are not sure what the 707 # direction is. 708 # 709 # For these we dont know what the direction is. But, since we dont 710 # specify anything, crosperf will assume higher is better: 711 # --percent_impl_scrolled--percent_impl_scrolled--percent 712 # --solid_color_tiles_analyzed--solid_color_tiles_analyzed--count 713 # --total_image_cache_hit_count--total_image_cache_hit_count--count 714 # --total_texture_upload_time_by_url 715 # 716 # About these we are doubtful but we made a guess: 717 # --average_num_missing_tiles_by_url--*--units (low is good) 718 # --experimental_mean_frame_time_by_url--*--units (low is good) 719 # --experimental_median_frame_time_by_url--*--units (low is good) 720 # --texture_upload_count--texture_upload_count--count (high is good) 721 # --total_deferred_image_decode_count--count (low is good) 722 # --total_tiles_analyzed--total_tiles_analyzed--count (high is good) 723 lower_is_better_keys = [ 724 "milliseconds", 725 "ms_", 726 "seconds_", 727 "KB", 728 "rdbytes", 729 "wrbytes", 730 "dropped_percent", 731 "(ms)", 732 "(seconds)", 733 "--ms", 734 "--average_num_missing_tiles", 735 "--experimental_jank", 736 "--experimental_mean_frame", 737 "--experimental_median_frame_time", 738 "--total_deferred_image_decode_count", 739 "--seconds", 740 "samples", 741 "bytes", 742 ] 743 744 return any([l in key for l in lower_is_better_keys]) 745 746 def _InvertIfLowerIsBetter(self, cell): 747 if self._IsLowerBetter(cell.name): 748 if cell.value: 749 cell.value = 1.0 / cell.value 750 751 752class AmeanRatioResult(KeyAwareComparisonResult): 753 """Ratio of arithmetic means of values vs. baseline values.""" 754 755 def __init__(self, ignore_min_max=False): 756 super(AmeanRatioResult, self).__init__() 757 self.ignore_min_max = ignore_min_max 758 759 def _ComputeFloat(self, cell, values, baseline_values): 760 if self.ignore_min_max: 761 values = _RemoveMinMax(cell, values) 762 baseline_values = _RemoveMinMax(cell, baseline_values) 763 764 baseline_mean = statistics.mean(baseline_values) 765 values_mean = statistics.mean(values) 766 if baseline_mean != 0: 767 cell.value = values_mean / baseline_mean 768 elif values_mean != 0: 769 cell.value = 0.00 770 # cell.value = 0 means the values and baseline_values have big difference 771 else: 772 cell.value = 1.00 773 # no difference if both values and baseline_values are 0 774 775 776class GmeanRatioResult(KeyAwareComparisonResult): 777 """Ratio of geometric means of values vs. baseline values.""" 778 779 def __init__(self, ignore_min_max=False): 780 super(GmeanRatioResult, self).__init__() 781 self.ignore_min_max = ignore_min_max 782 783 def _ComputeFloat(self, cell, values, baseline_values): 784 if self.ignore_min_max: 785 values = _RemoveMinMax(cell, values) 786 baseline_values = _RemoveMinMax(cell, baseline_values) 787 if self._GetGmean(baseline_values) != 0: 788 cell.value = self._GetGmean(values) / self._GetGmean( 789 baseline_values 790 ) 791 elif self._GetGmean(values) != 0: 792 cell.value = 0.00 793 else: 794 cell.value = 1.00 795 796 797class Color(object): 798 """Class that represents color in RGBA format.""" 799 800 def __init__(self, r=0, g=0, b=0, a=0): 801 self.r = r 802 self.g = g 803 self.b = b 804 self.a = a 805 806 def __str__(self): 807 return "r: %s g: %s: b: %s: a: %s" % (self.r, self.g, self.b, self.a) 808 809 def Round(self): 810 """Round RGBA values to the nearest integer.""" 811 self.r = int(self.r) 812 self.g = int(self.g) 813 self.b = int(self.b) 814 self.a = int(self.a) 815 816 def GetRGB(self): 817 """Get a hex representation of the color.""" 818 return "%02x%02x%02x" % (self.r, self.g, self.b) 819 820 @classmethod 821 def Lerp(cls, ratio, a, b): 822 """Perform linear interpolation between two colors. 823 824 Args: 825 ratio: The ratio to use for linear polation. 826 a: The first color object (used when ratio is 0). 827 b: The second color object (used when ratio is 1). 828 829 Returns: 830 Linearly interpolated color. 831 """ 832 ret = cls() 833 ret.r = (b.r - a.r) * ratio + a.r 834 ret.g = (b.g - a.g) * ratio + a.g 835 ret.b = (b.b - a.b) * ratio + a.b 836 ret.a = (b.a - a.a) * ratio + a.a 837 return ret 838 839 840class Format(object): 841 """A class that represents the format of a column.""" 842 843 def __init__(self): 844 pass 845 846 def Compute(self, cell): 847 """Computes the attributes of a cell based on its value. 848 849 Attributes typically are color, width, etc. 850 851 Args: 852 cell: The cell whose attributes are to be populated. 853 """ 854 if cell.value is None: 855 cell.string_value = "" 856 if isinstance(cell.value, float): 857 self._ComputeFloat(cell) 858 else: 859 self._ComputeString(cell) 860 861 def _ComputeFloat(self, cell): 862 cell.string_value = "{0:.2f}".format(cell.value) 863 864 def _ComputeString(self, cell): 865 cell.string_value = str(cell.value) 866 867 def _GetColor(self, value, low, mid, high, power=6, mid_value=1.0): 868 min_value = 0.0 869 max_value = 2.0 870 if math.isnan(value): 871 return mid 872 if value > mid_value: 873 value = max_value - mid_value / value 874 875 return self._GetColorBetweenRange( 876 value, min_value, mid_value, max_value, low, mid, high, power 877 ) 878 879 def _GetColorBetweenRange( 880 self, 881 value, 882 min_value, 883 mid_value, 884 max_value, 885 low_color, 886 mid_color, 887 high_color, 888 power, 889 ): 890 assert value <= max_value 891 assert value >= min_value 892 if value > mid_value: 893 value = (max_value - value) / (max_value - mid_value) 894 value **= power 895 ret = Color.Lerp(value, high_color, mid_color) 896 else: 897 value = (value - min_value) / (mid_value - min_value) 898 value **= power 899 ret = Color.Lerp(value, low_color, mid_color) 900 ret.Round() 901 return ret 902 903 904class PValueFormat(Format): 905 """Formatting for p-value.""" 906 907 def _ComputeFloat(self, cell): 908 cell.string_value = "%0.2f" % float(cell.value) 909 if float(cell.value) < 0.05: 910 cell.bgcolor = self._GetColor( 911 cell.value, 912 Color(255, 255, 0, 0), 913 Color(255, 255, 255, 0), 914 Color(255, 255, 255, 0), 915 mid_value=0.05, 916 power=1, 917 ) 918 919 920class WeightFormat(Format): 921 """Formatting for weight in cwp mode.""" 922 923 def _ComputeFloat(self, cell): 924 cell.string_value = "%0.4f" % float(cell.value) 925 926 927class StorageFormat(Format): 928 """Format the cell as a storage number. 929 930 Examples: 931 If the cell contains a value of 1024, the string_value will be 1.0K. 932 """ 933 934 def _ComputeFloat(self, cell): 935 base = 1024 936 suffices = ["K", "M", "G"] 937 v = float(cell.value) 938 current = 0 939 while v >= base ** (current + 1) and current < len(suffices): 940 current += 1 941 942 if current: 943 divisor = base ** current 944 cell.string_value = "%1.1f%s" % ( 945 (v / divisor), 946 suffices[current - 1], 947 ) 948 else: 949 cell.string_value = str(cell.value) 950 951 952class CoeffVarFormat(Format): 953 """Format the cell as a percent. 954 955 Examples: 956 If the cell contains a value of 1.5, the string_value will be +150%. 957 """ 958 959 def _ComputeFloat(self, cell): 960 cell.string_value = "%1.1f%%" % (float(cell.value) * 100) 961 cell.color = self._GetColor( 962 cell.value, 963 Color(0, 255, 0, 0), 964 Color(0, 0, 0, 0), 965 Color(255, 0, 0, 0), 966 mid_value=0.02, 967 power=1, 968 ) 969 970 971class PercentFormat(Format): 972 """Format the cell as a percent. 973 974 Examples: 975 If the cell contains a value of 1.5, the string_value will be +50%. 976 """ 977 978 def _ComputeFloat(self, cell): 979 cell.string_value = "%+1.1f%%" % ((float(cell.value) - 1) * 100) 980 cell.color = self._GetColor( 981 cell.value, 982 Color(255, 0, 0, 0), 983 Color(0, 0, 0, 0), 984 Color(0, 255, 0, 0), 985 ) 986 987 988class RatioFormat(Format): 989 """Format the cell as a ratio. 990 991 Examples: 992 If the cell contains a value of 1.5642, the string_value will be 1.56. 993 """ 994 995 def _ComputeFloat(self, cell): 996 cell.string_value = "%+1.1f%%" % ((cell.value - 1) * 100) 997 cell.color = self._GetColor( 998 cell.value, 999 Color(255, 0, 0, 0), 1000 Color(0, 0, 0, 0), 1001 Color(0, 255, 0, 0), 1002 ) 1003 1004 1005class ColorBoxFormat(Format): 1006 """Format the cell as a color box. 1007 1008 Examples: 1009 If the cell contains a value of 1.5, it will get a green color. 1010 If the cell contains a value of 0.5, it will get a red color. 1011 The intensity of the green/red will be determined by how much above or below 1012 1.0 the value is. 1013 """ 1014 1015 def _ComputeFloat(self, cell): 1016 cell.string_value = "--" 1017 bgcolor = self._GetColor( 1018 cell.value, 1019 Color(255, 0, 0, 0), 1020 Color(255, 255, 255, 0), 1021 Color(0, 255, 0, 0), 1022 ) 1023 cell.bgcolor = bgcolor 1024 cell.color = bgcolor 1025 1026 1027class Cell(object): 1028 """A class to represent a cell in a table. 1029 1030 Attributes: 1031 value: The raw value of the cell. 1032 color: The color of the cell. 1033 bgcolor: The background color of the cell. 1034 string_value: The string value of the cell. 1035 suffix: A string suffix to be attached to the value when displaying. 1036 prefix: A string prefix to be attached to the value when displaying. 1037 color_row: Indicates whether the whole row is to inherit this cell's color. 1038 bgcolor_row: Indicates whether the whole row is to inherit this cell's 1039 bgcolor. 1040 width: Optional specifier to make a column narrower than the usual width. 1041 The usual width of a column is the max of all its cells widths. 1042 colspan: Set the colspan of the cell in the HTML table, this is used for 1043 table headers. Default value is 1. 1044 name: the test name of the cell. 1045 header: Whether this is a header in html. 1046 """ 1047 1048 def __init__(self): 1049 self.value = None 1050 self.color = None 1051 self.bgcolor = None 1052 self.string_value = None 1053 self.suffix = None 1054 self.prefix = None 1055 # Entire row inherits this color. 1056 self.color_row = False 1057 self.bgcolor_row = False 1058 self.width = 0 1059 self.colspan = 1 1060 self.name = None 1061 self.header = False 1062 1063 def __str__(self): 1064 l = [] 1065 l.append("value: %s" % self.value) 1066 l.append("string_value: %s" % self.string_value) 1067 return " ".join(l) 1068 1069 1070class Column(object): 1071 """Class representing a column in a table. 1072 1073 Attributes: 1074 result: an object of the Result class. 1075 fmt: an object of the Format class. 1076 """ 1077 1078 def __init__(self, result, fmt, name=""): 1079 self.result = result 1080 self.fmt = fmt 1081 self.name = name 1082 1083 1084# Takes in: 1085# ["Key", "Label1", "Label2"] 1086# ["k", ["v", "v2"], [v3]] 1087# etc. 1088# Also takes in a format string. 1089# Returns a table like: 1090# ["Key", "Label1", "Label2"] 1091# ["k", avg("v", "v2"), stddev("v", "v2"), etc.]] 1092# according to format string 1093class TableFormatter(object): 1094 """Class to convert a plain table into a cell-table. 1095 1096 This class takes in a table generated by TableGenerator and a list of column 1097 formats to apply to the table and returns a table of cells. 1098 """ 1099 1100 def __init__(self, table, columns, samples_table=False): 1101 """The constructor takes in a table and a list of columns. 1102 1103 Args: 1104 table: A list of lists of values. 1105 columns: A list of column containing what to produce and how to format 1106 it. 1107 samples_table: A flag to check whether we are generating a table of 1108 samples in CWP apporximation mode. 1109 """ 1110 self._table = table 1111 self._columns = columns 1112 self._samples_table = samples_table 1113 self._table_columns = [] 1114 self._out_table = [] 1115 1116 def GenerateCellTable(self, table_type): 1117 row_index = 0 1118 all_failed = False 1119 1120 for row in self._table[1:]: 1121 # If we are generating samples_table, the second value will be weight 1122 # rather than values. 1123 start_col = 2 if self._samples_table else 1 1124 # It does not make sense to put retval in the summary table. 1125 if str(row[0]) == "retval" and table_type == "summary": 1126 # Check to see if any runs passed, and update all_failed. 1127 all_failed = True 1128 for values in row[start_col:]: 1129 if 0 in values: 1130 all_failed = False 1131 continue 1132 key = Cell() 1133 key.string_value = str(row[0]) 1134 out_row = [key] 1135 if self._samples_table: 1136 # Add one column for weight if in samples_table mode 1137 weight = Cell() 1138 weight.value = row[1] 1139 f = WeightFormat() 1140 f.Compute(weight) 1141 out_row.append(weight) 1142 baseline = None 1143 for results in row[start_col:]: 1144 column_start = 0 1145 values = None 1146 # If generating sample table, we will split a tuple of iterations info 1147 # from the results 1148 if isinstance(results, tuple): 1149 it, values = results 1150 column_start = 1 1151 cell = Cell() 1152 cell.string_value = "[%d: %d]" % (it[0], it[1]) 1153 out_row.append(cell) 1154 if not row_index: 1155 self._table_columns.append(self._columns[0]) 1156 else: 1157 values = results 1158 # Parse each column 1159 for column in self._columns[column_start:]: 1160 cell = Cell() 1161 cell.name = key.string_value 1162 if ( 1163 not column.result.NeedsBaseline() 1164 or baseline is not None 1165 ): 1166 column.result.Compute(cell, values, baseline) 1167 column.fmt.Compute(cell) 1168 out_row.append(cell) 1169 if not row_index: 1170 self._table_columns.append(column) 1171 1172 if baseline is None: 1173 baseline = values 1174 self._out_table.append(out_row) 1175 row_index += 1 1176 1177 # If this is a summary table, and the only row in it is 'retval', and 1178 # all the test runs failed, we need to a 'Results' row to the output 1179 # table. 1180 if table_type == "summary" and all_failed and len(self._table) == 2: 1181 labels_row = self._table[0] 1182 key = Cell() 1183 key.string_value = "Results" 1184 out_row = [key] 1185 baseline = None 1186 for _ in labels_row[1:]: 1187 for column in self._columns: 1188 cell = Cell() 1189 cell.name = key.string_value 1190 column.result.Compute(cell, ["Fail"], baseline) 1191 column.fmt.Compute(cell) 1192 out_row.append(cell) 1193 if not row_index: 1194 self._table_columns.append(column) 1195 self._out_table.append(out_row) 1196 1197 def AddColumnName(self): 1198 """Generate Column name at the top of table.""" 1199 key = Cell() 1200 key.header = True 1201 key.string_value = "Keys" if not self._samples_table else "Benchmarks" 1202 header = [key] 1203 if self._samples_table: 1204 weight = Cell() 1205 weight.header = True 1206 weight.string_value = "Weights" 1207 header.append(weight) 1208 for column in self._table_columns: 1209 cell = Cell() 1210 cell.header = True 1211 if column.name: 1212 cell.string_value = column.name 1213 else: 1214 result_name = column.result.__class__.__name__ 1215 format_name = column.fmt.__class__.__name__ 1216 1217 cell.string_value = "%s %s" % ( 1218 result_name.replace("Result", ""), 1219 format_name.replace("Format", ""), 1220 ) 1221 1222 header.append(cell) 1223 1224 self._out_table = [header] + self._out_table 1225 1226 def AddHeader(self, s): 1227 """Put additional string on the top of the table.""" 1228 cell = Cell() 1229 cell.header = True 1230 cell.string_value = str(s) 1231 header = [cell] 1232 colspan = max(1, max(len(row) for row in self._table)) 1233 cell.colspan = colspan 1234 self._out_table = [header] + self._out_table 1235 1236 def GetPassesAndFails(self, values): 1237 passes = 0 1238 fails = 0 1239 for val in values: 1240 if val == 0: 1241 passes = passes + 1 1242 else: 1243 fails = fails + 1 1244 return passes, fails 1245 1246 def AddLabelName(self): 1247 """Put label on the top of the table.""" 1248 top_header = [] 1249 base_colspan = len( 1250 [c for c in self._columns if not c.result.NeedsBaseline()] 1251 ) 1252 compare_colspan = len(self._columns) 1253 # Find the row with the key 'retval', if it exists. This 1254 # will be used to calculate the number of iterations that passed and 1255 # failed for each image label. 1256 retval_row = None 1257 for row in self._table: 1258 if row[0] == "retval": 1259 retval_row = row 1260 # The label is organized as follows 1261 # "keys" label_base, label_comparison1, label_comparison2 1262 # The first cell has colspan 1, the second is base_colspan 1263 # The others are compare_colspan 1264 column_position = 0 1265 for label in self._table[0]: 1266 cell = Cell() 1267 cell.header = True 1268 # Put the number of pass/fail iterations in the image label header. 1269 if column_position > 0 and retval_row: 1270 retval_values = retval_row[column_position] 1271 if isinstance(retval_values, list): 1272 passes, fails = self.GetPassesAndFails(retval_values) 1273 cell.string_value = str(label) + " (pass:%d fail:%d)" % ( 1274 passes, 1275 fails, 1276 ) 1277 else: 1278 cell.string_value = str(label) 1279 else: 1280 cell.string_value = str(label) 1281 if top_header: 1282 if not self._samples_table or ( 1283 self._samples_table and len(top_header) == 2 1284 ): 1285 cell.colspan = base_colspan 1286 if len(top_header) > 1: 1287 if not self._samples_table or ( 1288 self._samples_table and len(top_header) > 2 1289 ): 1290 cell.colspan = compare_colspan 1291 top_header.append(cell) 1292 column_position = column_position + 1 1293 self._out_table = [top_header] + self._out_table 1294 1295 def _PrintOutTable(self): 1296 o = "" 1297 for row in self._out_table: 1298 for cell in row: 1299 o += str(cell) + " " 1300 o += "\n" 1301 print(o) 1302 1303 def GetCellTable(self, table_type="full", headers=True): 1304 """Function to return a table of cells. 1305 1306 The table (list of lists) is converted into a table of cells by this 1307 function. 1308 1309 Args: 1310 table_type: Can be 'full' or 'summary' 1311 headers: A boolean saying whether we want default headers 1312 1313 Returns: 1314 A table of cells with each cell having the properties and string values as 1315 requiested by the columns passed in the constructor. 1316 """ 1317 # Generate the cell table, creating a list of dynamic columns on the fly. 1318 if not self._out_table: 1319 self.GenerateCellTable(table_type) 1320 if headers: 1321 self.AddColumnName() 1322 self.AddLabelName() 1323 return self._out_table 1324 1325 1326class TablePrinter(object): 1327 """Class to print a cell table to the console, file or html.""" 1328 1329 PLAIN = 0 1330 CONSOLE = 1 1331 HTML = 2 1332 TSV = 3 1333 EMAIL = 4 1334 1335 def __init__(self, table, output_type): 1336 """Constructor that stores the cell table and output type.""" 1337 self._table = table 1338 self._output_type = output_type 1339 self._row_styles = [] 1340 self._column_styles = [] 1341 1342 # Compute whole-table properties like max-size, etc. 1343 def _ComputeStyle(self): 1344 self._row_styles = [] 1345 for row in self._table: 1346 row_style = Cell() 1347 for cell in row: 1348 if cell.color_row: 1349 assert cell.color, "Cell color not set but color_row set!" 1350 assert ( 1351 not row_style.color 1352 ), "Multiple row_style.colors found!" 1353 row_style.color = cell.color 1354 if cell.bgcolor_row: 1355 assert ( 1356 cell.bgcolor 1357 ), "Cell bgcolor not set but bgcolor_row set!" 1358 assert ( 1359 not row_style.bgcolor 1360 ), "Multiple row_style.bgcolors found!" 1361 row_style.bgcolor = cell.bgcolor 1362 self._row_styles.append(row_style) 1363 1364 self._column_styles = [] 1365 if len(self._table) < 2: 1366 return 1367 1368 for i in range(max(len(row) for row in self._table)): 1369 column_style = Cell() 1370 for row in self._table: 1371 if not any([cell.colspan != 1 for cell in row]): 1372 column_style.width = max( 1373 column_style.width, len(row[i].string_value) 1374 ) 1375 self._column_styles.append(column_style) 1376 1377 def _GetBGColorFix(self, color): 1378 if self._output_type == self.CONSOLE: 1379 prefix = misc.rgb2short(color.r, color.g, color.b) 1380 # pylint: disable=anomalous-backslash-in-string 1381 prefix = "\033[48;5;%sm" % prefix 1382 suffix = "\033[0m" 1383 elif self._output_type in [self.EMAIL, self.HTML]: 1384 rgb = color.GetRGB() 1385 prefix = '<FONT style="BACKGROUND-COLOR:#{0}">'.format(rgb) 1386 suffix = "</FONT>" 1387 elif self._output_type in [self.PLAIN, self.TSV]: 1388 prefix = "" 1389 suffix = "" 1390 return prefix, suffix 1391 1392 def _GetColorFix(self, color): 1393 if self._output_type == self.CONSOLE: 1394 prefix = misc.rgb2short(color.r, color.g, color.b) 1395 # pylint: disable=anomalous-backslash-in-string 1396 prefix = "\033[38;5;%sm" % prefix 1397 suffix = "\033[0m" 1398 elif self._output_type in [self.EMAIL, self.HTML]: 1399 rgb = color.GetRGB() 1400 prefix = "<FONT COLOR=#{0}>".format(rgb) 1401 suffix = "</FONT>" 1402 elif self._output_type in [self.PLAIN, self.TSV]: 1403 prefix = "" 1404 suffix = "" 1405 return prefix, suffix 1406 1407 def Print(self): 1408 """Print the table to a console, html, etc. 1409 1410 Returns: 1411 A string that contains the desired representation of the table. 1412 """ 1413 self._ComputeStyle() 1414 return self._GetStringValue() 1415 1416 def _GetCellValue(self, i, j): 1417 cell = self._table[i][j] 1418 out = cell.string_value 1419 raw_width = len(out) 1420 1421 if cell.color: 1422 p, s = self._GetColorFix(cell.color) 1423 out = "%s%s%s" % (p, out, s) 1424 1425 if cell.bgcolor: 1426 p, s = self._GetBGColorFix(cell.bgcolor) 1427 out = "%s%s%s" % (p, out, s) 1428 1429 if self._output_type in [self.PLAIN, self.CONSOLE, self.EMAIL]: 1430 if cell.width: 1431 width = cell.width 1432 else: 1433 if self._column_styles: 1434 width = self._column_styles[j].width 1435 else: 1436 width = len(cell.string_value) 1437 if cell.colspan > 1: 1438 width = 0 1439 start = 0 1440 for k in range(j): 1441 start += self._table[i][k].colspan 1442 for k in range(cell.colspan): 1443 width += self._column_styles[start + k].width 1444 if width > raw_width: 1445 padding = ("%" + str(width - raw_width) + "s") % "" 1446 out = padding + out 1447 1448 if self._output_type == self.HTML: 1449 if cell.header: 1450 tag = "th" 1451 else: 1452 tag = "td" 1453 out = '<{0} colspan = "{2}"> {1} </{0}>'.format( 1454 tag, out, cell.colspan 1455 ) 1456 1457 return out 1458 1459 def _GetHorizontalSeparator(self): 1460 if self._output_type in [self.CONSOLE, self.PLAIN, self.EMAIL]: 1461 return " " 1462 if self._output_type == self.HTML: 1463 return "" 1464 if self._output_type == self.TSV: 1465 return "\t" 1466 1467 def _GetVerticalSeparator(self): 1468 if self._output_type in [ 1469 self.PLAIN, 1470 self.CONSOLE, 1471 self.TSV, 1472 self.EMAIL, 1473 ]: 1474 return "\n" 1475 if self._output_type == self.HTML: 1476 return "</tr>\n<tr>" 1477 1478 def _GetPrefix(self): 1479 if self._output_type in [ 1480 self.PLAIN, 1481 self.CONSOLE, 1482 self.TSV, 1483 self.EMAIL, 1484 ]: 1485 return "" 1486 if self._output_type == self.HTML: 1487 return '<p></p><table id="box-table-a">\n<tr>' 1488 1489 def _GetSuffix(self): 1490 if self._output_type in [ 1491 self.PLAIN, 1492 self.CONSOLE, 1493 self.TSV, 1494 self.EMAIL, 1495 ]: 1496 return "" 1497 if self._output_type == self.HTML: 1498 return "</tr>\n</table>" 1499 1500 def _GetStringValue(self): 1501 o = "" 1502 o += self._GetPrefix() 1503 for i in range(len(self._table)): 1504 row = self._table[i] 1505 # Apply row color and bgcolor. 1506 p = s = bgp = bgs = "" 1507 if self._row_styles[i].bgcolor: 1508 bgp, bgs = self._GetBGColorFix(self._row_styles[i].bgcolor) 1509 if self._row_styles[i].color: 1510 p, s = self._GetColorFix(self._row_styles[i].color) 1511 o += p + bgp 1512 for j in range(len(row)): 1513 out = self._GetCellValue(i, j) 1514 o += out + self._GetHorizontalSeparator() 1515 o += s + bgs 1516 o += self._GetVerticalSeparator() 1517 o += self._GetSuffix() 1518 return o 1519 1520 1521# Some common drivers 1522def GetSimpleTable(table, out_to=TablePrinter.CONSOLE): 1523 """Prints a simple table. 1524 1525 This is used by code that has a very simple list-of-lists and wants to 1526 produce a table with ameans, a percentage ratio of ameans and a colorbox. 1527 1528 Examples: 1529 GetSimpleConsoleTable([["binary", "b1", "b2"],["size", "300", "400"]]) 1530 will produce a colored table that can be printed to the console. 1531 1532 Args: 1533 table: a list of lists. 1534 out_to: specify the fomat of output. Currently it supports HTML and CONSOLE. 1535 1536 Returns: 1537 A string version of the table that can be printed to the console. 1538 """ 1539 columns = [ 1540 Column(AmeanResult(), Format()), 1541 Column(AmeanRatioResult(), PercentFormat()), 1542 Column(AmeanRatioResult(), ColorBoxFormat()), 1543 ] 1544 our_table = [table[0]] 1545 for row in table[1:]: 1546 our_row = [row[0]] 1547 for v in row[1:]: 1548 our_row.append([v]) 1549 our_table.append(our_row) 1550 1551 tf = TableFormatter(our_table, columns) 1552 cell_table = tf.GetCellTable() 1553 tp = TablePrinter(cell_table, out_to) 1554 return tp.Print() 1555 1556 1557# pylint: disable=redefined-outer-name 1558def GetComplexTable(runs, labels, out_to=TablePrinter.CONSOLE): 1559 """Prints a complex table. 1560 1561 This can be used to generate a table with arithmetic mean, standard deviation, 1562 coefficient of variation, p-values, etc. 1563 1564 Args: 1565 runs: A list of lists with data to tabulate. 1566 labels: A list of labels that correspond to the runs. 1567 out_to: specifies the format of the table (example CONSOLE or HTML). 1568 1569 Returns: 1570 A string table that can be printed to the console or put in an HTML file. 1571 """ 1572 tg = TableGenerator(runs, labels, TableGenerator.SORT_BY_VALUES_DESC) 1573 table = tg.GetTable() 1574 columns = [ 1575 Column(LiteralResult(), Format(), "Literal"), 1576 Column(AmeanResult(), Format()), 1577 Column(StdResult(), Format()), 1578 Column(CoeffVarResult(), CoeffVarFormat()), 1579 Column(NonEmptyCountResult(), Format()), 1580 Column(AmeanRatioResult(), PercentFormat()), 1581 Column(AmeanRatioResult(), RatioFormat()), 1582 Column(GmeanRatioResult(), RatioFormat()), 1583 Column(PValueResult(), PValueFormat()), 1584 ] 1585 tf = TableFormatter(table, columns) 1586 cell_table = tf.GetCellTable() 1587 tp = TablePrinter(cell_table, out_to) 1588 return tp.Print() 1589 1590 1591if __name__ == "__main__": 1592 # Run a few small tests here. 1593 run1 = { 1594 "k1": "10", 1595 "k2": "12", 1596 "k5": "40", 1597 "k6": "40", 1598 "ms_1": "20", 1599 "k7": "FAIL", 1600 "k8": "PASS", 1601 "k9": "PASS", 1602 "k10": "0", 1603 } 1604 run2 = { 1605 "k1": "13", 1606 "k2": "14", 1607 "k3": "15", 1608 "ms_1": "10", 1609 "k8": "PASS", 1610 "k9": "FAIL", 1611 "k10": "0", 1612 } 1613 run3 = { 1614 "k1": "50", 1615 "k2": "51", 1616 "k3": "52", 1617 "k4": "53", 1618 "k5": "35", 1619 "k6": "45", 1620 "ms_1": "200", 1621 "ms_2": "20", 1622 "k7": "FAIL", 1623 "k8": "PASS", 1624 "k9": "PASS", 1625 } 1626 runs = [[run1, run2], [run3]] 1627 labels = ["vanilla", "modified"] 1628 t = GetComplexTable(runs, labels, TablePrinter.CONSOLE) 1629 print(t) 1630 email = GetComplexTable(runs, labels, TablePrinter.EMAIL) 1631 1632 runs = [ 1633 [{"k1": "1"}, {"k1": "1.1"}, {"k1": "1.2"}], 1634 [{"k1": "5"}, {"k1": "5.1"}, {"k1": "5.2"}], 1635 ] 1636 t = GetComplexTable(runs, labels, TablePrinter.CONSOLE) 1637 print(t) 1638 1639 simple_table = [ 1640 ["binary", "b1", "b2", "b3"], 1641 ["size", 100, 105, 108], 1642 ["rodata", 100, 80, 70], 1643 ["data", 100, 100, 100], 1644 ["debug", 100, 140, 60], 1645 ] 1646 t = GetSimpleTable(simple_table) 1647 print(t) 1648 email += GetSimpleTable(simple_table, TablePrinter.HTML) 1649 email_to = [getpass.getuser()] 1650 email = "<pre style='font-size: 13px'>%s</pre>" % email 1651 EmailSender().SendEmail(email_to, "SimpleTableTest", email, msg_type="html") 1652