1# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5"""A module handling the logs. 6 7The structure of this module: 8 9 RoundLog: the test results of every round are saved in a log file. 10 includes: fw, and round_name (i.e., the date time of the round 11 12 --> GestureLogs: includes gesture name, and variation 13 14 --> ValidatorLogs: includes name, details, criteria, score, metrics 15 16 17 SummaryLog: derived from multiple RoundLogs 18 --> SimpleTable: (key, vlog) pairs 19 key: (fw, round_name, gesture_name, variation_name, validator_name) 20 vlog: name, details, criteria, score, metrics 21 22 TestResult: encapsulation of scores and metrics 23 used by a client program to query the test results 24 --> StatisticsScores: includes average, ssd, and count 25 --> StatisticsMetrics: includes average, min, max, and more 26 27 28How the logs work: 29 (1) ValidatorLogs are contained in a GestureLog. 30 (2) Multiple GestureLogs are packed in a RoundLog which is saved in a 31 separate pickle log file. 32 (3) To construct a SummaryLog, it reads RoundLogs from all pickle logs 33 in the specified log directory. It then creates a SimpleTable 34 consisting of (key, ValidatorLog) pairs, where 35 key is a 5-tuple: 36 (fw, round_name, gesture_name, variation_name, validator_name). 37 (4) The client program, i.e., firmware_summary module, contains a 38 SummaryLog, and queries all statistics using get_result() which returns 39 a TestResult object containing both StatisticsScores and 40 StatisticsMetrics. 41 42""" 43 44 45import glob 46import numpy as np 47import pickle 48import os 49 50import test_conf as conf 51import validators as val 52 53from collections import defaultdict, namedtuple 54 55from common_util import Debug, print_and_exit 56from firmware_constants import AXIS 57 58 59MetricProps = namedtuple('MetricProps', ['description', 'note', 'stat_func']) 60 61 62def _setup_debug(debug_flag): 63 """Set up the global debug_print function.""" 64 if 'debug_print' not in globals(): 65 global debug_print 66 debug = Debug(debug_flag) 67 debug_print = debug.print_msg 68 69 70def _calc_sample_standard_deviation(sample): 71 """Calculate the sample standard deviation (ssd) from a given sample. 72 73 To compute a sample standard deviation, the following formula is used: 74 sqrt(sum((x_i - x_average)^2) / N-1) 75 76 Note that N-1 is used in the denominator for sample standard deviation, 77 where N-1 is the degree of freedom. We need to set ddof=1 below; 78 otherwise, N would be used in the denominator as ddof's default value 79 is 0. 80 81 Reference: 82 http://en.wikipedia.org/wiki/Standard_deviation 83 """ 84 return np.std(np.array(sample), ddof=1) 85 86 87class float_d2(float): 88 """A float type with special __repr__ and __str__ methods that display 89 the float number to the 2nd decimal place.""" 90 template = '%.2f' 91 92 def __str__(self): 93 """Display the float to the 2nd decimal place.""" 94 return self.template % self.real 95 96 def __repr__(self): 97 """Display the float to the 2nd decimal place.""" 98 return self.template % self.real 99 100 101def convert_float_to_float_d2(value): 102 """Convert the float(s) in value to float_d2.""" 103 if isinstance(value, float): 104 return float_d2(value) 105 elif isinstance(value, tuple): 106 return tuple(float_d2(v) if isinstance(v, float) else v for v in value) 107 else: 108 return value 109 110 111class Metric: 112 """A class to handle the name and the value of a metric.""" 113 def __init__(self, name, value): 114 self.name = name 115 self.value = convert_float_to_float_d2(value) 116 117 def insert_key(self, key): 118 """Insert the key to this metric.""" 119 self.key = key 120 return self 121 122 123class MetricNameProps: 124 """A class keeping the information of metric name templates, descriptions, 125 and statistic functions. 126 """ 127 128 def __init__(self): 129 self._init_raw_metrics_props() 130 self._derive_metrics_props() 131 132 def _init_raw_metrics_props(self): 133 """Initialize raw_metrics_props. 134 135 The raw_metrics_props is a dictionary from metric attribute to the 136 corresponding metric properties. Take MAX_ERR as an example of metric 137 attribute. Its metric properties include 138 . metric name template: 'max error in {} (mm)' 139 The metric name template will be expanded later. For example, 140 with name variations ['x', 'y'], the above template will be 141 expanded to: 142 'max error in x (mm)', and 143 'max error in y (mm)' 144 . name variations: for example, ['x', 'y'] for MAX_ERR 145 . metric name description: 'The max err of all samples' 146 . metric note: None 147 . the stat function used to calculate the statistics for the metric: 148 we use max() to calculate MAX_ERR in x/y for linearity. 149 150 About metric note: 151 We show tuples instead of percentages if the metrics values are 152 percentages. This is because such a tuple unveils more information 153 (i.e., the values of the nominator and the denominator) than a mere 154 percentage value. For examples, 155 156 1f-click miss rate (%): 157 one_finger_physical_click.center (20130710_063117) : (0, 1) 158 the tuple means (the number of missed clicks, total clicks) 159 160 intervals > xxx ms (%) 161 one_finger_tap.top_left (20130710_063117) : (1, 6) 162 the tuple means (the number of long intervals, total packets) 163 """ 164 # stat_functions include: max, average, 165 # pct_by_numbers, pct_by_missed_numbers, 166 # pct_by_cases_neq, and pct_by_cases_less 167 average = lambda lst: float(sum(lst)) / len(lst) 168 get_sums = lambda lst: [sum(count) for count in zip(*lst)] 169 _pct = lambda lst: float(lst[0]) / lst[1] * 100 170 # The following lambda function is used to compute the missed pct of 171 # 172 # '(clicks with correct finger IDs, actual clicks)' 173 # 174 # In some cases when the number of actual clicks is 0, there are no 175 # missed finger IDs. So just return 0 for this special case to prevent 176 # the devision by 0 error. 177 _missed_pct = lambda lst: (float(lst[1] - lst[0]) / lst[1] * 100 178 if lst[1] != 0 else 0) 179 180 # pct by numbers: lst means [(incorrect number, total number), ...] 181 # E.g., lst = [(2, 10), (0, 10), (0, 10), (0, 10)] 182 # pct_by_numbers would be (2 + 0 + 0 + 0) / (10 + 10 + 10 + 10) * 100% 183 pct_by_numbers = lambda lst: _pct(get_sums(lst)) 184 185 # pct by misssed numbers: lst means 186 # [(actual number, expected number), ...] 187 # E.g., lst = [(0, 1), (1, 1), (1, 1), (1, 1)] 188 # pct_by_missed_numbers would be 189 # 0 + 1 + 1 + 1 = 3 190 # 1 + 1 + 1 + 1 = 4 191 # missed pct = (4 - 3) / 4 * 100% = 25% 192 pct_by_missed_numbers = lambda lst: _missed_pct(get_sums(lst)) 193 194 # pct of incorrect cases in [(acutal_value, expected_value), ...] 195 # E.g., lst = [(1, 1), (0, 1), (1, 1), (1, 1)] 196 # pct_by_cases_neq would be 1 / 4 * 100% 197 # This is used for CountTrackingIDValidator 198 pct_by_cases_neq = lambda lst: _pct( 199 [len([pair for pair in lst if pair[0] != pair[1]]), len(lst)]) 200 201 # pct of incorrect cases in [(acutal_value, min expected_value), ...] 202 # E.g., lst = [(3, 3), (4, 3)] 203 # pct_by_cases_less would be 0 / 2 * 100% 204 # E.g., lst = [(2, 3), (5, 3)] 205 # pct_by_cases_less would be 1 / 2 * 100% 206 # This is used for CountPacketsIDValidator and PinchValidator 207 pct_by_cases_less = lambda lst: _pct( 208 [len([pair for pair in lst if pair[0] < pair[1]]), len(lst)]) 209 210 self.max_report_interval_str = '%.2f' % conf.max_report_interval 211 212 # A dictionary from metric attribute to its properties: 213 # {metric_attr: (template, 214 # name_variations, 215 # description, 216 # metric_note, 217 # stat_func) 218 # } 219 # Ordered by validators 220 self.raw_metrics_props = { 221 # Count Packets Validator 222 'COUNT_PACKETS': ( 223 'pct of incorrect cases (%)--packets', 224 None, 225 'an incorrect case is one where a swipe has less than ' 226 '3 packets reported', 227 '(actual number of packets, expected number of packets)', 228 pct_by_cases_less), 229 # Count TrackingID Validator 230 'TID': ( 231 'pct of incorrect cases (%)--tids', 232 None, 233 'an incorrect case is one where there are an incorrect number ' 234 'of fingers detected', 235 '(actual tracking IDs, expected tracking IDs)', 236 pct_by_cases_neq), 237 # Drag Latency Validator 238 'AVG_LATENCY': ( 239 'average latency (ms)', 240 None, 241 'The average drag-latency in milliseconds', 242 None, 243 average), 244 # Drumroll Validator 245 'CIRCLE_RADIUS': ( 246 'circle radius (mm)', 247 None, 248 'the max radius of enclosing circles of tapping points', 249 None, 250 max), 251 # Hysteresis Validator 252 'MAX_INIT_GAP_RATIO': ( 253 'max init gap ratio', 254 None, 255 'the max ratio of dist(p0,p1) / dist(p1,p2)', 256 None, 257 max), 258 'AVE_INIT_GAP_RATIO': ( 259 'ave init gap ratio', 260 None, 261 'the average ratio of dist(p0,p1) / dist(p1,p2)', 262 None, 263 average), 264 # Linearity Validator 265 'MAX_ERR': ( 266 'max error in {} (mm)', 267 AXIS.LIST, 268 'The max err of all samples', 269 None, 270 max), 271 'RMS_ERR': ( 272 'rms error in {} (mm)', 273 AXIS.LIST, 274 'The mean of all rms means of all trials', 275 None, 276 average), 277 # MTB Sanity Validator 278 'MTB_SANITY_ERR': ( 279 'pct of MTB errors (%)', 280 None, 281 'pct of MTB errors', 282 '(MTB errors, expected errors)', 283 pct_by_cases_neq), 284 # No Ghost Finger Validator 285 'GHOST_FINGERS': ( 286 'pct of ghost fingers (%)', 287 None, 288 'pct of ghost fingers', 289 '(ghost fingers, expected fingers)', 290 pct_by_cases_neq), 291 # Physical Click Validator 292 'CLICK_CHECK_CLICK': ( 293 '{}f-click miss rate (%)', 294 conf.fingers_physical_click, 295 'the pct of finger IDs w/o a click', 296 '(acutual clicks, expected clicks)', 297 pct_by_missed_numbers), 298 'CLICK_CHECK_TIDS': ( 299 '{}f-click w/o finger IDs (%)', 300 conf.fingers_physical_click, 301 'the pct of clicks w/o correct finger IDs', 302 '(clicks with correct finger IDs, actual clicks)', 303 pct_by_missed_numbers), 304 # Pinch Validator 305 'PINCH': ( 306 'pct of incorrect cases (%)--pinch', 307 None, 308 'pct of incorrect cases over total cases', 309 '(actual relative motion (px), expected relative motion (px))', 310 pct_by_cases_less), 311 # Range Validator 312 'RANGE': ( 313 '{} edge not reached (mm)', 314 ['left', 'right', 'top', 'bottom'], 315 'Min unreachable distance', 316 None, 317 max), 318 # Report Rate Validator 319 'LONG_INTERVALS': ( 320 'pct of large intervals (%)', 321 None, 322 'pct of intervals larger than expected', 323 '(the number of long intervals, total packets)', 324 pct_by_numbers), 325 'AVE_TIME_INTERVAL': ( 326 'average time interval (ms)', 327 None, 328 'the average of report intervals', 329 None, 330 average), 331 'MAX_TIME_INTERVAL': ( 332 'max time interval (ms)', 333 None, 334 'the max report interval', 335 None, 336 max), 337 # Stationary Finger Validator 338 'MAX_DISTANCE': ( 339 'max distance (mm)', 340 None, 341 'max distance of any two points from any run', 342 None, 343 max), 344 } 345 346 # Set the metric attribute to its template 347 # E.g., self.MAX_ERR = 'max error in {} (mm)' 348 for key, props in self.raw_metrics_props.items(): 349 template = props[0] 350 setattr(self, key, template) 351 352 def _derive_metrics_props(self): 353 """Expand the metric name templates to the metric names, and then 354 derive the expanded metrics_props. 355 356 In _init_raw_metrics_props(): 357 The raw_metrics_props is defined as: 358 'MAX_ERR': ( 359 'max error in {} (mm)', # template 360 ['x', 'y'], # name variations 361 'The max err of all samples', # description 362 max), # stat_func 363 ... 364 365 By expanding the template with its corresponding name variations, 366 the names related with MAX_ERR will be: 367 'max error in x (mm)', and 368 'max error in y (mm)' 369 370 Here we are going to derive metrics_props as: 371 metrics_props = { 372 'max error in x (mm)': 373 MetricProps('The max err of all samples', max), 374 ... 375 } 376 """ 377 self.metrics_props = {} 378 for raw_props in self.raw_metrics_props.values(): 379 template, name_variations, description, note, stat_func = raw_props 380 metric_props = MetricProps(description, note, stat_func) 381 if name_variations: 382 # Expand the template with every variations. 383 # E.g., template = 'max error in {} (mm)' is expanded to 384 # name = 'max error in x (mm)' 385 for variation in name_variations: 386 name = template.format(variation) 387 self.metrics_props[name] = metric_props 388 else: 389 # Otherwise, the template is already the name. 390 # E.g., the template 'max distance (mm)' is same as the name. 391 self.metrics_props[template] = metric_props 392 393 394class ValidatorLog: 395 """A class handling the logs reported by validators.""" 396 def __init__(self): 397 self.name = None 398 self.details = [] 399 self.criteria = None 400 self.score = None 401 self.metrics = [] 402 self.error = None 403 404 def reset(self): 405 """Reset all attributes.""" 406 self.details = [] 407 self.score = None 408 self.metrics = [] 409 self.error = None 410 411 def insert_details(self, msg): 412 """Insert a msg into the details.""" 413 self.details.append(msg) 414 415 416class GestureLog: 417 """A class handling the logs related with a gesture.""" 418 def __init__(self): 419 self.name = None 420 self.variation = None 421 self.prompt = None 422 self.vlogs = [] 423 424 425class RoundLog: 426 """Manipulate the test result log generated in a single round.""" 427 def __init__(self, test_version, fw=None, round_name=None): 428 self._test_version = test_version 429 self._fw = fw 430 self._round_name = round_name 431 self._glogs = [] 432 433 def dump(self, filename): 434 """Dump the log to the specified filename.""" 435 try: 436 with open(filename, 'w') as log_file: 437 pickle.dump([self._fw, self._round_name, self._test_version, 438 self._glogs], log_file) 439 except Exception, e: 440 msg = 'Error in dumping to the log file (%s): %s' % (filename, e) 441 print_and_exit(msg) 442 443 @staticmethod 444 def load(filename): 445 """Load the log from the pickle file.""" 446 try: 447 with open(filename) as log_file: 448 return pickle.load(log_file) 449 except Exception, e: 450 msg = 'Error in loading the log file (%s): %s' % (filename, e) 451 print_and_exit(msg) 452 453 def insert_glog(self, glog): 454 """Insert the gesture log into the round log.""" 455 if glog.vlogs: 456 self._glogs.append(glog) 457 458 459class StatisticsScores: 460 """A statistics class to compute the average, ssd, and count of 461 aggregate scores. 462 """ 463 def __init__(self, scores): 464 self.all_data = () 465 if scores: 466 self.average = np.average(np.array(scores)) 467 self.ssd = _calc_sample_standard_deviation(scores) 468 self.count = len(scores) 469 self.all_data = (self.average, self.ssd, self.count) 470 471 472class StatisticsMetrics: 473 """A statistics class to compute the statistics including the min, max, or 474 average of aggregate metrics. 475 """ 476 477 def __init__(self, metrics): 478 """Collect all values for every metric. 479 480 @param metrics: a list of Metric objects. 481 """ 482 # metrics_values: the raw metrics values 483 self.metrics_values = defaultdict(list) 484 self.metrics_dict = defaultdict(list) 485 for metric in metrics: 486 self.metrics_values[metric.name].append(metric.value) 487 self.metrics_dict[metric.name].append(metric) 488 489 # Calculate the statistics of metrics using corresponding stat functions 490 self._calc_statistics(MetricNameProps().metrics_props) 491 492 def _calc_statistics(self, metrics_props): 493 """Calculate the desired statistics for every metric. 494 495 @param metrics_props: a dictionary mapping a metric name to a 496 metric props including the description and stat_func 497 """ 498 self.metrics_props = metrics_props 499 self.stats_values = {} 500 for metric_name, values in self.metrics_values.items(): 501 assert metric_name in metrics_props, ( 502 'The metric name "%s" cannot be found.' % metric_name) 503 stat_func = metrics_props[metric_name].stat_func 504 self.stats_values[metric_name] = stat_func(values) 505 506 507class TestResult: 508 """A class includes the statistics of the score and the metrics.""" 509 def __init__(self, scores, metrics): 510 self.stat_scores = StatisticsScores(scores) 511 self.stat_metrics = StatisticsMetrics(metrics) 512 513 514class SimpleTable: 515 """A very simple data table.""" 516 def __init__(self): 517 """This initializes a simple table.""" 518 self._table = defaultdict(list) 519 520 def insert(self, key, value): 521 """Insert a row. If the key exists already, the value is appended.""" 522 self._table[key].append(value) 523 debug_print(' key: %s' % str(key)) 524 525 def search(self, key): 526 """Search rows with the specified key. 527 528 A key is a list of attributes. 529 If any attribute is None, it means no need to match this attribute. 530 """ 531 match = lambda i, j: i == j or j is None 532 return filter(lambda (k, vlog): all(map(match, k, key)), 533 self._table.items()) 534 535 def items(self): 536 """Return the table items.""" 537 return self._table.items() 538 539 540class SummaryLog: 541 """A class to manipulate the summary logs. 542 543 A summary log may consist of result logs of different firmware versions 544 where every firmware version may consist of multiple rounds. 545 """ 546 def __init__(self, log_dir, segment_weights, validator_weights, 547 individual_round_flag, debug_flag): 548 self.log_dir = log_dir 549 self.segment_weights = segment_weights 550 self.validator_weights = validator_weights 551 self.individual_round_flag = individual_round_flag 552 _setup_debug(debug_flag) 553 self._read_logs() 554 self.ext_validator_weights = {} 555 for fw, validators in self.fw_validators.items(): 556 self.ext_validator_weights[fw] = \ 557 self._compute_extended_validator_weight(validators) 558 559 def _get_firmware_version(self, filename): 560 """Get the firmware version from the given filename.""" 561 return filename.split('-')[2] 562 563 def _read_logs(self): 564 """Read the result logs in the specified log directory.""" 565 # Get logs in the log_dir or its sub-directories. 566 log_filenames = glob.glob(os.path.join(self.log_dir, '*.log')) 567 if not log_filenames: 568 log_filenames = glob.glob(os.path.join(self.log_dir, '*', '*.log')) 569 570 if not log_filenames: 571 err_msg = 'Error: no log files in the test result directory: %s' 572 print_and_exit(err_msg % self.log_dir) 573 574 self.log_table = SimpleTable() 575 self.fws = set() 576 self.gestures = set() 577 # fw_validators keeps track of the validators of every firmware 578 self.fw_validators = defaultdict(set) 579 580 for i, log_filename in enumerate(log_filenames): 581 round_no = i if self.individual_round_flag else None 582 self._add_round_log(log_filename, round_no) 583 584 # Convert set to list below 585 self.fws = sorted(list(self.fws)) 586 self.gestures = sorted(list(self.gestures)) 587 # Construct validators by taking the union of the validators of 588 # all firmwares. 589 self.validators = sorted(list(set.union(*self.fw_validators.values()))) 590 591 for fw in self.fws: 592 self.fw_validators[fw] = sorted(list(self.fw_validators[fw])) 593 594 def _add_round_log(self, log_filename, round_no): 595 """Add the round log, decompose the validator logs, and build 596 a flat summary log. 597 """ 598 log_data = RoundLog.load(log_filename) 599 if len(log_data) == 3: 600 fw, round_name, glogs = log_data 601 self.test_version = 'test_version: NA' 602 elif len(log_data) == 4: 603 fw, round_name, self.test_version, glogs = log_data 604 else: 605 print 'Error: the log format is unknown.' 606 sys.exit(1) 607 608 if round_no is not None: 609 fw = '%s_%d' % (fw, round_no) 610 self.fws.add(fw) 611 debug_print(' fw(%s) round(%s)' % (fw, round_name)) 612 613 # Iterate through every gesture_variation of the round log, 614 # and generate a flat dictionary of the validator logs. 615 for glog in glogs: 616 self.gestures.add(glog.name) 617 for vlog in glog.vlogs: 618 self.fw_validators[fw].add(vlog.name) 619 key = (fw, round_name, glog.name, glog.variation, vlog.name) 620 self.log_table.insert(key, vlog) 621 622 def _compute_extended_validator_weight(self, validators): 623 """Compute extended validator weight from validator weight and segment 624 weight. The purpose is to merge the weights of split validators, e.g. 625 Linearity(*)Validator, so that their weights are not counted multiple 626 times. 627 628 Example: 629 validators = ['CountTrackingIDValidator', 630 'Linearity(BothEnds)Validator', 631 'Linearity(Middle)Validator', 632 'NoGapValidator'] 633 634 Note that both names of the validators 635 'Linearity(BothEnds)Validator' and 636 'Linearity(Middle)Validator' 637 are created at run time from LinearityValidator and use 638 the relative weights defined by segment_weights. 639 640 validator_weights = {'CountTrackingIDValidator': 12, 641 'LinearityValidator': 10, 642 'NoGapValidator': 10} 643 644 segment_weights = {'Middle': 0.7, 645 'BothEnds': 0.3} 646 647 split_validator = {'Linearity': ['BothEnds', 'Middle'],} 648 649 adjusted_weight of Lineary(*)Validator: 650 Linearity(BothEnds)Validator = 0.3 / (0.3 + 0.7) * 10 = 3 651 Linearity(Middle)Validator = 0.7 / (0.3 + 0.7) * 10 = 7 652 653 extended_validator_weights: {'CountTrackingIDValidator': 12, 654 'Linearity(BothEnds)Validator': 3, 655 'Linearity(Middle)Validator': 7, 656 'NoGapValidator': 10} 657 """ 658 extended_validator_weights = {} 659 split_validator = {} 660 661 # Copy the base validator weight into extended_validator_weights. 662 # For the split validators, collect them in split_validator. 663 for v in validators: 664 base_name, segment = val.get_base_name_and_segment(v) 665 if segment is None: 666 # It is a base validator. Just copy it into the 667 # extended_validaotr_weight dict. 668 extended_validator_weights[v] = self.validator_weights[v] 669 else: 670 # It is a derived validator, e.g., Linearity(BothEnds)Validator 671 # Needs to compute its adjusted weight. 672 673 # Initialize the split_validator for this base_name if not yet. 674 if split_validator.get(base_name) is None: 675 split_validator[base_name] = [] 676 677 # Append this segment name so that we know all segments for 678 # the base_name. 679 split_validator[base_name].append(segment) 680 681 # Compute the adjusted weight for split_validator 682 for base_name in split_validator: 683 name = val.get_validator_name(base_name) 684 weight_list = [self.segment_weights[segment] 685 for segment in split_validator[base_name]] 686 weight_sum = sum(weight_list) 687 for segment in split_validator[base_name]: 688 derived_name = val.get_derived_name(name, segment) 689 adjusted_weight = (self.segment_weights[segment] / weight_sum * 690 self.validator_weights[name]) 691 extended_validator_weights[derived_name] = adjusted_weight 692 693 return extended_validator_weights 694 695 def get_result(self, fw=None, round=None, gesture=None, variation=None, 696 validators=None): 697 """Get the result statistics of a validator which include both 698 the score and the metrics. 699 700 If validators is a list, every validator in the list is used to query 701 the log table, and all results are merged to get the final result. 702 For example, both StationaryFingerValidator and StationaryTapValidator 703 inherit StationaryValidator. The results of those two extended classes 704 will be merged into StationaryValidator. 705 """ 706 if not isinstance(validators, list): 707 validators = [validators,] 708 709 rows = [] 710 for validator in validators: 711 key = (fw, round, gesture, variation, validator) 712 rows.extend(self.log_table.search(key)) 713 714 scores = [vlog.score for _key, vlogs in rows for vlog in vlogs] 715 metrics = [metric.insert_key(_key) for _key, vlogs in rows 716 for vlog in vlogs 717 for metric in vlog.metrics] 718 return TestResult(scores, metrics) 719 720 def get_final_weighted_average(self): 721 """Calculate the final weighted average.""" 722 weighted_average = {} 723 # for fw in self.fws: 724 for fw, validators in self.fw_validators.items(): 725 scores = [self.get_result(fw=fw, validators=val).stat_scores.average 726 for val in validators] 727 _, weights = zip(*sorted(self.ext_validator_weights[fw].items())) 728 weighted_average[fw] = np.average(scores, weights=weights) 729 return weighted_average 730