• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""This module contains unit tests for firmware_summary module."""
6
7
8import os
9import unittest
10
11import common_unittest_utils
12
13from firmware_constants import VAL
14from firmware_summary import FirmwareSummary
15
16
17# Define the relative segment weights of a validator.
18segment_weights = {VAL.BEGIN: 0.15,
19                   VAL.MIDDLE: 0.7,
20                   VAL.END: 0.15,
21                   VAL.BOTH_ENDS: 0.15 + 0.15,
22                   VAL.WHOLE: 0.15 + 0.7 + 0.15}
23
24# Define the validator score weights
25weight_rare = 1
26weight_common = 2
27weight_critical = 3
28validator_weights = {'CountPacketsValidator': weight_common,
29                     'CountTrackingIDFatFingerValidator': weight_rare,
30                     'CountTrackingIDNormalFingerValidator': weight_critical,
31                     'DrumrollValidator': weight_rare,
32                     'LinearityFatFingerValidator': weight_rare,
33                     'LinearityNormalFingerValidator': weight_common,
34                     'NoGapValidator': weight_common,
35                     'NoLevelJumpValidator': weight_rare,
36                     'NoReversedMotionValidator': weight_common,
37                     'PhysicalClickValidator': weight_critical,
38                     'PinchValidator': weight_common,
39                     'RangeValidator': weight_common,
40                     'ReportRateValidator': weight_common,
41                     'StationaryFingerValidator': weight_common,
42                     'StationaryTapValidator': weight_common}
43
44
45class FirmwareSummaryTest(unittest.TestCase):
46    """A base class for FirwareSummary unit tests."""
47    @classmethod
48    def setUpClass(cls):
49        test_dir = os.path.join(os.getcwd(), 'tests')
50        log_dir = os.path.join(test_dir, 'logs', cls.log_category)
51        summary = FirmwareSummary(log_dir=log_dir,
52                                  validator_weights=validator_weights,
53                                  segment_weights=segment_weights)
54        cls.slog = summary.slog
55        cls._round_digits = 8
56
57    def _get_score(self, fw=None, gesture=None, validator=None):
58        """Score = sum / count, rounded to the 4th digit."""
59        result= self.slog.get_result(fw=fw, gesture=gesture,
60                                     validators=validator)
61        average = result.stat_scores.average
62        return round(average, self._round_digits)
63
64
65class FirmwareSummaryLumpyTest(FirmwareSummaryTest):
66    """Unit tests for firmware_summary.FirmwareSummary class using Lumpy logs.
67
68    Tests were conducted with both fw 11.23 and 11.26, and in combination of
69    single and multiple iterations.
70    """
71    @classmethod
72    def setUpClass(cls):
73        cls.log_category = 'lumpy'
74        cls.fws = ['fw_11.23', 'fw_11.27']
75        super(FirmwareSummaryLumpyTest, cls).setUpClass()
76
77    def _test_by_gesture(self, validator, expected_scores):
78        for fw, fw_expected_scores in expected_scores.items():
79            for gesture, expected_score in fw_expected_scores.items():
80                actual_score = self._get_score(fw=fw,
81                                               gesture=gesture,
82                                               validator=validator)
83                self.assertAlmostEqual(actual_score, expected_score)
84
85    def test_by_gesture_CountTrackingIDNormalFingerValidator(self):
86        validator = 'CountTrackingIDNormalFingerValidator'
87        expected_scores = {
88            'fw_11.23': {
89                'one_finger_tracking': 1.0,
90                'two_finger_tracking': 1.0,
91            },
92            'fw_11.27': {
93                'one_finger_tracking': 1.0,
94                'two_finger_tracking': 1.0,
95            }
96        }
97        self._test_by_gesture(validator, expected_scores)
98
99    def test_by_gesture_CountTrackingIDFatFingerValidator(self):
100        validator = 'CountTrackingIDFatFingerValidator'
101        expected_scores = {
102            'fw_11.23': {'drag_edge_thumb': 0.0,},
103            'fw_11.27': {'drag_edge_thumb': 0.5,}
104        }
105        self._test_by_gesture(validator, expected_scores)
106
107    def test_by_gesture_DrumrollValidator(self):
108        validator = 'DrumrollValidator'
109        expected_scores = {
110            'fw_11.23': {
111                'drumroll': 0.75,
112            },
113            'fw_11.27': {
114                'drumroll': 0.66666667,
115            }
116        }
117        self._test_by_gesture(validator, expected_scores)
118
119    def test_by_gesture_LinearityMiddleValidator(self):
120        validator = 'LinearityNormalFinger(Middle)Validator'
121        expected_scores = {
122            'fw_11.23': {
123                'one_finger_to_edge': 0.58086671000000001,
124                'one_finger_tracking': 0.42046572999999998,
125                'two_finger_tracking': 0.60548126000000002,
126            },
127            'fw_11.27': {
128                'one_finger_to_edge': 0.36506074999999999,
129                'one_finger_tracking': 0.73313022999999999,
130                'two_finger_tracking': 0.70906895999999997,
131            }
132        }
133        self._test_by_gesture(validator, expected_scores)
134
135    def test_by_gesture_NoGapValidator(self):
136        validator = 'NoGapValidator'
137        expected_scores = {
138            'fw_11.23': {
139                'one_finger_to_edge': 0.16022362,
140                'one_finger_tracking': 0.11006574,
141                'two_finger_tracking': 0.09455679,
142            },
143            'fw_11.27': {
144                'one_finger_to_edge': 0.00000000,
145                'one_finger_tracking': 0.86488696,
146                'two_finger_tracking': 0.76206434,
147            }
148        }
149        self._test_by_gesture(validator, expected_scores)
150
151    def test_by_gesture_PhysicalClickValidator(self):
152        validator = 'PhysicalClickValidator'
153        expected_scores = {
154            'fw_11.23': {
155                'one_finger_physical_click': 0.875,
156                'two_fingers_physical_click': 0.25,
157            },
158            'fw_11.27': {
159                'one_finger_physical_click': 1.0,
160                'two_fingers_physical_click': 1.0,
161            }
162        }
163        self._test_by_gesture(validator, expected_scores)
164
165    def test_by_gesture_StationaryTapValidator(self):
166        validator = 'StationaryTapValidator'
167        expected_scores = {
168            'fw_11.23': {
169                'one_finger_physical_click': 0.1875,
170                'two_fingers_physical_click': 0.125,
171            },
172            'fw_11.27': {
173                'one_finger_physical_click': 0.58333332999999998,
174                'two_fingers_physical_click': 0.16666666999999999,
175            }
176        }
177        self._test_by_gesture(validator, expected_scores)
178
179    def test_by_validator(self):
180        expected_scores = {
181            'fw_11.23': {
182                'CountPacketsValidator': 0.895833333333,
183                'CountTrackingIDFatFingerValidator': 0.0,
184                'CountTrackingIDNormalFingerValidator': 0.96666666999999995,
185                'DrumrollValidator': 0.75,
186                'LinearityNormalFinger(Middle)Validator': 0.54910331999999995,
187                'NoGapValidator': 0.101144302433,
188                'PhysicalClickValidator': 0.75,
189                'PinchValidator': 0.875,
190                'StationaryTapValidator': 0.16666666999999999,
191            },
192            'fw_11.27': {
193                'CountPacketsValidator': 1.0,
194                'CountTrackingIDFatFingerValidator': 0.5,
195                'CountTrackingIDNormalFingerValidator': 0.95555555999999997,
196                'DrumrollValidator': 0.666666666667,
197                'LinearityNormalFinger(Middle)Validator': 0.66679957999999995,
198                'NoGapValidator': 0.623221473233,
199                'PhysicalClickValidator': 1.0,
200                'PinchValidator': 1.0,
201                'StationaryTapValidator': 0.44444444,
202            }
203        }
204        for fw, fw_expected_scores in expected_scores.items():
205            for validator, expected_score in fw_expected_scores.items():
206                actual_score = self._get_score(fw=fw, validator=validator)
207                actual_score = round(actual_score, self._round_digits)
208                self.assertAlmostEqual(actual_score, expected_score)
209
210    def test_stat_metrics(self):
211        """Test the statistics of metrics."""
212        expected_stats_values = {
213            'fw_11.23': {
214                'CountPacketsValidator': [
215                    ('pct of incorrect cases (%)--packets', 25.00)],
216                'PhysicalClickValidator': [
217                    ('1f-click miss rate (%)', 12.50),
218                    ('2f-click miss rate (%)', 75.00)],
219                'PinchValidator': [
220                    ('pct of incorrect cases (%)--pinch', 12.50)],
221            },
222            'fw_11.27': {
223                'CountPacketsValidator': [
224                    ('pct of incorrect cases (%)--packets', 0.00)],
225                'PhysicalClickValidator': [
226                    ('1f-click miss rate (%)', 0.00),
227                    ('2f-click miss rate (%)', 0.00)],
228                'PinchValidator': [
229                    ('pct of incorrect cases (%)--pinch', 0.00)],
230            },
231        }
232
233        for fw, fw_stats_values in expected_stats_values.items():
234            for validator, stats_metrics in fw_stats_values.items():
235                result = self.slog.get_result(fw=fw, validators=validator)
236                for metric_name, expected_value in stats_metrics:
237                    actual_value = result.stat_metrics.stats_values[metric_name]
238                    self.assertAlmostEqual(actual_value, expected_value)
239
240    def test_final_weighted_average(self):
241        expected_weighted_averages = {
242            'fw_11.23': 0.68406327,
243            'fw_11.27': 0.83886367,
244        }
245        final_weighted_average = self.slog.get_final_weighted_average()
246        for fw, expected_value in expected_weighted_averages.items():
247            actual_value = final_weighted_average[fw]
248            actual_value = round(actual_value, self._round_digits)
249            self.assertAlmostEqual(actual_value, expected_value)
250
251
252if __name__ == '__main__':
253  unittest.main()
254