• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2#
3#   Copyright 2016 - The Android Open Source Project
4#
5#   Licensed under the Apache License, Version 2.0 (the "License");
6#   you may not use this file except in compliance with the License.
7#   You may obtain a copy of the License at
8#
9#       http://www.apache.org/licenses/LICENSE-2.0
10#
11#   Unless required by applicable law or agreed to in writing, software
12#   distributed under the License is distributed on an "AS IS" BASIS,
13#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14#   See the License for the specific language governing permissions and
15#   limitations under the License.
16
17from mock import Mock
18from mock import patch
19import tempfile
20import unittest
21from unittest import TestCase
22from acts.base_test import BaseTestClass
23from acts.metrics.loggers.blackbox import BlackboxMetricLogger
24from acts.test_runner import TestRunner
25
26COMPILE_IMPORT_PROTO = 'acts.metrics.logger.compile_import_proto'
27GET_CONTEXT_FOR_EVENT = 'acts.metrics.logger.get_context_for_event'
28PROTO_METRIC_PUBLISHER = 'acts.metrics.logger.ProtoMetricPublisher'
29
30
31class BlackboxMetricLoggerTest(TestCase):
32    """Unit tests for BlackboxMetricLogger."""
33
34    TEST_METRIC_NAME = "metric_name"
35    TEST_FILE_NAME = "blackbox_metric_name"
36
37    def setUp(self):
38        self.proto_module = Mock()
39        self.event = Mock()
40        self.context = Mock()
41        self.publisher = Mock()
42        self._get_blackbox_identifier = lambda: str(id(self.context))
43
44    @patch(COMPILE_IMPORT_PROTO)
45    def test_default_init_attributes(self, compile_import_proto):
46        metric_name = Mock()
47        compile_import_proto.return_value = self.proto_module
48
49        logger = BlackboxMetricLogger(metric_name)
50
51        self.assertEqual(logger.metric_name, metric_name)
52        self.assertEqual(logger.proto_module, self.proto_module)
53        self.assertEqual(logger.result_attr, 'result')
54        self.assertIsNone(logger.metric_key)
55
56    @patch(COMPILE_IMPORT_PROTO)
57    def test_init_with_params(self, compile_import_proto):
58        metric_name = Mock()
59        result_attr = Mock()
60        metric_key = Mock()
61
62        logger = BlackboxMetricLogger(metric_name,
63                                      result_attr=result_attr,
64                                      metric_key=metric_key)
65
66        self.assertEqual(logger.result_attr, result_attr)
67        self.assertEqual(logger.metric_key, metric_key)
68
69    @patch(PROTO_METRIC_PUBLISHER)
70    @patch(GET_CONTEXT_FOR_EVENT)
71    @patch(COMPILE_IMPORT_PROTO)
72    def test_init_with_event(self,
73                             compile_import_proto,
74                             get_context,
75                             publisher_cls):
76        metric_name = Mock()
77
78        logger = BlackboxMetricLogger(metric_name, event=self.event)
79
80        self.assertIsNotNone(logger.context)
81        self.assertIsNotNone(logger.publisher)
82
83    @patch(COMPILE_IMPORT_PROTO)
84    def test_end_populates_result(self, compile_import_proto):
85        result = Mock()
86        compile_import_proto.return_value = self.proto_module
87        self.proto_module.ActsBlackboxMetricResult.return_value = result
88
89        logger = BlackboxMetricLogger(self.TEST_METRIC_NAME)
90        logger.context = self.context
91        logger.publisher = self.publisher
92        logger.context.identifier = 'Class.test'
93
94        logger.end(self.event)
95
96        self.assertEqual(result.test_identifier, 'Class#test')
97        self.assertEqual(result.metric_key, '%s.%s' % ('Class#test',
98                                                       self.TEST_METRIC_NAME))
99        self.assertEqual(result.metric_value, self.context.test_class.result)
100
101    @patch(COMPILE_IMPORT_PROTO)
102    def test_end_uses_custom_result_attr(self, compile_import_proto):
103        result = Mock()
104        compile_import_proto.return_value = self.proto_module
105        self.proto_module.ActsBlackboxMetricResult.return_value = result
106        result_attr = 'result_attr'
107
108        logger = BlackboxMetricLogger(self.TEST_METRIC_NAME,
109                                      result_attr=result_attr)
110        logger.context = self.context
111        logger.publisher = self.publisher
112        logger._get_blackbox_identifier = self._get_blackbox_identifier
113
114        logger.end(self.event)
115
116        self.assertEqual(result.metric_value,
117                         getattr(self.context.test_class, result_attr))
118
119    @patch(COMPILE_IMPORT_PROTO)
120    def test_end_uses_metric_value_on_result_attr_none(self,
121                                                       compile_import_proto):
122        result = Mock()
123        expected_result = Mock()
124        compile_import_proto.return_value = self.proto_module
125        self.proto_module.ActsBlackboxMetricResult.return_value = result
126        result_attr = None
127
128        logger = BlackboxMetricLogger(self.TEST_METRIC_NAME,
129                                      result_attr=result_attr)
130        logger.context = self.context
131        logger.publisher = self.publisher
132        logger._get_blackbox_identifier = self._get_blackbox_identifier
133        logger.metric_value = expected_result
134        logger.end(self.event)
135
136        self.assertEqual(result.metric_value, expected_result)
137
138    @patch(COMPILE_IMPORT_PROTO)
139    def test_end_uses_metric_value_on_metric_value_not_none(
140            self, compile_import_proto):
141        result = Mock()
142        expected_result = Mock()
143        compile_import_proto.return_value = self.proto_module
144        self.proto_module.ActsBlackboxMetricResult.return_value = result
145        result_attr = 'result_attr'
146
147        logger = BlackboxMetricLogger(self.TEST_METRIC_NAME,
148                                      result_attr=result_attr)
149        logger.context = self.context
150        logger.context.identifier = 'Class.test'
151        logger.publisher = self.publisher
152        logger.metric_value = expected_result
153        logger.end(self.event)
154
155        self.assertEqual(result.metric_value, expected_result)
156
157    @patch(COMPILE_IMPORT_PROTO)
158    def test_end_uses_custom_metric_key(self, compile_import_proto):
159        result = Mock()
160        compile_import_proto.return_value = self.proto_module
161        self.proto_module.ActsBlackboxMetricResult.return_value = result
162        metric_key = 'metric_key'
163
164        logger = BlackboxMetricLogger(self.TEST_METRIC_NAME,
165                                      metric_key=metric_key)
166        logger.context = self.context
167        logger.publisher = self.publisher
168        logger._get_blackbox_identifier = self._get_blackbox_identifier
169
170        logger.end(self.event)
171
172        expected_metric_key = '%s.%s' % (metric_key, self.TEST_METRIC_NAME)
173        self.assertEqual(result.metric_key, expected_metric_key)
174
175    @patch('acts.metrics.loggers.blackbox.ProtoMetric')
176    @patch(COMPILE_IMPORT_PROTO)
177    def test_end_does_publish(self, compile_import_proto, proto_metric_cls):
178        result = Mock()
179        compile_import_proto.return_value = self.proto_module
180        self.proto_module.ActsBlackboxMetricResult.return_value = result
181        metric_key = 'metric_key'
182
183        logger = BlackboxMetricLogger(self.TEST_METRIC_NAME,
184                                      metric_key=metric_key)
185        logger.context = self.context
186        logger.publisher = self.publisher
187        logger._get_blackbox_identifier = self._get_blackbox_identifier
188
189        logger.end(self.event)
190
191        proto_metric_cls.assert_called_once_with(name=self.TEST_FILE_NAME,
192                                                 data=result)
193        self.publisher.publish.assert_called_once_with(
194            proto_metric_cls.return_value)
195
196
197class BlackboxMetricLoggerIntegrationTest(TestCase):
198    """Integration tests for BlackboxMetricLogger."""
199
200    @patch('acts.test_runner.sys')
201    @patch('acts.test_runner.utils')
202    @patch('acts.test_runner.importlib')
203    def run_acts_test(self, test_class, importlib, utils, sys):
204        config = {
205            "testbed": {
206                "name": "SampleTestBed",
207            },
208            "logpath": tempfile.mkdtemp(),
209            "cli_args": None,
210            "testpaths": ["./"],
211        }
212        mockModule = Mock()
213        setattr(mockModule, test_class.__name__, test_class)
214        utils.find_files.return_value = [(None, None, None)]
215        importlib.import_module.return_value = mockModule
216        runner = TestRunner(config, [(test_class.__name__, None,)])
217
218        runner.run()
219        runner.stop()
220        return runner
221
222    @patch('acts.metrics.logger.ProtoMetricPublisher')
223    def test_test_case_metric(self, publisher_cls):
224        result = 5.0
225
226        class MyTest(BaseTestClass):
227            def __init__(self, controllers):
228                BaseTestClass.__init__(self, controllers)
229                self.tests = ('test_case',)
230                BlackboxMetricLogger.for_test_case('my_metric')
231
232            def test_case(self):
233                self.result = result
234
235        self.run_acts_test(MyTest)
236
237        args_list = publisher_cls().publish.call_args_list
238        self.assertEqual(len(args_list), 1)
239        metric = self.__get_only_arg(args_list[0])
240        self.assertEqual(metric.name, 'blackbox_my_metric')
241        self.assertEqual(metric.data.test_identifier, 'MyTest#test_case')
242        self.assertEqual(metric.data.metric_key, 'MyTest#test_case.my_metric')
243        self.assertEqual(metric.data.metric_value, result)
244
245    @patch('acts.metrics.logger.ProtoMetricPublisher')
246    def test_multiple_test_case_metrics(self, publisher_cls):
247        result = 5.0
248
249        class MyTest(BaseTestClass):
250            def __init__(self, controllers):
251                BaseTestClass.__init__(self, controllers)
252                self.tests = ('test_case',)
253                BlackboxMetricLogger.for_test_case('my_metric_1')
254                BlackboxMetricLogger.for_test_case('my_metric_2')
255
256            def test_case(self):
257                self.result = result
258
259        self.run_acts_test(MyTest)
260
261        args_list = publisher_cls().publish.call_args_list
262        self.assertEqual(len(args_list), 2)
263        metrics = [self.__get_only_arg(args) for args in args_list]
264        self.assertEqual(
265            {metric.name for metric in metrics},
266            {'blackbox_my_metric_1', 'blackbox_my_metric_2'})
267        self.assertEqual(
268            {metric.data.test_identifier for metric in metrics},
269            {'MyTest#test_case'})
270        self.assertEqual(
271            {metric.data.metric_key for metric in metrics},
272            {'MyTest#test_case.my_metric_1', 'MyTest#test_case.my_metric_2'})
273        self.assertEqual(
274            {metric.data.metric_value for metric in metrics},
275            {result})
276
277    @patch('acts.metrics.logger.ProtoMetricPublisher')
278    def test_test_case_metric_with_custom_key(self, publisher_cls):
279        result = 5.0
280
281        class MyTest(BaseTestClass):
282            def __init__(self, controllers):
283                BaseTestClass.__init__(self, controllers)
284                self.tests = ('test_case',)
285                BlackboxMetricLogger.for_test_case('my_metric',
286                                                   metric_key='my_metric_key')
287
288            def test_case(self):
289                self.result = result
290
291        self.run_acts_test(MyTest)
292
293        args_list = publisher_cls().publish.call_args_list
294        self.assertEqual(len(args_list), 1)
295        metric = self.__get_only_arg(args_list[0])
296        self.assertEqual(metric.data.metric_key, 'my_metric_key.my_metric')
297
298    @patch('acts.metrics.logger.ProtoMetricPublisher')
299    def test_test_case_metric_with_custom_result_attr(self, publisher_cls):
300        true_result = 5.0
301        other_result = 10.0
302
303        class MyTest(BaseTestClass):
304            def __init__(self, controllers):
305                BaseTestClass.__init__(self, controllers)
306                self.tests = ('test_case',)
307                BlackboxMetricLogger.for_test_case('my_metric',
308                                                   result_attr='true_result')
309
310            def test_case(self):
311                self.true_result = true_result
312                self.result = other_result
313
314        self.run_acts_test(MyTest)
315
316        args_list = publisher_cls().publish.call_args_list
317        self.assertEqual(len(args_list), 1)
318        metric = self.__get_only_arg(args_list[0])
319        self.assertEqual(metric.data.metric_value, true_result)
320
321    @patch('acts.metrics.logger.ProtoMetricPublisher')
322    def test_test_class_metric(self, publisher_cls):
323        publisher_cls().publish = Mock()
324        result_1 = 5.0
325        result_2 = 8.0
326
327        class MyTest(BaseTestClass):
328            def __init__(self, controllers):
329                BaseTestClass.__init__(self, controllers)
330                self.tests = ('test_case_1', 'test_case_2',)
331                BlackboxMetricLogger.for_test_class('my_metric')
332                self.result = 0
333
334            def test_case_1(self):
335                self.result += result_1
336
337            def test_case_2(self):
338                self.result += result_2
339
340        self.run_acts_test(MyTest)
341
342        args_list = publisher_cls().publish.call_args_list
343        self.assertEqual(len(args_list), 1)
344        metric = self.__get_only_arg(args_list[0])
345        self.assertEqual(metric.data.metric_value, result_1 + result_2)
346        self.assertEqual(metric.data.test_identifier, MyTest.__name__)
347
348    def __get_only_arg(self, call_args):
349        self.assertEqual(len(call_args[0]) + len(call_args[1]), 1)
350        if len(call_args[0]) == 1:
351            return call_args[0][0]
352        return next(iter(call_args[1].values()))
353
354
355if __name__ == '__main__':
356    unittest.main()
357