1#!/usr/bin/env python3 2# 3# Copyright 2016 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17import shutil 18import tempfile 19import unittest 20from unittest import TestCase 21import warnings 22 23from acts.base_test import BaseTestClass 24from acts.metrics.loggers.blackbox import BlackboxMetricLogger 25from acts.test_runner import TestRunner 26from mobly.config_parser import TestRunConfig 27from mock import call 28from mock import Mock 29from mock import patch 30 31GET_CONTEXT_FOR_EVENT = 'acts.metrics.logger.get_context_for_event' 32PROTO_METRIC_PUBLISHER = 'acts.metrics.logger.ProtoMetricPublisher' 33 34 35class BlackboxMetricLoggerTest(TestCase): 36 """Unit tests for BlackboxMetricLogger.""" 37 38 TEST_METRIC_NAME = "metric_name" 39 40 def setUp(self): 41 self.event = Mock() 42 self.context = Mock() 43 self.publisher = Mock() 44 self._get_blackbox_identifier = lambda: str(id(self.context)) 45 46 def test_default_init_attributes(self): 47 metric_name = Mock() 48 49 logger = BlackboxMetricLogger(metric_name) 50 51 self.assertEqual(logger.metric_name, metric_name) 52 self.assertIsNone(logger.metric_key) 53 54 def test_init_with_params(self): 55 metric_name = Mock() 56 metric_key = Mock() 57 58 logger = BlackboxMetricLogger(metric_name, metric_key=metric_key) 59 60 self.assertEqual(logger.metric_key, metric_key) 61 62 @patch(PROTO_METRIC_PUBLISHER) 63 @patch(GET_CONTEXT_FOR_EVENT) 64 def test_init_with_event(self, _get_context, _publisher_cls): 65 metric_name = Mock() 66 67 logger = BlackboxMetricLogger(metric_name, event=self.event) 68 69 self.assertIsNotNone(logger.context) 70 self.assertIsNotNone(logger.publisher) 71 72 @patch('acts.metrics.loggers.protos.gen.acts_blackbox_pb2' 73 '.ActsBlackboxMetricResultsBundle') 74 @patch('acts.metrics.loggers.protos.gen.acts_blackbox_pb2' 75 '.ActsBlackboxMetricResult') 76 def test_end_populates_result(self, mock_acts_blackbox, 77 _mock_acts_blackbox_bundle): 78 result = Mock() 79 mock_acts_blackbox.return_value = result 80 81 logger = BlackboxMetricLogger(self.TEST_METRIC_NAME) 82 logger.context = self.context 83 logger.publisher = self.publisher 84 logger.context.identifier = 'Class.test' 85 logger.metric_value = 'foo' 86 87 logger.end(self.event) 88 89 self.assertEqual(result.test_identifier, 'Class#test') 90 self.assertEqual(result.metric_key, 91 '%s.%s' % ('Class#test', self.TEST_METRIC_NAME)) 92 self.assertEqual(result.metric_value, logger.metric_value) 93 94 @patch('acts.metrics.loggers.protos.gen.acts_blackbox_pb2' 95 '.ActsBlackboxMetricResultsBundle') 96 @patch('acts.metrics.loggers.protos.gen.acts_blackbox_pb2' 97 '.ActsBlackboxMetricResult') 98 def test_end_uses_metric_value_on_metric_value_not_none( 99 self, mock_acts_blackbox, _mock_acts_blackbox_bundle): 100 result = Mock() 101 expected_result = Mock() 102 mock_acts_blackbox.return_value = result 103 104 logger = BlackboxMetricLogger(self.TEST_METRIC_NAME) 105 logger.context = self.context 106 logger.context.identifier = 'Class.test' 107 logger.publisher = self.publisher 108 logger.metric_value = expected_result 109 logger.end(self.event) 110 111 self.assertEqual(result.metric_value, expected_result) 112 113 @patch('acts.metrics.loggers.protos.gen.acts_blackbox_pb2' 114 '.ActsBlackboxMetricResultsBundle') 115 @patch('acts.metrics.loggers.protos.gen.acts_blackbox_pb2' 116 '.ActsBlackboxMetricResult') 117 def test_end_uses_custom_metric_key(self, mock_acts_blackbox, 118 _mock_acts_blackbox_bundle): 119 result = Mock() 120 mock_acts_blackbox.return_value = result 121 metric_key = 'metric_key' 122 123 logger = BlackboxMetricLogger(self.TEST_METRIC_NAME, 124 metric_key=metric_key) 125 logger.context = self.context 126 logger.publisher = self.publisher 127 logger._get_blackbox_identifier = self._get_blackbox_identifier 128 logger.metric_value = 'foo' 129 130 logger.end(self.event) 131 132 expected_metric_key = '%s.%s' % (metric_key, self.TEST_METRIC_NAME) 133 self.assertEqual(result.metric_key, expected_metric_key) 134 135 @patch('acts.metrics.loggers.protos.gen.acts_blackbox_pb2' 136 '.ActsBlackboxMetricResultsBundle') 137 @patch('acts.metrics.loggers.blackbox.ProtoMetric') 138 @patch('acts.metrics.loggers.blackbox.md5_proto') 139 def test_end_does_publish(self, mock_md5_proto, 140 proto_metric_cls, 141 mock_acts_blackbox_bundle): 142 result_bundle = Mock() 143 mock_acts_blackbox_bundle.return_value = result_bundle 144 mock_md5_proto.return_value = '<123456>' 145 146 logger = BlackboxMetricLogger(self.TEST_METRIC_NAME) 147 logger.context = self.context 148 logger.publisher = self.publisher 149 logger._get_blackbox_identifier = self._get_blackbox_identifier 150 logger.metric_value = 42 151 152 logger.end(self.event) 153 154 self.assertEqual(1, proto_metric_cls.call_count, 155 'expected exactly 1 call') 156 proto_metric_cls.assert_has_calls([ 157 call(name='blackbox_metrics_bundle.<123456>', 158 data=result_bundle)]) 159 160 self.publisher.publish.assert_called_once_with( 161 [proto_metric_cls.return_value]) 162 163 164class BlackboxMetricLoggerIntegrationTest(TestCase): 165 """Integration tests for BlackboxMetricLogger.""" 166 167 def setUp(self): 168 warnings.simplefilter('ignore', ResourceWarning) 169 170 @patch('acts.test_runner.sys') 171 @patch('acts.test_runner.utils') 172 @patch('acts.test_runner.importlib') 173 def run_acts_test(self, test_class, importlib, utils, sys): 174 test_run_config = TestRunConfig() 175 test_run_config.testbed_name = 'SampleTestBed' 176 test_run_config.log_path = tempfile.mkdtemp() 177 test_run_config.controller_configs = {'testpaths': ['./']} 178 179 mock_module = Mock() 180 setattr(mock_module, test_class.__name__, test_class) 181 utils.find_files.return_value = [(None, None, None)] 182 importlib.import_module.return_value = mock_module 183 runner = TestRunner(test_run_config, [( 184 test_class.__name__, 185 None, 186 )]) 187 188 runner.run() 189 runner.stop() 190 shutil.rmtree(test_run_config.log_path) 191 return runner 192 193 @patch('acts.metrics.logger.ProtoMetricPublisher') 194 def test_test_case_metric(self, publisher_cls): 195 result = 5.0 196 197 class MyTest(BaseTestClass): 198 def __init__(self, controllers): 199 super().__init__(controllers) 200 self.tests = ('test_case',) 201 self.metric = BlackboxMetricLogger.for_test_case('my_metric') 202 203 def test_case(self): 204 self.metric.metric_value = result 205 206 self.run_acts_test(MyTest) 207 208 args_list = publisher_cls().publish.call_args_list 209 self.assertEqual(len(args_list), 1) 210 published = self.__get_only_arg(args_list[0])[0] 211 bundle = published.data 212 metric = bundle.acts_blackbox_metric_results[0] 213 self.assertIn('blackbox_metrics_bundle', published.name) 214 self.assertEqual(metric.test_identifier, 'MyTest#test_case') 215 self.assertEqual(metric.metric_key, 'MyTest#test_case.my_metric') 216 self.assertEqual(metric.metric_value, result) 217 218 @patch('acts.metrics.logger.ProtoMetricPublisher') 219 def test_multiple_test_case_metrics(self, publisher_cls): 220 result = 5.0 221 222 class MyTest(BaseTestClass): 223 def __init__(self, controllers): 224 super().__init__(controllers) 225 self.tests = ('test_case',) 226 self.metric_1 = (BlackboxMetricLogger.for_test_case( 227 'my_metric_1')) 228 self.metric_2 = (BlackboxMetricLogger.for_test_case( 229 'my_metric_2')) 230 231 def test_case(self): 232 self.metric_1.metric_value = result 233 self.metric_2.metric_value = result 234 235 self.run_acts_test(MyTest) 236 237 args_list = publisher_cls().publish.call_args_list 238 self.assertEqual(len(args_list), 2) 239 all_published = [self.__get_only_arg(args)[0] for args in args_list] 240 bundles = [published_instance.data for 241 published_instance in all_published] 242 243 flattened_metrics = [metric for bundle in bundles for metric in 244 bundle.acts_blackbox_metric_results] 245 246 self.assertEqual({metric.test_identifier 247 for metric in 248 flattened_metrics}, 249 {'MyTest#test_case'}) 250 self.assertEqual({metric.metric_key 251 for metric in flattened_metrics}, 252 {'MyTest#test_case.my_metric_1', 253 'MyTest#test_case.my_metric_2'}) 254 self.assertEqual({metric.metric_value 255 for metric in 256 flattened_metrics}, {result}) 257 258 @patch('acts.metrics.logger.ProtoMetricPublisher') 259 def test_test_case_metric_with_custom_key(self, publisher_cls): 260 result = 5.0 261 262 class MyTest(BaseTestClass): 263 def __init__(self, controllers): 264 super().__init__(controllers) 265 self.tests = ('test_case',) 266 self.metrics = BlackboxMetricLogger.for_test_case( 267 'my_metric', metric_key='my_metric_key') 268 269 def test_case(self): 270 self.metrics.metric_value = result 271 272 self.run_acts_test(MyTest) 273 274 args_list = publisher_cls().publish.call_args_list 275 self.assertEqual(len(args_list), 1) 276 bundle = self.__get_only_arg(args_list[0])[0].data 277 self.assertEqual(bundle.acts_blackbox_metric_results[0].metric_key, 278 'my_metric_key.my_metric') 279 280 @patch('acts.metrics.logger.ProtoMetricPublisher') 281 def test_test_class_metric(self, publisher_cls): 282 publisher_cls().publish = Mock() 283 result_1 = 5.0 284 result_2 = 8.0 285 286 class MyTest(BaseTestClass): 287 def __init__(self, controllers): 288 super().__init__(controllers) 289 self.tests = ( 290 'test_case_1', 291 'test_case_2', 292 ) 293 self.metric = BlackboxMetricLogger.for_test_class('my_metric') 294 295 def setup_class(self): 296 self.metric.metric_value = 0 297 298 def test_case_1(self): 299 self.metric.metric_value += result_1 300 301 def test_case_2(self): 302 self.metric.metric_value += result_2 303 304 self.run_acts_test(MyTest) 305 306 args_list = publisher_cls().publish.call_args_list 307 self.assertEqual(len(args_list), 1) 308 bundle = self.__get_only_arg(args_list[0])[0].data 309 metric = bundle.acts_blackbox_metric_results[0] 310 self.assertEqual(metric.metric_value, result_1 + result_2) 311 self.assertEqual(metric.test_identifier, MyTest.__name__) 312 313 def __get_only_arg(self, call_args): 314 self.assertEqual(len(call_args[0]) + len(call_args[1]), 1) 315 if len(call_args[0]) == 1: 316 return call_args[0][0] 317 return next(iter(call_args[1].values())) 318 319 320if __name__ == '__main__': 321 unittest.main() 322