1#!/usr/bin/python2 2#pylint: disable-msg=C0111 3"""Unit Tests for autotest.client.common_lib.test""" 4 5__author__ = 'gps@google.com (Gregory P. Smith)' 6 7import json 8import tempfile 9import unittest 10import common 11from autotest_lib.client.common_lib import test 12from autotest_lib.client.common_lib.test_utils import mock 13 14class TestTestCase(unittest.TestCase): 15 class _neutered_base_test(test.base_test): 16 """A child class of base_test to avoid calling the constructor.""" 17 def __init__(self, *args, **kwargs): 18 class MockJob(object): 19 pass 20 class MockProfilerManager(object): 21 def active(self): 22 return False 23 def present(self): 24 return True 25 self.job = MockJob() 26 self.job.default_profile_only = False 27 self.job.profilers = MockProfilerManager() 28 self.job.test_retry = 0 29 self.job.fast = False 30 self._new_keyval = False 31 self.iteration = 0 32 self.tagged_testname = 'neutered_base_test' 33 self.before_iteration_hooks = [] 34 self.after_iteration_hooks = [] 35 36 37 def setUp(self): 38 self.god = mock.mock_god() 39 self.test = self._neutered_base_test() 40 41 42 def tearDown(self): 43 self.god.unstub_all() 44 45 46 47class Test_base_test_execute(TestTestCase): 48 # Test the various behaviors of the base_test.execute() method. 49 def setUp(self): 50 TestTestCase.setUp(self) 51 self.god.stub_function(self.test, 'run_once_profiling') 52 self.god.stub_function(self.test, 'postprocess') 53 self.god.stub_function(self.test, 'process_failed_constraints') 54 55 56 def test_call_run_once(self): 57 # setup 58 self.god.stub_function(self.test, 'drop_caches_between_iterations') 59 self.god.stub_function(self.test, 'run_once') 60 self.god.stub_function(self.test, 'postprocess_iteration') 61 self.god.stub_function(self.test, 'analyze_perf_constraints') 62 before_hook = self.god.create_mock_function('before_hook') 63 after_hook = self.god.create_mock_function('after_hook') 64 self.test.register_before_iteration_hook(before_hook) 65 self.test.register_after_iteration_hook(after_hook) 66 67 # tests the test._call_run_once implementation 68 self.test.drop_caches_between_iterations.expect_call() 69 before_hook.expect_call(self.test) 70 self.test.run_once.expect_call(1, 2, arg='val') 71 self.test.postprocess_iteration.expect_call() 72 self.test.analyze_perf_constraints.expect_call([]) 73 after_hook.expect_call(self.test) 74 self.test._call_run_once([], False, None, (1, 2), {'arg': 'val'}) 75 self.god.check_playback() 76 77 78 def test_call_run_once_with_exception(self): 79 # setup 80 self.god.stub_function(self.test, 'drop_caches_between_iterations') 81 self.god.stub_function(self.test, 'run_once') 82 before_hook = self.god.create_mock_function('before_hook') 83 after_hook = self.god.create_mock_function('after_hook') 84 self.test.register_before_iteration_hook(before_hook) 85 self.test.register_after_iteration_hook(after_hook) 86 error = Exception('fail') 87 88 # tests the test._call_run_once implementation 89 self.test.drop_caches_between_iterations.expect_call() 90 before_hook.expect_call(self.test) 91 self.test.run_once.expect_call(1, 2, arg='val').and_raises(error) 92 after_hook.expect_call(self.test) 93 try: 94 self.test._call_run_once([], False, None, (1, 2), {'arg': 'val'}) 95 except: 96 pass 97 self.god.check_playback() 98 99 100 def _expect_call_run_once(self): 101 self.test._call_run_once.expect_call((), False, None, (), {}) 102 103 104 def test_execute_test_length(self): 105 # test that test_length overrides iterations and works. 106 self.god.stub_function(self.test, '_call_run_once') 107 108 self._expect_call_run_once() 109 self._expect_call_run_once() 110 self._expect_call_run_once() 111 self.test.run_once_profiling.expect_call(None) 112 self.test.postprocess.expect_call() 113 self.test.process_failed_constraints.expect_call() 114 115 fake_time = iter(xrange(4)).next 116 self.test.execute(iterations=1, test_length=3, _get_time=fake_time) 117 self.god.check_playback() 118 119 120 def test_execute_iterations(self): 121 # test that iterations works. 122 self.god.stub_function(self.test, '_call_run_once') 123 124 iterations = 2 125 for _ in range(iterations): 126 self._expect_call_run_once() 127 self.test.run_once_profiling.expect_call(None) 128 self.test.postprocess.expect_call() 129 self.test.process_failed_constraints.expect_call() 130 131 self.test.execute(iterations=iterations) 132 self.god.check_playback() 133 134 135 def _mock_calls_for_execute_no_iterations(self): 136 self.test.run_once_profiling.expect_call(None) 137 self.test.postprocess.expect_call() 138 self.test.process_failed_constraints.expect_call() 139 140 141 def test_execute_iteration_zero(self): 142 # test that iterations=0 works. 143 self._mock_calls_for_execute_no_iterations() 144 145 self.test.execute(iterations=0) 146 self.god.check_playback() 147 148 149 def test_execute_profile_only(self): 150 # test that profile_only=True works. 151 self.god.stub_function(self.test, 'drop_caches_between_iterations') 152 self.test.drop_caches_between_iterations.expect_call() 153 self.test.run_once_profiling.expect_call(None) 154 self.test.drop_caches_between_iterations.expect_call() 155 self.test.run_once_profiling.expect_call(None) 156 self.test.postprocess.expect_call() 157 self.test.process_failed_constraints.expect_call() 158 self.test.execute(profile_only=True, iterations=2) 159 self.god.check_playback() 160 161 162 def test_execute_default_profile_only(self): 163 # test that profile_only=True works. 164 self.god.stub_function(self.test, 'drop_caches_between_iterations') 165 for _ in xrange(3): 166 self.test.drop_caches_between_iterations.expect_call() 167 self.test.run_once_profiling.expect_call(None) 168 self.test.postprocess.expect_call() 169 self.test.process_failed_constraints.expect_call() 170 self.test.job.default_profile_only = True 171 self.test.execute(iterations=3) 172 self.god.check_playback() 173 174 175 def test_execute_postprocess_profiled_false(self): 176 # test that postprocess_profiled_run=False works 177 self.god.stub_function(self.test, '_call_run_once') 178 179 self.test._call_run_once.expect_call((), False, False, (), {}) 180 self.test.run_once_profiling.expect_call(False) 181 self.test.postprocess.expect_call() 182 self.test.process_failed_constraints.expect_call() 183 184 self.test.execute(postprocess_profiled_run=False, iterations=1) 185 self.god.check_playback() 186 187 188 def test_execute_postprocess_profiled_true(self): 189 # test that postprocess_profiled_run=True works 190 self.god.stub_function(self.test, '_call_run_once') 191 192 self.test._call_run_once.expect_call((), False, True, (), {}) 193 self.test.run_once_profiling.expect_call(True) 194 self.test.postprocess.expect_call() 195 self.test.process_failed_constraints.expect_call() 196 197 self.test.execute(postprocess_profiled_run=True, iterations=1) 198 self.god.check_playback() 199 200 201 def test_output_single_perf_value(self): 202 self.test.resultsdir = tempfile.mkdtemp() 203 204 self.test.output_perf_value("Test", 1, units="ms", higher_is_better=True) 205 206 f = open(self.test.resultsdir + "/results-chart.json") 207 expected_result = {"Test": {"summary": {"units": "ms", "type": "scalar", 208 "value": 1, "improvement_direction": "up"}}} 209 self.assertDictEqual(expected_result, json.loads(f.read())) 210 211 def test_output_perf_value_with_custom_resultsdir(self): 212 self.test.resultsdir = tempfile.mkdtemp() 213 214 resultsdir = self.test.resultsdir + "/tests/tmp" 215 self.test.output_perf_value("Test", 1, units="ms",higher_is_better=True, 216 resultsdir=resultsdir) 217 218 f = open(self.test.resultsdir + "/tests/tmp/results-chart.json") 219 expected_result = {"Test": {"summary": {"units": "ms", "type": "scalar", 220 "value": 1, "improvement_direction": "up"}}} 221 self.assertDictEqual(expected_result, json.loads(f.read())) 222 223 224 def test_output_single_perf_value_twice(self): 225 self.test.resultsdir = tempfile.mkdtemp() 226 227 self.test.output_perf_value("Test", 1, units="ms", higher_is_better=True) 228 self.test.output_perf_value("Test", 2, units="ms", higher_is_better=True) 229 230 f = open(self.test.resultsdir + "/results-chart.json") 231 expected_result = {"Test": {"summary": {"units": "ms", 232 "type": "list_of_scalar_values", "values": [1, 2], 233 "improvement_direction": "up"}}} 234 self.assertDictEqual(expected_result, json.loads(f.read())) 235 236 237 def test_output_single_perf_value_three_times(self): 238 self.test.resultsdir = tempfile.mkdtemp() 239 240 self.test.output_perf_value("Test", 1, units="ms", 241 higher_is_better=True) 242 self.test.output_perf_value("Test", 2, units="ms", higher_is_better=True) 243 self.test.output_perf_value("Test", 3, units="ms", higher_is_better=True) 244 245 f = open(self.test.resultsdir + "/results-chart.json") 246 expected_result = {"Test": {"summary": {"units": "ms", 247 "type": "list_of_scalar_values", "values": [1, 2, 3], 248 "improvement_direction": "up"}}} 249 self.assertDictEqual(expected_result, json.loads(f.read())) 250 251 252 def test_output_list_perf_value(self): 253 self.test.resultsdir = tempfile.mkdtemp() 254 255 self.test.output_perf_value("Test", [1, 2, 3], units="ms", 256 higher_is_better=False) 257 258 f = open(self.test.resultsdir + "/results-chart.json") 259 expected_result = {"Test": {"summary": {"units": "ms", 260 "type": "list_of_scalar_values", "values": [1, 2, 3], 261 "improvement_direction": "down"}}} 262 self.assertDictEqual(expected_result, json.loads(f.read())) 263 264 265 def test_output_single_then_list_perf_value(self): 266 self.test.resultsdir = tempfile.mkdtemp() 267 self.test.output_perf_value("Test", 1, units="ms", 268 higher_is_better=False) 269 self.test.output_perf_value("Test", [4, 3, 2], units="ms", 270 higher_is_better=False) 271 f = open(self.test.resultsdir + "/results-chart.json") 272 expected_result = {"Test": {"summary": {"units": "ms", 273 "type": "list_of_scalar_values", 274 "values": [1, 4, 3, 2], 275 "improvement_direction": "down"}}} 276 self.assertDictEqual(expected_result, json.loads(f.read())) 277 278 279 def test_output_list_then_list_perf_value(self): 280 self.test.resultsdir = tempfile.mkdtemp() 281 self.test.output_perf_value("Test", [1, 2, 3], units="ms", 282 higher_is_better=False) 283 self.test.output_perf_value("Test", [4, 3, 2], units="ms", 284 higher_is_better=False) 285 f = open(self.test.resultsdir + "/results-chart.json") 286 expected_result = {"Test": {"summary": {"units": "ms", 287 "type": "list_of_scalar_values", 288 "values": [1, 2, 3, 4, 3, 2], 289 "improvement_direction": "down"}}} 290 self.assertDictEqual(expected_result, json.loads(f.read())) 291 292 293 def test_output_single_perf_value_input_string(self): 294 self.test.resultsdir = tempfile.mkdtemp() 295 296 self.test.output_perf_value("Test", u'-0.34', units="ms", 297 higher_is_better=True) 298 299 f = open(self.test.resultsdir + "/results-chart.json") 300 expected_result = {"Test": {"summary": {"units": "ms", "type": "scalar", 301 "value": -0.34, "improvement_direction": "up"}}} 302 self.assertDictEqual(expected_result, json.loads(f.read())) 303 304 305 def test_output_single_perf_value_input_list_of_string(self): 306 self.test.resultsdir = tempfile.mkdtemp() 307 308 self.test.output_perf_value("Test", [0, u'-0.34', 1], units="ms", 309 higher_is_better=True) 310 311 f = open(self.test.resultsdir + "/results-chart.json") 312 expected_result = {"Test": {"summary": {"units": "ms", 313 "type": "list_of_scalar_values", 314 "values": [0, -0.34, 1], 315 "improvement_direction": "up"}}} 316 self.assertDictEqual(expected_result, json.loads(f.read())) 317 318 def test_output_list_then_replace_list_perf_value(self): 319 self.test.resultsdir = tempfile.mkdtemp() 320 self.test.output_perf_value("Test", [1, 2, 3], units="ms", 321 higher_is_better=False) 322 self.test.output_perf_value("Test", [4, 5, 6], units="ms", 323 higher_is_better=False, 324 replace_existing_values=True) 325 f = open(self.test.resultsdir + "/results-chart.json") 326 expected_result = {"Test": {"summary": {"units": "ms", 327 "type": "list_of_scalar_values", 328 "values": [4, 5, 6], 329 "improvement_direction": "down"}}} 330 self.assertDictEqual(expected_result, json.loads(f.read())) 331 332 def test_output_single_then_replace_list_perf_value(self): 333 self.test.resultsdir = tempfile.mkdtemp() 334 self.test.output_perf_value("Test", 3, units="ms", 335 higher_is_better=False) 336 self.test.output_perf_value("Test", [4, 5, 6], units="ms", 337 higher_is_better=False, 338 replace_existing_values=True) 339 f = open(self.test.resultsdir + "/results-chart.json") 340 expected_result = {"Test": {"summary": {"units": "ms", 341 "type": "list_of_scalar_values", 342 "values": [4, 5, 6], 343 "improvement_direction": "down"}}} 344 self.assertDictEqual(expected_result, json.loads(f.read())) 345 346 def test_output_list_then_replace_single_perf_value(self): 347 self.test.resultsdir = tempfile.mkdtemp() 348 self.test.output_perf_value("Test", [1,2,3], units="ms", 349 higher_is_better=False) 350 self.test.output_perf_value("Test", 4, units="ms", 351 higher_is_better=False, 352 replace_existing_values=True) 353 f = open(self.test.resultsdir + "/results-chart.json") 354 expected_result = {"Test": {"summary": {"units": "ms", 355 "type": "scalar", 356 "value": 4, 357 "improvement_direction": "down"}}} 358 self.assertDictEqual(expected_result, json.loads(f.read())) 359 360 def test_output_single_then_replace_single_perf_value(self): 361 self.test.resultsdir = tempfile.mkdtemp() 362 self.test.output_perf_value("Test", 1, units="ms", 363 higher_is_better=False) 364 self.test.output_perf_value("Test", 2, units="ms", 365 higher_is_better=False, 366 replace_existing_values=True) 367 f = open(self.test.resultsdir + "/results-chart.json") 368 expected_result = {"Test": {"summary": {"units": "ms", 369 "type": "scalar", 370 "value": 2, 371 "improvement_direction": "down"}}} 372 self.assertDictEqual(expected_result, json.loads(f.read())) 373 374 def test_output_perf_then_replace_certain_perf_value(self): 375 self.test.resultsdir = tempfile.mkdtemp() 376 self.test.output_perf_value("Test1", 1, units="ms", 377 higher_is_better=False) 378 self.test.output_perf_value("Test2", 2, units="ms", 379 higher_is_better=False) 380 self.test.output_perf_value("Test3", 3, units="ms", 381 higher_is_better=False) 382 self.test.output_perf_value("Test2", -1, units="ms", 383 higher_is_better=False, 384 replace_existing_values=True) 385 f = open(self.test.resultsdir + "/results-chart.json") 386 expected_result = {"Test1": {"summary": 387 {"units": "ms", 388 "type": "scalar", 389 "value": 1, 390 "improvement_direction": "down"}}, 391 "Test2": {"summary": 392 {"units": "ms", 393 "type": "scalar", 394 "value": -1, 395 "improvement_direction": "down"}}, 396 "Test3": {"summary": 397 {"units": "ms", 398 "type": "scalar", 399 "value": 3, 400 "improvement_direction": "down"}}} 401 self.assertDictEqual(expected_result, json.loads(f.read())) 402 403 def test_chart_supplied(self): 404 self.test.resultsdir = tempfile.mkdtemp() 405 406 test_data = [("tcp_tx", "ch006_mode11B_none", "BT_connected_but_not_streaming", 0), 407 ("tcp_tx", "ch006_mode11B_none", "BT_streaming_audiofile", 5), 408 ("tcp_tx", "ch006_mode11B_none", "BT_disconnected_again", 0), 409 ("tcp_rx", "ch006_mode11B_none", "BT_connected_but_not_streaming", 0), 410 ("tcp_rx", "ch006_mode11B_none", "BT_streaming_audiofile", 8), 411 ("tcp_rx", "ch006_mode11B_none", "BT_disconnected_again", 0), 412 ("udp_tx", "ch006_mode11B_none", "BT_connected_but_not_streaming", 0), 413 ("udp_tx", "ch006_mode11B_none", "BT_streaming_audiofile", 6), 414 ("udp_tx", "ch006_mode11B_none", "BT_disconnected_again", 0), 415 ("udp_rx", "ch006_mode11B_none", "BT_connected_but_not_streaming", 0), 416 ("udp_rx", "ch006_mode11B_none", "BT_streaming_audiofile", 8), 417 ("udp_rx", "ch006_mode11B_none", "BT_streaming_audiofile", 9), 418 ("udp_rx", "ch006_mode11B_none", "BT_disconnected_again", 0)] 419 420 421 for (config_tag, ap_config_tag, bt_tag, drop) in test_data: 422 self.test.output_perf_value(config_tag + '_' + bt_tag + '_drop', 423 drop, units='percent_drop', 424 higher_is_better=False, 425 graph=ap_config_tag + '_drop') 426 f = open(self.test.resultsdir + "/results-chart.json") 427 expected_result = { 428 "ch006_mode11B_none_drop": { 429 "udp_tx_BT_streaming_audiofile_drop": { 430 "units": "percent_drop", 431 "type": "scalar", 432 "value": 6.0, 433 "improvement_direction": "down" 434 }, 435 "udp_rx_BT_disconnected_again_drop": { 436 "units": "percent_drop", 437 "type": "scalar", 438 "value": 0.0, 439 "improvement_direction": "down" 440 }, 441 "tcp_tx_BT_disconnected_again_drop": { 442 "units": "percent_drop", 443 "type": "scalar", 444 "value": 0.0, 445 "improvement_direction": "down" 446 }, 447 "tcp_rx_BT_streaming_audiofile_drop": { 448 "units": "percent_drop", 449 "type": "scalar", 450 "value": 8.0, 451 "improvement_direction": "down" 452 }, 453 "udp_tx_BT_connected_but_not_streaming_drop": { 454 "units": "percent_drop", 455 "type": "scalar", 456 "value": 0.0, 457 "improvement_direction": "down" 458 }, 459 "tcp_tx_BT_connected_but_not_streaming_drop": { 460 "units": "percent_drop", 461 "type": "scalar", 462 "value": 0.0, 463 "improvement_direction": "down" 464 }, 465 "udp_tx_BT_disconnected_again_drop": { 466 "units": "percent_drop", 467 "type": "scalar", 468 "value": 0.0, 469 "improvement_direction": "down" 470 }, 471 "tcp_tx_BT_streaming_audiofile_drop": { 472 "units": "percent_drop", 473 "type": "scalar", 474 "value": 5.0, 475 "improvement_direction": "down" 476 }, 477 "tcp_rx_BT_connected_but_not_streaming_drop": { 478 "units": "percent_drop", 479 "type": "scalar", 480 "value": 0.0, 481 "improvement_direction": "down" 482 }, 483 "udp_rx_BT_connected_but_not_streaming_drop": { 484 "units": "percent_drop", 485 "type": "scalar", 486 "value": 0.0, 487 "improvement_direction": "down" 488 }, 489 "udp_rx_BT_streaming_audiofile_drop": { 490 "units": "percent_drop", 491 "type": "list_of_scalar_values", 492 "values": [ 493 8.0, 494 9.0 495 ], 496 "improvement_direction": "down" 497 }, 498 "tcp_rx_BT_disconnected_again_drop": { 499 "units": "percent_drop", 500 "type": "scalar", 501 "value": 0.0, 502 "improvement_direction": "down" 503 } 504 } 505 } 506 self.maxDiff = None 507 self.assertDictEqual(expected_result, json.loads(f.read())) 508 509if __name__ == '__main__': 510 unittest.main() 511