1#!/usr/bin/env python2 2 3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6"""Testing of benchmark_run.""" 7 8from __future__ import print_function 9 10import mock 11import unittest 12import inspect 13 14from cros_utils import logger 15 16import benchmark_run 17 18from suite_runner import MockSuiteRunner 19from suite_runner import SuiteRunner 20from label import MockLabel 21from benchmark import Benchmark 22from machine_manager import MockMachineManager 23from machine_manager import MachineManager 24from machine_manager import MockCrosMachine 25from results_cache import MockResultsCache 26from results_cache import CacheConditions 27from results_cache import Result 28from results_cache import ResultsCache 29 30 31class BenchmarkRunTest(unittest.TestCase): 32 """Unit tests for the BenchmarkRun class and all of its methods.""" 33 34 def setUp(self): 35 self.status = [] 36 self.called_ReadCache = None 37 self.log_error = [] 38 self.log_output = [] 39 self.err_msg = None 40 self.test_benchmark = Benchmark( 41 'page_cycler.netsim.top_10', # name 42 'page_cycler.netsim.top_10', # test_name 43 '', # test_args 44 1, # iterations 45 False, # rm_chroot_tmp 46 '', # perf_args 47 suite='telemetry_Crosperf') # suite 48 49 self.test_label = MockLabel( 50 'test1', 51 'image1', 52 'autotest_dir', 53 '/tmp/test_benchmark_run', 54 'x86-alex', 55 'chromeos2-row1-rack4-host9.cros', 56 image_args='', 57 cache_dir='', 58 cache_only=False, 59 log_level='average', 60 compiler='gcc') 61 62 self.test_cache_conditions = [ 63 CacheConditions.CACHE_FILE_EXISTS, CacheConditions.CHECKSUMS_MATCH 64 ] 65 66 self.mock_logger = logger.GetLogger(log_dir='', mock=True) 67 68 self.mock_machine_manager = mock.Mock(spec=MachineManager) 69 70 def testDryRun(self): 71 my_label = MockLabel( 72 'test1', 73 'image1', 74 'autotest_dir', 75 '/tmp/test_benchmark_run', 76 'x86-alex', 77 'chromeos2-row1-rack4-host9.cros', 78 image_args='', 79 cache_dir='', 80 cache_only=False, 81 log_level='average', 82 compiler='gcc') 83 84 logging_level = 'average' 85 m = MockMachineManager('/tmp/chromeos_root', 0, logging_level, '') 86 m.AddMachine('chromeos2-row1-rack4-host9.cros') 87 bench = Benchmark( 88 'page_cycler.netsim.top_10', # name 89 'page_cycler.netsim.top_10', # test_name 90 '', # test_args 91 1, # iterations 92 False, # rm_chroot_tmp 93 '', # perf_args 94 suite='telemetry_Crosperf') # suite 95 b = benchmark_run.MockBenchmarkRun('test run', bench, my_label, 1, [], m, 96 logger.GetLogger(), logging_level, '') 97 b.cache = MockResultsCache() 98 b.suite_runner = MockSuiteRunner() 99 b.start() 100 101 # Make sure the arguments to BenchmarkRun.__init__ have not changed 102 # since the last time this test was updated: 103 args_list = [ 104 'self', 'name', 'benchmark', 'label', 'iteration', 'cache_conditions', 105 'machine_manager', 'logger_to_use', 'log_level', 'share_cache' 106 ] 107 arg_spec = inspect.getargspec(benchmark_run.BenchmarkRun.__init__) 108 self.assertEqual(len(arg_spec.args), len(args_list)) 109 self.assertEqual(arg_spec.args, args_list) 110 111 def test_init(self): 112 # Nothing really worth testing here; just field assignments. 113 pass 114 115 def test_read_cache(self): 116 # Nothing really worth testing here, either. 117 pass 118 119 def test_run(self): 120 br = benchmark_run.BenchmarkRun('test_run', self.test_benchmark, 121 self.test_label, 1, 122 self.test_cache_conditions, 123 self.mock_machine_manager, self.mock_logger, 124 'average', '') 125 126 def MockLogOutput(msg, print_to_console=False): 127 'Helper function for test_run.' 128 del print_to_console 129 self.log_output.append(msg) 130 131 def MockLogError(msg, print_to_console=False): 132 'Helper function for test_run.' 133 del print_to_console 134 self.log_error.append(msg) 135 136 def MockRecordStatus(msg): 137 'Helper function for test_run.' 138 self.status.append(msg) 139 140 def FakeReadCache(): 141 'Helper function for test_run.' 142 br.cache = mock.Mock(spec=ResultsCache) 143 self.called_ReadCache = True 144 return 0 145 146 def FakeReadCacheSucceed(): 147 'Helper function for test_run.' 148 br.cache = mock.Mock(spec=ResultsCache) 149 br.result = mock.Mock(spec=Result) 150 br.result.out = 'result.out stuff' 151 br.result.err = 'result.err stuff' 152 br.result.retval = 0 153 self.called_ReadCache = True 154 return 0 155 156 def FakeReadCacheException(): 157 'Helper function for test_run.' 158 raise RuntimeError('This is an exception test; it is supposed to happen') 159 160 def FakeAcquireMachine(): 161 'Helper function for test_run.' 162 mock_machine = MockCrosMachine('chromeos1-row3-rack5-host7.cros', 163 'chromeos', 'average') 164 return mock_machine 165 166 def FakeRunTest(_machine): 167 'Helper function for test_run.' 168 mock_result = mock.Mock(spec=Result) 169 mock_result.retval = 0 170 return mock_result 171 172 def FakeRunTestFail(_machine): 173 'Helper function for test_run.' 174 mock_result = mock.Mock(spec=Result) 175 mock_result.retval = 1 176 return mock_result 177 178 def ResetTestValues(): 179 'Helper function for test_run.' 180 self.log_output = [] 181 self.log_error = [] 182 self.status = [] 183 br.result = None 184 self.called_ReadCache = False 185 186 # Assign all the fake functions to the appropriate objects. 187 br.logger().LogOutput = MockLogOutput 188 br.logger().LogError = MockLogError 189 br.timeline.Record = MockRecordStatus 190 br.ReadCache = FakeReadCache 191 br.RunTest = FakeRunTest 192 br.AcquireMachine = FakeAcquireMachine 193 194 # First test: No cache hit, all goes well. 195 ResetTestValues() 196 br.run() 197 self.assertTrue(self.called_ReadCache) 198 self.assertEqual(self.log_output, [ 199 'test_run: No cache hit.', 200 'Releasing machine: chromeos1-row3-rack5-host7.cros', 201 'Released machine: chromeos1-row3-rack5-host7.cros' 202 ]) 203 self.assertEqual(len(self.log_error), 0) 204 self.assertEqual(self.status, ['WAITING', 'SUCCEEDED']) 205 206 # Second test: No cached result found; test run was "terminated" for some 207 # reason. 208 ResetTestValues() 209 br.terminated = True 210 br.run() 211 self.assertTrue(self.called_ReadCache) 212 self.assertEqual(self.log_output, [ 213 'test_run: No cache hit.', 214 'Releasing machine: chromeos1-row3-rack5-host7.cros', 215 'Released machine: chromeos1-row3-rack5-host7.cros' 216 ]) 217 self.assertEqual(len(self.log_error), 0) 218 self.assertEqual(self.status, ['WAITING']) 219 220 # Third test. No cached result found; RunTest failed for some reason. 221 ResetTestValues() 222 br.terminated = False 223 br.RunTest = FakeRunTestFail 224 br.run() 225 self.assertTrue(self.called_ReadCache) 226 self.assertEqual(self.log_output, [ 227 'test_run: No cache hit.', 228 'Releasing machine: chromeos1-row3-rack5-host7.cros', 229 'Released machine: chromeos1-row3-rack5-host7.cros' 230 ]) 231 self.assertEqual(len(self.log_error), 0) 232 self.assertEqual(self.status, ['WAITING', 'FAILED']) 233 234 # Fourth test: ReadCache found a cached result. 235 ResetTestValues() 236 br.RunTest = FakeRunTest 237 br.ReadCache = FakeReadCacheSucceed 238 br.run() 239 self.assertTrue(self.called_ReadCache) 240 self.assertEqual(self.log_output, [ 241 'test_run: Cache hit.', 'result.out stuff', 242 'Releasing machine: chromeos1-row3-rack5-host7.cros', 243 'Released machine: chromeos1-row3-rack5-host7.cros' 244 ]) 245 self.assertEqual(self.log_error, ['result.err stuff']) 246 self.assertEqual(self.status, ['SUCCEEDED']) 247 248 # Fifth test: ReadCache generates an exception; does the try/finally block 249 # work? 250 ResetTestValues() 251 br.ReadCache = FakeReadCacheException 252 br.machine = FakeAcquireMachine() 253 br.run() 254 self.assertEqual(self.log_error, [ 255 "Benchmark run: 'test_run' failed: This is an exception test; it is " 256 'supposed to happen' 257 ]) 258 self.assertEqual(self.status, ['FAILED']) 259 260 def test_terminate_pass(self): 261 br = benchmark_run.BenchmarkRun('test_run', self.test_benchmark, 262 self.test_label, 1, 263 self.test_cache_conditions, 264 self.mock_machine_manager, self.mock_logger, 265 'average', '') 266 267 def GetLastEventPassed(): 268 'Helper function for test_terminate_pass' 269 return benchmark_run.STATUS_SUCCEEDED 270 271 def RecordStub(status): 272 'Helper function for test_terminate_pass' 273 self.status = status 274 275 self.status = benchmark_run.STATUS_SUCCEEDED 276 self.assertFalse(br.terminated) 277 self.assertFalse(br.suite_runner.CommandTerminator().IsTerminated()) 278 279 br.timeline.GetLastEvent = GetLastEventPassed 280 br.timeline.Record = RecordStub 281 282 br.Terminate() 283 284 self.assertTrue(br.terminated) 285 self.assertTrue(br.suite_runner.CommandTerminator().IsTerminated()) 286 self.assertEqual(self.status, benchmark_run.STATUS_FAILED) 287 288 def test_terminate_fail(self): 289 br = benchmark_run.BenchmarkRun('test_run', self.test_benchmark, 290 self.test_label, 1, 291 self.test_cache_conditions, 292 self.mock_machine_manager, self.mock_logger, 293 'average', '') 294 295 def GetLastEventFailed(): 296 'Helper function for test_terminate_fail' 297 return benchmark_run.STATUS_FAILED 298 299 def RecordStub(status): 300 'Helper function for test_terminate_fail' 301 self.status = status 302 303 self.status = benchmark_run.STATUS_SUCCEEDED 304 self.assertFalse(br.terminated) 305 self.assertFalse(br.suite_runner.CommandTerminator().IsTerminated()) 306 307 br.timeline.GetLastEvent = GetLastEventFailed 308 br.timeline.Record = RecordStub 309 310 br.Terminate() 311 312 self.assertTrue(br.terminated) 313 self.assertTrue(br.suite_runner.CommandTerminator().IsTerminated()) 314 self.assertEqual(self.status, benchmark_run.STATUS_SUCCEEDED) 315 316 def test_acquire_machine(self): 317 br = benchmark_run.BenchmarkRun('test_run', self.test_benchmark, 318 self.test_label, 1, 319 self.test_cache_conditions, 320 self.mock_machine_manager, self.mock_logger, 321 'average', '') 322 323 br.terminated = True 324 self.assertRaises(Exception, br.AcquireMachine) 325 326 br.terminated = False 327 mock_machine = MockCrosMachine('chromeos1-row3-rack5-host7.cros', 328 'chromeos', 'average') 329 self.mock_machine_manager.AcquireMachine.return_value = mock_machine 330 331 machine = br.AcquireMachine() 332 self.assertEqual(machine.name, 'chromeos1-row3-rack5-host7.cros') 333 334 def test_get_extra_autotest_args(self): 335 br = benchmark_run.BenchmarkRun('test_run', self.test_benchmark, 336 self.test_label, 1, 337 self.test_cache_conditions, 338 self.mock_machine_manager, self.mock_logger, 339 'average', '') 340 341 def MockLogError(err_msg): 342 'Helper function for test_get_extra_autotest_args' 343 self.err_msg = err_msg 344 345 self.mock_logger.LogError = MockLogError 346 347 result = br.GetExtraAutotestArgs() 348 self.assertEqual(result, '') 349 350 self.test_benchmark.perf_args = 'record -e cycles' 351 result = br.GetExtraAutotestArgs() 352 self.assertEqual( 353 result, 354 "--profiler=custom_perf --profiler_args='perf_options=\"record -a -e " 355 "cycles\"'") 356 357 self.test_benchmark.suite = 'telemetry' 358 result = br.GetExtraAutotestArgs() 359 self.assertEqual(result, '') 360 self.assertEqual(self.err_msg, 'Telemetry does not support profiler.') 361 362 self.test_benchmark.perf_args = 'record -e cycles' 363 self.test_benchmark.suite = 'test_that' 364 result = br.GetExtraAutotestArgs() 365 self.assertEqual(result, '') 366 self.assertEqual(self.err_msg, 'test_that does not support profiler.') 367 368 self.test_benchmark.perf_args = 'junk args' 369 self.test_benchmark.suite = 'telemetry_Crosperf' 370 self.assertRaises(Exception, br.GetExtraAutotestArgs) 371 372 @mock.patch.object(SuiteRunner, 'Run') 373 @mock.patch.object(Result, 'CreateFromRun') 374 def test_run_test(self, mock_result, mock_runner): 375 br = benchmark_run.BenchmarkRun('test_run', self.test_benchmark, 376 self.test_label, 1, 377 self.test_cache_conditions, 378 self.mock_machine_manager, self.mock_logger, 379 'average', '') 380 381 self.status = [] 382 383 def MockRecord(status): 384 self.status.append(status) 385 386 br.timeline.Record = MockRecord 387 mock_machine = MockCrosMachine('chromeos1-row3-rack5-host7.cros', 388 'chromeos', 'average') 389 mock_runner.return_value = [0, "{'Score':100}", ''] 390 391 br.RunTest(mock_machine) 392 393 self.assertTrue(br.run_completed) 394 self.assertEqual( 395 self.status, 396 [benchmark_run.STATUS_IMAGING, benchmark_run.STATUS_RUNNING]) 397 398 self.assertEqual(br.machine_manager.ImageMachine.call_count, 1) 399 br.machine_manager.ImageMachine.assert_called_with(mock_machine, 400 self.test_label) 401 self.assertEqual(mock_runner.call_count, 1) 402 mock_runner.assert_called_with(mock_machine.name, br.label, br.benchmark, 403 '', br.profiler_args) 404 405 self.assertEqual(mock_result.call_count, 1) 406 mock_result.assert_called_with(self.mock_logger, 'average', self.test_label, 407 None, "{'Score':100}", '', 0, 408 'page_cycler.netsim.top_10', 409 'telemetry_Crosperf') 410 411 def test_set_cache_conditions(self): 412 br = benchmark_run.BenchmarkRun('test_run', self.test_benchmark, 413 self.test_label, 1, 414 self.test_cache_conditions, 415 self.mock_machine_manager, self.mock_logger, 416 'average', '') 417 418 phony_cache_conditions = [123, 456, True, False] 419 420 self.assertEqual(br.cache_conditions, self.test_cache_conditions) 421 422 br.SetCacheConditions(phony_cache_conditions) 423 self.assertEqual(br.cache_conditions, phony_cache_conditions) 424 425 br.SetCacheConditions(self.test_cache_conditions) 426 self.assertEqual(br.cache_conditions, self.test_cache_conditions) 427 428 429if __name__ == '__main__': 430 unittest.main() 431