1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3 4# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 5# Use of this source code is governed by a BSD-style license that can be 6# found in the LICENSE file. 7 8"""Testing of benchmark_run.""" 9 10from __future__ import print_function 11 12import inspect 13import unittest 14import unittest.mock as mock 15 16import benchmark_run 17 18from cros_utils import logger 19from suite_runner import MockSuiteRunner 20from suite_runner import SuiteRunner 21from label import MockLabel 22from benchmark import Benchmark 23from machine_manager import MockMachineManager 24from machine_manager import MachineManager 25from machine_manager import MockCrosMachine 26from results_cache import MockResultsCache 27from results_cache import CacheConditions 28from results_cache import Result 29from results_cache import ResultsCache 30 31 32class BenchmarkRunTest(unittest.TestCase): 33 """Unit tests for the BenchmarkRun class and all of its methods.""" 34 35 def setUp(self): 36 self.status = [] 37 self.called_ReadCache = None 38 self.log_error = [] 39 self.log_output = [] 40 self.err_msg = None 41 self.test_benchmark = Benchmark( 42 'page_cycler.netsim.top_10', # name 43 'page_cycler.netsim.top_10', # test_name 44 '', # test_args 45 1, # iterations 46 False, # rm_chroot_tmp 47 '', # perf_args 48 suite='telemetry_Crosperf') # suite 49 50 self.test_label = MockLabel( 51 'test1', 52 'build', 53 'image1', 54 'autotest_dir', 55 'debug_dir', 56 '/tmp/test_benchmark_run', 57 'x86-alex', 58 'chromeos2-row1-rack4-host9.cros', 59 image_args='', 60 cache_dir='', 61 cache_only=False, 62 log_level='average', 63 compiler='gcc', 64 crosfleet=False) 65 66 self.test_cache_conditions = [ 67 CacheConditions.CACHE_FILE_EXISTS, CacheConditions.CHECKSUMS_MATCH 68 ] 69 70 self.mock_logger = logger.GetLogger(log_dir='', mock=True) 71 72 self.mock_machine_manager = mock.Mock(spec=MachineManager) 73 74 def testDryRun(self): 75 my_label = MockLabel( 76 'test1', 77 'build', 78 'image1', 79 'autotest_dir', 80 'debug_dir', 81 '/tmp/test_benchmark_run', 82 'x86-alex', 83 'chromeos2-row1-rack4-host9.cros', 84 image_args='', 85 cache_dir='', 86 cache_only=False, 87 log_level='average', 88 compiler='gcc', 89 crosfleet=False) 90 91 logging_level = 'average' 92 m = MockMachineManager('/tmp/chromeos_root', 0, logging_level, '') 93 m.AddMachine('chromeos2-row1-rack4-host9.cros') 94 bench = Benchmark( 95 'page_cycler.netsim.top_10', # name 96 'page_cycler.netsim.top_10', # test_name 97 '', # test_args 98 1, # iterations 99 False, # rm_chroot_tmp 100 '', # perf_args 101 suite='telemetry_Crosperf') # suite 102 dut_conf = { 103 'cooldown_time': 5, 104 'cooldown_temp': 45, 105 'governor': 'powersave', 106 'cpu_usage': 'big_only', 107 'cpu_freq_pct': 80, 108 } 109 b = benchmark_run.MockBenchmarkRun('test run', bench, my_label, 1, [], m, 110 logger.GetLogger(), logging_level, '', 111 dut_conf) 112 b.cache = MockResultsCache() 113 b.suite_runner = MockSuiteRunner() 114 b.start() 115 116 # Make sure the arguments to BenchmarkRun.__init__ have not changed 117 # since the last time this test was updated: 118 args_list = [ 119 'self', 'name', 'benchmark', 'label', 'iteration', 'cache_conditions', 120 'machine_manager', 'logger_to_use', 'log_level', 'share_cache', 121 'dut_config' 122 ] 123 arg_spec = inspect.getfullargspec(benchmark_run.BenchmarkRun.__init__) 124 self.assertEqual(len(arg_spec.args), len(args_list)) 125 self.assertEqual(arg_spec.args, args_list) 126 127 def test_init(self): 128 # Nothing really worth testing here; just field assignments. 129 pass 130 131 def test_read_cache(self): 132 # Nothing really worth testing here, either. 133 pass 134 135 def test_run(self): 136 br = benchmark_run.BenchmarkRun('test_run', self.test_benchmark, 137 self.test_label, 1, 138 self.test_cache_conditions, 139 self.mock_machine_manager, self.mock_logger, 140 'average', '', {}) 141 142 def MockLogOutput(msg, print_to_console=False): 143 """Helper function for test_run.""" 144 del print_to_console 145 self.log_output.append(msg) 146 147 def MockLogError(msg, print_to_console=False): 148 """Helper function for test_run.""" 149 del print_to_console 150 self.log_error.append(msg) 151 152 def MockRecordStatus(msg): 153 """Helper function for test_run.""" 154 self.status.append(msg) 155 156 def FakeReadCache(): 157 """Helper function for test_run.""" 158 br.cache = mock.Mock(spec=ResultsCache) 159 self.called_ReadCache = True 160 return 0 161 162 def FakeReadCacheSucceed(): 163 """Helper function for test_run.""" 164 br.cache = mock.Mock(spec=ResultsCache) 165 br.result = mock.Mock(spec=Result) 166 br.result.out = 'result.out stuff' 167 br.result.err = 'result.err stuff' 168 br.result.retval = 0 169 self.called_ReadCache = True 170 return 0 171 172 def FakeReadCacheException(): 173 """Helper function for test_run.""" 174 raise RuntimeError('This is an exception test; it is supposed to happen') 175 176 def FakeAcquireMachine(): 177 """Helper function for test_run.""" 178 mock_machine = MockCrosMachine('chromeos1-row3-rack5-host7.cros', 179 'chromeos', 'average') 180 return mock_machine 181 182 def FakeRunTest(_machine): 183 """Helper function for test_run.""" 184 mock_result = mock.Mock(spec=Result) 185 mock_result.retval = 0 186 return mock_result 187 188 def FakeRunTestFail(_machine): 189 """Helper function for test_run.""" 190 mock_result = mock.Mock(spec=Result) 191 mock_result.retval = 1 192 return mock_result 193 194 def ResetTestValues(): 195 """Helper function for test_run.""" 196 self.log_output = [] 197 self.log_error = [] 198 self.status = [] 199 br.result = None 200 self.called_ReadCache = False 201 202 # Assign all the fake functions to the appropriate objects. 203 br.logger().LogOutput = MockLogOutput 204 br.logger().LogError = MockLogError 205 br.timeline.Record = MockRecordStatus 206 br.ReadCache = FakeReadCache 207 br.RunTest = FakeRunTest 208 br.AcquireMachine = FakeAcquireMachine 209 210 # First test: No cache hit, all goes well. 211 ResetTestValues() 212 br.run() 213 self.assertTrue(self.called_ReadCache) 214 self.assertEqual(self.log_output, [ 215 'test_run: No cache hit.', 216 'Releasing machine: chromeos1-row3-rack5-host7.cros', 217 'Released machine: chromeos1-row3-rack5-host7.cros' 218 ]) 219 self.assertEqual(len(self.log_error), 0) 220 self.assertEqual(self.status, ['WAITING', 'SUCCEEDED']) 221 222 # Second test: No cached result found; test run was "terminated" for some 223 # reason. 224 ResetTestValues() 225 br.terminated = True 226 br.run() 227 self.assertTrue(self.called_ReadCache) 228 self.assertEqual(self.log_output, [ 229 'test_run: No cache hit.', 230 'Releasing machine: chromeos1-row3-rack5-host7.cros', 231 'Released machine: chromeos1-row3-rack5-host7.cros' 232 ]) 233 self.assertEqual(len(self.log_error), 0) 234 self.assertEqual(self.status, ['WAITING']) 235 236 # Third test. No cached result found; RunTest failed for some reason. 237 ResetTestValues() 238 br.terminated = False 239 br.RunTest = FakeRunTestFail 240 br.run() 241 self.assertTrue(self.called_ReadCache) 242 self.assertEqual(self.log_output, [ 243 'test_run: No cache hit.', 244 'Releasing machine: chromeos1-row3-rack5-host7.cros', 245 'Released machine: chromeos1-row3-rack5-host7.cros' 246 ]) 247 self.assertEqual(len(self.log_error), 0) 248 self.assertEqual(self.status, ['WAITING', 'FAILED']) 249 250 # Fourth test: ReadCache found a cached result. 251 ResetTestValues() 252 br.RunTest = FakeRunTest 253 br.ReadCache = FakeReadCacheSucceed 254 br.run() 255 self.assertTrue(self.called_ReadCache) 256 self.assertEqual(self.log_output, [ 257 'test_run: Cache hit.', 'result.out stuff', 258 'Releasing machine: chromeos1-row3-rack5-host7.cros', 259 'Released machine: chromeos1-row3-rack5-host7.cros' 260 ]) 261 self.assertEqual(self.log_error, ['result.err stuff']) 262 self.assertEqual(self.status, ['SUCCEEDED']) 263 264 # Fifth test: ReadCache generates an exception; does the try/finally block 265 # work? 266 ResetTestValues() 267 br.ReadCache = FakeReadCacheException 268 br.machine = FakeAcquireMachine() 269 br.run() 270 self.assertEqual(self.log_error, [ 271 "Benchmark run: 'test_run' failed: This is an exception test; it is " 272 'supposed to happen' 273 ]) 274 self.assertEqual(self.status, ['FAILED']) 275 276 def test_terminate_pass(self): 277 br = benchmark_run.BenchmarkRun('test_run', self.test_benchmark, 278 self.test_label, 1, 279 self.test_cache_conditions, 280 self.mock_machine_manager, self.mock_logger, 281 'average', '', {}) 282 283 def GetLastEventPassed(): 284 """Helper function for test_terminate_pass""" 285 return benchmark_run.STATUS_SUCCEEDED 286 287 def RecordStub(status): 288 """Helper function for test_terminate_pass""" 289 self.status = status 290 291 self.status = benchmark_run.STATUS_SUCCEEDED 292 self.assertFalse(br.terminated) 293 self.assertFalse(br.suite_runner.CommandTerminator().IsTerminated()) 294 295 br.timeline.GetLastEvent = GetLastEventPassed 296 br.timeline.Record = RecordStub 297 298 br.Terminate() 299 300 self.assertTrue(br.terminated) 301 self.assertTrue(br.suite_runner.CommandTerminator().IsTerminated()) 302 self.assertEqual(self.status, benchmark_run.STATUS_FAILED) 303 304 def test_terminate_fail(self): 305 br = benchmark_run.BenchmarkRun('test_run', self.test_benchmark, 306 self.test_label, 1, 307 self.test_cache_conditions, 308 self.mock_machine_manager, self.mock_logger, 309 'average', '', {}) 310 311 def GetLastEventFailed(): 312 """Helper function for test_terminate_fail""" 313 return benchmark_run.STATUS_FAILED 314 315 def RecordStub(status): 316 """Helper function for test_terminate_fail""" 317 self.status = status 318 319 self.status = benchmark_run.STATUS_SUCCEEDED 320 self.assertFalse(br.terminated) 321 self.assertFalse(br.suite_runner.CommandTerminator().IsTerminated()) 322 323 br.timeline.GetLastEvent = GetLastEventFailed 324 br.timeline.Record = RecordStub 325 326 br.Terminate() 327 328 self.assertTrue(br.terminated) 329 self.assertTrue(br.suite_runner.CommandTerminator().IsTerminated()) 330 self.assertEqual(self.status, benchmark_run.STATUS_SUCCEEDED) 331 332 def test_acquire_machine(self): 333 br = benchmark_run.BenchmarkRun('test_run', self.test_benchmark, 334 self.test_label, 1, 335 self.test_cache_conditions, 336 self.mock_machine_manager, self.mock_logger, 337 'average', '', {}) 338 339 br.terminated = True 340 self.assertRaises(Exception, br.AcquireMachine) 341 342 br.terminated = False 343 mock_machine = MockCrosMachine('chromeos1-row3-rack5-host7.cros', 344 'chromeos', 'average') 345 self.mock_machine_manager.AcquireMachine.return_value = mock_machine 346 347 machine = br.AcquireMachine() 348 self.assertEqual(machine.name, 'chromeos1-row3-rack5-host7.cros') 349 350 def test_get_extra_autotest_args(self): 351 br = benchmark_run.BenchmarkRun('test_run', self.test_benchmark, 352 self.test_label, 1, 353 self.test_cache_conditions, 354 self.mock_machine_manager, self.mock_logger, 355 'average', '', {}) 356 357 def MockLogError(err_msg): 358 """Helper function for test_get_extra_autotest_args""" 359 self.err_msg = err_msg 360 361 self.mock_logger.LogError = MockLogError 362 363 result = br.GetExtraAutotestArgs() 364 self.assertEqual(result, '') 365 366 self.test_benchmark.perf_args = 'record -e cycles' 367 result = br.GetExtraAutotestArgs() 368 self.assertEqual( 369 result, 370 '--profiler=custom_perf --profiler_args=\'perf_options="record -a -e ' 371 'cycles"\'') 372 373 self.test_benchmark.perf_args = 'record -e cycles' 374 self.test_benchmark.suite = 'test_that' 375 result = br.GetExtraAutotestArgs() 376 self.assertEqual(result, '') 377 self.assertEqual(self.err_msg, 378 'Non-telemetry benchmark does not support profiler.') 379 380 self.test_benchmark.perf_args = 'junk args' 381 self.test_benchmark.suite = 'telemetry_Crosperf' 382 self.assertRaises(Exception, br.GetExtraAutotestArgs) 383 384 @mock.patch.object(SuiteRunner, 'Run') 385 @mock.patch.object(Result, 'CreateFromRun') 386 def test_run_test(self, mock_result, mock_runner): 387 br = benchmark_run.BenchmarkRun('test_run', self.test_benchmark, 388 self.test_label, 1, 389 self.test_cache_conditions, 390 self.mock_machine_manager, self.mock_logger, 391 'average', '', {}) 392 393 self.status = [] 394 395 def MockRecord(status): 396 self.status.append(status) 397 398 br.timeline.Record = MockRecord 399 mock_machine = MockCrosMachine('chromeos1-row3-rack5-host7.cros', 400 'chromeos', 'average') 401 mock_runner.return_value = [0, "{'Score':100}", ''] 402 403 br.RunTest(mock_machine) 404 405 self.assertTrue(br.run_completed) 406 self.assertEqual( 407 self.status, 408 [benchmark_run.STATUS_IMAGING, benchmark_run.STATUS_RUNNING]) 409 410 self.assertEqual(br.machine_manager.ImageMachine.call_count, 1) 411 br.machine_manager.ImageMachine.assert_called_with(mock_machine, 412 self.test_label) 413 self.assertEqual(mock_runner.call_count, 1) 414 mock_runner.assert_called_with(mock_machine, br.label, br.benchmark, '', 415 br.profiler_args) 416 417 self.assertEqual(mock_result.call_count, 1) 418 mock_result.assert_called_with(self.mock_logger, 'average', self.test_label, 419 None, "{'Score':100}", '', 0, 420 'page_cycler.netsim.top_10', 421 'telemetry_Crosperf', '') 422 423 def test_set_cache_conditions(self): 424 br = benchmark_run.BenchmarkRun('test_run', self.test_benchmark, 425 self.test_label, 1, 426 self.test_cache_conditions, 427 self.mock_machine_manager, self.mock_logger, 428 'average', '', {}) 429 430 phony_cache_conditions = [123, 456, True, False] 431 432 self.assertEqual(br.cache_conditions, self.test_cache_conditions) 433 434 br.SetCacheConditions(phony_cache_conditions) 435 self.assertEqual(br.cache_conditions, phony_cache_conditions) 436 437 br.SetCacheConditions(self.test_cache_conditions) 438 self.assertEqual(br.cache_conditions, self.test_cache_conditions) 439 440 441if __name__ == '__main__': 442 unittest.main() 443