• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3
4# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
5# Use of this source code is governed by a BSD-style license that can be
6# found in the LICENSE file.
7
8"""Testing of benchmark_run."""
9
10from __future__ import print_function
11
12import inspect
13import unittest
14import unittest.mock as mock
15
16import benchmark_run
17
18from cros_utils import logger
19from suite_runner import MockSuiteRunner
20from suite_runner import SuiteRunner
21from label import MockLabel
22from benchmark import Benchmark
23from machine_manager import MockMachineManager
24from machine_manager import MachineManager
25from machine_manager import MockCrosMachine
26from results_cache import MockResultsCache
27from results_cache import CacheConditions
28from results_cache import Result
29from results_cache import ResultsCache
30
31
32class BenchmarkRunTest(unittest.TestCase):
33  """Unit tests for the BenchmarkRun class and all of its methods."""
34
35  def setUp(self):
36    self.status = []
37    self.called_ReadCache = None
38    self.log_error = []
39    self.log_output = []
40    self.err_msg = None
41    self.test_benchmark = Benchmark(
42        'page_cycler.netsim.top_10',  # name
43        'page_cycler.netsim.top_10',  # test_name
44        '',  # test_args
45        1,  # iterations
46        False,  # rm_chroot_tmp
47        '',  # perf_args
48        suite='telemetry_Crosperf')  # suite
49
50    self.test_label = MockLabel(
51        'test1',
52        'build',
53        'image1',
54        'autotest_dir',
55        'debug_dir',
56        '/tmp/test_benchmark_run',
57        'x86-alex',
58        'chromeos2-row1-rack4-host9.cros',
59        image_args='',
60        cache_dir='',
61        cache_only=False,
62        log_level='average',
63        compiler='gcc',
64        skylab=False)
65
66    self.test_cache_conditions = [
67        CacheConditions.CACHE_FILE_EXISTS, CacheConditions.CHECKSUMS_MATCH
68    ]
69
70    self.mock_logger = logger.GetLogger(log_dir='', mock=True)
71
72    self.mock_machine_manager = mock.Mock(spec=MachineManager)
73
74  def testDryRun(self):
75    my_label = MockLabel(
76        'test1',
77        'build',
78        'image1',
79        'autotest_dir',
80        'debug_dir',
81        '/tmp/test_benchmark_run',
82        'x86-alex',
83        'chromeos2-row1-rack4-host9.cros',
84        image_args='',
85        cache_dir='',
86        cache_only=False,
87        log_level='average',
88        compiler='gcc',
89        skylab=False)
90
91    logging_level = 'average'
92    m = MockMachineManager('/tmp/chromeos_root', 0, logging_level, '')
93    m.AddMachine('chromeos2-row1-rack4-host9.cros')
94    bench = Benchmark(
95        'page_cycler.netsim.top_10',  # name
96        'page_cycler.netsim.top_10',  # test_name
97        '',  # test_args
98        1,  # iterations
99        False,  # rm_chroot_tmp
100        '',  # perf_args
101        suite='telemetry_Crosperf')  # suite
102    dut_conf = {
103        'cooldown_time': 5,
104        'cooldown_temp': 45,
105        'governor': 'powersave',
106        'cpu_usage': 'big_only',
107        'cpu_freq_pct': 80,
108    }
109    b = benchmark_run.MockBenchmarkRun('test run', bench, my_label, 1, [], m,
110                                       logger.GetLogger(), logging_level, '',
111                                       dut_conf)
112    b.cache = MockResultsCache()
113    b.suite_runner = MockSuiteRunner()
114    b.start()
115
116    # Make sure the arguments to BenchmarkRun.__init__ have not changed
117    # since the last time this test was updated:
118    args_list = [
119        'self', 'name', 'benchmark', 'label', 'iteration', 'cache_conditions',
120        'machine_manager', 'logger_to_use', 'log_level', 'share_cache',
121        'dut_config'
122    ]
123    arg_spec = inspect.getfullargspec(benchmark_run.BenchmarkRun.__init__)
124    self.assertEqual(len(arg_spec.args), len(args_list))
125    self.assertEqual(arg_spec.args, args_list)
126
127  def test_init(self):
128    # Nothing really worth testing here; just field assignments.
129    pass
130
131  def test_read_cache(self):
132    # Nothing really worth testing here, either.
133    pass
134
135  def test_run(self):
136    br = benchmark_run.BenchmarkRun(
137        'test_run', self.test_benchmark, self.test_label, 1,
138        self.test_cache_conditions, self.mock_machine_manager, self.mock_logger,
139        'average', '', {})
140
141    def MockLogOutput(msg, print_to_console=False):
142      """Helper function for test_run."""
143      del print_to_console
144      self.log_output.append(msg)
145
146    def MockLogError(msg, print_to_console=False):
147      """Helper function for test_run."""
148      del print_to_console
149      self.log_error.append(msg)
150
151    def MockRecordStatus(msg):
152      """Helper function for test_run."""
153      self.status.append(msg)
154
155    def FakeReadCache():
156      """Helper function for test_run."""
157      br.cache = mock.Mock(spec=ResultsCache)
158      self.called_ReadCache = True
159      return 0
160
161    def FakeReadCacheSucceed():
162      """Helper function for test_run."""
163      br.cache = mock.Mock(spec=ResultsCache)
164      br.result = mock.Mock(spec=Result)
165      br.result.out = 'result.out stuff'
166      br.result.err = 'result.err stuff'
167      br.result.retval = 0
168      self.called_ReadCache = True
169      return 0
170
171    def FakeReadCacheException():
172      """Helper function for test_run."""
173      raise RuntimeError('This is an exception test; it is supposed to happen')
174
175    def FakeAcquireMachine():
176      """Helper function for test_run."""
177      mock_machine = MockCrosMachine('chromeos1-row3-rack5-host7.cros',
178                                     'chromeos', 'average')
179      return mock_machine
180
181    def FakeRunTest(_machine):
182      """Helper function for test_run."""
183      mock_result = mock.Mock(spec=Result)
184      mock_result.retval = 0
185      return mock_result
186
187    def FakeRunTestFail(_machine):
188      """Helper function for test_run."""
189      mock_result = mock.Mock(spec=Result)
190      mock_result.retval = 1
191      return mock_result
192
193    def ResetTestValues():
194      """Helper function for test_run."""
195      self.log_output = []
196      self.log_error = []
197      self.status = []
198      br.result = None
199      self.called_ReadCache = False
200
201    # Assign all the fake functions to the appropriate objects.
202    br.logger().LogOutput = MockLogOutput
203    br.logger().LogError = MockLogError
204    br.timeline.Record = MockRecordStatus
205    br.ReadCache = FakeReadCache
206    br.RunTest = FakeRunTest
207    br.AcquireMachine = FakeAcquireMachine
208
209    # First test:  No cache hit, all goes well.
210    ResetTestValues()
211    br.run()
212    self.assertTrue(self.called_ReadCache)
213    self.assertEqual(self.log_output, [
214        'test_run: No cache hit.',
215        'Releasing machine: chromeos1-row3-rack5-host7.cros',
216        'Released machine: chromeos1-row3-rack5-host7.cros'
217    ])
218    self.assertEqual(len(self.log_error), 0)
219    self.assertEqual(self.status, ['WAITING', 'SUCCEEDED'])
220
221    # Second test: No cached result found; test run was "terminated" for some
222    # reason.
223    ResetTestValues()
224    br.terminated = True
225    br.run()
226    self.assertTrue(self.called_ReadCache)
227    self.assertEqual(self.log_output, [
228        'test_run: No cache hit.',
229        'Releasing machine: chromeos1-row3-rack5-host7.cros',
230        'Released machine: chromeos1-row3-rack5-host7.cros'
231    ])
232    self.assertEqual(len(self.log_error), 0)
233    self.assertEqual(self.status, ['WAITING'])
234
235    # Third test.  No cached result found; RunTest failed for some reason.
236    ResetTestValues()
237    br.terminated = False
238    br.RunTest = FakeRunTestFail
239    br.run()
240    self.assertTrue(self.called_ReadCache)
241    self.assertEqual(self.log_output, [
242        'test_run: No cache hit.',
243        'Releasing machine: chromeos1-row3-rack5-host7.cros',
244        'Released machine: chromeos1-row3-rack5-host7.cros'
245    ])
246    self.assertEqual(len(self.log_error), 0)
247    self.assertEqual(self.status, ['WAITING', 'FAILED'])
248
249    # Fourth test: ReadCache found a cached result.
250    ResetTestValues()
251    br.RunTest = FakeRunTest
252    br.ReadCache = FakeReadCacheSucceed
253    br.run()
254    self.assertTrue(self.called_ReadCache)
255    self.assertEqual(self.log_output, [
256        'test_run: Cache hit.', 'result.out stuff',
257        'Releasing machine: chromeos1-row3-rack5-host7.cros',
258        'Released machine: chromeos1-row3-rack5-host7.cros'
259    ])
260    self.assertEqual(self.log_error, ['result.err stuff'])
261    self.assertEqual(self.status, ['SUCCEEDED'])
262
263    # Fifth test: ReadCache generates an exception; does the try/finally block
264    # work?
265    ResetTestValues()
266    br.ReadCache = FakeReadCacheException
267    br.machine = FakeAcquireMachine()
268    br.run()
269    self.assertEqual(self.log_error, [
270        "Benchmark run: 'test_run' failed: This is an exception test; it is "
271        'supposed to happen'
272    ])
273    self.assertEqual(self.status, ['FAILED'])
274
275  def test_terminate_pass(self):
276    br = benchmark_run.BenchmarkRun(
277        'test_run', self.test_benchmark, self.test_label, 1,
278        self.test_cache_conditions, self.mock_machine_manager, self.mock_logger,
279        'average', '', {})
280
281    def GetLastEventPassed():
282      """Helper function for test_terminate_pass"""
283      return benchmark_run.STATUS_SUCCEEDED
284
285    def RecordStub(status):
286      """Helper function for test_terminate_pass"""
287      self.status = status
288
289    self.status = benchmark_run.STATUS_SUCCEEDED
290    self.assertFalse(br.terminated)
291    self.assertFalse(br.suite_runner.CommandTerminator().IsTerminated())
292
293    br.timeline.GetLastEvent = GetLastEventPassed
294    br.timeline.Record = RecordStub
295
296    br.Terminate()
297
298    self.assertTrue(br.terminated)
299    self.assertTrue(br.suite_runner.CommandTerminator().IsTerminated())
300    self.assertEqual(self.status, benchmark_run.STATUS_FAILED)
301
302  def test_terminate_fail(self):
303    br = benchmark_run.BenchmarkRun(
304        'test_run', self.test_benchmark, self.test_label, 1,
305        self.test_cache_conditions, self.mock_machine_manager, self.mock_logger,
306        'average', '', {})
307
308    def GetLastEventFailed():
309      """Helper function for test_terminate_fail"""
310      return benchmark_run.STATUS_FAILED
311
312    def RecordStub(status):
313      """Helper function for test_terminate_fail"""
314      self.status = status
315
316    self.status = benchmark_run.STATUS_SUCCEEDED
317    self.assertFalse(br.terminated)
318    self.assertFalse(br.suite_runner.CommandTerminator().IsTerminated())
319
320    br.timeline.GetLastEvent = GetLastEventFailed
321    br.timeline.Record = RecordStub
322
323    br.Terminate()
324
325    self.assertTrue(br.terminated)
326    self.assertTrue(br.suite_runner.CommandTerminator().IsTerminated())
327    self.assertEqual(self.status, benchmark_run.STATUS_SUCCEEDED)
328
329  def test_acquire_machine(self):
330    br = benchmark_run.BenchmarkRun(
331        'test_run', self.test_benchmark, self.test_label, 1,
332        self.test_cache_conditions, self.mock_machine_manager, self.mock_logger,
333        'average', '', {})
334
335    br.terminated = True
336    self.assertRaises(Exception, br.AcquireMachine)
337
338    br.terminated = False
339    mock_machine = MockCrosMachine('chromeos1-row3-rack5-host7.cros',
340                                   'chromeos', 'average')
341    self.mock_machine_manager.AcquireMachine.return_value = mock_machine
342
343    machine = br.AcquireMachine()
344    self.assertEqual(machine.name, 'chromeos1-row3-rack5-host7.cros')
345
346  def test_get_extra_autotest_args(self):
347    br = benchmark_run.BenchmarkRun(
348        'test_run', self.test_benchmark, self.test_label, 1,
349        self.test_cache_conditions, self.mock_machine_manager, self.mock_logger,
350        'average', '', {})
351
352    def MockLogError(err_msg):
353      """Helper function for test_get_extra_autotest_args"""
354      self.err_msg = err_msg
355
356    self.mock_logger.LogError = MockLogError
357
358    result = br.GetExtraAutotestArgs()
359    self.assertEqual(result, '')
360
361    self.test_benchmark.perf_args = 'record -e cycles'
362    result = br.GetExtraAutotestArgs()
363    self.assertEqual(
364        result,
365        '--profiler=custom_perf --profiler_args=\'perf_options="record -a -e '
366        'cycles"\'')
367
368    self.test_benchmark.perf_args = 'record -e cycles'
369    self.test_benchmark.suite = 'test_that'
370    result = br.GetExtraAutotestArgs()
371    self.assertEqual(result, '')
372    self.assertEqual(self.err_msg,
373                     'Non-telemetry benchmark does not support profiler.')
374
375    self.test_benchmark.perf_args = 'junk args'
376    self.test_benchmark.suite = 'telemetry_Crosperf'
377    self.assertRaises(Exception, br.GetExtraAutotestArgs)
378
379  @mock.patch.object(SuiteRunner, 'Run')
380  @mock.patch.object(Result, 'CreateFromRun')
381  def test_run_test(self, mock_result, mock_runner):
382    br = benchmark_run.BenchmarkRun(
383        'test_run', self.test_benchmark, self.test_label, 1,
384        self.test_cache_conditions, self.mock_machine_manager, self.mock_logger,
385        'average', '', {})
386
387    self.status = []
388
389    def MockRecord(status):
390      self.status.append(status)
391
392    br.timeline.Record = MockRecord
393    mock_machine = MockCrosMachine('chromeos1-row3-rack5-host7.cros',
394                                   'chromeos', 'average')
395    mock_runner.return_value = [0, "{'Score':100}", '']
396
397    br.RunTest(mock_machine)
398
399    self.assertTrue(br.run_completed)
400    self.assertEqual(
401        self.status,
402        [benchmark_run.STATUS_IMAGING, benchmark_run.STATUS_RUNNING])
403
404    self.assertEqual(br.machine_manager.ImageMachine.call_count, 1)
405    br.machine_manager.ImageMachine.assert_called_with(mock_machine,
406                                                       self.test_label)
407    self.assertEqual(mock_runner.call_count, 1)
408    mock_runner.assert_called_with(mock_machine, br.label, br.benchmark, '',
409                                   br.profiler_args)
410
411    self.assertEqual(mock_result.call_count, 1)
412    mock_result.assert_called_with(
413        self.mock_logger, 'average', self.test_label, None, "{'Score':100}", '',
414        0, 'page_cycler.netsim.top_10', 'telemetry_Crosperf', '')
415
416  def test_set_cache_conditions(self):
417    br = benchmark_run.BenchmarkRun(
418        'test_run', self.test_benchmark, self.test_label, 1,
419        self.test_cache_conditions, self.mock_machine_manager, self.mock_logger,
420        'average', '', {})
421
422    phony_cache_conditions = [123, 456, True, False]
423
424    self.assertEqual(br.cache_conditions, self.test_cache_conditions)
425
426    br.SetCacheConditions(phony_cache_conditions)
427    self.assertEqual(br.cache_conditions, phony_cache_conditions)
428
429    br.SetCacheConditions(self.test_cache_conditions)
430    self.assertEqual(br.cache_conditions, self.test_cache_conditions)
431
432
433if __name__ == '__main__':
434  unittest.main()
435