• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python2
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6"""Testing of benchmark_run."""
7
8from __future__ import print_function
9
10import mock
11import unittest
12import inspect
13
14from cros_utils import logger
15
16import benchmark_run
17
18from suite_runner import MockSuiteRunner
19from suite_runner import SuiteRunner
20from label import MockLabel
21from benchmark import Benchmark
22from machine_manager import MockMachineManager
23from machine_manager import MachineManager
24from machine_manager import MockCrosMachine
25from results_cache import MockResultsCache
26from results_cache import CacheConditions
27from results_cache import Result
28from results_cache import ResultsCache
29
30
31class BenchmarkRunTest(unittest.TestCase):
32  """Unit tests for the BenchmarkRun class and all of its methods."""
33
34  def setUp(self):
35    self.status = []
36    self.called_ReadCache = None
37    self.log_error = []
38    self.log_output = []
39    self.err_msg = None
40    self.test_benchmark = Benchmark(
41        'page_cycler.netsim.top_10',  # name
42        'page_cycler.netsim.top_10',  # test_name
43        '',  # test_args
44        1,  # iterations
45        False,  # rm_chroot_tmp
46        '',  # perf_args
47        suite='telemetry_Crosperf')  # suite
48
49    self.test_label = MockLabel(
50        'test1',
51        'image1',
52        'autotest_dir',
53        '/tmp/test_benchmark_run',
54        'x86-alex',
55        'chromeos2-row1-rack4-host9.cros',
56        image_args='',
57        cache_dir='',
58        cache_only=False,
59        log_level='average',
60        compiler='gcc')
61
62    self.test_cache_conditions = [
63        CacheConditions.CACHE_FILE_EXISTS, CacheConditions.CHECKSUMS_MATCH
64    ]
65
66    self.mock_logger = logger.GetLogger(log_dir='', mock=True)
67
68    self.mock_machine_manager = mock.Mock(spec=MachineManager)
69
70  def testDryRun(self):
71    my_label = MockLabel(
72        'test1',
73        'image1',
74        'autotest_dir',
75        '/tmp/test_benchmark_run',
76        'x86-alex',
77        'chromeos2-row1-rack4-host9.cros',
78        image_args='',
79        cache_dir='',
80        cache_only=False,
81        log_level='average',
82        compiler='gcc')
83
84    logging_level = 'average'
85    m = MockMachineManager('/tmp/chromeos_root', 0, logging_level, '')
86    m.AddMachine('chromeos2-row1-rack4-host9.cros')
87    bench = Benchmark(
88        'page_cycler.netsim.top_10',  # name
89        'page_cycler.netsim.top_10',  # test_name
90        '',  # test_args
91        1,  # iterations
92        False,  # rm_chroot_tmp
93        '',  # perf_args
94        suite='telemetry_Crosperf')  # suite
95    b = benchmark_run.MockBenchmarkRun('test run', bench, my_label, 1, [], m,
96                                       logger.GetLogger(), logging_level, '')
97    b.cache = MockResultsCache()
98    b.suite_runner = MockSuiteRunner()
99    b.start()
100
101    # Make sure the arguments to BenchmarkRun.__init__ have not changed
102    # since the last time this test was updated:
103    args_list = [
104        'self', 'name', 'benchmark', 'label', 'iteration', 'cache_conditions',
105        'machine_manager', 'logger_to_use', 'log_level', 'share_cache'
106    ]
107    arg_spec = inspect.getargspec(benchmark_run.BenchmarkRun.__init__)
108    self.assertEqual(len(arg_spec.args), len(args_list))
109    self.assertEqual(arg_spec.args, args_list)
110
111  def test_init(self):
112    # Nothing really worth testing here; just field assignments.
113    pass
114
115  def test_read_cache(self):
116    # Nothing really worth testing here, either.
117    pass
118
119  def test_run(self):
120    br = benchmark_run.BenchmarkRun(
121        'test_run', self.test_benchmark, self.test_label, 1,
122        self.test_cache_conditions, self.mock_machine_manager, self.mock_logger,
123        'average', '')
124
125    def MockLogOutput(msg, print_to_console=False):
126      'Helper function for test_run.'
127      del print_to_console
128      self.log_output.append(msg)
129
130    def MockLogError(msg, print_to_console=False):
131      'Helper function for test_run.'
132      del print_to_console
133      self.log_error.append(msg)
134
135    def MockRecordStatus(msg):
136      'Helper function for test_run.'
137      self.status.append(msg)
138
139    def FakeReadCache():
140      'Helper function for test_run.'
141      br.cache = mock.Mock(spec=ResultsCache)
142      self.called_ReadCache = True
143      return 0
144
145    def FakeReadCacheSucceed():
146      'Helper function for test_run.'
147      br.cache = mock.Mock(spec=ResultsCache)
148      br.result = mock.Mock(spec=Result)
149      br.result.out = 'result.out stuff'
150      br.result.err = 'result.err stuff'
151      br.result.retval = 0
152      self.called_ReadCache = True
153      return 0
154
155    def FakeReadCacheException():
156      'Helper function for test_run.'
157      raise RuntimeError('This is an exception test; it is supposed to happen')
158
159    def FakeAcquireMachine():
160      'Helper function for test_run.'
161      mock_machine = MockCrosMachine('chromeos1-row3-rack5-host7.cros',
162                                     'chromeos', 'average')
163      return mock_machine
164
165    def FakeRunTest(_machine):
166      'Helper function for test_run.'
167      mock_result = mock.Mock(spec=Result)
168      mock_result.retval = 0
169      return mock_result
170
171    def FakeRunTestFail(_machine):
172      'Helper function for test_run.'
173      mock_result = mock.Mock(spec=Result)
174      mock_result.retval = 1
175      return mock_result
176
177    def ResetTestValues():
178      'Helper function for test_run.'
179      self.log_output = []
180      self.log_error = []
181      self.status = []
182      br.result = None
183      self.called_ReadCache = False
184
185    # Assign all the fake functions to the appropriate objects.
186    br.logger().LogOutput = MockLogOutput
187    br.logger().LogError = MockLogError
188    br.timeline.Record = MockRecordStatus
189    br.ReadCache = FakeReadCache
190    br.RunTest = FakeRunTest
191    br.AcquireMachine = FakeAcquireMachine
192
193    # First test:  No cache hit, all goes well.
194    ResetTestValues()
195    br.run()
196    self.assertTrue(self.called_ReadCache)
197    self.assertEqual(self.log_output, [
198        'test_run: No cache hit.',
199        'Releasing machine: chromeos1-row3-rack5-host7.cros',
200        'Released machine: chromeos1-row3-rack5-host7.cros'
201    ])
202    self.assertEqual(len(self.log_error), 0)
203    self.assertEqual(self.status, ['WAITING', 'SUCCEEDED'])
204
205    # Second test: No cached result found; test run was "terminated" for some
206    # reason.
207    ResetTestValues()
208    br.terminated = True
209    br.run()
210    self.assertTrue(self.called_ReadCache)
211    self.assertEqual(self.log_output, [
212        'test_run: No cache hit.',
213        'Releasing machine: chromeos1-row3-rack5-host7.cros',
214        'Released machine: chromeos1-row3-rack5-host7.cros'
215    ])
216    self.assertEqual(len(self.log_error), 0)
217    self.assertEqual(self.status, ['WAITING'])
218
219    # Third test.  No cached result found; RunTest failed for some reason.
220    ResetTestValues()
221    br.terminated = False
222    br.RunTest = FakeRunTestFail
223    br.run()
224    self.assertTrue(self.called_ReadCache)
225    self.assertEqual(self.log_output, [
226        'test_run: No cache hit.',
227        'Releasing machine: chromeos1-row3-rack5-host7.cros',
228        'Released machine: chromeos1-row3-rack5-host7.cros'
229    ])
230    self.assertEqual(len(self.log_error), 0)
231    self.assertEqual(self.status, ['WAITING', 'FAILED'])
232
233    # Fourth test: ReadCache found a cached result.
234    ResetTestValues()
235    br.RunTest = FakeRunTest
236    br.ReadCache = FakeReadCacheSucceed
237    br.run()
238    self.assertTrue(self.called_ReadCache)
239    self.assertEqual(self.log_output, [
240        'test_run: Cache hit.', 'result.out stuff',
241        'Releasing machine: chromeos1-row3-rack5-host7.cros',
242        'Released machine: chromeos1-row3-rack5-host7.cros'
243    ])
244    self.assertEqual(self.log_error, ['result.err stuff'])
245    self.assertEqual(self.status, ['SUCCEEDED'])
246
247    # Fifth test: ReadCache generates an exception; does the try/finally block
248    # work?
249    ResetTestValues()
250    br.ReadCache = FakeReadCacheException
251    br.machine = FakeAcquireMachine()
252    br.run()
253    self.assertEqual(self.log_error, [
254        "Benchmark run: 'test_run' failed: This is an exception test; it is "
255        'supposed to happen'
256    ])
257    self.assertEqual(self.status, ['FAILED'])
258
259  def test_terminate_pass(self):
260    br = benchmark_run.BenchmarkRun(
261        'test_run', self.test_benchmark, self.test_label, 1,
262        self.test_cache_conditions, self.mock_machine_manager, self.mock_logger,
263        'average', '')
264
265    def GetLastEventPassed():
266      'Helper function for test_terminate_pass'
267      return benchmark_run.STATUS_SUCCEEDED
268
269    def RecordStub(status):
270      'Helper function for test_terminate_pass'
271      self.status = status
272
273    self.status = benchmark_run.STATUS_SUCCEEDED
274    self.assertFalse(br.terminated)
275    self.assertFalse(br.suite_runner.CommandTerminator().IsTerminated())
276
277    br.timeline.GetLastEvent = GetLastEventPassed
278    br.timeline.Record = RecordStub
279
280    br.Terminate()
281
282    self.assertTrue(br.terminated)
283    self.assertTrue(br.suite_runner.CommandTerminator().IsTerminated())
284    self.assertEqual(self.status, benchmark_run.STATUS_FAILED)
285
286  def test_terminate_fail(self):
287    br = benchmark_run.BenchmarkRun(
288        'test_run', self.test_benchmark, self.test_label, 1,
289        self.test_cache_conditions, self.mock_machine_manager, self.mock_logger,
290        'average', '')
291
292    def GetLastEventFailed():
293      'Helper function for test_terminate_fail'
294      return benchmark_run.STATUS_FAILED
295
296    def RecordStub(status):
297      'Helper function for test_terminate_fail'
298      self.status = status
299
300    self.status = benchmark_run.STATUS_SUCCEEDED
301    self.assertFalse(br.terminated)
302    self.assertFalse(br.suite_runner.CommandTerminator().IsTerminated())
303
304    br.timeline.GetLastEvent = GetLastEventFailed
305    br.timeline.Record = RecordStub
306
307    br.Terminate()
308
309    self.assertTrue(br.terminated)
310    self.assertTrue(br.suite_runner.CommandTerminator().IsTerminated())
311    self.assertEqual(self.status, benchmark_run.STATUS_SUCCEEDED)
312
313  def test_acquire_machine(self):
314    br = benchmark_run.BenchmarkRun(
315        'test_run', self.test_benchmark, self.test_label, 1,
316        self.test_cache_conditions, self.mock_machine_manager, self.mock_logger,
317        'average', '')
318
319    br.terminated = True
320    self.assertRaises(Exception, br.AcquireMachine)
321
322    br.terminated = False
323    mock_machine = MockCrosMachine('chromeos1-row3-rack5-host7.cros',
324                                   'chromeos', 'average')
325    self.mock_machine_manager.AcquireMachine.return_value = mock_machine
326
327    machine = br.AcquireMachine()
328    self.assertEqual(machine.name, 'chromeos1-row3-rack5-host7.cros')
329
330  def test_get_extra_autotest_args(self):
331    br = benchmark_run.BenchmarkRun(
332        'test_run', self.test_benchmark, self.test_label, 1,
333        self.test_cache_conditions, self.mock_machine_manager, self.mock_logger,
334        'average', '')
335
336    def MockLogError(err_msg):
337      'Helper function for test_get_extra_autotest_args'
338      self.err_msg = err_msg
339
340    self.mock_logger.LogError = MockLogError
341
342    result = br.GetExtraAutotestArgs()
343    self.assertEqual(result, '')
344
345    self.test_benchmark.perf_args = 'record -e cycles'
346    result = br.GetExtraAutotestArgs()
347    self.assertEqual(
348        result,
349        "--profiler=custom_perf --profiler_args='perf_options=\"record -a -e "
350        "cycles\"'")
351
352    self.test_benchmark.suite = 'telemetry'
353    result = br.GetExtraAutotestArgs()
354    self.assertEqual(result, '')
355    self.assertEqual(self.err_msg, 'Telemetry does not support profiler.')
356
357    self.test_benchmark.perf_args = 'record -e cycles'
358    self.test_benchmark.suite = 'test_that'
359    result = br.GetExtraAutotestArgs()
360    self.assertEqual(result, '')
361    self.assertEqual(self.err_msg, 'test_that does not support profiler.')
362
363    self.test_benchmark.perf_args = 'junk args'
364    self.test_benchmark.suite = 'telemetry_Crosperf'
365    self.assertRaises(Exception, br.GetExtraAutotestArgs)
366
367  @mock.patch.object(SuiteRunner, 'Run')
368  @mock.patch.object(Result, 'CreateFromRun')
369  def test_run_test(self, mock_result, mock_runner):
370    br = benchmark_run.BenchmarkRun(
371        'test_run', self.test_benchmark, self.test_label, 1,
372        self.test_cache_conditions, self.mock_machine_manager, self.mock_logger,
373        'average', '')
374
375    self.status = []
376
377    def MockRecord(status):
378      self.status.append(status)
379
380    br.timeline.Record = MockRecord
381    mock_machine = MockCrosMachine('chromeos1-row3-rack5-host7.cros',
382                                   'chromeos', 'average')
383    mock_runner.return_value = [0, "{'Score':100}", '']
384
385    br.RunTest(mock_machine)
386
387    self.assertTrue(br.run_completed)
388    self.assertEqual(self.status, [
389        benchmark_run.STATUS_IMAGING, benchmark_run.STATUS_RUNNING
390    ])
391
392    self.assertEqual(br.machine_manager.ImageMachine.call_count, 1)
393    br.machine_manager.ImageMachine.assert_called_with(mock_machine,
394                                                       self.test_label)
395    self.assertEqual(mock_runner.call_count, 1)
396    mock_runner.assert_called_with(mock_machine.name, br.label, br.benchmark,
397                                   '', br.profiler_args)
398
399    self.assertEqual(mock_result.call_count, 1)
400    mock_result.assert_called_with(
401        self.mock_logger, 'average', self.test_label, None, "{'Score':100}", '',
402        0, 'page_cycler.netsim.top_10', 'telemetry_Crosperf')
403
404  def test_set_cache_conditions(self):
405    br = benchmark_run.BenchmarkRun(
406        'test_run', self.test_benchmark, self.test_label, 1,
407        self.test_cache_conditions, self.mock_machine_manager, self.mock_logger,
408        'average', '')
409
410    phony_cache_conditions = [123, 456, True, False]
411
412    self.assertEqual(br.cache_conditions, self.test_cache_conditions)
413
414    br.SetCacheConditions(phony_cache_conditions)
415    self.assertEqual(br.cache_conditions, phony_cache_conditions)
416
417    br.SetCacheConditions(self.test_cache_conditions)
418    self.assertEqual(br.cache_conditions, self.test_cache_conditions)
419
420
421if __name__ == '__main__':
422  unittest.main()
423