• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python2
2#
3# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6"""Tests for the experiment runner module."""
7
8from __future__ import print_function
9
10import StringIO
11import getpass
12import os
13
14import mock
15import unittest
16
17import experiment_runner
18import experiment_status
19import machine_manager
20import config
21import test_flag
22
23from experiment_factory import ExperimentFactory
24from experiment_file import ExperimentFile
25from results_cache import Result
26from results_report import HTMLResultsReport
27from results_report import TextResultsReport
28
29from cros_utils import command_executer
30from cros_utils.email_sender import EmailSender
31from cros_utils.file_utils import FileUtils
32
33EXPERIMENT_FILE_1 = """
34  board: parrot
35  remote: chromeos-parrot1.cros chromreos-parrot2.cros
36
37  benchmark: kraken {
38    suite: telemetry_Crosperf
39    iterations: 3
40  }
41
42  image1 {
43    chromeos_root: /usr/local/google/chromeos
44    chromeos_image: /usr/local/google/chromeos/src/build/images/parrot/latest/cros_image1.bin
45  }
46
47  image2 {
48    chromeos_image: /usr/local/google/chromeos/src/build/imaages/parrot/latest/cros_image2.bin
49  }
50  """
51
52# pylint: disable=protected-access
53
54
55class FakeLogger(object):
56  """Fake logger for tests."""
57
58  def __init__(self):
59    self.LogOutputCount = 0
60    self.LogErrorCount = 0
61    self.output_msgs = []
62    self.error_msgs = []
63    self.dot_count = 0
64    self.LogStartDotsCount = 0
65    self.LogEndDotsCount = 0
66    self.LogAppendDotCount = 0
67
68  def LogOutput(self, msg):
69    self.LogOutputCount += 1
70    self.output_msgs.append(msg)
71
72  def LogError(self, msg):
73    self.LogErrorCount += 1
74    self.error_msgs.append(msg)
75
76  def LogStartDots(self):
77    self.LogStartDotsCount += 1
78    self.dot_count += 1
79
80  def LogAppendDot(self):
81    self.LogAppendDotCount += 1
82    self.dot_count += 1
83
84  def LogEndDots(self):
85    self.LogEndDotsCount += 1
86
87  def Reset(self):
88    self.LogOutputCount = 0
89    self.LogErrorCount = 0
90    self.output_msgs = []
91    self.error_msgs = []
92    self.dot_count = 0
93    self.LogStartDotsCount = 0
94    self.LogEndDotsCount = 0
95    self.LogAppendDotCount = 0
96
97
98class ExperimentRunnerTest(unittest.TestCase):
99  """Test for experiment runner class."""
100
101  run_count = 0
102  is_complete_count = 0
103  mock_logger = FakeLogger()
104  mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter)
105
106  def make_fake_experiment(self):
107    test_flag.SetTestMode(True)
108    experiment_file = ExperimentFile(StringIO.StringIO(EXPERIMENT_FILE_1))
109    experiment = ExperimentFactory().GetExperiment(
110        experiment_file, working_directory='', log_dir='')
111    return experiment
112
113  @mock.patch.object(machine_manager.MachineManager, 'AddMachine')
114  @mock.patch.object(os.path, 'isfile')
115
116  # pylint: disable=arguments-differ
117  def setUp(self, mock_isfile, _mock_addmachine):
118    mock_isfile.return_value = True
119    self.exp = self.make_fake_experiment()
120
121  def test_init(self):
122    er = experiment_runner.ExperimentRunner(
123        self.exp,
124        json_report=False,
125        using_schedv2=False,
126        log=self.mock_logger,
127        cmd_exec=self.mock_cmd_exec)
128    self.assertFalse(er._terminated)
129    self.assertEqual(er.STATUS_TIME_DELAY, 10)
130
131    self.exp.log_level = 'verbose'
132    er = experiment_runner.ExperimentRunner(
133        self.exp,
134        json_report=False,
135        using_schedv2=False,
136        log=self.mock_logger,
137        cmd_exec=self.mock_cmd_exec)
138    self.assertEqual(er.STATUS_TIME_DELAY, 30)
139
140  @mock.patch.object(experiment_status.ExperimentStatus, 'GetStatusString')
141  @mock.patch.object(experiment_status.ExperimentStatus, 'GetProgressString')
142  def test_run(self, mock_progress_string, mock_status_string):
143
144    self.run_count = 0
145    self.is_complete_count = 0
146
147    def reset():
148      self.run_count = 0
149      self.is_complete_count = 0
150
151    def FakeRun():
152      self.run_count += 1
153      return 0
154
155    def FakeIsComplete():
156      self.is_complete_count += 1
157      if self.is_complete_count < 3:
158        return False
159      else:
160        return True
161
162    self.mock_logger.Reset()
163    self.exp.Run = FakeRun
164    self.exp.IsComplete = FakeIsComplete
165
166    # Test 1: log_level == "quiet"
167    self.exp.log_level = 'quiet'
168    er = experiment_runner.ExperimentRunner(
169        self.exp,
170        json_report=False,
171        using_schedv2=False,
172        log=self.mock_logger,
173        cmd_exec=self.mock_cmd_exec)
174    er.STATUS_TIME_DELAY = 2
175    mock_status_string.return_value = 'Fake status string'
176    er._Run(self.exp)
177    self.assertEqual(self.run_count, 1)
178    self.assertTrue(self.is_complete_count > 0)
179    self.assertEqual(self.mock_logger.LogStartDotsCount, 1)
180    self.assertEqual(self.mock_logger.LogAppendDotCount, 1)
181    self.assertEqual(self.mock_logger.LogEndDotsCount, 1)
182    self.assertEqual(self.mock_logger.dot_count, 2)
183    self.assertEqual(mock_progress_string.call_count, 0)
184    self.assertEqual(mock_status_string.call_count, 2)
185    self.assertEqual(self.mock_logger.output_msgs, [
186        '==============================', 'Fake status string',
187        '=============================='
188    ])
189    self.assertEqual(len(self.mock_logger.error_msgs), 0)
190
191    # Test 2: log_level == "average"
192    self.mock_logger.Reset()
193    reset()
194    self.exp.log_level = 'average'
195    mock_status_string.call_count = 0
196    er = experiment_runner.ExperimentRunner(
197        self.exp,
198        json_report=False,
199        using_schedv2=False,
200        log=self.mock_logger,
201        cmd_exec=self.mock_cmd_exec)
202    er.STATUS_TIME_DELAY = 2
203    mock_status_string.return_value = 'Fake status string'
204    er._Run(self.exp)
205    self.assertEqual(self.run_count, 1)
206    self.assertTrue(self.is_complete_count > 0)
207    self.assertEqual(self.mock_logger.LogStartDotsCount, 1)
208    self.assertEqual(self.mock_logger.LogAppendDotCount, 1)
209    self.assertEqual(self.mock_logger.LogEndDotsCount, 1)
210    self.assertEqual(self.mock_logger.dot_count, 2)
211    self.assertEqual(mock_progress_string.call_count, 0)
212    self.assertEqual(mock_status_string.call_count, 2)
213    self.assertEqual(self.mock_logger.output_msgs, [
214        '==============================', 'Fake status string',
215        '=============================='
216    ])
217    self.assertEqual(len(self.mock_logger.error_msgs), 0)
218
219    # Test 3: log_level == "verbose"
220    self.mock_logger.Reset()
221    reset()
222    self.exp.log_level = 'verbose'
223    mock_status_string.call_count = 0
224    er = experiment_runner.ExperimentRunner(
225        self.exp,
226        json_report=False,
227        using_schedv2=False,
228        log=self.mock_logger,
229        cmd_exec=self.mock_cmd_exec)
230    er.STATUS_TIME_DELAY = 2
231    mock_status_string.return_value = 'Fake status string'
232    mock_progress_string.return_value = 'Fake progress string'
233    er._Run(self.exp)
234    self.assertEqual(self.run_count, 1)
235    self.assertTrue(self.is_complete_count > 0)
236    self.assertEqual(self.mock_logger.LogStartDotsCount, 0)
237    self.assertEqual(self.mock_logger.LogAppendDotCount, 0)
238    self.assertEqual(self.mock_logger.LogEndDotsCount, 0)
239    self.assertEqual(self.mock_logger.dot_count, 0)
240    self.assertEqual(mock_progress_string.call_count, 2)
241    self.assertEqual(mock_status_string.call_count, 2)
242    self.assertEqual(self.mock_logger.output_msgs, [
243        '==============================', 'Fake progress string',
244        'Fake status string', '==============================',
245        '==============================', 'Fake progress string',
246        'Fake status string', '=============================='
247    ])
248    self.assertEqual(len(self.mock_logger.error_msgs), 0)
249
250  @mock.patch.object(TextResultsReport, 'GetReport')
251  def test_print_table(self, mock_report):
252    self.mock_logger.Reset()
253    mock_report.return_value = 'This is a fake experiment report.'
254    er = experiment_runner.ExperimentRunner(
255        self.exp,
256        json_report=False,
257        using_schedv2=False,
258        log=self.mock_logger,
259        cmd_exec=self.mock_cmd_exec)
260    er._PrintTable(self.exp)
261    self.assertEqual(mock_report.call_count, 1)
262    self.assertEqual(self.mock_logger.output_msgs,
263                     ['This is a fake experiment report.'])
264
265  @mock.patch.object(HTMLResultsReport, 'GetReport')
266  @mock.patch.object(TextResultsReport, 'GetReport')
267  @mock.patch.object(EmailSender, 'Attachment')
268  @mock.patch.object(EmailSender, 'SendEmail')
269  @mock.patch.object(getpass, 'getuser')
270  def test_email(self, mock_getuser, mock_emailer, mock_attachment,
271                 mock_text_report, mock_html_report):
272
273    mock_getuser.return_value = 'john.smith@google.com'
274    mock_text_report.return_value = 'This is a fake text report.'
275    mock_html_report.return_value = 'This is a fake html report.'
276
277    self.mock_logger.Reset()
278    config.AddConfig('no_email', True)
279    self.exp.email_to = ['jane.doe@google.com']
280    er = experiment_runner.ExperimentRunner(
281        self.exp,
282        json_report=False,
283        using_schedv2=False,
284        log=self.mock_logger,
285        cmd_exec=self.mock_cmd_exec)
286    # Test 1. Config:no_email; exp.email_to set ==> no email sent
287    er._Email(self.exp)
288    self.assertEqual(mock_getuser.call_count, 0)
289    self.assertEqual(mock_emailer.call_count, 0)
290    self.assertEqual(mock_attachment.call_count, 0)
291    self.assertEqual(mock_text_report.call_count, 0)
292    self.assertEqual(mock_html_report.call_count, 0)
293
294    # Test 2. Config: email. exp.email_to set; cache hit.  => send email
295    self.mock_logger.Reset()
296    config.AddConfig('no_email', False)
297    for r in self.exp.benchmark_runs:
298      r.cache_hit = True
299    er._Email(self.exp)
300    self.assertEqual(mock_getuser.call_count, 1)
301    self.assertEqual(mock_emailer.call_count, 1)
302    self.assertEqual(mock_attachment.call_count, 1)
303    self.assertEqual(mock_text_report.call_count, 1)
304    self.assertEqual(mock_html_report.call_count, 1)
305    self.assertEqual(len(mock_emailer.call_args), 2)
306    self.assertEqual(mock_emailer.call_args[0],
307                     (['jane.doe@google.com',
308                       'john.smith@google.com'], ': image1 vs. image2',
309                      "<pre style='font-size: 13px'>This is a fake text "
310                      'report.\nResults are stored in _results.\n</pre>'))
311    self.assertTrue(type(mock_emailer.call_args[1]) is dict)
312    self.assertEqual(len(mock_emailer.call_args[1]), 2)
313    self.assertTrue('attachments' in mock_emailer.call_args[1].keys())
314    self.assertEqual(mock_emailer.call_args[1]['msg_type'], 'html')
315
316    mock_attachment.assert_called_with('report.html',
317                                       'This is a fake html report.')
318
319    # Test 3. Config: email; exp.mail_to set; no cache hit.  => send email
320    self.mock_logger.Reset()
321    mock_getuser.reset_mock()
322    mock_emailer.reset_mock()
323    mock_attachment.reset_mock()
324    mock_text_report.reset_mock()
325    mock_html_report.reset_mock()
326    config.AddConfig('no_email', False)
327    for r in self.exp.benchmark_runs:
328      r.cache_hit = False
329    er._Email(self.exp)
330    self.assertEqual(mock_getuser.call_count, 1)
331    self.assertEqual(mock_emailer.call_count, 1)
332    self.assertEqual(mock_attachment.call_count, 1)
333    self.assertEqual(mock_text_report.call_count, 1)
334    self.assertEqual(mock_html_report.call_count, 1)
335    self.assertEqual(len(mock_emailer.call_args), 2)
336    self.assertEqual(mock_emailer.call_args[0],
337                     ([
338                         'jane.doe@google.com', 'john.smith@google.com',
339                         'john.smith@google.com'
340                     ], ': image1 vs. image2',
341                      "<pre style='font-size: 13px'>This is a fake text "
342                      'report.\nResults are stored in _results.\n</pre>'))
343    self.assertTrue(type(mock_emailer.call_args[1]) is dict)
344    self.assertEqual(len(mock_emailer.call_args[1]), 2)
345    self.assertTrue('attachments' in mock_emailer.call_args[1].keys())
346    self.assertEqual(mock_emailer.call_args[1]['msg_type'], 'html')
347
348    mock_attachment.assert_called_with('report.html',
349                                       'This is a fake html report.')
350
351    # Test 4. Config: email; exp.mail_to = None; no cache hit. => send email
352    self.mock_logger.Reset()
353    mock_getuser.reset_mock()
354    mock_emailer.reset_mock()
355    mock_attachment.reset_mock()
356    mock_text_report.reset_mock()
357    mock_html_report.reset_mock()
358    self.exp.email_to = []
359    er._Email(self.exp)
360    self.assertEqual(mock_getuser.call_count, 1)
361    self.assertEqual(mock_emailer.call_count, 1)
362    self.assertEqual(mock_attachment.call_count, 1)
363    self.assertEqual(mock_text_report.call_count, 1)
364    self.assertEqual(mock_html_report.call_count, 1)
365    self.assertEqual(len(mock_emailer.call_args), 2)
366    self.assertEqual(mock_emailer.call_args[0],
367                     (['john.smith@google.com'], ': image1 vs. image2',
368                      "<pre style='font-size: 13px'>This is a fake text "
369                      'report.\nResults are stored in _results.\n</pre>'))
370    self.assertTrue(type(mock_emailer.call_args[1]) is dict)
371    self.assertEqual(len(mock_emailer.call_args[1]), 2)
372    self.assertTrue('attachments' in mock_emailer.call_args[1].keys())
373    self.assertEqual(mock_emailer.call_args[1]['msg_type'], 'html')
374
375    mock_attachment.assert_called_with('report.html',
376                                       'This is a fake html report.')
377
378    # Test 5. Config: email; exp.mail_to = None; cache hit => no email sent
379    self.mock_logger.Reset()
380    mock_getuser.reset_mock()
381    mock_emailer.reset_mock()
382    mock_attachment.reset_mock()
383    mock_text_report.reset_mock()
384    mock_html_report.reset_mock()
385    for r in self.exp.benchmark_runs:
386      r.cache_hit = True
387    er._Email(self.exp)
388    self.assertEqual(mock_getuser.call_count, 0)
389    self.assertEqual(mock_emailer.call_count, 0)
390    self.assertEqual(mock_attachment.call_count, 0)
391    self.assertEqual(mock_text_report.call_count, 0)
392    self.assertEqual(mock_html_report.call_count, 0)
393
394  @mock.patch.object(FileUtils, 'RmDir')
395  @mock.patch.object(FileUtils, 'MkDirP')
396  @mock.patch.object(FileUtils, 'WriteFile')
397  @mock.patch.object(HTMLResultsReport, 'FromExperiment')
398  @mock.patch.object(TextResultsReport, 'FromExperiment')
399  @mock.patch.object(Result, 'CopyResultsTo')
400  @mock.patch.object(Result, 'CleanUp')
401  def test_store_results(self, mock_cleanup, mock_copy, _mock_text_report,
402                         mock_report, mock_writefile, mock_mkdir, mock_rmdir):
403
404    self.mock_logger.Reset()
405    self.exp.results_directory = '/usr/local/crosperf-results'
406    bench_run = self.exp.benchmark_runs[5]
407    bench_path = '/usr/local/crosperf-results/' + filter(
408        str.isalnum, bench_run.name)
409    self.assertEqual(len(self.exp.benchmark_runs), 6)
410
411    er = experiment_runner.ExperimentRunner(
412        self.exp,
413        json_report=False,
414        using_schedv2=False,
415        log=self.mock_logger,
416        cmd_exec=self.mock_cmd_exec)
417
418    # Test 1. Make sure nothing is done if _terminated is true.
419    er._terminated = True
420    er._StoreResults(self.exp)
421    self.assertEqual(mock_cleanup.call_count, 0)
422    self.assertEqual(mock_copy.call_count, 0)
423    self.assertEqual(mock_report.call_count, 0)
424    self.assertEqual(mock_writefile.call_count, 0)
425    self.assertEqual(mock_mkdir.call_count, 0)
426    self.assertEqual(mock_rmdir.call_count, 0)
427    self.assertEqual(self.mock_logger.LogOutputCount, 0)
428
429    # Test 2. _terminated is false; everything works properly.
430    fake_result = Result(self.mock_logger, self.exp.labels[0], 'average',
431                         'daisy1')
432    for r in self.exp.benchmark_runs:
433      r.result = fake_result
434    er._terminated = False
435    er._StoreResults(self.exp)
436    self.assertEqual(mock_cleanup.call_count, 6)
437    mock_cleanup.called_with(bench_run.benchmark.rm_chroot_tmp)
438    self.assertEqual(mock_copy.call_count, 6)
439    mock_copy.called_with(bench_path)
440    self.assertEqual(mock_writefile.call_count, 3)
441    self.assertEqual(len(mock_writefile.call_args_list), 3)
442    first_args = mock_writefile.call_args_list[0]
443    second_args = mock_writefile.call_args_list[1]
444    self.assertEqual(first_args[0][0],
445                     '/usr/local/crosperf-results/experiment.exp')
446    self.assertEqual(second_args[0][0],
447                     '/usr/local/crosperf-results/results.html')
448    self.assertEqual(mock_mkdir.call_count, 1)
449    mock_mkdir.called_with('/usr/local/crosperf-results')
450    self.assertEqual(mock_rmdir.call_count, 1)
451    mock_rmdir.called_with('/usr/local/crosperf-results')
452    self.assertEqual(self.mock_logger.LogOutputCount, 4)
453    self.assertEqual(self.mock_logger.output_msgs, [
454        'Storing experiment file in /usr/local/crosperf-results.',
455        'Storing results report in /usr/local/crosperf-results.',
456        'Storing email message body in /usr/local/crosperf-results.',
457        'Storing results of each benchmark run.'
458    ])
459
460
461if __name__ == '__main__':
462  unittest.main()
463