• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3#
4# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
5# Use of this source code is governed by a BSD-style license that can be
6# found in the LICENSE file.
7
8"""Tests for the experiment runner module."""
9
10from __future__ import print_function
11
12import getpass
13import io
14import os
15import time
16
17import unittest
18import unittest.mock as mock
19
20import experiment_runner
21import experiment_status
22import machine_manager
23import config
24import test_flag
25
26from experiment_factory import ExperimentFactory
27from experiment_file import ExperimentFile
28from results_cache import Result
29from results_report import HTMLResultsReport
30from results_report import TextResultsReport
31
32from cros_utils import command_executer
33from cros_utils.email_sender import EmailSender
34from cros_utils.file_utils import FileUtils
35
36EXPERIMENT_FILE_1 = """
37  board: parrot
38  remote: chromeos-parrot1.cros chromreos-parrot2.cros
39  locks_dir: /tmp
40
41  benchmark: kraken {
42    suite: telemetry_Crosperf
43    iterations: 3
44  }
45
46  image1 {
47    chromeos_root: /usr/local/google/chromeos
48    chromeos_image: /usr/local/google/chromeos/src/build/images/parrot/latest/cros_image1.bin
49  }
50
51  image2 {
52    chromeos_image: /usr/local/google/chromeos/src/build/imaages/parrot/latest/cros_image2.bin
53  }
54  """
55
56# pylint: disable=protected-access
57
58
59class FakeLogger(object):
60  """Fake logger for tests."""
61
62  def __init__(self):
63    self.LogOutputCount = 0
64    self.LogErrorCount = 0
65    self.output_msgs = []
66    self.error_msgs = []
67    self.dot_count = 0
68    self.LogStartDotsCount = 0
69    self.LogEndDotsCount = 0
70    self.LogAppendDotCount = 0
71
72  def LogOutput(self, msg):
73    self.LogOutputCount += 1
74    self.output_msgs.append(msg)
75
76  def LogError(self, msg):
77    self.LogErrorCount += 1
78    self.error_msgs.append(msg)
79
80  def LogStartDots(self):
81    self.LogStartDotsCount += 1
82    self.dot_count += 1
83
84  def LogAppendDot(self):
85    self.LogAppendDotCount += 1
86    self.dot_count += 1
87
88  def LogEndDots(self):
89    self.LogEndDotsCount += 1
90
91  def Reset(self):
92    self.LogOutputCount = 0
93    self.LogErrorCount = 0
94    self.output_msgs = []
95    self.error_msgs = []
96    self.dot_count = 0
97    self.LogStartDotsCount = 0
98    self.LogEndDotsCount = 0
99    self.LogAppendDotCount = 0
100
101
102class ExperimentRunnerTest(unittest.TestCase):
103  """Test for experiment runner class."""
104
105  run_count = 0
106  is_complete_count = 0
107  mock_logger = FakeLogger()
108  mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter)
109
110  def make_fake_experiment(self):
111    test_flag.SetTestMode(True)
112    experiment_file = ExperimentFile(io.StringIO(EXPERIMENT_FILE_1))
113    experiment = ExperimentFactory().GetExperiment(
114        experiment_file, working_directory='', log_dir='')
115    return experiment
116
117  @mock.patch.object(machine_manager.MachineManager, 'AddMachine')
118  @mock.patch.object(os.path, 'isfile')
119
120  # pylint: disable=arguments-differ
121  def setUp(self, mock_isfile, _mock_addmachine):
122    mock_isfile.return_value = True
123    self.exp = self.make_fake_experiment()
124
125  def test_init(self):
126    er = experiment_runner.ExperimentRunner(
127        self.exp,
128        json_report=False,
129        using_schedv2=False,
130        log=self.mock_logger,
131        cmd_exec=self.mock_cmd_exec)
132    self.assertFalse(er._terminated)
133    self.assertEqual(er.STATUS_TIME_DELAY, 10)
134
135    self.exp.log_level = 'verbose'
136    er = experiment_runner.ExperimentRunner(
137        self.exp,
138        json_report=False,
139        using_schedv2=False,
140        log=self.mock_logger,
141        cmd_exec=self.mock_cmd_exec)
142    self.assertEqual(er.STATUS_TIME_DELAY, 30)
143
144  @mock.patch.object(time, 'time')
145  @mock.patch.object(time, 'sleep')
146  @mock.patch.object(experiment_status.ExperimentStatus, 'GetStatusString')
147  @mock.patch.object(experiment_status.ExperimentStatus, 'GetProgressString')
148  def test_run(self, mock_progress_string, mock_status_string, mock_sleep,
149               mock_time):
150
151    self.run_count = 0
152    self.is_complete_count = 0
153    mock_sleep.return_value = None
154    # pylint: disable=range-builtin-not-iterating
155    mock_time.side_effect = range(1, 50, 1)
156
157    def reset():
158      self.run_count = 0
159      self.is_complete_count = 0
160
161    def FakeRun():
162      self.run_count += 1
163      return 0
164
165    def FakeIsComplete():
166      self.is_complete_count += 1
167      if self.is_complete_count < 6:
168        return False
169      else:
170        return True
171
172    self.mock_logger.Reset()
173    self.exp.Run = FakeRun
174    self.exp.IsComplete = FakeIsComplete
175
176    # Test 1: log_level == "quiet"
177    self.exp.log_level = 'quiet'
178    er = experiment_runner.ExperimentRunner(
179        self.exp,
180        json_report=False,
181        using_schedv2=False,
182        log=self.mock_logger,
183        cmd_exec=self.mock_cmd_exec)
184    er.STATUS_TIME_DELAY = 2
185    mock_status_string.return_value = 'Fake status string'
186    er._Run(self.exp)
187    self.assertEqual(self.run_count, 1)
188    self.assertTrue(self.is_complete_count > 0)
189    self.assertEqual(self.mock_logger.LogStartDotsCount, 1)
190    self.assertEqual(self.mock_logger.LogAppendDotCount, 1)
191    self.assertEqual(self.mock_logger.LogEndDotsCount, 1)
192    self.assertEqual(self.mock_logger.dot_count, 2)
193    self.assertEqual(mock_progress_string.call_count, 0)
194    self.assertEqual(mock_status_string.call_count, 2)
195    self.assertEqual(self.mock_logger.output_msgs, [
196        '==============================', 'Fake status string',
197        '=============================='
198    ])
199    self.assertEqual(len(self.mock_logger.error_msgs), 0)
200
201    # Test 2: log_level == "average"
202    self.mock_logger.Reset()
203    reset()
204    self.exp.log_level = 'average'
205    mock_status_string.call_count = 0
206    er = experiment_runner.ExperimentRunner(
207        self.exp,
208        json_report=False,
209        using_schedv2=False,
210        log=self.mock_logger,
211        cmd_exec=self.mock_cmd_exec)
212    er.STATUS_TIME_DELAY = 2
213    mock_status_string.return_value = 'Fake status string'
214    er._Run(self.exp)
215    self.assertEqual(self.run_count, 1)
216    self.assertTrue(self.is_complete_count > 0)
217    self.assertEqual(self.mock_logger.LogStartDotsCount, 1)
218    self.assertEqual(self.mock_logger.LogAppendDotCount, 1)
219    self.assertEqual(self.mock_logger.LogEndDotsCount, 1)
220    self.assertEqual(self.mock_logger.dot_count, 2)
221    self.assertEqual(mock_progress_string.call_count, 0)
222    self.assertEqual(mock_status_string.call_count, 2)
223    self.assertEqual(self.mock_logger.output_msgs, [
224        '==============================', 'Fake status string',
225        '=============================='
226    ])
227    self.assertEqual(len(self.mock_logger.error_msgs), 0)
228
229    # Test 3: log_level == "verbose"
230    self.mock_logger.Reset()
231    reset()
232    self.exp.log_level = 'verbose'
233    mock_status_string.call_count = 0
234    er = experiment_runner.ExperimentRunner(
235        self.exp,
236        json_report=False,
237        using_schedv2=False,
238        log=self.mock_logger,
239        cmd_exec=self.mock_cmd_exec)
240    er.STATUS_TIME_DELAY = 2
241    mock_status_string.return_value = 'Fake status string'
242    mock_progress_string.return_value = 'Fake progress string'
243    er._Run(self.exp)
244    self.assertEqual(self.run_count, 1)
245    self.assertTrue(self.is_complete_count > 0)
246    self.assertEqual(self.mock_logger.LogStartDotsCount, 0)
247    self.assertEqual(self.mock_logger.LogAppendDotCount, 0)
248    self.assertEqual(self.mock_logger.LogEndDotsCount, 0)
249    self.assertEqual(self.mock_logger.dot_count, 0)
250    self.assertEqual(mock_progress_string.call_count, 2)
251    self.assertEqual(mock_status_string.call_count, 2)
252    self.assertEqual(self.mock_logger.output_msgs, [
253        '==============================', 'Fake progress string',
254        'Fake status string', '==============================',
255        '==============================', 'Fake progress string',
256        'Fake status string', '=============================='
257    ])
258    self.assertEqual(len(self.mock_logger.error_msgs), 0)
259
260  @mock.patch.object(TextResultsReport, 'GetReport')
261  def test_print_table(self, mock_report):
262    self.mock_logger.Reset()
263    mock_report.return_value = 'This is a fake experiment report.'
264    er = experiment_runner.ExperimentRunner(
265        self.exp,
266        json_report=False,
267        using_schedv2=False,
268        log=self.mock_logger,
269        cmd_exec=self.mock_cmd_exec)
270    er._PrintTable(self.exp)
271    self.assertEqual(mock_report.call_count, 1)
272    self.assertEqual(self.mock_logger.output_msgs,
273                     ['This is a fake experiment report.'])
274
275  @mock.patch.object(HTMLResultsReport, 'GetReport')
276  @mock.patch.object(TextResultsReport, 'GetReport')
277  @mock.patch.object(EmailSender, 'Attachment')
278  @mock.patch.object(EmailSender, 'SendEmail')
279  @mock.patch.object(getpass, 'getuser')
280  def test_email(self, mock_getuser, mock_emailer, mock_attachment,
281                 mock_text_report, mock_html_report):
282
283    mock_getuser.return_value = 'john.smith@google.com'
284    mock_text_report.return_value = 'This is a fake text report.'
285    mock_html_report.return_value = 'This is a fake html report.'
286
287    self.mock_logger.Reset()
288    config.AddConfig('no_email', True)
289    self.exp.email_to = ['jane.doe@google.com']
290    er = experiment_runner.ExperimentRunner(
291        self.exp,
292        json_report=False,
293        using_schedv2=False,
294        log=self.mock_logger,
295        cmd_exec=self.mock_cmd_exec)
296    # Test 1. Config:no_email; exp.email_to set ==> no email sent
297    er._Email(self.exp)
298    self.assertEqual(mock_getuser.call_count, 0)
299    self.assertEqual(mock_emailer.call_count, 0)
300    self.assertEqual(mock_attachment.call_count, 0)
301    self.assertEqual(mock_text_report.call_count, 0)
302    self.assertEqual(mock_html_report.call_count, 0)
303
304    # Test 2. Config: email. exp.email_to set; cache hit.  => send email
305    self.mock_logger.Reset()
306    config.AddConfig('no_email', False)
307    for r in self.exp.benchmark_runs:
308      r.cache_hit = True
309    er._Email(self.exp)
310    self.assertEqual(mock_getuser.call_count, 1)
311    self.assertEqual(mock_emailer.call_count, 1)
312    self.assertEqual(mock_attachment.call_count, 1)
313    self.assertEqual(mock_text_report.call_count, 1)
314    self.assertEqual(mock_html_report.call_count, 1)
315    self.assertEqual(len(mock_emailer.call_args), 2)
316    self.assertEqual(mock_emailer.call_args[0],
317                     (['jane.doe@google.com', 'john.smith@google.com'
318                      ], ': image1 vs. image2',
319                      "<pre style='font-size: 13px'>This is a fake text "
320                      'report.\nResults are stored in _results.\n</pre>'))
321    self.assertTrue(isinstance(mock_emailer.call_args[1], dict))
322    self.assertEqual(len(mock_emailer.call_args[1]), 2)
323    self.assertTrue('attachments' in mock_emailer.call_args[1].keys())
324    self.assertEqual(mock_emailer.call_args[1]['msg_type'], 'html')
325
326    mock_attachment.assert_called_with('report.html',
327                                       'This is a fake html report.')
328
329    # Test 3. Config: email; exp.mail_to set; no cache hit.  => send email
330    self.mock_logger.Reset()
331    mock_getuser.reset_mock()
332    mock_emailer.reset_mock()
333    mock_attachment.reset_mock()
334    mock_text_report.reset_mock()
335    mock_html_report.reset_mock()
336    config.AddConfig('no_email', False)
337    for r in self.exp.benchmark_runs:
338      r.cache_hit = False
339    er._Email(self.exp)
340    self.assertEqual(mock_getuser.call_count, 1)
341    self.assertEqual(mock_emailer.call_count, 1)
342    self.assertEqual(mock_attachment.call_count, 1)
343    self.assertEqual(mock_text_report.call_count, 1)
344    self.assertEqual(mock_html_report.call_count, 1)
345    self.assertEqual(len(mock_emailer.call_args), 2)
346    self.assertEqual(mock_emailer.call_args[0],
347                     ([
348                         'jane.doe@google.com', 'john.smith@google.com',
349                         'john.smith@google.com'
350                     ], ': image1 vs. image2',
351                      "<pre style='font-size: 13px'>This is a fake text "
352                      'report.\nResults are stored in _results.\n</pre>'))
353    self.assertTrue(isinstance(mock_emailer.call_args[1], dict))
354    self.assertEqual(len(mock_emailer.call_args[1]), 2)
355    self.assertTrue('attachments' in mock_emailer.call_args[1].keys())
356    self.assertEqual(mock_emailer.call_args[1]['msg_type'], 'html')
357
358    mock_attachment.assert_called_with('report.html',
359                                       'This is a fake html report.')
360
361    # Test 4. Config: email; exp.mail_to = None; no cache hit. => send email
362    self.mock_logger.Reset()
363    mock_getuser.reset_mock()
364    mock_emailer.reset_mock()
365    mock_attachment.reset_mock()
366    mock_text_report.reset_mock()
367    mock_html_report.reset_mock()
368    self.exp.email_to = []
369    er._Email(self.exp)
370    self.assertEqual(mock_getuser.call_count, 1)
371    self.assertEqual(mock_emailer.call_count, 1)
372    self.assertEqual(mock_attachment.call_count, 1)
373    self.assertEqual(mock_text_report.call_count, 1)
374    self.assertEqual(mock_html_report.call_count, 1)
375    self.assertEqual(len(mock_emailer.call_args), 2)
376    self.assertEqual(mock_emailer.call_args[0],
377                     (['john.smith@google.com'], ': image1 vs. image2',
378                      "<pre style='font-size: 13px'>This is a fake text "
379                      'report.\nResults are stored in _results.\n</pre>'))
380    self.assertTrue(isinstance(mock_emailer.call_args[1], dict))
381    self.assertEqual(len(mock_emailer.call_args[1]), 2)
382    self.assertTrue('attachments' in mock_emailer.call_args[1].keys())
383    self.assertEqual(mock_emailer.call_args[1]['msg_type'], 'html')
384
385    mock_attachment.assert_called_with('report.html',
386                                       'This is a fake html report.')
387
388    # Test 5. Config: email; exp.mail_to = None; cache hit => no email sent
389    self.mock_logger.Reset()
390    mock_getuser.reset_mock()
391    mock_emailer.reset_mock()
392    mock_attachment.reset_mock()
393    mock_text_report.reset_mock()
394    mock_html_report.reset_mock()
395    for r in self.exp.benchmark_runs:
396      r.cache_hit = True
397    er._Email(self.exp)
398    self.assertEqual(mock_getuser.call_count, 0)
399    self.assertEqual(mock_emailer.call_count, 0)
400    self.assertEqual(mock_attachment.call_count, 0)
401    self.assertEqual(mock_text_report.call_count, 0)
402    self.assertEqual(mock_html_report.call_count, 0)
403
404  @mock.patch.object(FileUtils, 'RmDir')
405  @mock.patch.object(FileUtils, 'MkDirP')
406  @mock.patch.object(FileUtils, 'WriteFile')
407  @mock.patch.object(HTMLResultsReport, 'FromExperiment')
408  @mock.patch.object(TextResultsReport, 'FromExperiment')
409  @mock.patch.object(Result, 'CopyResultsTo')
410  @mock.patch.object(Result, 'CleanUp')
411  @mock.patch.object(Result, 'FormatStringTop5')
412  @mock.patch('builtins.open', new_callable=mock.mock_open)
413  def test_store_results(self, mock_open, mock_top5, mock_cleanup, mock_copy,
414                         _mock_text_report, mock_report, mock_writefile,
415                         mock_mkdir, mock_rmdir):
416
417    self.mock_logger.Reset()
418    self.exp.results_directory = '/usr/local/crosperf-results'
419    bench_run = self.exp.benchmark_runs[5]
420    bench_path = '/usr/local/crosperf-results/' + ''.join(
421        ch for ch in bench_run.name if ch.isalnum())
422    self.assertEqual(len(self.exp.benchmark_runs), 6)
423
424    er = experiment_runner.ExperimentRunner(
425        self.exp,
426        json_report=False,
427        using_schedv2=False,
428        log=self.mock_logger,
429        cmd_exec=self.mock_cmd_exec)
430
431    # Test 1. Make sure nothing is done if _terminated is true.
432    er._terminated = True
433    er._StoreResults(self.exp)
434    self.assertEqual(mock_cleanup.call_count, 0)
435    self.assertEqual(mock_copy.call_count, 0)
436    self.assertEqual(mock_report.call_count, 0)
437    self.assertEqual(mock_writefile.call_count, 0)
438    self.assertEqual(mock_mkdir.call_count, 0)
439    self.assertEqual(mock_rmdir.call_count, 0)
440    self.assertEqual(self.mock_logger.LogOutputCount, 0)
441    self.assertEqual(mock_open.call_count, 0)
442    self.assertEqual(mock_top5.call_count, 0)
443
444    # Test 2. _terminated is false; everything works properly.
445    fake_result = Result(self.mock_logger, self.exp.labels[0], 'average',
446                         'daisy1')
447    for r in self.exp.benchmark_runs:
448      r.result = fake_result
449    er._terminated = False
450    er._StoreResults(self.exp)
451    self.assertEqual(mock_cleanup.call_count, 6)
452    mock_cleanup.assert_called_with(bench_run.benchmark.rm_chroot_tmp)
453    self.assertEqual(mock_copy.call_count, 6)
454    mock_copy.assert_called_with(bench_path)
455    self.assertEqual(mock_writefile.call_count, 3)
456    self.assertEqual(len(mock_writefile.call_args_list), 3)
457    first_args = mock_writefile.call_args_list[0]
458    second_args = mock_writefile.call_args_list[1]
459    self.assertEqual(first_args[0][0],
460                     '/usr/local/crosperf-results/experiment.exp')
461    self.assertEqual(second_args[0][0],
462                     '/usr/local/crosperf-results/results.html')
463    self.assertEqual(mock_mkdir.call_count, 1)
464    mock_mkdir.assert_called_with('/usr/local/crosperf-results')
465    self.assertEqual(mock_rmdir.call_count, 1)
466    mock_rmdir.assert_called_with('/usr/local/crosperf-results')
467    self.assertEqual(self.mock_logger.LogOutputCount, 5)
468    self.assertEqual(self.mock_logger.output_msgs, [
469        'Storing experiment file in /usr/local/crosperf-results.',
470        'Storing results report in /usr/local/crosperf-results.',
471        'Storing email message body in /usr/local/crosperf-results.',
472        'Storing results of each benchmark run.',
473        'Storing top5 statistics of each benchmark run into'
474        ' /usr/local/crosperf-results/topstats.log.',
475    ])
476    self.assertEqual(mock_open.call_count, 1)
477    # Check write to a topstats.log file.
478    mock_open.assert_called_with('/usr/local/crosperf-results/topstats.log',
479                                 'w')
480    mock_open().write.assert_called()
481
482    # Check top5 calls with no arguments.
483    top5calls = [mock.call()] * 6
484    self.assertEqual(mock_top5.call_args_list, top5calls)
485
486
487if __name__ == '__main__':
488  unittest.main()
489