• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python2
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6"""Unit test for experiment_factory.py"""
7
8from __future__ import print_function
9
10import StringIO
11import socket
12import mock
13import unittest
14
15from cros_utils.file_utils import FileUtils
16
17from experiment_factory import ExperimentFactory
18from experiment_file import ExperimentFile
19import test_flag
20import benchmark
21import experiment_factory
22import settings_factory
23
24EXPERIMENT_FILE_1 = """
25  board: x86-alex
26  remote: chromeos-alex3
27
28  benchmark: PageCycler {
29    iterations: 3
30  }
31
32  image1 {
33    chromeos_image: /usr/local/google/cros_image1.bin
34  }
35
36  image2 {
37    chromeos_image: /usr/local/google/cros_image2.bin
38  }
39  """
40
41# pylint: disable=too-many-function-args
42
43
44class ExperimentFactoryTest(unittest.TestCase):
45  """Class for running experiment factory unittests."""
46
47  def setUp(self):
48    self.append_benchmark_call_args = []
49
50  def testLoadExperimentFile1(self):
51    experiment_file = ExperimentFile(StringIO.StringIO(EXPERIMENT_FILE_1))
52    exp = ExperimentFactory().GetExperiment(
53        experiment_file, working_directory='', log_dir='')
54    self.assertEqual(exp.remote, ['chromeos-alex3'])
55
56    self.assertEqual(len(exp.benchmarks), 1)
57    self.assertEqual(exp.benchmarks[0].name, 'PageCycler')
58    self.assertEqual(exp.benchmarks[0].test_name, 'PageCycler')
59    self.assertEqual(exp.benchmarks[0].iterations, 3)
60
61    self.assertEqual(len(exp.labels), 2)
62    self.assertEqual(exp.labels[0].chromeos_image,
63                     '/usr/local/google/cros_image1.bin')
64    self.assertEqual(exp.labels[0].board, 'x86-alex')
65
66  def test_append_benchmark_set(self):
67    ef = ExperimentFactory()
68
69    bench_list = []
70    ef.AppendBenchmarkSet(bench_list, experiment_factory.telemetry_perfv2_tests,
71                          '', 1, False, '', 'telemetry_Crosperf', False, 0,
72                          False)
73    self.assertEqual(
74        len(bench_list), len(experiment_factory.telemetry_perfv2_tests))
75    self.assertTrue(type(bench_list[0]) is benchmark.Benchmark)
76
77    bench_list = []
78    ef.AppendBenchmarkSet(bench_list,
79                          experiment_factory.telemetry_pagecycler_tests, '', 1,
80                          False, '', 'telemetry_Crosperf', False, 0, False)
81    self.assertEqual(
82        len(bench_list), len(experiment_factory.telemetry_pagecycler_tests))
83    self.assertTrue(type(bench_list[0]) is benchmark.Benchmark)
84
85    bench_list = []
86    ef.AppendBenchmarkSet(bench_list,
87                          experiment_factory.telemetry_toolchain_perf_tests, '',
88                          1, False, '', 'telemetry_Crosperf', False, 0, False)
89    self.assertEqual(
90        len(bench_list), len(experiment_factory.telemetry_toolchain_perf_tests))
91    self.assertTrue(type(bench_list[0]) is benchmark.Benchmark)
92
93  @mock.patch.object(socket, 'gethostname')
94  def test_get_experiment(self, mock_socket):
95
96    test_flag.SetTestMode(False)
97    self.append_benchmark_call_args = []
98
99    def FakeAppendBenchmarkSet(bench_list, set_list, args, iters, rm_ch,
100                               perf_args, suite, show_all):
101      'Helper function for test_get_experiment'
102      arg_list = [
103          bench_list, set_list, args, iters, rm_ch, perf_args, suite, show_all
104      ]
105      self.append_benchmark_call_args.append(arg_list)
106
107    def FakeGetDefaultRemotes(board):
108      if not board:
109        return []
110      return ['fake_chromeos_machine1.cros', 'fake_chromeos_machine2.cros']
111
112    def FakeGetXbuddyPath(build, autotest_dir, board, chroot, log_level):
113      autotest_path = autotest_dir
114      if not autotest_path:
115        autotest_path = 'fake_autotest_path'
116      if not build or not board or not chroot or not log_level:
117        return '', autotest_path
118      return 'fake_image_path', autotest_path
119
120    ef = ExperimentFactory()
121    ef.AppendBenchmarkSet = FakeAppendBenchmarkSet
122    ef.GetDefaultRemotes = FakeGetDefaultRemotes
123
124    label_settings = settings_factory.LabelSettings('image_label')
125    benchmark_settings = settings_factory.BenchmarkSettings('bench_test')
126    global_settings = settings_factory.GlobalSettings('test_name')
127
128    label_settings.GetXbuddyPath = FakeGetXbuddyPath
129
130    mock_experiment_file = ExperimentFile(StringIO.StringIO(''))
131    mock_experiment_file.all_settings = []
132
133    test_flag.SetTestMode(True)
134    # Basic test.
135    global_settings.SetField('name', 'unittest_test')
136    global_settings.SetField('board', 'lumpy')
137    global_settings.SetField('remote', '123.45.67.89 123.45.76.80')
138    benchmark_settings.SetField('test_name', 'kraken')
139    benchmark_settings.SetField('suite', 'telemetry_Crosperf')
140    benchmark_settings.SetField('iterations', 1)
141    label_settings.SetField(
142        'chromeos_image',
143        'chromeos/src/build/images/lumpy/latest/chromiumos_test_image.bin')
144    label_settings.SetField('chrome_src', '/usr/local/google/home/chrome-top')
145    label_settings.SetField('autotest_path', '/tmp/autotest')
146
147    mock_experiment_file.global_settings = global_settings
148    mock_experiment_file.all_settings.append(label_settings)
149    mock_experiment_file.all_settings.append(benchmark_settings)
150    mock_experiment_file.all_settings.append(global_settings)
151
152    mock_socket.return_value = ''
153
154    # First test. General test.
155    exp = ef.GetExperiment(mock_experiment_file, '', '')
156    self.assertEqual(exp.remote, ['123.45.67.89', '123.45.76.80'])
157    self.assertEqual(exp.cache_conditions, [0, 2, 1])
158    self.assertEqual(exp.log_level, 'average')
159
160    self.assertEqual(len(exp.benchmarks), 1)
161    self.assertEqual(exp.benchmarks[0].name, 'kraken')
162    self.assertEqual(exp.benchmarks[0].test_name, 'kraken')
163    self.assertEqual(exp.benchmarks[0].iterations, 1)
164    self.assertEqual(exp.benchmarks[0].suite, 'telemetry_Crosperf')
165    self.assertFalse(exp.benchmarks[0].show_all_results)
166
167    self.assertEqual(len(exp.labels), 1)
168    self.assertEqual(exp.labels[0].chromeos_image,
169                     'chromeos/src/build/images/lumpy/latest/'
170                     'chromiumos_test_image.bin')
171    self.assertEqual(exp.labels[0].autotest_path, '/tmp/autotest')
172    self.assertEqual(exp.labels[0].board, 'lumpy')
173
174    # Second test: Remotes listed in labels.
175    test_flag.SetTestMode(True)
176    label_settings.SetField('remote', 'chromeos1.cros chromeos2.cros')
177    exp = ef.GetExperiment(mock_experiment_file, '', '')
178    self.assertEqual(
179        exp.remote,
180        ['chromeos1.cros', 'chromeos2.cros', '123.45.67.89', '123.45.76.80'])
181
182    # Third test: Automatic fixing of bad  logging_level param:
183    global_settings.SetField('logging_level', 'really loud!')
184    exp = ef.GetExperiment(mock_experiment_file, '', '')
185    self.assertEqual(exp.log_level, 'verbose')
186
187    # Fourth test: Setting cache conditions; only 1 remote with "same_machine"
188    global_settings.SetField('rerun_if_failed', 'true')
189    global_settings.SetField('rerun', 'true')
190    global_settings.SetField('same_machine', 'true')
191    global_settings.SetField('same_specs', 'true')
192
193    self.assertRaises(Exception, ef.GetExperiment, mock_experiment_file, '', '')
194    label_settings.SetField('remote', '')
195    global_settings.SetField('remote', '123.45.67.89')
196    exp = ef.GetExperiment(mock_experiment_file, '', '')
197    self.assertEqual(exp.cache_conditions, [0, 2, 3, 4, 6, 1])
198
199    # Fifth Test: Adding a second label; calling GetXbuddyPath; omitting all
200    # remotes (Call GetDefaultRemotes).
201    mock_socket.return_value = 'test.corp.google.com'
202    global_settings.SetField('remote', '')
203    global_settings.SetField('same_machine', 'false')
204
205    label_settings_2 = settings_factory.LabelSettings('official_image_label')
206    label_settings_2.SetField('chromeos_root', 'chromeos')
207    label_settings_2.SetField('build', 'official-dev')
208    label_settings_2.SetField('autotest_path', '')
209    label_settings_2.GetXbuddyPath = FakeGetXbuddyPath
210
211    mock_experiment_file.all_settings.append(label_settings_2)
212    exp = ef.GetExperiment(mock_experiment_file, '', '')
213    self.assertEqual(len(exp.labels), 2)
214    self.assertEqual(exp.labels[1].chromeos_image, 'fake_image_path')
215    self.assertEqual(exp.labels[1].autotest_path, 'fake_autotest_path')
216    self.assertEqual(
217        exp.remote,
218        ['fake_chromeos_machine1.cros', 'fake_chromeos_machine2.cros'])
219
220  def test_get_default_remotes(self):
221    board_list = [
222        'x86-alex', 'lumpy', 'elm', 'parrot', 'daisy', 'peach_pit', 'peppy',
223        'squawks'
224    ]
225
226    ef = ExperimentFactory()
227    self.assertRaises(Exception, ef.GetDefaultRemotes, 'bad-board')
228
229    # Verify that we have entries for every board, and that we get three
230    # machines back for each board.
231    for b in board_list:
232      remotes = ef.GetDefaultRemotes(b)
233      if b == 'elm':
234        self.assertEqual(len(remotes), 2)
235      else:
236        self.assertEqual(len(remotes), 3)
237
238
239if __name__ == '__main__':
240  FileUtils.Configure(True)
241  test_flag.SetTestMode(True)
242  unittest.main()
243