• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3
4# Copyright 2013 The ChromiumOS Authors
5# Use of this source code is governed by a BSD-style license that can be
6# found in the LICENSE file.
7
8"""Unit test for experiment_factory.py"""
9
10
11import io
12import os
13import socket
14import unittest
15import unittest.mock as mock
16
17import benchmark
18from cros_utils import command_executer
19from cros_utils.file_utils import FileUtils
20import experiment_factory
21from experiment_factory import ExperimentFactory
22from experiment_file import ExperimentFile
23import settings_factory
24import test_flag
25
26
27EXPERIMENT_FILE_1 = """
28  board: x86-alex
29  remote: chromeos-alex3
30  locks_dir: /tmp
31
32  benchmark: PageCycler {
33    iterations: 3
34  }
35
36  benchmark: webrtc {
37    iterations: 1
38    test_args: --story-filter=datachannel
39  }
40
41  image1 {
42    chromeos_image: /usr/local/google/cros_image1.bin
43  }
44
45  image2 {
46    chromeos_image: /usr/local/google/cros_image2.bin
47  }
48  """
49
50EXPERIMENT_FILE_2 = """
51  board: x86-alex
52  remote: chromeos-alex3
53  locks_dir: /tmp
54
55  cwp_dso: kallsyms
56
57  benchmark: Octane {
58    iterations: 1
59    suite: telemetry_Crosperf
60    run_local: False
61    weight: 0.8
62  }
63
64  benchmark: Kraken {
65    iterations: 1
66    suite: telemetry_Crosperf
67    run_local: False
68    weight: 0.2
69  }
70
71  image1 {
72    chromeos_image: /usr/local/google/cros_image1.bin
73  }
74  """
75
76# pylint: disable=too-many-function-args
77
78
79class ExperimentFactoryTest(unittest.TestCase):
80    """Class for running experiment factory unittests."""
81
82    def setUp(self):
83        self.append_benchmark_call_args = []
84
85    def testLoadExperimentFile1(self):
86        experiment_file = ExperimentFile(io.StringIO(EXPERIMENT_FILE_1))
87        exp = ExperimentFactory().GetExperiment(
88            experiment_file, working_directory="", log_dir=""
89        )
90        self.assertEqual(exp.remote, ["chromeos-alex3"])
91
92        self.assertEqual(len(exp.benchmarks), 2)
93        self.assertEqual(exp.benchmarks[0].name, "PageCycler")
94        self.assertEqual(exp.benchmarks[0].test_name, "PageCycler")
95        self.assertEqual(exp.benchmarks[0].iterations, 3)
96        self.assertEqual(exp.benchmarks[1].name, "webrtc@@datachannel")
97        self.assertEqual(exp.benchmarks[1].test_name, "webrtc")
98        self.assertEqual(exp.benchmarks[1].iterations, 1)
99
100        self.assertEqual(len(exp.labels), 2)
101        self.assertEqual(
102            exp.labels[0].chromeos_image, "/usr/local/google/cros_image1.bin"
103        )
104        self.assertEqual(exp.labels[0].board, "x86-alex")
105
106    def testLoadExperimentFile2CWP(self):
107        experiment_file = ExperimentFile(io.StringIO(EXPERIMENT_FILE_2))
108        exp = ExperimentFactory().GetExperiment(
109            experiment_file, working_directory="", log_dir=""
110        )
111        self.assertEqual(exp.cwp_dso, "kallsyms")
112        self.assertEqual(len(exp.benchmarks), 2)
113        self.assertEqual(exp.benchmarks[0].weight, 0.8)
114        self.assertEqual(exp.benchmarks[1].weight, 0.2)
115
116    def testDuplecateBenchmark(self):
117        mock_experiment_file = ExperimentFile(io.StringIO(EXPERIMENT_FILE_1))
118        mock_experiment_file.all_settings = []
119        benchmark_settings1 = settings_factory.BenchmarkSettings("name")
120        mock_experiment_file.all_settings.append(benchmark_settings1)
121        benchmark_settings2 = settings_factory.BenchmarkSettings("name")
122        mock_experiment_file.all_settings.append(benchmark_settings2)
123
124        with self.assertRaises(SyntaxError):
125            ef = ExperimentFactory()
126            ef.GetExperiment(mock_experiment_file, "", "")
127
128    def testCWPExceptions(self):
129        mock_experiment_file = ExperimentFile(io.StringIO(""))
130        mock_experiment_file.all_settings = []
131        global_settings = settings_factory.GlobalSettings("test_name")
132        global_settings.SetField("locks_dir", "/tmp")
133
134        # Test 1: DSO type not supported
135        global_settings.SetField("cwp_dso", "test")
136        self.assertEqual(global_settings.GetField("cwp_dso"), "test")
137        mock_experiment_file.global_settings = global_settings
138        with self.assertRaises(RuntimeError) as msg:
139            ef = ExperimentFactory()
140            ef.GetExperiment(mock_experiment_file, "", "")
141        self.assertEqual(
142            "The DSO specified is not supported", str(msg.exception)
143        )
144
145        # Test 2: No weight after DSO specified
146        global_settings.SetField("cwp_dso", "kallsyms")
147        mock_experiment_file.global_settings = global_settings
148        benchmark_settings = settings_factory.BenchmarkSettings("name")
149        mock_experiment_file.all_settings.append(benchmark_settings)
150        with self.assertRaises(RuntimeError) as msg:
151            ef = ExperimentFactory()
152            ef.GetExperiment(mock_experiment_file, "", "")
153        self.assertEqual(
154            "With DSO specified, each benchmark should have a weight",
155            str(msg.exception),
156        )
157
158        # Test 3: Weight is set, but no dso specified
159        global_settings.SetField("cwp_dso", "")
160        mock_experiment_file.global_settings = global_settings
161        benchmark_settings = settings_factory.BenchmarkSettings("name")
162        benchmark_settings.SetField("weight", "0.8")
163        mock_experiment_file.all_settings = []
164        mock_experiment_file.all_settings.append(benchmark_settings)
165        with self.assertRaises(RuntimeError) as msg:
166            ef = ExperimentFactory()
167            ef.GetExperiment(mock_experiment_file, "", "")
168        self.assertEqual(
169            "Weight can only be set when DSO specified", str(msg.exception)
170        )
171
172        # Test 4: cwp_dso only works for telemetry_Crosperf benchmarks
173        global_settings.SetField("cwp_dso", "kallsyms")
174        mock_experiment_file.global_settings = global_settings
175        benchmark_settings = settings_factory.BenchmarkSettings("name")
176        benchmark_settings.SetField("weight", "0.8")
177        mock_experiment_file.all_settings = []
178        mock_experiment_file.all_settings.append(benchmark_settings)
179        with self.assertRaises(RuntimeError) as msg:
180            ef = ExperimentFactory()
181            ef.GetExperiment(mock_experiment_file, "", "")
182        self.assertEqual(
183            "CWP approximation weight only works with "
184            "telemetry_Crosperf suite",
185            str(msg.exception),
186        )
187
188        # Test 5: cwp_dso does not work for local run
189        benchmark_settings = settings_factory.BenchmarkSettings("name")
190        benchmark_settings.SetField("weight", "0.8")
191        benchmark_settings.SetField("suite", "telemetry_Crosperf")
192        benchmark_settings.SetField("run_local", "True")
193        mock_experiment_file.all_settings = []
194        mock_experiment_file.all_settings.append(benchmark_settings)
195        with self.assertRaises(RuntimeError) as msg:
196            ef = ExperimentFactory()
197            ef.GetExperiment(mock_experiment_file, "", "")
198        self.assertEqual(
199            "run_local must be set to False to use CWP approximation",
200            str(msg.exception),
201        )
202
203        # Test 6: weight should be float >=0
204        benchmark_settings = settings_factory.BenchmarkSettings("name")
205        benchmark_settings.SetField("weight", "-1.2")
206        benchmark_settings.SetField("suite", "telemetry_Crosperf")
207        benchmark_settings.SetField("run_local", "False")
208        mock_experiment_file.all_settings = []
209        mock_experiment_file.all_settings.append(benchmark_settings)
210        with self.assertRaises(RuntimeError) as msg:
211            ef = ExperimentFactory()
212            ef.GetExperiment(mock_experiment_file, "", "")
213        self.assertEqual("Weight should be a float >=0", str(msg.exception))
214
215        # Test 7: more than one story tag in test_args
216        benchmark_settings = settings_factory.BenchmarkSettings("name")
217        benchmark_settings.SetField(
218            "test_args", "--story-filter=a --story-tag-filter=b"
219        )
220        benchmark_settings.SetField("weight", "1.2")
221        benchmark_settings.SetField("suite", "telemetry_Crosperf")
222        mock_experiment_file.all_settings = []
223        mock_experiment_file.all_settings.append(benchmark_settings)
224        with self.assertRaises(RuntimeError) as msg:
225            ef = ExperimentFactory()
226            ef.GetExperiment(mock_experiment_file, "", "")
227        self.assertEqual(
228            "Only one story or story-tag filter allowed in a single "
229            "benchmark run",
230            str(msg.exception),
231        )
232
233        # Test 8: Iterations of each benchmark run are not same in cwp mode
234        mock_experiment_file.all_settings = []
235        benchmark_settings = settings_factory.BenchmarkSettings("name1")
236        benchmark_settings.SetField("iterations", "4")
237        benchmark_settings.SetField("weight", "1.2")
238        benchmark_settings.SetField("suite", "telemetry_Crosperf")
239        benchmark_settings.SetField("run_local", "False")
240        mock_experiment_file.all_settings.append(benchmark_settings)
241        benchmark_settings = settings_factory.BenchmarkSettings("name2")
242        benchmark_settings.SetField("iterations", "3")
243        benchmark_settings.SetField("weight", "1.2")
244        benchmark_settings.SetField("suite", "telemetry_Crosperf")
245        benchmark_settings.SetField("run_local", "False")
246        mock_experiment_file.all_settings.append(benchmark_settings)
247        with self.assertRaises(RuntimeError) as msg:
248            ef = ExperimentFactory()
249            ef.GetExperiment(mock_experiment_file, "", "")
250        self.assertEqual(
251            "Iterations of each benchmark run are not the same",
252            str(msg.exception),
253        )
254
255    def test_append_benchmark_set(self):
256        ef = ExperimentFactory()
257
258        bench_list = []
259        ef.AppendBenchmarkSet(
260            bench_list,
261            experiment_factory.telemetry_perfv2_tests,
262            "",
263            1,
264            False,
265            "",
266            "telemetry_Crosperf",
267            False,
268            0,
269            False,
270            "",
271            0,
272        )
273        self.assertEqual(
274            len(bench_list), len(experiment_factory.telemetry_perfv2_tests)
275        )
276        self.assertTrue(isinstance(bench_list[0], benchmark.Benchmark))
277
278        bench_list = []
279        ef.AppendBenchmarkSet(
280            bench_list,
281            experiment_factory.telemetry_pagecycler_tests,
282            "",
283            1,
284            False,
285            "",
286            "telemetry_Crosperf",
287            False,
288            0,
289            False,
290            "",
291            0,
292        )
293        self.assertEqual(
294            len(bench_list), len(experiment_factory.telemetry_pagecycler_tests)
295        )
296        self.assertTrue(isinstance(bench_list[0], benchmark.Benchmark))
297
298        bench_list = []
299        ef.AppendBenchmarkSet(
300            bench_list,
301            experiment_factory.telemetry_toolchain_perf_tests,
302            "",
303            1,
304            False,
305            "",
306            "telemetry_Crosperf",
307            False,
308            0,
309            False,
310            "",
311            0,
312        )
313        self.assertEqual(
314            len(bench_list),
315            len(experiment_factory.telemetry_toolchain_perf_tests),
316        )
317        self.assertTrue(isinstance(bench_list[0], benchmark.Benchmark))
318
319    @mock.patch.object(socket, "gethostname")
320    def test_get_experiment(self, mock_socket):
321
322        test_flag.SetTestMode(False)
323        self.append_benchmark_call_args = []
324
325        def FakeAppendBenchmarkSet(
326            bench_list, set_list, args, iters, rm_ch, perf_args, suite, show_all
327        ):
328            "Helper function for test_get_experiment"
329            arg_list = [
330                bench_list,
331                set_list,
332                args,
333                iters,
334                rm_ch,
335                perf_args,
336                suite,
337                show_all,
338            ]
339            self.append_benchmark_call_args.append(arg_list)
340
341        def FakeGetDefaultRemotes(board):
342            if not board:
343                return []
344            return [
345                "fake_chromeos_machine1.cros",
346                "fake_chromeos_machine2.cros",
347            ]
348
349        def FakeGetXbuddyPath(
350            build, autotest_dir, debug_dir, board, chroot, log_level, perf_args
351        ):
352            autotest_path = autotest_dir
353            if not autotest_path:
354                autotest_path = "fake_autotest_path"
355            debug_path = debug_dir
356            if not debug_path and perf_args:
357                debug_path = "fake_debug_path"
358            if not build or not board or not chroot or not log_level:
359                return "", autotest_path, debug_path
360            return "fake_image_path", autotest_path, debug_path
361
362        ef = ExperimentFactory()
363        ef.AppendBenchmarkSet = FakeAppendBenchmarkSet
364        ef.GetDefaultRemotes = FakeGetDefaultRemotes
365
366        label_settings = settings_factory.LabelSettings("image_label")
367        benchmark_settings = settings_factory.BenchmarkSettings("bench_test")
368        global_settings = settings_factory.GlobalSettings("test_name")
369
370        label_settings.GetXbuddyPath = FakeGetXbuddyPath
371
372        mock_experiment_file = ExperimentFile(io.StringIO(""))
373        mock_experiment_file.all_settings = []
374
375        test_flag.SetTestMode(True)
376        # Basic test.
377        global_settings.SetField("name", "unittest_test")
378        global_settings.SetField("board", "lumpy")
379        global_settings.SetField("locks_dir", "/tmp")
380        global_settings.SetField("remote", "123.45.67.89 123.45.76.80")
381        benchmark_settings.SetField("test_name", "kraken")
382        benchmark_settings.SetField("suite", "telemetry_Crosperf")
383        benchmark_settings.SetField("iterations", 1)
384        label_settings.SetField(
385            "chromeos_image",
386            "chromeos/src/build/images/lumpy/latest/chromiumos_test_image.bin",
387        )
388        label_settings.SetField(
389            "chrome_src", "/usr/local/google/home/chrome-top"
390        )
391        label_settings.SetField("autotest_path", "/tmp/autotest")
392
393        mock_experiment_file.global_settings = global_settings
394        mock_experiment_file.all_settings.append(label_settings)
395        mock_experiment_file.all_settings.append(benchmark_settings)
396        mock_experiment_file.all_settings.append(global_settings)
397
398        mock_socket.return_value = ""
399
400        # First test. General test.
401        exp = ef.GetExperiment(mock_experiment_file, "", "")
402        self.assertCountEqual(exp.remote, ["123.45.67.89", "123.45.76.80"])
403        self.assertEqual(exp.cache_conditions, [0, 2, 1])
404        self.assertEqual(exp.log_level, "average")
405
406        self.assertEqual(len(exp.benchmarks), 1)
407        self.assertEqual(exp.benchmarks[0].name, "bench_test")
408        self.assertEqual(exp.benchmarks[0].test_name, "kraken")
409        self.assertEqual(exp.benchmarks[0].iterations, 1)
410        self.assertEqual(exp.benchmarks[0].suite, "telemetry_Crosperf")
411        self.assertFalse(exp.benchmarks[0].show_all_results)
412
413        self.assertEqual(len(exp.labels), 1)
414        self.assertEqual(
415            exp.labels[0].chromeos_image,
416            "chromeos/src/build/images/lumpy/latest/"
417            "chromiumos_test_image.bin",
418        )
419        self.assertEqual(exp.labels[0].autotest_path, "/tmp/autotest")
420        self.assertEqual(exp.labels[0].board, "lumpy")
421
422        # Second test: Remotes listed in labels.
423        test_flag.SetTestMode(True)
424        label_settings.SetField("remote", "chromeos1.cros chromeos2.cros")
425        exp = ef.GetExperiment(mock_experiment_file, "", "")
426        self.assertCountEqual(
427            exp.remote,
428            [
429                "123.45.67.89",
430                "123.45.76.80",
431                "chromeos1.cros",
432                "chromeos2.cros",
433            ],
434        )
435
436        # Third test: Automatic fixing of bad  logging_level param:
437        global_settings.SetField("logging_level", "really loud!")
438        exp = ef.GetExperiment(mock_experiment_file, "", "")
439        self.assertEqual(exp.log_level, "verbose")
440
441        # Fourth test: Setting cache conditions; only 1 remote with "same_machine"
442        global_settings.SetField("rerun_if_failed", "true")
443        global_settings.SetField("rerun", "true")
444        global_settings.SetField("same_machine", "true")
445        global_settings.SetField("same_specs", "true")
446
447        self.assertRaises(
448            Exception, ef.GetExperiment, mock_experiment_file, "", ""
449        )
450        label_settings.SetField("remote", "")
451        global_settings.SetField("remote", "123.45.67.89")
452        exp = ef.GetExperiment(mock_experiment_file, "", "")
453        self.assertEqual(exp.cache_conditions, [0, 2, 3, 4, 6, 1])
454
455        # Fifth Test: Adding a second label; calling GetXbuddyPath; omitting all
456        # remotes (Call GetDefaultRemotes).
457        mock_socket.return_value = "test.corp.google.com"
458        global_settings.SetField("remote", "")
459        global_settings.SetField("same_machine", "false")
460
461        label_settings_2 = settings_factory.LabelSettings(
462            "official_image_label"
463        )
464        label_settings_2.SetField("chromeos_root", "chromeos")
465        label_settings_2.SetField("build", "official-dev")
466        label_settings_2.SetField("autotest_path", "")
467        label_settings_2.GetXbuddyPath = FakeGetXbuddyPath
468
469        mock_experiment_file.all_settings.append(label_settings_2)
470        exp = ef.GetExperiment(mock_experiment_file, "", "")
471        self.assertEqual(len(exp.labels), 2)
472        self.assertEqual(exp.labels[1].chromeos_image, "fake_image_path")
473        self.assertEqual(exp.labels[1].autotest_path, "fake_autotest_path")
474        self.assertCountEqual(
475            exp.remote,
476            ["fake_chromeos_machine1.cros", "fake_chromeos_machine2.cros"],
477        )
478
479    def test_get_default_remotes(self):
480        board_list = [
481            "bob",
482            "chell",
483            "coral",
484            "elm",
485            "nautilus",
486            "snappy",
487        ]
488
489        ef = ExperimentFactory()
490        self.assertRaises(Exception, ef.GetDefaultRemotes, "bad-board")
491
492        # Verify that we have entries for every board
493        for b in board_list:
494            remotes = ef.GetDefaultRemotes(b)
495            self.assertGreaterEqual(len(remotes), 1)
496
497    @mock.patch.object(command_executer.CommandExecuter, "RunCommand")
498    @mock.patch.object(os.path, "exists")
499    def test_check_crosfleet_tool(self, mock_exists, mock_runcmd):
500        ef = ExperimentFactory()
501        chromeos_root = "/tmp/chromeos"
502        log_level = "average"
503
504        mock_exists.return_value = True
505        ret = ef.CheckCrosfleetTool(chromeos_root, log_level)
506        self.assertTrue(ret)
507
508        mock_exists.return_value = False
509        mock_runcmd.return_value = 1
510        with self.assertRaises(RuntimeError) as err:
511            ef.CheckCrosfleetTool(chromeos_root, log_level)
512        self.assertEqual(mock_runcmd.call_count, 1)
513        self.assertEqual(
514            str(err.exception),
515            "Crosfleet tool not installed "
516            "correctly, please try to manually install it from "
517            "/tmp/chromeos/chromeos-admin/lab-tools/setup_lab_tools",
518        )
519
520        mock_runcmd.return_value = 0
521        mock_runcmd.call_count = 0
522        ret = ef.CheckCrosfleetTool(chromeos_root, log_level)
523        self.assertEqual(mock_runcmd.call_count, 1)
524        self.assertFalse(ret)
525
526
527if __name__ == "__main__":
528    FileUtils.Configure(True)
529    test_flag.SetTestMode(True)
530    unittest.main()
531