• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3
4# Copyright 2015 The Chromium OS Authors. All rights reserved.
5# Use of this source code is governed by a BSD-style license that can be
6# found in the LICENSE file.
7
8"""This contains the unit tests for the new Crosperf task scheduler."""
9
10from __future__ import print_function
11
12import functools
13import io
14import unittest
15import unittest.mock as mock
16
17import benchmark_run
18import test_flag
19from experiment_factory import ExperimentFactory
20from experiment_file import ExperimentFile
21from cros_utils.command_executer import CommandExecuter
22from experiment_runner_unittest import FakeLogger
23from schedv2 import Schedv2
24
25EXPERIMENT_FILE_1 = """\
26board: daisy
27remote: chromeos-daisy1.cros chromeos-daisy2.cros
28locks_dir: /tmp
29
30benchmark: kraken {
31  suite: telemetry_Crosperf
32  iterations: 3
33}
34
35image1 {
36  chromeos_image: /chromeos/src/build/images/daisy/latest/cros_image1.bin
37  remote: chromeos-daisy3.cros
38}
39
40image2 {
41  chromeos_image: /chromeos/src/build/imaages/daisy/latest/cros_image2.bin
42  remote: chromeos-daisy4.cros chromeos-daisy5.cros
43}
44"""
45
46EXPERIMENT_FILE_WITH_FORMAT = """\
47board: daisy
48remote: chromeos-daisy1.cros chromeos-daisy2.cros
49locks_dir: /tmp
50
51benchmark: kraken {{
52  suite: telemetry_Crosperf
53  iterations: {kraken_iterations}
54}}
55
56image1 {{
57  chromeos_image: /chromeos/src/build/images/daisy/latest/cros_image1.bin
58  remote: chromeos-daisy3.cros
59}}
60
61image2 {{
62  chromeos_image: /chromeos/src/build/imaages/daisy/latest/cros_image2.bin
63  remote: chromeos-daisy4.cros chromeos-daisy5.cros
64}}
65"""
66
67
68class Schedv2Test(unittest.TestCase):
69  """Class for setting up and running the unit tests."""
70
71  def setUp(self):
72    self.exp = None
73
74  mock_logger = FakeLogger()
75  mock_cmd_exec = mock.Mock(spec=CommandExecuter)
76
77  @mock.patch('benchmark_run.BenchmarkRun', new=benchmark_run.MockBenchmarkRun)
78  def _make_fake_experiment(self, expstr):
79    """Create fake experiment from string.
80
81        Note - we mock out BenchmarkRun in this step.
82    """
83    experiment_file = ExperimentFile(io.StringIO(expstr))
84    experiment = ExperimentFactory().GetExperiment(
85        experiment_file, working_directory='', log_dir='')
86    return experiment
87
88  def test_remote(self):
89    """Test that remotes in labels are aggregated into experiment.remote."""
90
91    self.exp = self._make_fake_experiment(EXPERIMENT_FILE_1)
92    self.exp.log_level = 'verbose'
93    my_schedv2 = Schedv2(self.exp)
94    self.assertFalse(my_schedv2.is_complete())
95    self.assertIn('chromeos-daisy1.cros', self.exp.remote)
96    self.assertIn('chromeos-daisy2.cros', self.exp.remote)
97    self.assertIn('chromeos-daisy3.cros', self.exp.remote)
98    self.assertIn('chromeos-daisy4.cros', self.exp.remote)
99    self.assertIn('chromeos-daisy5.cros', self.exp.remote)
100
101  def test_unreachable_remote(self):
102    """Test unreachable remotes are removed from experiment and label."""
103
104    def MockIsReachable(cm):
105      return (cm.name != 'chromeos-daisy3.cros' and
106              cm.name != 'chromeos-daisy5.cros')
107
108    with mock.patch(
109        'machine_manager.MockCrosMachine.IsReachable', new=MockIsReachable):
110      self.exp = self._make_fake_experiment(EXPERIMENT_FILE_1)
111      self.assertIn('chromeos-daisy1.cros', self.exp.remote)
112      self.assertIn('chromeos-daisy2.cros', self.exp.remote)
113      self.assertNotIn('chromeos-daisy3.cros', self.exp.remote)
114      self.assertIn('chromeos-daisy4.cros', self.exp.remote)
115      self.assertNotIn('chromeos-daisy5.cros', self.exp.remote)
116
117      for l in self.exp.labels:
118        if l.name == 'image2':
119          self.assertNotIn('chromeos-daisy5.cros', l.remote)
120          self.assertIn('chromeos-daisy4.cros', l.remote)
121        elif l.name == 'image1':
122          self.assertNotIn('chromeos-daisy3.cros', l.remote)
123
124  @mock.patch('schedv2.BenchmarkRunCacheReader')
125  def test_BenchmarkRunCacheReader_1(self, reader):
126    """Test benchmarkrun set is split into 5 segments."""
127
128    self.exp = self._make_fake_experiment(
129        EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=9))
130    my_schedv2 = Schedv2(self.exp)
131    self.assertFalse(my_schedv2.is_complete())
132    # We have 9 * 2 == 18 brs, we use 5 threads, each reading 4, 4, 4,
133    # 4, 2 brs respectively.
134    # Assert that BenchmarkRunCacheReader() is called 5 times.
135    self.assertEqual(reader.call_count, 5)
136    # reader.call_args_list[n] - nth call.
137    # reader.call_args_list[n][0] - positioned args in nth call.
138    # reader.call_args_list[n][0][1] - the 2nd arg in nth call,
139    # that is 'br_list' in 'schedv2.BenchmarkRunCacheReader'.
140    self.assertEqual(len(reader.call_args_list[0][0][1]), 4)
141    self.assertEqual(len(reader.call_args_list[1][0][1]), 4)
142    self.assertEqual(len(reader.call_args_list[2][0][1]), 4)
143    self.assertEqual(len(reader.call_args_list[3][0][1]), 4)
144    self.assertEqual(len(reader.call_args_list[4][0][1]), 2)
145
146  @mock.patch('schedv2.BenchmarkRunCacheReader')
147  def test_BenchmarkRunCacheReader_2(self, reader):
148    """Test benchmarkrun set is split into 4 segments."""
149
150    self.exp = self._make_fake_experiment(
151        EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=8))
152    my_schedv2 = Schedv2(self.exp)
153    self.assertFalse(my_schedv2.is_complete())
154    # We have 8 * 2 == 16 brs, we use 4 threads, each reading 4 brs.
155    self.assertEqual(reader.call_count, 4)
156    self.assertEqual(len(reader.call_args_list[0][0][1]), 4)
157    self.assertEqual(len(reader.call_args_list[1][0][1]), 4)
158    self.assertEqual(len(reader.call_args_list[2][0][1]), 4)
159    self.assertEqual(len(reader.call_args_list[3][0][1]), 4)
160
161  @mock.patch('schedv2.BenchmarkRunCacheReader')
162  def test_BenchmarkRunCacheReader_3(self, reader):
163    """Test benchmarkrun set is split into 2 segments."""
164
165    self.exp = self._make_fake_experiment(
166        EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=3))
167    my_schedv2 = Schedv2(self.exp)
168    self.assertFalse(my_schedv2.is_complete())
169    # We have 3 * 2 == 6 brs, we use 2 threads.
170    self.assertEqual(reader.call_count, 2)
171    self.assertEqual(len(reader.call_args_list[0][0][1]), 3)
172    self.assertEqual(len(reader.call_args_list[1][0][1]), 3)
173
174  @mock.patch('schedv2.BenchmarkRunCacheReader')
175  def test_BenchmarkRunCacheReader_4(self, reader):
176    """Test benchmarkrun set is not splitted."""
177
178    self.exp = self._make_fake_experiment(
179        EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=1))
180    my_schedv2 = Schedv2(self.exp)
181    self.assertFalse(my_schedv2.is_complete())
182    # We have 1 * 2 == 2 br, so only 1 instance.
183    self.assertEqual(reader.call_count, 1)
184    self.assertEqual(len(reader.call_args_list[0][0][1]), 2)
185
186  def test_cachehit(self):
187    """Test cache-hit and none-cache-hit brs are properly organized."""
188
189    def MockReadCache(br):
190      br.cache_hit = (br.label.name == 'image2')
191
192    with mock.patch(
193        'benchmark_run.MockBenchmarkRun.ReadCache', new=MockReadCache):
194      # We have 2 * 30 brs, half of which are put into _cached_br_list.
195      self.exp = self._make_fake_experiment(
196          EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=30))
197      my_schedv2 = Schedv2(self.exp)
198      self.assertEqual(len(my_schedv2.get_cached_run_list()), 30)
199      # The non-cache-hit brs are put into Schedv2._label_brl_map.
200      self.assertEqual(
201          functools.reduce(lambda a, x: a + len(x[1]),
202                           my_schedv2.get_label_map().items(), 0), 30)
203
204  def test_nocachehit(self):
205    """Test no cache-hit."""
206
207    def MockReadCache(br):
208      br.cache_hit = False
209
210    with mock.patch(
211        'benchmark_run.MockBenchmarkRun.ReadCache', new=MockReadCache):
212      # We have 2 * 30 brs, none of which are put into _cached_br_list.
213      self.exp = self._make_fake_experiment(
214          EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=30))
215      my_schedv2 = Schedv2(self.exp)
216      self.assertEqual(len(my_schedv2.get_cached_run_list()), 0)
217      # The non-cache-hit brs are put into Schedv2._label_brl_map.
218      self.assertEqual(
219          functools.reduce(lambda a, x: a + len(x[1]),
220                           my_schedv2.get_label_map().items(), 0), 60)
221
222
223if __name__ == '__main__':
224  test_flag.SetTestMode(True)
225  unittest.main()
226