• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4"""Module of benchmark runs."""
5from __future__ import print_function
6
7import datetime
8import threading
9import time
10import traceback
11
12from cros_utils import command_executer
13from cros_utils import timeline
14
15from suite_runner import SuiteRunner
16from results_cache import MockResult
17from results_cache import MockResultsCache
18from results_cache import Result
19from results_cache import ResultsCache
20
21STATUS_FAILED = 'FAILED'
22STATUS_SUCCEEDED = 'SUCCEEDED'
23STATUS_IMAGING = 'IMAGING'
24STATUS_RUNNING = 'RUNNING'
25STATUS_WAITING = 'WAITING'
26STATUS_PENDING = 'PENDING'
27
28
29class BenchmarkRun(threading.Thread):
30  """The benchmarkrun class."""
31
32  def __init__(self, name, benchmark, label, iteration, cache_conditions,
33               machine_manager, logger_to_use, log_level, share_cache):
34    threading.Thread.__init__(self)
35    self.name = name
36    self._logger = logger_to_use
37    self.log_level = log_level
38    self.benchmark = benchmark
39    self.iteration = iteration
40    self.label = label
41    self.result = None
42    self.terminated = False
43    self.retval = None
44    self.run_completed = False
45    self.machine_manager = machine_manager
46    self.suite_runner = SuiteRunner(self._logger, self.log_level)
47    self.machine = None
48    self.cache_conditions = cache_conditions
49    self.runs_complete = 0
50    self.cache_hit = False
51    self.failure_reason = ''
52    self.test_args = benchmark.test_args
53    self.cache = None
54    self.profiler_args = self.GetExtraAutotestArgs()
55    self._ce = command_executer.GetCommandExecuter(
56        self._logger, log_level=self.log_level)
57    self.timeline = timeline.Timeline()
58    self.timeline.Record(STATUS_PENDING)
59    self.share_cache = share_cache
60    self.cache_has_been_read = False
61
62    # This is used by schedv2.
63    self.owner_thread = None
64
65  def ReadCache(self):
66    # Just use the first machine for running the cached version,
67    # without locking it.
68    self.cache = ResultsCache()
69    self.cache.Init(self.label.chromeos_image, self.label.chromeos_root,
70                    self.benchmark.test_name, self.iteration, self.test_args,
71                    self.profiler_args, self.machine_manager, self.machine,
72                    self.label.board, self.cache_conditions, self._logger,
73                    self.log_level, self.label, self.share_cache,
74                    self.benchmark.suite, self.benchmark.show_all_results,
75                    self.benchmark.run_local)
76
77    self.result = self.cache.ReadResult()
78    self.cache_hit = (self.result is not None)
79    self.cache_has_been_read = True
80
81  def run(self):
82    try:
83      if not self.cache_has_been_read:
84        self.ReadCache()
85
86      if self.result:
87        self._logger.LogOutput('%s: Cache hit.' % self.name)
88        self._logger.LogOutput(self.result.out, print_to_console=False)
89        self._logger.LogError(self.result.err, print_to_console=False)
90
91      elif self.label.cache_only:
92        self._logger.LogOutput('%s: No cache hit.' % self.name)
93        output = '%s: No Cache hit.' % self.name
94        retval = 1
95        err = 'No cache hit.'
96        self.result = Result.CreateFromRun(
97            self._logger, self.log_level, self.label, self.machine, output, err,
98            retval, self.benchmark.test_name, self.benchmark.suite)
99
100      else:
101        self._logger.LogOutput('%s: No cache hit.' % self.name)
102        self.timeline.Record(STATUS_WAITING)
103        # Try to acquire a machine now.
104        self.machine = self.AcquireMachine()
105        self.cache.machine = self.machine
106        self.result = self.RunTest(self.machine)
107
108        self.cache.remote = self.machine.name
109        self.label.chrome_version = self.machine_manager.GetChromeVersion(
110            self.machine)
111        self.cache.StoreResult(self.result)
112
113      if not self.label.chrome_version:
114        if self.machine:
115          self.label.chrome_version = self.machine_manager.GetChromeVersion(
116              self.machine)
117        elif self.result.chrome_version:
118          self.label.chrome_version = self.result.chrome_version
119
120      if self.terminated:
121        return
122
123      if not self.result.retval:
124        self.timeline.Record(STATUS_SUCCEEDED)
125      else:
126        if self.timeline.GetLastEvent() != STATUS_FAILED:
127          self.failure_reason = 'Return value of test suite was non-zero.'
128          self.timeline.Record(STATUS_FAILED)
129
130    except Exception, e:
131      self._logger.LogError("Benchmark run: '%s' failed: %s" % (self.name, e))
132      traceback.print_exc()
133      if self.timeline.GetLastEvent() != STATUS_FAILED:
134        self.timeline.Record(STATUS_FAILED)
135        self.failure_reason = str(e)
136    finally:
137      if self.owner_thread is not None:
138        # In schedv2 mode, we do not lock machine locally. So noop here.
139        pass
140      elif self.machine:
141        if not self.machine.IsReachable():
142          self._logger.LogOutput(
143              'Machine %s is not reachable, removing it.' % self.machine.name)
144          self.machine_manager.RemoveMachine(self.machine.name)
145        self._logger.LogOutput('Releasing machine: %s' % self.machine.name)
146        self.machine_manager.ReleaseMachine(self.machine)
147        self._logger.LogOutput('Released machine: %s' % self.machine.name)
148
149  def Terminate(self):
150    self.terminated = True
151    self.suite_runner.Terminate()
152    if self.timeline.GetLastEvent() != STATUS_FAILED:
153      self.timeline.Record(STATUS_FAILED)
154      self.failure_reason = 'Thread terminated.'
155
156  def AcquireMachine(self):
157    if self.owner_thread is not None:
158      # No need to lock machine locally, DutWorker, which is a thread, is
159      # responsible for running br.
160      return self.owner_thread.dut()
161    while True:
162      machine = None
163      if self.terminated:
164        raise RuntimeError('Thread terminated while trying to acquire machine.')
165
166      machine = self.machine_manager.AcquireMachine(self.label)
167
168      if machine:
169        self._logger.LogOutput('%s: Machine %s acquired at %s' %
170                               (self.name, machine.name,
171                                datetime.datetime.now()))
172        break
173      time.sleep(10)
174    return machine
175
176  def GetExtraAutotestArgs(self):
177    if self.benchmark.perf_args and self.benchmark.suite == 'telemetry':
178      self._logger.LogError('Telemetry does not support profiler.')
179      self.benchmark.perf_args = ''
180
181    if self.benchmark.perf_args and self.benchmark.suite == 'test_that':
182      self._logger.LogError('test_that does not support profiler.')
183      self.benchmark.perf_args = ''
184
185    if self.benchmark.perf_args:
186      perf_args_list = self.benchmark.perf_args.split(' ')
187      perf_args_list = [perf_args_list[0]] + ['-a'] + perf_args_list[1:]
188      perf_args = ' '.join(perf_args_list)
189      if not perf_args_list[0] in ['record', 'stat']:
190        raise SyntaxError('perf_args must start with either record or stat')
191      extra_test_args = [
192          '--profiler=custom_perf',
193          ("--profiler_args='perf_options=\"%s\"'" % perf_args)
194      ]
195      return ' '.join(extra_test_args)
196    else:
197      return ''
198
199  def RunTest(self, machine):
200    self.timeline.Record(STATUS_IMAGING)
201    if self.owner_thread is not None:
202      # In schedv2 mode, do not even call ImageMachine. Machine image is
203      # guarenteed.
204      pass
205    else:
206      self.machine_manager.ImageMachine(machine, self.label)
207    self.timeline.Record(STATUS_RUNNING)
208    retval, out, err = self.suite_runner.Run(machine.name, self.label,
209                                             self.benchmark, self.test_args,
210                                             self.profiler_args)
211    self.run_completed = True
212    return Result.CreateFromRun(self._logger, self.log_level, self.label,
213                                self.machine, out, err, retval,
214                                self.benchmark.test_name, self.benchmark.suite)
215
216  def SetCacheConditions(self, cache_conditions):
217    self.cache_conditions = cache_conditions
218
219  def logger(self):
220    """Return the logger, only used by unittest.
221
222    Returns:
223      self._logger
224    """
225
226    return self._logger
227
228  def __str__(self):
229    """For better debugging."""
230
231    return 'BenchmarkRun[name="{}"]'.format(self.name)
232
233
234class MockBenchmarkRun(BenchmarkRun):
235  """Inherited from BenchmarkRun."""
236
237  def ReadCache(self):
238    # Just use the first machine for running the cached version,
239    # without locking it.
240    self.cache = MockResultsCache()
241    self.cache.Init(self.label.chromeos_image, self.label.chromeos_root,
242                    self.benchmark.test_name, self.iteration, self.test_args,
243                    self.profiler_args, self.machine_manager, self.machine,
244                    self.label.board, self.cache_conditions, self._logger,
245                    self.log_level, self.label, self.share_cache,
246                    self.benchmark.suite, self.benchmark.show_all_results,
247                    self.benchmark.run_local)
248
249    self.result = self.cache.ReadResult()
250    self.cache_hit = (self.result is not None)
251
252  def RunTest(self, machine):
253    """Remove Result.CreateFromRun for testing."""
254    self.timeline.Record(STATUS_IMAGING)
255    self.machine_manager.ImageMachine(machine, self.label)
256    self.timeline.Record(STATUS_RUNNING)
257    [retval, out,
258     err] = self.suite_runner.Run(machine.name, self.label, self.benchmark,
259                                  self.test_args, self.profiler_args)
260    self.run_completed = True
261    rr = MockResult('logger', self.label, self.log_level, machine)
262    rr.out = out
263    rr.err = err
264    rr.retval = retval
265    return rr
266