• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# -*- coding: utf-8 -*-
2# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Module of benchmark runs."""
7from __future__ import print_function
8
9import datetime
10import threading
11import time
12import traceback
13
14from cros_utils import command_executer
15from cros_utils import timeline
16
17from suite_runner import SuiteRunner
18from results_cache import MockResult
19from results_cache import MockResultsCache
20from results_cache import Result
21from results_cache import ResultsCache
22
23STATUS_FAILED = 'FAILED'
24STATUS_SUCCEEDED = 'SUCCEEDED'
25STATUS_IMAGING = 'IMAGING'
26STATUS_RUNNING = 'RUNNING'
27STATUS_WAITING = 'WAITING'
28STATUS_PENDING = 'PENDING'
29
30
31class BenchmarkRun(threading.Thread):
32  """The benchmarkrun class."""
33
34  def __init__(self, name, benchmark, label, iteration, cache_conditions,
35               machine_manager, logger_to_use, log_level, share_cache,
36               dut_config):
37    threading.Thread.__init__(self)
38    self.name = name
39    self._logger = logger_to_use
40    self.log_level = log_level
41    self.benchmark = benchmark
42    self.iteration = iteration
43    self.label = label
44    self.result = None
45    self.terminated = False
46    self.retval = None
47    self.run_completed = False
48    self.machine_manager = machine_manager
49    self.suite_runner = SuiteRunner(dut_config, self._logger, self.log_level)
50    self.machine = None
51    self.cache_conditions = cache_conditions
52    self.runs_complete = 0
53    self.cache_hit = False
54    self.failure_reason = ''
55    self.test_args = benchmark.test_args
56    self.cache = None
57    self.profiler_args = self.GetExtraAutotestArgs()
58    self._ce = command_executer.GetCommandExecuter(
59        self._logger, log_level=self.log_level)
60    self.timeline = timeline.Timeline()
61    self.timeline.Record(STATUS_PENDING)
62    self.share_cache = share_cache
63    self.cache_has_been_read = False
64
65    # This is used by schedv2.
66    self.owner_thread = None
67
68  def ReadCache(self):
69    # Just use the first machine for running the cached version,
70    # without locking it.
71    self.cache = ResultsCache()
72    self.cache.Init(self.label.chromeos_image, self.label.chromeos_root,
73                    self.benchmark.test_name, self.iteration, self.test_args,
74                    self.profiler_args, self.machine_manager, self.machine,
75                    self.label.board, self.cache_conditions, self._logger,
76                    self.log_level, self.label, self.share_cache,
77                    self.benchmark.suite, self.benchmark.show_all_results,
78                    self.benchmark.run_local, self.benchmark.cwp_dso)
79
80    self.result = self.cache.ReadResult()
81    self.cache_hit = (self.result is not None)
82    self.cache_has_been_read = True
83
84  def run(self):
85    try:
86      if not self.cache_has_been_read:
87        self.ReadCache()
88
89      if self.result:
90        self._logger.LogOutput('%s: Cache hit.' % self.name)
91        self._logger.LogOutput(self.result.out, print_to_console=False)
92        self._logger.LogError(self.result.err, print_to_console=False)
93
94      elif self.label.cache_only:
95        self._logger.LogOutput('%s: No cache hit.' % self.name)
96        output = '%s: No Cache hit.' % self.name
97        retval = 1
98        err = 'No cache hit.'
99        self.result = Result.CreateFromRun(
100            self._logger, self.log_level, self.label, self.machine, output, err,
101            retval, self.benchmark.test_name, self.benchmark.suite,
102            self.benchmark.cwp_dso)
103
104      else:
105        self._logger.LogOutput('%s: No cache hit.' % self.name)
106        self.timeline.Record(STATUS_WAITING)
107        # Try to acquire a machine now.
108        self.machine = self.AcquireMachine()
109        self.cache.machine = self.machine
110        self.result = self.RunTest(self.machine)
111
112        self.cache.remote = self.machine.name
113        self.label.chrome_version = self.machine_manager.GetChromeVersion(
114            self.machine)
115        self.cache.StoreResult(self.result)
116
117      if not self.label.chrome_version:
118        if self.machine:
119          self.label.chrome_version = self.machine_manager.GetChromeVersion(
120              self.machine)
121        elif self.result.chrome_version:
122          self.label.chrome_version = self.result.chrome_version
123
124      if self.terminated:
125        return
126
127      if not self.result.retval:
128        self.timeline.Record(STATUS_SUCCEEDED)
129      else:
130        if self.timeline.GetLastEvent() != STATUS_FAILED:
131          self.failure_reason = 'Return value of test suite was non-zero.'
132          self.timeline.Record(STATUS_FAILED)
133
134    except Exception as e:
135      self._logger.LogError("Benchmark run: '%s' failed: %s" % (self.name, e))
136      traceback.print_exc()
137      if self.timeline.GetLastEvent() != STATUS_FAILED:
138        self.timeline.Record(STATUS_FAILED)
139        self.failure_reason = str(e)
140    finally:
141      if self.owner_thread is not None:
142        # In schedv2 mode, we do not lock machine locally. So noop here.
143        pass
144      elif self.machine:
145        if not self.machine.IsReachable():
146          self._logger.LogOutput(
147              'Machine %s is not reachable, removing it.' % self.machine.name)
148          self.machine_manager.RemoveMachine(self.machine.name)
149        self._logger.LogOutput('Releasing machine: %s' % self.machine.name)
150        self.machine_manager.ReleaseMachine(self.machine)
151        self._logger.LogOutput('Released machine: %s' % self.machine.name)
152
153  def Terminate(self):
154    self.terminated = True
155    self.suite_runner.Terminate()
156    if self.timeline.GetLastEvent() != STATUS_FAILED:
157      self.timeline.Record(STATUS_FAILED)
158      self.failure_reason = 'Thread terminated.'
159
160  def AcquireMachine(self):
161    if self.owner_thread is not None:
162      # No need to lock machine locally, DutWorker, which is a thread, is
163      # responsible for running br.
164      return self.owner_thread.dut()
165    while True:
166      machine = None
167      if self.terminated:
168        raise RuntimeError('Thread terminated while trying to acquire machine.')
169
170      machine = self.machine_manager.AcquireMachine(self.label)
171
172      if machine:
173        self._logger.LogOutput(
174            '%s: Machine %s acquired at %s' % (self.name, machine.name,
175                                               datetime.datetime.now()))
176        break
177      time.sleep(10)
178    return machine
179
180  def GetExtraAutotestArgs(self):
181    if (self.benchmark.perf_args and
182        self.benchmark.suite != 'telemetry_Crosperf'):
183      self._logger.LogError(
184          'Non-telemetry benchmark does not support profiler.')
185      self.benchmark.perf_args = ''
186
187    if self.benchmark.perf_args:
188      perf_args_list = self.benchmark.perf_args.split(' ')
189      perf_args_list = [perf_args_list[0]] + ['-a'] + perf_args_list[1:]
190      perf_args = ' '.join(perf_args_list)
191      if not perf_args_list[0] in ['record', 'stat']:
192        raise SyntaxError('perf_args must start with either record or stat')
193      extra_test_args = [
194          '--profiler=custom_perf',
195          ('--profiler_args=\'perf_options="%s"\'' % perf_args)
196      ]
197      return ' '.join(extra_test_args)
198    else:
199      return ''
200
201  def RunTest(self, machine):
202    self.timeline.Record(STATUS_IMAGING)
203    if self.owner_thread is not None:
204      # In schedv2 mode, do not even call ImageMachine. Machine image is
205      # guarenteed.
206      pass
207    else:
208      self.machine_manager.ImageMachine(machine, self.label)
209    self.timeline.Record(STATUS_RUNNING)
210    retval, out, err = self.suite_runner.Run(
211        machine, self.label, self.benchmark, self.test_args, self.profiler_args)
212    self.run_completed = True
213    return Result.CreateFromRun(self._logger, self.log_level, self.label,
214                                self.machine, out, err, retval,
215                                self.benchmark.test_name, self.benchmark.suite,
216                                self.benchmark.cwp_dso)
217
218  def SetCacheConditions(self, cache_conditions):
219    self.cache_conditions = cache_conditions
220
221  def logger(self):
222    """Return the logger, only used by unittest.
223
224    Returns:
225      self._logger
226    """
227
228    return self._logger
229
230  def __str__(self):
231    """For better debugging."""
232
233    return 'BenchmarkRun[name="{}"]'.format(self.name)
234
235
236class MockBenchmarkRun(BenchmarkRun):
237  """Inherited from BenchmarkRun."""
238
239  def ReadCache(self):
240    # Just use the first machine for running the cached version,
241    # without locking it.
242    self.cache = MockResultsCache()
243    self.cache.Init(self.label.chromeos_image, self.label.chromeos_root,
244                    self.benchmark.test_name, self.iteration, self.test_args,
245                    self.profiler_args, self.machine_manager, self.machine,
246                    self.label.board, self.cache_conditions, self._logger,
247                    self.log_level, self.label, self.share_cache,
248                    self.benchmark.suite, self.benchmark.show_all_results,
249                    self.benchmark.run_local, self.benchmark.cwp_dso)
250
251    self.result = self.cache.ReadResult()
252    self.cache_hit = (self.result is not None)
253
254  def RunTest(self, machine):
255    """Remove Result.CreateFromRun for testing."""
256    self.timeline.Record(STATUS_IMAGING)
257    self.machine_manager.ImageMachine(machine, self.label)
258    self.timeline.Record(STATUS_RUNNING)
259    [retval, out, err] = self.suite_runner.Run(
260        machine, self.label, self.benchmark, self.test_args, self.profiler_args)
261    self.run_completed = True
262    rr = MockResult('logger', self.label, self.log_level, machine)
263    rr.out = out
264    rr.err = err
265    rr.retval = retval
266    return rr
267