• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4"""The experiment runner module."""
5from __future__ import print_function
6
7import getpass
8import os
9import shutil
10import time
11
12import afe_lock_machine
13import test_flag
14
15from cros_utils import command_executer
16from cros_utils import logger
17from cros_utils.email_sender import EmailSender
18from cros_utils.file_utils import FileUtils
19
20import config
21from experiment_status import ExperimentStatus
22from results_cache import CacheConditions
23from results_cache import ResultsCache
24from results_report import HTMLResultsReport
25from results_report import TextResultsReport
26from results_report import JSONResultsReport
27from schedv2 import Schedv2
28
29def _WriteJSONReportToFile(experiment, results_dir, json_report):
30  """Writes a JSON report to a file in results_dir."""
31  has_llvm = any('llvm' in l.compiler for l in experiment.labels)
32  compiler_string = 'llvm' if has_llvm else 'gcc'
33  board = experiment.labels[0].board
34  filename = 'report_%s_%s_%s.%s.json' % (
35      board, json_report.date, json_report.time.replace(':', '.'),
36      compiler_string)
37  fullname = os.path.join(results_dir, filename)
38  report_text = json_report.GetReport()
39  with open(fullname, 'w') as out_file:
40    out_file.write(report_text)
41
42
43class ExperimentRunner(object):
44  """ExperimentRunner Class."""
45
46  STATUS_TIME_DELAY = 30
47  THREAD_MONITOR_DELAY = 2
48
49  def __init__(self,
50               experiment,
51               json_report,
52               using_schedv2=False,
53               log=None,
54               cmd_exec=None):
55    self._experiment = experiment
56    self.l = log or logger.GetLogger(experiment.log_dir)
57    self._ce = cmd_exec or command_executer.GetCommandExecuter(self.l)
58    self._terminated = False
59    self.json_report = json_report
60    self.locked_machines = []
61    if experiment.log_level != 'verbose':
62      self.STATUS_TIME_DELAY = 10
63
64    # Setting this to True will use crosperf sched v2 (feature in progress).
65    self._using_schedv2 = using_schedv2
66
67  def _GetMachineList(self):
68    """Return a list of all requested machines.
69
70    Create a list of all the requested machines, both global requests and
71    label-specific requests, and return the list.
72    """
73    machines = self._experiment.remote
74    # All Label.remote is a sublist of experiment.remote.
75    for l in self._experiment.labels:
76      for r in l.remote:
77        assert r in machines
78    return machines
79
80  def _UpdateMachineList(self, locked_machines):
81    """Update machines lists to contain only locked machines.
82
83    Go through all the lists of requested machines, both global and
84    label-specific requests, and remove any machine that we were not
85    able to lock.
86
87    Args:
88      locked_machines: A list of the machines we successfully locked.
89    """
90    for m in self._experiment.remote:
91      if m not in locked_machines:
92        self._experiment.remote.remove(m)
93
94    for l in self._experiment.labels:
95      for m in l.remote:
96        if m not in locked_machines:
97          l.remote.remove(m)
98
99  def _LockAllMachines(self, experiment):
100    """Attempt to globally lock all of the machines requested for run.
101
102    This method will use the AFE server to globally lock all of the machines
103    requested for this crosperf run, to prevent any other crosperf runs from
104    being able to update/use the machines while this experiment is running.
105    """
106    if test_flag.GetTestMode():
107      self.locked_machines = self._GetMachineList()
108      self._experiment.locked_machines = self.locked_machines
109    else:
110      lock_mgr = afe_lock_machine.AFELockManager(
111          self._GetMachineList(),
112          '',
113          experiment.labels[0].chromeos_root,
114          None,
115          log=self.l,)
116      for m in lock_mgr.machines:
117        if not lock_mgr.MachineIsKnown(m):
118          lock_mgr.AddLocalMachine(m)
119      machine_states = lock_mgr.GetMachineStates('lock')
120      lock_mgr.CheckMachineLocks(machine_states, 'lock')
121      self.locked_machines = lock_mgr.UpdateMachines(True)
122      self._experiment.locked_machines = self.locked_machines
123      self._UpdateMachineList(self.locked_machines)
124      self._experiment.machine_manager.RemoveNonLockedMachines(
125          self.locked_machines)
126      if len(self.locked_machines) == 0:
127        raise RuntimeError('Unable to lock any machines.')
128
129  def _UnlockAllMachines(self, experiment):
130    """Attempt to globally unlock all of the machines requested for run.
131
132    The method will use the AFE server to globally unlock all of the machines
133    requested for this crosperf run.
134    """
135    if not self.locked_machines or test_flag.GetTestMode():
136      return
137
138    lock_mgr = afe_lock_machine.AFELockManager(
139        self.locked_machines,
140        '',
141        experiment.labels[0].chromeos_root,
142        None,
143        log=self.l,)
144    machine_states = lock_mgr.GetMachineStates('unlock')
145    lock_mgr.CheckMachineLocks(machine_states, 'unlock')
146    lock_mgr.UpdateMachines(False)
147
148  def _ClearCacheEntries(self, experiment):
149    for br in experiment.benchmark_runs:
150      cache = ResultsCache()
151      cache.Init(br.label.chromeos_image, br.label.chromeos_root,
152                 br.benchmark.test_name, br.iteration, br.test_args,
153                 br.profiler_args, br.machine_manager, br.machine,
154                 br.label.board, br.cache_conditions, br._logger, br.log_level,
155                 br.label, br.share_cache, br.benchmark.suite,
156                 br.benchmark.show_all_results, br.benchmark.run_local)
157      cache_dir = cache.GetCacheDirForWrite()
158      if os.path.exists(cache_dir):
159        self.l.LogOutput('Removing cache dir: %s' % cache_dir)
160        shutil.rmtree(cache_dir)
161
162  def _Run(self, experiment):
163    try:
164      if not experiment.locks_dir:
165        self._LockAllMachines(experiment)
166      if self._using_schedv2:
167        schedv2 = Schedv2(experiment)
168        experiment.set_schedv2(schedv2)
169      if CacheConditions.FALSE in experiment.cache_conditions:
170        self._ClearCacheEntries(experiment)
171      status = ExperimentStatus(experiment)
172      experiment.Run()
173      last_status_time = 0
174      last_status_string = ''
175      try:
176        if experiment.log_level != 'verbose':
177          self.l.LogStartDots()
178        while not experiment.IsComplete():
179          if last_status_time + self.STATUS_TIME_DELAY < time.time():
180            last_status_time = time.time()
181            border = '=============================='
182            if experiment.log_level == 'verbose':
183              self.l.LogOutput(border)
184              self.l.LogOutput(status.GetProgressString())
185              self.l.LogOutput(status.GetStatusString())
186              self.l.LogOutput(border)
187            else:
188              current_status_string = status.GetStatusString()
189              if current_status_string != last_status_string:
190                self.l.LogEndDots()
191                self.l.LogOutput(border)
192                self.l.LogOutput(current_status_string)
193                self.l.LogOutput(border)
194                last_status_string = current_status_string
195              else:
196                self.l.LogAppendDot()
197          time.sleep(self.THREAD_MONITOR_DELAY)
198      except KeyboardInterrupt:
199        self._terminated = True
200        self.l.LogError('Ctrl-c pressed. Cleaning up...')
201        experiment.Terminate()
202        raise
203      except SystemExit:
204        self._terminated = True
205        self.l.LogError('Unexpected exit. Cleaning up...')
206        experiment.Terminate()
207        raise
208    finally:
209      if not experiment.locks_dir:
210        self._UnlockAllMachines(experiment)
211
212  def _PrintTable(self, experiment):
213    self.l.LogOutput(TextResultsReport.FromExperiment(experiment).GetReport())
214
215  def _Email(self, experiment):
216    # Only email by default if a new run was completed.
217    send_mail = False
218    for benchmark_run in experiment.benchmark_runs:
219      if not benchmark_run.cache_hit:
220        send_mail = True
221        break
222    if (not send_mail and not experiment.email_to or
223        config.GetConfig('no_email')):
224      return
225
226    label_names = []
227    for label in experiment.labels:
228      label_names.append(label.name)
229    subject = '%s: %s' % (experiment.name, ' vs. '.join(label_names))
230
231    text_report = TextResultsReport.FromExperiment(experiment, True).GetReport()
232    text_report += ('\nResults are stored in %s.\n' %
233                    experiment.results_directory)
234    text_report = "<pre style='font-size: 13px'>%s</pre>" % text_report
235    html_report = HTMLResultsReport.FromExperiment(experiment).GetReport()
236    attachment = EmailSender.Attachment('report.html', html_report)
237    email_to = experiment.email_to or []
238    email_to.append(getpass.getuser())
239    EmailSender().SendEmail(email_to,
240                            subject,
241                            text_report,
242                            attachments=[attachment],
243                            msg_type='html')
244
245  def _StoreResults(self, experiment):
246    if self._terminated:
247      return
248    results_directory = experiment.results_directory
249    FileUtils().RmDir(results_directory)
250    FileUtils().MkDirP(results_directory)
251    self.l.LogOutput('Storing experiment file in %s.' % results_directory)
252    experiment_file_path = os.path.join(results_directory, 'experiment.exp')
253    FileUtils().WriteFile(experiment_file_path, experiment.experiment_file)
254
255    self.l.LogOutput('Storing results report in %s.' % results_directory)
256    results_table_path = os.path.join(results_directory, 'results.html')
257    report = HTMLResultsReport.FromExperiment(experiment).GetReport()
258    if self.json_report:
259      json_report = JSONResultsReport.FromExperiment(experiment,
260                                                     json_args={'indent': 2})
261      _WriteJSONReportToFile(experiment, results_directory, json_report)
262
263    FileUtils().WriteFile(results_table_path, report)
264
265    self.l.LogOutput('Storing email message body in %s.' % results_directory)
266    msg_file_path = os.path.join(results_directory, 'msg_body.html')
267    text_report = TextResultsReport.FromExperiment(experiment, True).GetReport()
268    text_report += ('\nResults are stored in %s.\n' %
269                    experiment.results_directory)
270    msg_body = "<pre style='font-size: 13px'>%s</pre>" % text_report
271    FileUtils().WriteFile(msg_file_path, msg_body)
272
273    self.l.LogOutput('Storing results of each benchmark run.')
274    for benchmark_run in experiment.benchmark_runs:
275      if benchmark_run.result:
276        benchmark_run_name = filter(str.isalnum, benchmark_run.name)
277        benchmark_run_path = os.path.join(results_directory, benchmark_run_name)
278        benchmark_run.result.CopyResultsTo(benchmark_run_path)
279        benchmark_run.result.CleanUp(benchmark_run.benchmark.rm_chroot_tmp)
280
281  def Run(self):
282    try:
283      self._Run(self._experiment)
284    finally:
285      # Always print the report at the end of the run.
286      self._PrintTable(self._experiment)
287      if not self._terminated:
288        self._StoreResults(self._experiment)
289        self._Email(self._experiment)
290
291
292class MockExperimentRunner(ExperimentRunner):
293  """Mocked ExperimentRunner for testing."""
294
295  def __init__(self, experiment, json_report):
296    super(MockExperimentRunner, self).__init__(experiment, json_report)
297
298  def _Run(self, experiment):
299    self.l.LogOutput("Would run the following experiment: '%s'." %
300                     experiment.name)
301
302  def _PrintTable(self, experiment):
303    self.l.LogOutput('Would print the experiment table.')
304
305  def _Email(self, experiment):
306    self.l.LogOutput('Would send result email.')
307
308  def _StoreResults(self, experiment):
309    self.l.LogOutput('Would store the results.')
310