• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# -*- coding: utf-8 -*-
2# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""The experiment runner module."""
7from __future__ import print_function
8
9import getpass
10import os
11import shutil
12import time
13
14import lock_machine
15import test_flag
16
17from cros_utils import command_executer
18from cros_utils import logger
19from cros_utils.email_sender import EmailSender
20from cros_utils.file_utils import FileUtils
21
22import config
23from experiment_status import ExperimentStatus
24from results_cache import CacheConditions
25from results_cache import ResultsCache
26from results_report import HTMLResultsReport
27from results_report import TextResultsReport
28from results_report import JSONResultsReport
29from schedv2 import Schedv2
30
31
32def _WriteJSONReportToFile(experiment, results_dir, json_report):
33  """Writes a JSON report to a file in results_dir."""
34  has_llvm = any('llvm' in l.compiler for l in experiment.labels)
35  compiler_string = 'llvm' if has_llvm else 'gcc'
36  board = experiment.labels[0].board
37  filename = 'report_%s_%s_%s.%s.json' % (board, json_report.date,
38                                          json_report.time.replace(':', '.'),
39                                          compiler_string)
40  fullname = os.path.join(results_dir, filename)
41  report_text = json_report.GetReport()
42  with open(fullname, 'w') as out_file:
43    out_file.write(report_text)
44
45
46class ExperimentRunner(object):
47  """ExperimentRunner Class."""
48
49  STATUS_TIME_DELAY = 30
50  THREAD_MONITOR_DELAY = 2
51
52  def __init__(self,
53               experiment,
54               json_report,
55               using_schedv2=False,
56               log=None,
57               cmd_exec=None):
58    self._experiment = experiment
59    self.l = log or logger.GetLogger(experiment.log_dir)
60    self._ce = cmd_exec or command_executer.GetCommandExecuter(self.l)
61    self._terminated = False
62    self.json_report = json_report
63    self.locked_machines = []
64    if experiment.log_level != 'verbose':
65      self.STATUS_TIME_DELAY = 10
66
67    # Setting this to True will use crosperf sched v2 (feature in progress).
68    self._using_schedv2 = using_schedv2
69
70  def _GetMachineList(self):
71    """Return a list of all requested machines.
72
73    Create a list of all the requested machines, both global requests and
74    label-specific requests, and return the list.
75    """
76    machines = self._experiment.remote
77    # All Label.remote is a sublist of experiment.remote.
78    for l in self._experiment.labels:
79      for r in l.remote:
80        assert r in machines
81    return machines
82
83  def _UpdateMachineList(self, locked_machines):
84    """Update machines lists to contain only locked machines.
85
86    Go through all the lists of requested machines, both global and
87    label-specific requests, and remove any machine that we were not
88    able to lock.
89
90    Args:
91      locked_machines: A list of the machines we successfully locked.
92    """
93    for m in self._experiment.remote:
94      if m not in locked_machines:
95        self._experiment.remote.remove(m)
96
97    for l in self._experiment.labels:
98      for m in l.remote:
99        if m not in locked_machines:
100          l.remote.remove(m)
101
102  def _GetMachineType(self, lock_mgr, machine):
103    """Get where is the machine from.
104
105    Returns:
106      The location of the machine: local or skylab
107    """
108    # We assume that lab machine always starts with chromeos*, and local
109    # machines are ip address.
110    if 'chromeos' in machine:
111      if lock_mgr.CheckMachineInSkylab(machine):
112        return 'skylab'
113      else:
114        raise RuntimeError('Lab machine not in Skylab.')
115    return 'local'
116
117  def _LockAllMachines(self, experiment):
118    """Attempt to globally lock all of the machines requested for run.
119
120    This method tries to lock all machines requested for this crosperf run
121    in three different modes automatically, to prevent any other crosperf runs
122    from being able to update/use the machines while this experiment is
123    running:
124      - Skylab machines: Use skylab lease-dut mechanism to lease
125      - Local machines: Use file lock mechanism to lock
126    """
127    if test_flag.GetTestMode():
128      self.locked_machines = self._GetMachineList()
129      experiment.locked_machines = self.locked_machines
130    else:
131      experiment.lock_mgr = lock_machine.LockManager(
132          self._GetMachineList(),
133          '',
134          experiment.labels[0].chromeos_root,
135          experiment.locks_dir,
136          log=self.l,
137      )
138      for m in experiment.lock_mgr.machines:
139        machine_type = self._GetMachineType(experiment.lock_mgr, m)
140        if machine_type == 'local':
141          experiment.lock_mgr.AddMachineToLocal(m)
142        elif machine_type == 'skylab':
143          experiment.lock_mgr.AddMachineToSkylab(m)
144      machine_states = experiment.lock_mgr.GetMachineStates('lock')
145      experiment.lock_mgr.CheckMachineLocks(machine_states, 'lock')
146      self.locked_machines = experiment.lock_mgr.UpdateMachines(True)
147      experiment.locked_machines = self.locked_machines
148      self._UpdateMachineList(self.locked_machines)
149      experiment.machine_manager.RemoveNonLockedMachines(self.locked_machines)
150      if not self.locked_machines:
151        raise RuntimeError('Unable to lock any machines.')
152
153  def _ClearCacheEntries(self, experiment):
154    for br in experiment.benchmark_runs:
155      cache = ResultsCache()
156      cache.Init(
157          br.label.chromeos_image, br.label.chromeos_root,
158          br.benchmark.test_name, br.iteration, br.test_args, br.profiler_args,
159          br.machine_manager, br.machine, br.label.board, br.cache_conditions,
160          br.logger(), br.log_level, br.label, br.share_cache,
161          br.benchmark.suite, br.benchmark.show_all_results,
162          br.benchmark.run_local, br.benchmark.cwp_dso)
163      cache_dir = cache.GetCacheDirForWrite()
164      if os.path.exists(cache_dir):
165        self.l.LogOutput('Removing cache dir: %s' % cache_dir)
166        shutil.rmtree(cache_dir)
167
168  def _Run(self, experiment):
169    try:
170      # We should not lease machines if tests are launched via `skylab
171      # create-test`. This is because leasing DUT in skylab will create a
172      # dummy task on the DUT and new test created will be hanging there.
173      # TODO(zhizhouy): Need to check whether machine is ready or not before
174      # assigning a test to it.
175      if not experiment.skylab:
176        self._LockAllMachines(experiment)
177      # Calculate all checksums of avaiable/locked machines, to ensure same
178      # label has same machines for testing
179      experiment.SetCheckSums(forceSameImage=True)
180      if self._using_schedv2:
181        schedv2 = Schedv2(experiment)
182        experiment.set_schedv2(schedv2)
183      if CacheConditions.FALSE in experiment.cache_conditions:
184        self._ClearCacheEntries(experiment)
185      status = ExperimentStatus(experiment)
186      experiment.Run()
187      last_status_time = 0
188      last_status_string = ''
189      try:
190        if experiment.log_level != 'verbose':
191          self.l.LogStartDots()
192        while not experiment.IsComplete():
193          if last_status_time + self.STATUS_TIME_DELAY < time.time():
194            last_status_time = time.time()
195            border = '=============================='
196            if experiment.log_level == 'verbose':
197              self.l.LogOutput(border)
198              self.l.LogOutput(status.GetProgressString())
199              self.l.LogOutput(status.GetStatusString())
200              self.l.LogOutput(border)
201            else:
202              current_status_string = status.GetStatusString()
203              if current_status_string != last_status_string:
204                self.l.LogEndDots()
205                self.l.LogOutput(border)
206                self.l.LogOutput(current_status_string)
207                self.l.LogOutput(border)
208                last_status_string = current_status_string
209              else:
210                self.l.LogAppendDot()
211          time.sleep(self.THREAD_MONITOR_DELAY)
212      except KeyboardInterrupt:
213        self._terminated = True
214        self.l.LogError('Ctrl-c pressed. Cleaning up...')
215        experiment.Terminate()
216        raise
217      except SystemExit:
218        self._terminated = True
219        self.l.LogError('Unexpected exit. Cleaning up...')
220        experiment.Terminate()
221        raise
222    finally:
223      experiment.Cleanup()
224
225  def _PrintTable(self, experiment):
226    self.l.LogOutput(TextResultsReport.FromExperiment(experiment).GetReport())
227
228  def _Email(self, experiment):
229    # Only email by default if a new run was completed.
230    send_mail = False
231    for benchmark_run in experiment.benchmark_runs:
232      if not benchmark_run.cache_hit:
233        send_mail = True
234        break
235    if (not send_mail and not experiment.email_to or
236        config.GetConfig('no_email')):
237      return
238
239    label_names = []
240    for label in experiment.labels:
241      label_names.append(label.name)
242    subject = '%s: %s' % (experiment.name, ' vs. '.join(label_names))
243
244    text_report = TextResultsReport.FromExperiment(experiment, True).GetReport()
245    text_report += (
246        '\nResults are stored in %s.\n' % experiment.results_directory)
247    text_report = "<pre style='font-size: 13px'>%s</pre>" % text_report
248    html_report = HTMLResultsReport.FromExperiment(experiment).GetReport()
249    attachment = EmailSender.Attachment('report.html', html_report)
250    email_to = experiment.email_to or []
251    email_to.append(getpass.getuser())
252    EmailSender().SendEmail(
253        email_to,
254        subject,
255        text_report,
256        attachments=[attachment],
257        msg_type='html')
258
259  def _StoreResults(self, experiment):
260    if self._terminated:
261      return
262    results_directory = experiment.results_directory
263    FileUtils().RmDir(results_directory)
264    FileUtils().MkDirP(results_directory)
265    self.l.LogOutput('Storing experiment file in %s.' % results_directory)
266    experiment_file_path = os.path.join(results_directory, 'experiment.exp')
267    FileUtils().WriteFile(experiment_file_path, experiment.experiment_file)
268
269    self.l.LogOutput('Storing results report in %s.' % results_directory)
270    results_table_path = os.path.join(results_directory, 'results.html')
271    report = HTMLResultsReport.FromExperiment(experiment).GetReport()
272    if self.json_report:
273      json_report = JSONResultsReport.FromExperiment(
274          experiment, json_args={'indent': 2})
275      _WriteJSONReportToFile(experiment, results_directory, json_report)
276
277    FileUtils().WriteFile(results_table_path, report)
278
279    self.l.LogOutput('Storing email message body in %s.' % results_directory)
280    msg_file_path = os.path.join(results_directory, 'msg_body.html')
281    text_report = TextResultsReport.FromExperiment(experiment, True).GetReport()
282    text_report += (
283        '\nResults are stored in %s.\n' % experiment.results_directory)
284    msg_body = "<pre style='font-size: 13px'>%s</pre>" % text_report
285    FileUtils().WriteFile(msg_file_path, msg_body)
286
287    self.l.LogOutput('Storing results of each benchmark run.')
288    for benchmark_run in experiment.benchmark_runs:
289      if benchmark_run.result:
290        benchmark_run_name = ''.join(
291            ch for ch in benchmark_run.name if ch.isalnum())
292        benchmark_run_path = os.path.join(results_directory, benchmark_run_name)
293        benchmark_run.result.CopyResultsTo(benchmark_run_path)
294        benchmark_run.result.CleanUp(benchmark_run.benchmark.rm_chroot_tmp)
295
296    topstats_file = os.path.join(results_directory, 'topstats.log')
297    self.l.LogOutput('Storing top5 statistics of each benchmark run into %s.' %
298                     topstats_file)
299    with open(topstats_file, 'w') as top_fd:
300      for benchmark_run in experiment.benchmark_runs:
301        if benchmark_run.result:
302          # Header with benchmark run name.
303          top_fd.write('%s\n' % str(benchmark_run))
304          # Formatted string with top statistics.
305          top_fd.write(benchmark_run.result.FormatStringTop5())
306          top_fd.write('\n\n')
307
308  def Run(self):
309    try:
310      self._Run(self._experiment)
311    finally:
312      # Always print the report at the end of the run.
313      self._PrintTable(self._experiment)
314      if not self._terminated:
315        self._StoreResults(self._experiment)
316        self._Email(self._experiment)
317
318
319class MockExperimentRunner(ExperimentRunner):
320  """Mocked ExperimentRunner for testing."""
321
322  def __init__(self, experiment, json_report):
323    super(MockExperimentRunner, self).__init__(experiment, json_report)
324
325  def _Run(self, experiment):
326    self.l.LogOutput(
327        "Would run the following experiment: '%s'." % experiment.name)
328
329  def _PrintTable(self, experiment):
330    self.l.LogOutput('Would print the experiment table.')
331
332  def _Email(self, experiment):
333    self.l.LogOutput('Would send result email.')
334
335  def _StoreResults(self, experiment):
336    self.l.LogOutput('Would store the results.')
337