• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2013 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Runs perf tests.
6
7Our buildbot infrastructure requires each slave to run steps serially.
8This is sub-optimal for android, where these steps can run independently on
9multiple connected devices.
10
11The buildbots will run this script multiple times per cycle:
12- First: all steps listed in --steps in will be executed in parallel using all
13connected devices. Step results will be pickled to disk. Each step has a unique
14name. The result code will be ignored if the step name is listed in
15--flaky-steps.
16The buildbot will treat this step as a regular step, and will not process any
17graph data.
18
19- Then, with -print-step STEP_NAME: at this stage, we'll simply print the file
20with the step results previously saved. The buildbot will then process the graph
21data accordingly.
22
23
24The JSON steps file contains a dictionary in the format:
25[
26  ["step_name_foo", "script_to_execute foo"],
27  ["step_name_bar", "script_to_execute bar"]
28]
29
30This preserves the order in which the steps are executed.
31
32The JSON flaky steps file contains a list with step names which results should
33be ignored:
34[
35  "step_name_foo",
36  "step_name_bar"
37]
38
39Note that script_to_execute necessarily have to take at least the following
40option:
41  --device: the serial number to be passed to all adb commands.
42"""
43
44import datetime
45import logging
46import os
47import pickle
48import sys
49import threading
50import time
51
52from pylib import constants
53from pylib import forwarder
54from pylib import pexpect
55from pylib.base import base_test_result
56from pylib.base import base_test_runner
57
58
59def PrintTestOutput(test_name):
60  """Helper method to print the output of previously executed test_name.
61
62  Args:
63    test_name: name of the test that has been previously executed.
64
65  Returns:
66    exit code generated by the test step.
67  """
68  file_name = os.path.join(constants.PERF_OUTPUT_DIR, test_name)
69  if not os.path.exists(file_name):
70    logging.error('File not found %s', file_name)
71    return 1
72
73  with file(file_name, 'r') as f:
74    persisted_result = pickle.loads(f.read())
75  logging.info('*' * 80)
76  logging.info('Output from:')
77  logging.info(persisted_result['cmd'])
78  logging.info('*' * 80)
79  print persisted_result['output']
80
81  return persisted_result['exit_code']
82
83
84class _HeartBeatLogger(object):
85  # How often to print the heartbeat on flush().
86  _PRINT_INTERVAL = 30.0
87
88  def __init__(self):
89    """A file-like class for keeping the buildbot alive."""
90    self._len = 0
91    self._tick = time.time()
92    self._stopped = threading.Event()
93    self._timer = threading.Thread(target=self._runner)
94    self._timer.start()
95
96  def _runner(self):
97    while not self._stopped.is_set():
98      self.flush()
99      self._stopped.wait(_HeartBeatLogger._PRINT_INTERVAL)
100
101  def write(self, data):
102    self._len += len(data)
103
104  def flush(self):
105    now = time.time()
106    if now - self._tick >= _HeartBeatLogger._PRINT_INTERVAL:
107      self._tick = now
108      print '--single-step output length %d' % self._len
109      sys.stdout.flush()
110
111  def stop(self):
112    self._stopped.set()
113
114
115class TestRunner(base_test_runner.BaseTestRunner):
116  def __init__(self, test_options, device, tests, flaky_tests):
117    """A TestRunner instance runs a perf test on a single device.
118
119    Args:
120      test_options: A PerfOptions object.
121      device: Device to run the tests.
122      tests: a dict mapping test_name to command.
123      flaky_tests: a list of flaky test_name.
124    """
125    super(TestRunner, self).__init__(device, None, 'Release')
126    self._options = test_options
127    self._tests = tests
128    self._flaky_tests = flaky_tests
129
130  @staticmethod
131  def _IsBetter(result):
132    if result['actual_exit_code'] == 0:
133      return True
134    pickled = os.path.join(constants.PERF_OUTPUT_DIR,
135                           result['name'])
136    if not os.path.exists(pickled):
137      return True
138    with file(pickled, 'r') as f:
139      previous = pickle.loads(f.read())
140    return result['actual_exit_code'] < previous['actual_exit_code']
141
142  @staticmethod
143  def _SaveResult(result):
144    if TestRunner._IsBetter(result):
145      with file(os.path.join(constants.PERF_OUTPUT_DIR,
146                             result['name']), 'w') as f:
147        f.write(pickle.dumps(result))
148
149  def _LaunchPerfTest(self, test_name):
150    """Runs a perf test.
151
152    Args:
153      test_name: the name of the test to be executed.
154
155    Returns:
156      A tuple containing (Output, base_test_result.ResultType)
157    """
158    try:
159      logging.warning('Unmapping device ports')
160      forwarder.Forwarder.UnmapAllDevicePorts(self.adb)
161      self.adb.RestartAdbdOnDevice()
162    except Exception as e:
163      logging.error('Exception when tearing down device %s', e)
164
165    cmd = ('%s --device %s' %
166           (self._tests[test_name], self.device))
167    logging.info('%s : %s', test_name, cmd)
168    start_time = datetime.datetime.now()
169
170    timeout = 5400
171    if self._options.no_timeout:
172      timeout = None
173    full_cmd = cmd
174    if self._options.dry_run:
175      full_cmd = 'echo %s' % cmd
176
177    logfile = sys.stdout
178    if self._options.single_step:
179      # Just print a heart-beat so that the outer buildbot scripts won't timeout
180      # without response.
181      logfile = _HeartBeatLogger()
182    cwd = os.path.abspath(constants.DIR_SOURCE_ROOT)
183    if full_cmd.startswith('src/'):
184      cwd = os.path.abspath(os.path.join(constants.DIR_SOURCE_ROOT, os.pardir))
185    output, exit_code = pexpect.run(
186        full_cmd, cwd=cwd,
187        withexitstatus=True, logfile=logfile, timeout=timeout,
188        env=os.environ)
189    if self._options.single_step:
190      # Stop the logger.
191      logfile.stop()
192    end_time = datetime.datetime.now()
193    if exit_code is None:
194      exit_code = -1
195    logging.info('%s : exit_code=%d in %d secs at %s',
196                 test_name, exit_code, (end_time - start_time).seconds,
197                 self.device)
198    result_type = base_test_result.ResultType.FAIL
199    if exit_code == 0:
200      result_type = base_test_result.ResultType.PASS
201    actual_exit_code = exit_code
202    if test_name in self._flaky_tests:
203      # The exit_code is used at the second stage when printing the
204      # test output. If the test is flaky, force to "0" to get that step green
205      # whilst still gathering data to the perf dashboards.
206      # The result_type is used by the test_dispatcher to retry the test.
207      exit_code = 0
208
209    persisted_result = {
210        'name': test_name,
211        'output': output,
212        'exit_code': exit_code,
213        'actual_exit_code': actual_exit_code,
214        'result_type': result_type,
215        'total_time': (end_time - start_time).seconds,
216        'device': self.device,
217        'cmd': cmd,
218    }
219    self._SaveResult(persisted_result)
220
221    return (output, result_type)
222
223  def RunTest(self, test_name):
224    """Run a perf test on the device.
225
226    Args:
227      test_name: String to use for logging the test result.
228
229    Returns:
230      A tuple of (TestRunResults, retry).
231    """
232    output, result_type = self._LaunchPerfTest(test_name)
233    results = base_test_result.TestRunResults()
234    results.AddResult(base_test_result.BaseTestResult(test_name, result_type))
235    retry = None
236    if not results.DidRunPass():
237      retry = test_name
238    return results, retry
239