• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python
2# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Do all the steps required to build and test against nacl."""
7
8
9import optparse
10import os.path
11import re
12import shutil
13import subprocess
14import sys
15
16import find_chrome
17
18
19# Copied from buildbot/buildbot_lib.py
20def TryToCleanContents(path, file_name_filter=lambda fn: True):
21  """
22  Remove the contents of a directory without touching the directory itself.
23  Ignores all failures.
24  """
25  if os.path.exists(path):
26    for fn in os.listdir(path):
27      TryToCleanPath(os.path.join(path, fn), file_name_filter)
28
29
30# Copied from buildbot/buildbot_lib.py
31def TryToCleanPath(path, file_name_filter=lambda fn: True):
32  """
33  Removes a file or directory.
34  Ignores all failures.
35  """
36  if os.path.exists(path):
37    if file_name_filter(path):
38      print 'Trying to remove %s' % path
39      if os.path.isdir(path):
40        shutil.rmtree(path, ignore_errors=True)
41      else:
42        try:
43          os.remove(path)
44        except Exception:
45          pass
46    else:
47      print 'Skipping %s' % path
48
49
50# TODO(ncbray): this is somewhat unsafe.  We should fix the underlying problem.
51def CleanTempDir():
52  # Only delete files and directories like:
53  # a) C:\temp\83C4.tmp
54  # b) /tmp/.org.chromium.Chromium.EQrEzl
55  file_name_re = re.compile(
56      r'[\\/]([0-9a-fA-F]+\.tmp|\.org\.chrom\w+\.Chrom\w+\..+)$')
57  file_name_filter = lambda fn: file_name_re.search(fn) is not None
58
59  path = os.environ.get('TMP', os.environ.get('TEMP', '/tmp'))
60  if len(path) >= 4 and os.path.isdir(path):
61    print
62    print "Cleaning out the temp directory."
63    print
64    TryToCleanContents(path, file_name_filter)
65  else:
66    print
67    print "Cannot find temp directory, not cleaning it."
68    print
69
70
71def RunCommand(cmd, cwd, env):
72  sys.stdout.write('\nRunning %s\n\n' % ' '.join(cmd))
73  sys.stdout.flush()
74  retcode = subprocess.call(cmd, cwd=cwd, env=env)
75  if retcode != 0:
76    sys.stdout.write('\nFailed: %s\n\n' % ' '.join(cmd))
77    sys.exit(retcode)
78
79
80def RunTests(name, cmd, nacl_dir, env):
81  sys.stdout.write('\n\nBuilding files needed for %s testing...\n\n' % name)
82  RunCommand(cmd + ['do_not_run_tests=1', '-j8'], nacl_dir, env)
83  sys.stdout.write('\n\nRunning %s tests...\n\n' % name)
84  RunCommand(cmd, nacl_dir, env)
85
86
87def BuildAndTest(options):
88  # Refuse to run under cygwin.
89  if sys.platform == 'cygwin':
90    raise Exception('I do not work under cygwin, sorry.')
91
92  # By default, use the version of Python is being used to run this script.
93  python = sys.executable
94  if sys.platform == 'darwin':
95    # Mac 10.5 bots tend to use a particularlly old version of Python, look for
96    # a newer version.
97    macpython27 = '/Library/Frameworks/Python.framework/Versions/2.7/bin/python'
98    if os.path.exists(macpython27):
99      python = macpython27
100
101  script_dir = os.path.dirname(os.path.abspath(__file__))
102  src_dir = os.path.dirname(os.path.dirname(os.path.dirname(script_dir)))
103  nacl_dir = os.path.join(src_dir, 'native_client')
104
105  # Decide platform specifics.
106  if options.browser_path:
107    chrome_filename = options.browser_path
108  else:
109    chrome_filename = find_chrome.FindChrome(src_dir, [options.mode])
110    if chrome_filename is None:
111      raise Exception('Cannot find a chome binary - specify one with '
112                      '--browser_path?')
113
114  env = dict(os.environ)
115  if sys.platform in ['win32', 'cygwin']:
116    if options.bits == 64:
117      bits = 64
118    elif options.bits == 32:
119      bits = 32
120    elif '64' in os.environ.get('PROCESSOR_ARCHITECTURE', '') or \
121         '64' in os.environ.get('PROCESSOR_ARCHITEW6432', ''):
122      bits = 64
123    else:
124      bits = 32
125    msvs_path = ';'.join([
126        r'c:\Program Files\Microsoft Visual Studio 9.0\VC',
127        r'c:\Program Files (x86)\Microsoft Visual Studio 9.0\VC',
128        r'c:\Program Files\Microsoft Visual Studio 9.0\Common7\Tools',
129        r'c:\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\Tools',
130        r'c:\Program Files\Microsoft Visual Studio 8\VC',
131        r'c:\Program Files (x86)\Microsoft Visual Studio 8\VC',
132        r'c:\Program Files\Microsoft Visual Studio 8\Common7\Tools',
133        r'c:\Program Files (x86)\Microsoft Visual Studio 8\Common7\Tools',
134    ])
135    env['PATH'] += ';' + msvs_path
136    scons = [python, 'scons.py']
137  elif sys.platform == 'darwin':
138    if options.bits == 64:
139      bits = 64
140    elif options.bits == 32:
141      bits = 32
142    else:
143      p = subprocess.Popen(['file', chrome_filename], stdout=subprocess.PIPE)
144      (p_stdout, _) = p.communicate()
145      assert p.returncode == 0
146      if p_stdout.find('executable x86_64') >= 0:
147        bits = 64
148      else:
149        bits = 32
150    scons = [python, 'scons.py']
151  else:
152    p = subprocess.Popen(
153        'uname -m | '
154        'sed -e "s/i.86/ia32/;s/x86_64/x64/;s/amd64/x64/;s/arm.*/arm/"',
155        shell=True, stdout=subprocess.PIPE)
156    (p_stdout, _) = p.communicate()
157    assert p.returncode == 0
158    if options.bits == 64:
159      bits = 64
160    elif options.bits == 32:
161      bits = 32
162    elif p_stdout.find('64') >= 0:
163      bits = 64
164    else:
165      bits = 32
166    # xvfb-run has a 2-second overhead per invocation, so it is cheaper to wrap
167    # the entire build step rather than each test (browser_headless=1).
168    # We also need to make sure that there are at least 24 bits per pixel.
169    # https://code.google.com/p/chromium/issues/detail?id=316687
170    scons = [
171        'xvfb-run',
172        '--auto-servernum',
173        '--server-args', '-screen 0 1024x768x24',
174        python, 'scons.py',
175    ]
176
177  if options.jobs > 1:
178    scons.append('-j%d' % options.jobs)
179
180  scons.append('disable_tests=%s' % options.disable_tests)
181
182  if options.buildbot is not None:
183    scons.append('buildbot=%s' % (options.buildbot,))
184
185  # Clean the output of the previous build.
186  # Incremental builds can get wedged in weird ways, so we're trading speed
187  # for reliability.
188  shutil.rmtree(os.path.join(nacl_dir, 'scons-out'), True)
189
190  # check that the HOST (not target) is 64bit
191  # this is emulating what msvs_env.bat is doing
192  if '64' in os.environ.get('PROCESSOR_ARCHITECTURE', '') or \
193     '64' in os.environ.get('PROCESSOR_ARCHITEW6432', ''):
194    # 64bit HOST
195    env['VS90COMNTOOLS'] = ('c:\\Program Files (x86)\\'
196                            'Microsoft Visual Studio 9.0\\Common7\\Tools\\')
197    env['VS80COMNTOOLS'] = ('c:\\Program Files (x86)\\'
198                            'Microsoft Visual Studio 8.0\\Common7\\Tools\\')
199  else:
200    # 32bit HOST
201    env['VS90COMNTOOLS'] = ('c:\\Program Files\\Microsoft Visual Studio 9.0\\'
202                            'Common7\\Tools\\')
203    env['VS80COMNTOOLS'] = ('c:\\Program Files\\Microsoft Visual Studio 8.0\\'
204                            'Common7\\Tools\\')
205
206  # Run nacl/chrome integration tests.
207  # Note that we have to add nacl_irt_test to --mode in order to get
208  # inbrowser_test_runner to run.
209  # TODO(mseaborn): Change it so that inbrowser_test_runner is not a
210  # special case.
211  cmd = scons + ['--verbose', '-k', 'platform=x86-%d' % bits,
212      '--mode=opt-host,nacl,nacl_irt_test',
213      'chrome_browser_path=%s' % chrome_filename,
214  ]
215  if not options.integration_bot and not options.morenacl_bot:
216    cmd.append('disable_flaky_tests=1')
217  cmd.append('chrome_browser_tests')
218
219  # Propagate path to JSON output if present.
220  # Note that RunCommand calls sys.exit on errors, so potential errors
221  # from one command won't be overwritten by another one. Overwriting
222  # a successful results file with either success or failure is fine.
223  if options.json_build_results_output_file:
224    cmd.append('json_build_results_output_file=%s' %
225               options.json_build_results_output_file)
226
227  # Download the toolchain(s).
228  pkg_ver_dir = os.path.join(nacl_dir, 'build', 'package_version')
229  RunCommand([python, os.path.join(pkg_ver_dir, 'package_version.py'),
230              '--exclude', 'arm_trusted',
231              '--exclude', 'pnacl_newlib',
232              '--exclude', 'nacl_arm_newlib',
233              'sync', '--extract'],
234             nacl_dir, os.environ)
235
236  CleanTempDir()
237
238  if options.enable_newlib:
239    RunTests('nacl-newlib', cmd, nacl_dir, env)
240
241  if options.enable_glibc:
242    RunTests('nacl-glibc', cmd + ['--nacl_glibc'], nacl_dir, env)
243
244
245def MakeCommandLineParser():
246  parser = optparse.OptionParser()
247  parser.add_option('-m', '--mode', dest='mode', default='Debug',
248                    help='Debug/Release mode')
249  parser.add_option('-j', dest='jobs', default=1, type='int',
250                    help='Number of parallel jobs')
251
252  parser.add_option('--enable_newlib', dest='enable_newlib', default=-1,
253                    type='int', help='Run newlib tests?')
254  parser.add_option('--enable_glibc', dest='enable_glibc', default=-1,
255                    type='int', help='Run glibc tests?')
256
257  parser.add_option('--json_build_results_output_file',
258                    help='Path to a JSON file for machine-readable output.')
259
260  # Deprecated, but passed to us by a script in the Chrome repo.
261  # Replaced by --enable_glibc=0
262  parser.add_option('--disable_glibc', dest='disable_glibc',
263                    action='store_true', default=False,
264                    help='Do not test using glibc.')
265
266  parser.add_option('--disable_tests', dest='disable_tests',
267                    type='string', default='',
268                    help='Comma-separated list of tests to omit')
269  builder_name = os.environ.get('BUILDBOT_BUILDERNAME', '')
270  is_integration_bot = 'nacl-chrome' in builder_name
271  parser.add_option('--integration_bot', dest='integration_bot',
272                    type='int', default=int(is_integration_bot),
273                    help='Is this an integration bot?')
274  is_morenacl_bot = (
275      'More NaCl' in builder_name or
276      'naclmore' in builder_name)
277  parser.add_option('--morenacl_bot', dest='morenacl_bot',
278                    type='int', default=int(is_morenacl_bot),
279                    help='Is this a morenacl bot?')
280
281  # Not used on the bots, but handy for running the script manually.
282  parser.add_option('--bits', dest='bits', action='store',
283                    type='int', default=None,
284                    help='32/64')
285  parser.add_option('--browser_path', dest='browser_path', action='store',
286                    type='string', default=None,
287                    help='Path to the chrome browser.')
288  parser.add_option('--buildbot', dest='buildbot', action='store',
289                    type='string', default=None,
290                    help='Value passed to scons as buildbot= option.')
291  return parser
292
293
294def Main():
295  parser = MakeCommandLineParser()
296  options, args = parser.parse_args()
297  if options.integration_bot and options.morenacl_bot:
298    parser.error('ERROR: cannot be both an integration bot and a morenacl bot')
299
300  # Set defaults for enabling newlib.
301  if options.enable_newlib == -1:
302    options.enable_newlib = 1
303
304  # Set defaults for enabling glibc.
305  if options.enable_glibc == -1:
306    if options.integration_bot or options.morenacl_bot:
307      options.enable_glibc = 1
308    else:
309      options.enable_glibc = 0
310
311  if args:
312    parser.error('ERROR: invalid argument')
313  BuildAndTest(options)
314
315
316if __name__ == '__main__':
317  Main()
318