• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python
2# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Do all the steps required to build and test against nacl."""
7
8
9import optparse
10import os.path
11import re
12import shutil
13import subprocess
14import sys
15
16import find_chrome
17
18THIS_DIR = os.path.abspath(os.path.dirname(__file__))
19CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..'))
20sys.path.append(os.path.join(CHROMIUM_DIR, 'build'))
21import detect_host_arch
22
23
24# Copied from buildbot/buildbot_lib.py
25def TryToCleanContents(path, file_name_filter=lambda fn: True):
26  """
27  Remove the contents of a directory without touching the directory itself.
28  Ignores all failures.
29  """
30  if os.path.exists(path):
31    for fn in os.listdir(path):
32      TryToCleanPath(os.path.join(path, fn), file_name_filter)
33
34
35# Copied from buildbot/buildbot_lib.py
36def TryToCleanPath(path, file_name_filter=lambda fn: True):
37  """
38  Removes a file or directory.
39  Ignores all failures.
40  """
41  if os.path.exists(path):
42    if file_name_filter(path):
43      print 'Trying to remove %s' % path
44      if os.path.isdir(path):
45        shutil.rmtree(path, ignore_errors=True)
46      else:
47        try:
48          os.remove(path)
49        except Exception:
50          pass
51    else:
52      print 'Skipping %s' % path
53
54
55# TODO(ncbray): this is somewhat unsafe.  We should fix the underlying problem.
56def CleanTempDir():
57  # Only delete files and directories like:
58  # a) C:\temp\83C4.tmp
59  # b) /tmp/.org.chromium.Chromium.EQrEzl
60  file_name_re = re.compile(
61      r'[\\/]([0-9a-fA-F]+\.tmp|\.org\.chrom\w+\.Chrom\w+\..+)$')
62  file_name_filter = lambda fn: file_name_re.search(fn) is not None
63
64  path = os.environ.get('TMP', os.environ.get('TEMP', '/tmp'))
65  if len(path) >= 4 and os.path.isdir(path):
66    print
67    print "Cleaning out the temp directory."
68    print
69    TryToCleanContents(path, file_name_filter)
70  else:
71    print
72    print "Cannot find temp directory, not cleaning it."
73    print
74
75
76def RunCommand(cmd, cwd, env):
77  sys.stdout.write('\nRunning %s\n\n' % ' '.join(cmd))
78  sys.stdout.flush()
79  retcode = subprocess.call(cmd, cwd=cwd, env=env)
80  if retcode != 0:
81    sys.stdout.write('\nFailed: %s\n\n' % ' '.join(cmd))
82    sys.exit(retcode)
83
84
85def RunTests(name, cmd, nacl_dir, env):
86  sys.stdout.write('\n\nBuilding files needed for %s testing...\n\n' % name)
87  RunCommand(cmd + ['do_not_run_tests=1', '-j8'], nacl_dir, env)
88  sys.stdout.write('\n\nRunning %s tests...\n\n' % name)
89  RunCommand(cmd, nacl_dir, env)
90
91
92def BuildAndTest(options):
93  # Refuse to run under cygwin.
94  if sys.platform == 'cygwin':
95    raise Exception('I do not work under cygwin, sorry.')
96
97  # By default, use the version of Python is being used to run this script.
98  python = sys.executable
99  if sys.platform == 'darwin':
100    # Mac 10.5 bots tend to use a particularlly old version of Python, look for
101    # a newer version.
102    macpython27 = '/Library/Frameworks/Python.framework/Versions/2.7/bin/python'
103    if os.path.exists(macpython27):
104      python = macpython27
105
106  script_dir = os.path.dirname(os.path.abspath(__file__))
107  src_dir = os.path.dirname(os.path.dirname(os.path.dirname(script_dir)))
108  nacl_dir = os.path.join(src_dir, 'native_client')
109
110  # Decide platform specifics.
111  if options.browser_path:
112    chrome_filename = options.browser_path
113  else:
114    chrome_filename = find_chrome.FindChrome(src_dir, [options.mode])
115    if chrome_filename is None:
116      raise Exception('Cannot find a chrome binary - specify one with '
117                      '--browser_path?')
118
119  env = dict(os.environ)
120  if sys.platform in ['win32', 'cygwin']:
121    if options.bits == 64:
122      bits = 64
123    elif options.bits == 32:
124      bits = 32
125    elif '64' in os.environ.get('PROCESSOR_ARCHITECTURE', '') or \
126         '64' in os.environ.get('PROCESSOR_ARCHITEW6432', ''):
127      bits = 64
128    else:
129      bits = 32
130    msvs_path = ';'.join([
131        r'c:\Program Files\Microsoft Visual Studio 9.0\VC',
132        r'c:\Program Files (x86)\Microsoft Visual Studio 9.0\VC',
133        r'c:\Program Files\Microsoft Visual Studio 9.0\Common7\Tools',
134        r'c:\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\Tools',
135        r'c:\Program Files\Microsoft Visual Studio 8\VC',
136        r'c:\Program Files (x86)\Microsoft Visual Studio 8\VC',
137        r'c:\Program Files\Microsoft Visual Studio 8\Common7\Tools',
138        r'c:\Program Files (x86)\Microsoft Visual Studio 8\Common7\Tools',
139    ])
140    env['PATH'] += ';' + msvs_path
141    scons = [python, 'scons.py']
142  elif sys.platform == 'darwin':
143    if options.bits == 64:
144      bits = 64
145    elif options.bits == 32:
146      bits = 32
147    else:
148      p = subprocess.Popen(['file', chrome_filename], stdout=subprocess.PIPE)
149      (p_stdout, _) = p.communicate()
150      assert p.returncode == 0
151      if p_stdout.find('executable x86_64') >= 0:
152        bits = 64
153      else:
154        bits = 32
155    scons = [python, 'scons.py']
156  else:
157    if options.bits == 64:
158      bits = 64
159    elif options.bits == 32:
160      bits = 32
161    elif '64' in detect_host_arch.HostArch():
162      bits = 64
163    else:
164      bits = 32
165    # xvfb-run has a 2-second overhead per invocation, so it is cheaper to wrap
166    # the entire build step rather than each test (browser_headless=1).
167    # We also need to make sure that there are at least 24 bits per pixel.
168    # https://code.google.com/p/chromium/issues/detail?id=316687
169    scons = [
170        'xvfb-run',
171        '--auto-servernum',
172        '--server-args', '-screen 0 1024x768x24',
173        python, 'scons.py',
174    ]
175
176  if options.jobs > 1:
177    scons.append('-j%d' % options.jobs)
178
179  scons.append('disable_tests=%s' % options.disable_tests)
180
181  if options.buildbot is not None:
182    scons.append('buildbot=%s' % (options.buildbot,))
183
184  # Clean the output of the previous build.
185  # Incremental builds can get wedged in weird ways, so we're trading speed
186  # for reliability.
187  shutil.rmtree(os.path.join(nacl_dir, 'scons-out'), True)
188
189  # check that the HOST (not target) is 64bit
190  # this is emulating what msvs_env.bat is doing
191  if '64' in os.environ.get('PROCESSOR_ARCHITECTURE', '') or \
192     '64' in os.environ.get('PROCESSOR_ARCHITEW6432', ''):
193    # 64bit HOST
194    env['VS90COMNTOOLS'] = ('c:\\Program Files (x86)\\'
195                            'Microsoft Visual Studio 9.0\\Common7\\Tools\\')
196    env['VS80COMNTOOLS'] = ('c:\\Program Files (x86)\\'
197                            'Microsoft Visual Studio 8.0\\Common7\\Tools\\')
198  else:
199    # 32bit HOST
200    env['VS90COMNTOOLS'] = ('c:\\Program Files\\Microsoft Visual Studio 9.0\\'
201                            'Common7\\Tools\\')
202    env['VS80COMNTOOLS'] = ('c:\\Program Files\\Microsoft Visual Studio 8.0\\'
203                            'Common7\\Tools\\')
204
205  # Run nacl/chrome integration tests.
206  # Note that we have to add nacl_irt_test to --mode in order to get
207  # inbrowser_test_runner to run.
208  # TODO(mseaborn): Change it so that inbrowser_test_runner is not a
209  # special case.
210  cmd = scons + ['--verbose', '-k', 'platform=x86-%d' % bits,
211      '--mode=opt-host,nacl,nacl_irt_test',
212      'chrome_browser_path=%s' % chrome_filename,
213  ]
214  if not options.integration_bot and not options.morenacl_bot:
215    cmd.append('disable_flaky_tests=1')
216  cmd.append('chrome_browser_tests')
217
218  # Propagate path to JSON output if present.
219  # Note that RunCommand calls sys.exit on errors, so potential errors
220  # from one command won't be overwritten by another one. Overwriting
221  # a successful results file with either success or failure is fine.
222  if options.json_build_results_output_file:
223    cmd.append('json_build_results_output_file=%s' %
224               options.json_build_results_output_file)
225
226  # Download the toolchain(s).
227  pkg_ver_dir = os.path.join(nacl_dir, 'build', 'package_version')
228  RunCommand([python, os.path.join(pkg_ver_dir, 'package_version.py'),
229              '--exclude', 'arm_trusted',
230              '--exclude', 'pnacl_newlib',
231              '--exclude', 'nacl_arm_newlib',
232              'sync', '--extract'],
233             nacl_dir, os.environ)
234
235  CleanTempDir()
236
237  if options.enable_newlib:
238    RunTests('nacl-newlib', cmd, nacl_dir, env)
239
240  if options.enable_glibc:
241    RunTests('nacl-glibc', cmd + ['--nacl_glibc'], nacl_dir, env)
242
243
244def MakeCommandLineParser():
245  parser = optparse.OptionParser()
246  parser.add_option('-m', '--mode', dest='mode', default='Debug',
247                    help='Debug/Release mode')
248  parser.add_option('-j', dest='jobs', default=1, type='int',
249                    help='Number of parallel jobs')
250
251  parser.add_option('--enable_newlib', dest='enable_newlib', default=-1,
252                    type='int', help='Run newlib tests?')
253  parser.add_option('--enable_glibc', dest='enable_glibc', default=-1,
254                    type='int', help='Run glibc tests?')
255
256  parser.add_option('--json_build_results_output_file',
257                    help='Path to a JSON file for machine-readable output.')
258
259  # Deprecated, but passed to us by a script in the Chrome repo.
260  # Replaced by --enable_glibc=0
261  parser.add_option('--disable_glibc', dest='disable_glibc',
262                    action='store_true', default=False,
263                    help='Do not test using glibc.')
264
265  parser.add_option('--disable_tests', dest='disable_tests',
266                    type='string', default='',
267                    help='Comma-separated list of tests to omit')
268  builder_name = os.environ.get('BUILDBOT_BUILDERNAME', '')
269  is_integration_bot = 'nacl-chrome' in builder_name
270  parser.add_option('--integration_bot', dest='integration_bot',
271                    type='int', default=int(is_integration_bot),
272                    help='Is this an integration bot?')
273  is_morenacl_bot = (
274      'More NaCl' in builder_name or
275      'naclmore' in builder_name)
276  parser.add_option('--morenacl_bot', dest='morenacl_bot',
277                    type='int', default=int(is_morenacl_bot),
278                    help='Is this a morenacl bot?')
279
280  # Not used on the bots, but handy for running the script manually.
281  parser.add_option('--bits', dest='bits', action='store',
282                    type='int', default=None,
283                    help='32/64')
284  parser.add_option('--browser_path', dest='browser_path', action='store',
285                    type='string', default=None,
286                    help='Path to the chrome browser.')
287  parser.add_option('--buildbot', dest='buildbot', action='store',
288                    type='string', default=None,
289                    help='Value passed to scons as buildbot= option.')
290  return parser
291
292
293def Main():
294  parser = MakeCommandLineParser()
295  options, args = parser.parse_args()
296  if options.integration_bot and options.morenacl_bot:
297    parser.error('ERROR: cannot be both an integration bot and a morenacl bot')
298
299  # Set defaults for enabling newlib.
300  if options.enable_newlib == -1:
301    options.enable_newlib = 1
302
303  # Set defaults for enabling glibc.
304  if options.enable_glibc == -1:
305    if options.integration_bot or options.morenacl_bot:
306      options.enable_glibc = 1
307    else:
308      options.enable_glibc = 0
309
310  if args:
311    parser.error('ERROR: invalid argument')
312  BuildAndTest(options)
313
314
315if __name__ == '__main__':
316  Main()
317