• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright 2016 the V8 project authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Script to generate V8's gn arguments based on common developer defaults
7or builder configurations.
8
9Goma is used by default if detected. The compiler proxy is assumed to run.
10
11This script can be added to the PATH and be used on other checkouts. It always
12runs for the checkout nesting the CWD.
13
14Configurations of this script live in infra/mb/mb_config.pyl.
15
16Available actions are: {gen,list}. Omitting the action defaults to "gen".
17
18-------------------------------------------------------------------------------
19
20Examples:
21
22# Generate the ia32.release config in out.gn/ia32.release.
23v8gen.py ia32.release
24
25# Generate into out.gn/foo without goma auto-detect.
26v8gen.py gen -b ia32.release foo --no-goma
27
28# Pass additional gn arguments after -- (don't use spaces within gn args).
29v8gen.py ia32.optdebug -- v8_enable_slow_dchecks=true
30
31# Generate gn arguments of 'V8 Linux64 - builder' from 'client.v8'. To switch
32# off goma usage here, the args.gn file must be edited manually.
33v8gen.py -m client.v8 -b 'V8 Linux64 - builder'
34
35# Show available configurations.
36v8gen.py list
37
38-------------------------------------------------------------------------------
39"""
40
41import argparse
42import os
43import re
44import subprocess
45import sys
46
47CONFIG = os.path.join('infra', 'mb', 'mb_config.pyl')
48GOMA_DEFAULT = os.path.join(os.path.expanduser("~"), 'goma')
49OUT_DIR = 'out.gn'
50
51TOOLS_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
52sys.path.append(os.path.join(TOOLS_PATH, 'mb'))
53
54import mb
55
56
57def _sanitize_nonalpha(text):
58  return re.sub(r'[^a-zA-Z0-9.]', '_', text)
59
60
61class GenerateGnArgs(object):
62  def __init__(self, args):
63    # Split args into this script's arguments and gn args passed to the
64    # wrapped gn.
65    index = args.index('--') if '--' in args else len(args)
66    self._options = self._parse_arguments(args[:index])
67    self._gn_args = args[index + 1:]
68
69  def _parse_arguments(self, args):
70    self.parser = argparse.ArgumentParser(
71      description=__doc__,
72      formatter_class=argparse.RawTextHelpFormatter,
73    )
74
75    def add_common_options(p):
76      p.add_argument(
77          '-m', '--master', default='developer_default',
78          help='config group or master from mb_config.pyl - default: '
79               'developer_default')
80      p.add_argument(
81          '-v', '--verbosity', action='count',
82          help='print wrapped commands (use -vv to print output of wrapped '
83               'commands)')
84
85    subps = self.parser.add_subparsers()
86
87    # Command: gen.
88    gen_cmd = subps.add_parser(
89        'gen', help='generate a new set of build files (default)')
90    gen_cmd.set_defaults(func=self.cmd_gen)
91    add_common_options(gen_cmd)
92    gen_cmd.add_argument(
93        'outdir', nargs='?',
94        help='optional gn output directory')
95    gen_cmd.add_argument(
96        '-b', '--builder',
97        help='build configuration or builder name from mb_config.pyl, e.g. '
98             'x64.release')
99    gen_cmd.add_argument(
100        '-p', '--pedantic', action='store_true',
101        help='run gn over command-line gn args to catch errors early')
102
103    goma = gen_cmd.add_mutually_exclusive_group()
104    goma.add_argument(
105        '-g' , '--goma',
106        action='store_true', default=None, dest='goma',
107        help='force using goma')
108    goma.add_argument(
109        '--nogoma', '--no-goma',
110        action='store_false', default=None, dest='goma',
111        help='don\'t use goma auto detection - goma might still be used if '
112             'specified as a gn arg')
113
114    # Command: list.
115    list_cmd = subps.add_parser(
116        'list', help='list available configurations')
117    list_cmd.set_defaults(func=self.cmd_list)
118    add_common_options(list_cmd)
119
120    # Default to "gen" unless global help is requested.
121    if not args or args[0] not in subps.choices.keys() + ['-h', '--help']:
122      args = ['gen'] + args
123
124    return self.parser.parse_args(args)
125
126  def cmd_gen(self):
127    if not self._options.outdir and not self._options.builder:
128      self.parser.error('please specify either an output directory or '
129                        'a builder/config name (-b), e.g. x64.release')
130
131    if not self._options.outdir:
132      # Derive output directory from builder name.
133      self._options.outdir = _sanitize_nonalpha(self._options.builder)
134    else:
135      # Also, if this should work on windows, we might need to use \ where
136      # outdir is used as path, while using / if it's used in a gn context.
137      if self._options.outdir.startswith('/'):
138        self.parser.error(
139            'only output directories relative to %s are supported' % OUT_DIR)
140
141    if not self._options.builder:
142      # Derive builder from output directory.
143      self._options.builder = self._options.outdir
144
145    # Check for builder/config in mb config.
146    if self._options.builder not in self._mbw.masters[self._options.master]:
147      print '%s does not exist in %s for %s' % (
148          self._options.builder, CONFIG, self._options.master)
149      return 1
150
151    # TODO(machenbach): Check if the requested configurations has switched to
152    # gn at all.
153
154    # The directories are separated with slashes in a gn context (platform
155    # independent).
156    gn_outdir = '/'.join([OUT_DIR, self._options.outdir])
157
158    # Call MB to generate the basic configuration.
159    self._call_cmd([
160      sys.executable,
161      '-u', os.path.join('tools', 'mb', 'mb.py'),
162      'gen',
163      '-f', CONFIG,
164      '-m', self._options.master,
165      '-b', self._options.builder,
166      gn_outdir,
167    ])
168
169    # Handle extra gn arguments.
170    gn_args_path = os.path.join(OUT_DIR, self._options.outdir, 'args.gn')
171
172    # Append command-line args.
173    modified = self._append_gn_args(
174        'command-line', gn_args_path, '\n'.join(self._gn_args))
175
176    # Append goma args.
177    # TODO(machenbach): We currently can't remove existing goma args from the
178    # original config. E.g. to build like a bot that uses goma, but switch
179    # goma off.
180    modified |= self._append_gn_args(
181        'goma', gn_args_path, self._goma_args)
182
183    # Regenerate ninja files to check for errors in the additional gn args.
184    if modified and self._options.pedantic:
185      self._call_cmd(['gn', 'gen', gn_outdir])
186    return 0
187
188  def cmd_list(self):
189    print '\n'.join(sorted(self._mbw.masters[self._options.master]))
190    return 0
191
192  def verbose_print_1(self, text):
193    if self._options.verbosity >= 1:
194      print '#' * 80
195      print text
196
197  def verbose_print_2(self, text):
198    if self._options.verbosity >= 2:
199      indent = ' ' * 2
200      for l in text.splitlines():
201        print indent + l
202
203  def _call_cmd(self, args):
204    self.verbose_print_1(' '.join(args))
205    try:
206      output = subprocess.check_output(
207        args=args,
208        stderr=subprocess.STDOUT,
209      )
210      self.verbose_print_2(output)
211    except subprocess.CalledProcessError as e:
212      self.verbose_print_2(e.output)
213      raise
214
215  def _find_work_dir(self, path):
216    """Find the closest v8 root to `path`."""
217    if os.path.exists(os.path.join(path, 'tools', 'dev', 'v8gen.py')):
218      # Approximate the v8 root dir by a folder where this script exists
219      # in the expected place.
220      return path
221    elif os.path.dirname(path) == path:
222      raise Exception(
223          'This appears to not be called from a recent v8 checkout')
224    else:
225      return self._find_work_dir(os.path.dirname(path))
226
227  @property
228  def _goma_dir(self):
229    return os.path.normpath(os.environ.get('GOMA_DIR') or GOMA_DEFAULT)
230
231  @property
232  def _need_goma_dir(self):
233    return self._goma_dir != GOMA_DEFAULT
234
235  @property
236  def _use_goma(self):
237    if self._options.goma is None:
238      # Auto-detect.
239      return os.path.exists(self._goma_dir) and os.path.isdir(self._goma_dir)
240    else:
241      return self._options.goma
242
243  @property
244  def _goma_args(self):
245    """Gn args for using goma."""
246    # Specify goma args if we want to use goma and if goma isn't specified
247    # via command line already. The command-line always has precedence over
248    # any other specification.
249    if (self._use_goma and
250        not any(re.match(r'use_goma\s*=.*', x) for x in self._gn_args)):
251      if self._need_goma_dir:
252        return 'use_goma=true\ngoma_dir="%s"' % self._goma_dir
253      else:
254        return 'use_goma=true'
255    else:
256      return ''
257
258  def _append_gn_args(self, type, gn_args_path, more_gn_args):
259    """Append extra gn arguments to the generated args.gn file."""
260    if not more_gn_args:
261      return False
262    self.verbose_print_1('Appending """\n%s\n""" to %s.' % (
263        more_gn_args, os.path.abspath(gn_args_path)))
264    with open(gn_args_path, 'a') as f:
265      f.write('\n# Additional %s args:\n' % type)
266      f.write(more_gn_args)
267      f.write('\n')
268
269    # Artificially increment modification time as our modifications happen too
270    # fast. This makes sure that gn is properly rebuilding the ninja files.
271    mtime = os.path.getmtime(gn_args_path) + 1
272    with open(gn_args_path, 'a'):
273      os.utime(gn_args_path, (mtime, mtime))
274
275    return True
276
277  def main(self):
278    # Always operate relative to the base directory for better relative-path
279    # handling. This script can be used in any v8 checkout.
280    workdir = self._find_work_dir(os.getcwd())
281    if workdir != os.getcwd():
282      self.verbose_print_1('cd ' + workdir)
283      os.chdir(workdir)
284
285    # Initialize MB as a library.
286    self._mbw = mb.MetaBuildWrapper()
287
288    # TODO(machenbach): Factor out common methods independent of mb arguments.
289    self._mbw.ParseArgs(['lookup', '-f', CONFIG])
290    self._mbw.ReadConfigFile()
291
292    if not self._options.master in self._mbw.masters:
293      print '%s not found in %s\n' % (self._options.master, CONFIG)
294      print 'Choose one of:\n%s\n' % (
295          '\n'.join(sorted(self._mbw.masters.keys())))
296      return 1
297
298    return self._options.func()
299
300
301if __name__ == "__main__":
302  gen = GenerateGnArgs(sys.argv[1:])
303  try:
304    sys.exit(gen.main())
305  except Exception:
306    if gen._options.verbosity < 2:
307      print ('\nHint: You can raise verbosity (-vv) to see the output of '
308             'failed commands.\n')
309    raise
310