• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright 2016 the V8 project authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Script to generate V8's gn arguments based on common developer defaults
7or builder configurations.
8
9Goma is used by default if detected. The compiler proxy is assumed to run.
10
11This script can be added to the PATH and be used on other checkouts. It always
12runs for the checkout nesting the CWD.
13
14Configurations of this script live in infra/mb/mb_config.pyl.
15
16Available actions are: {gen,list}. Omitting the action defaults to "gen".
17
18-------------------------------------------------------------------------------
19
20Examples:
21
22# Generate the ia32.release config in out.gn/ia32.release.
23v8gen.py ia32.release
24
25# Generate into out.gn/foo without goma auto-detect.
26v8gen.py gen -b ia32.release foo --no-goma
27
28# Pass additional gn arguments after -- (don't use spaces within gn args).
29v8gen.py ia32.optdebug -- v8_enable_slow_dchecks=true
30
31# Generate gn arguments of 'V8 Linux64 - builder' from 'client.v8'. To switch
32# off goma usage here, the args.gn file must be edited manually.
33v8gen.py -m client.v8 -b 'V8 Linux64 - builder'
34
35# Show available configurations.
36v8gen.py list
37
38-------------------------------------------------------------------------------
39"""
40
41# for py2/py3 compatibility
42from __future__ import print_function
43
44import argparse
45import os
46import re
47import subprocess
48import sys
49
50CONFIG = os.path.join('infra', 'mb', 'mb_config.pyl')
51GOMA_DEFAULT = os.path.join(os.path.expanduser("~"), 'goma')
52OUT_DIR = 'out.gn'
53
54TOOLS_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
55sys.path.append(os.path.join(TOOLS_PATH, 'mb'))
56
57import mb
58
59
60def _sanitize_nonalpha(text):
61  return re.sub(r'[^a-zA-Z0-9.]', '_', text)
62
63
64class GenerateGnArgs(object):
65  def __init__(self, args):
66    # Split args into this script's arguments and gn args passed to the
67    # wrapped gn.
68    index = args.index('--') if '--' in args else len(args)
69    self._options = self._parse_arguments(args[:index])
70    self._gn_args = args[index + 1:]
71
72  def _parse_arguments(self, args):
73    self.parser = argparse.ArgumentParser(
74      description=__doc__,
75      formatter_class=argparse.RawTextHelpFormatter,
76    )
77
78    def add_common_options(p):
79      p.add_argument(
80          '-m', '--master', default='developer_default',
81          help='config group or master from mb_config.pyl - default: '
82               'developer_default')
83      p.add_argument(
84          '-v', '--verbosity', action='count',
85          help='print wrapped commands (use -vv to print output of wrapped '
86               'commands)')
87
88    subps = self.parser.add_subparsers()
89
90    # Command: gen.
91    gen_cmd = subps.add_parser(
92        'gen', help='generate a new set of build files (default)')
93    gen_cmd.set_defaults(func=self.cmd_gen)
94    add_common_options(gen_cmd)
95    gen_cmd.add_argument(
96        'outdir', nargs='?',
97        help='optional gn output directory')
98    gen_cmd.add_argument(
99        '-b', '--builder',
100        help='build configuration or builder name from mb_config.pyl, e.g. '
101             'x64.release')
102    gen_cmd.add_argument(
103        '-p', '--pedantic', action='store_true',
104        help='run gn over command-line gn args to catch errors early')
105
106    goma = gen_cmd.add_mutually_exclusive_group()
107    goma.add_argument(
108        '-g' , '--goma',
109        action='store_true', default=None, dest='goma',
110        help='force using goma')
111    goma.add_argument(
112        '--nogoma', '--no-goma',
113        action='store_false', default=None, dest='goma',
114        help='don\'t use goma auto detection - goma might still be used if '
115             'specified as a gn arg')
116
117    # Command: list.
118    list_cmd = subps.add_parser(
119        'list', help='list available configurations')
120    list_cmd.set_defaults(func=self.cmd_list)
121    add_common_options(list_cmd)
122
123    # Default to "gen" unless global help is requested.
124    if not args or args[0] not in list(subps.choices) + ['-h', '--help']:
125      args = ['gen'] + args
126
127    return self.parser.parse_args(args)
128
129  def cmd_gen(self):
130    if not self._options.outdir and not self._options.builder:
131      self.parser.error('please specify either an output directory or '
132                        'a builder/config name (-b), e.g. x64.release')
133
134    if not self._options.outdir:
135      # Derive output directory from builder name.
136      self._options.outdir = _sanitize_nonalpha(self._options.builder)
137    else:
138      # Also, if this should work on windows, we might need to use \ where
139      # outdir is used as path, while using / if it's used in a gn context.
140      if self._options.outdir.startswith('/'):
141        self.parser.error(
142            'only output directories relative to %s are supported' % OUT_DIR)
143
144    if not self._options.builder:
145      # Derive builder from output directory.
146      self._options.builder = self._options.outdir
147
148    # Check for builder/config in mb config.
149    if self._options.builder not in self._mbw.builder_groups[self._options.master]:
150      print('%s does not exist in %s for %s' % (
151          self._options.builder, CONFIG, self._options.master))
152      return 1
153
154    # TODO(machenbach): Check if the requested configurations has switched to
155    # gn at all.
156
157    # The directories are separated with slashes in a gn context (platform
158    # independent).
159    gn_outdir = '/'.join([OUT_DIR, self._options.outdir])
160
161    # Call MB to generate the basic configuration.
162    self._call_cmd([
163      sys.executable,
164      '-u', os.path.join('tools', 'mb', 'mb.py'),
165      'gen',
166      '-f', CONFIG,
167      '-m', self._options.master,
168      '-b', self._options.builder,
169      gn_outdir,
170    ])
171
172    # Handle extra gn arguments.
173    gn_args_path = os.path.join(OUT_DIR, self._options.outdir, 'args.gn')
174
175    # Append command-line args.
176    modified = self._append_gn_args(
177        'command-line', gn_args_path, '\n'.join(self._gn_args))
178
179    # Append goma args.
180    # TODO(machenbach): We currently can't remove existing goma args from the
181    # original config. E.g. to build like a bot that uses goma, but switch
182    # goma off.
183    modified |= self._append_gn_args(
184        'goma', gn_args_path, self._goma_args)
185
186    # Regenerate ninja files to check for errors in the additional gn args.
187    if modified and self._options.pedantic:
188      self._call_cmd(['gn', 'gen', gn_outdir])
189    return 0
190
191  def cmd_list(self):
192    print('\n'.join(sorted(self._mbw.builder_groups[self._options.master])))
193    return 0
194
195  def verbose_print_1(self, text):
196    if self._options.verbosity and self._options.verbosity >= 1:
197      print('#' * 80)
198      print(text)
199
200  def verbose_print_2(self, text):
201    if self._options.verbosity and self._options.verbosity >= 2:
202      indent = ' ' * 2
203      for l in text.splitlines():
204        if type(l) == bytes:
205          l = l.decode()
206        print(indent + l)
207
208  def _call_cmd(self, args):
209    self.verbose_print_1(' '.join(args))
210    try:
211      output = subprocess.check_output(
212        args=args,
213        stderr=subprocess.STDOUT,
214      )
215      self.verbose_print_2(output)
216    except subprocess.CalledProcessError as e:
217      self.verbose_print_2(e.output)
218      raise
219
220  def _find_work_dir(self, path):
221    """Find the closest v8 root to `path`."""
222    if os.path.exists(os.path.join(path, 'tools', 'dev', 'v8gen.py')):
223      # Approximate the v8 root dir by a folder where this script exists
224      # in the expected place.
225      return path
226    elif os.path.dirname(path) == path:
227      raise Exception(
228          'This appears to not be called from a recent v8 checkout')
229    else:
230      return self._find_work_dir(os.path.dirname(path))
231
232  @property
233  def _goma_dir(self):
234    return os.path.normpath(os.environ.get('GOMA_DIR') or GOMA_DEFAULT)
235
236  @property
237  def _need_goma_dir(self):
238    return self._goma_dir != GOMA_DEFAULT
239
240  @property
241  def _use_goma(self):
242    if self._options.goma is None:
243      # Auto-detect.
244      return os.path.exists(self._goma_dir) and os.path.isdir(self._goma_dir)
245    else:
246      return self._options.goma
247
248  @property
249  def _goma_args(self):
250    """Gn args for using goma."""
251    # Specify goma args if we want to use goma and if goma isn't specified
252    # via command line already. The command-line always has precedence over
253    # any other specification.
254    if (self._use_goma and
255        not any(re.match(r'use_goma\s*=.*', x) for x in self._gn_args)):
256      if self._need_goma_dir:
257        return 'use_goma=true\ngoma_dir="%s"' % self._goma_dir
258      else:
259        return 'use_goma=true'
260    else:
261      return ''
262
263  def _append_gn_args(self, type, gn_args_path, more_gn_args):
264    """Append extra gn arguments to the generated args.gn file."""
265    if not more_gn_args:
266      return False
267    self.verbose_print_1('Appending """\n%s\n""" to %s.' % (
268        more_gn_args, os.path.abspath(gn_args_path)))
269    with open(gn_args_path, 'a') as f:
270      f.write('\n# Additional %s args:\n' % type)
271      f.write(more_gn_args)
272      f.write('\n')
273
274    # Artificially increment modification time as our modifications happen too
275    # fast. This makes sure that gn is properly rebuilding the ninja files.
276    mtime = os.path.getmtime(gn_args_path) + 1
277    with open(gn_args_path, 'a'):
278      os.utime(gn_args_path, (mtime, mtime))
279
280    return True
281
282  def main(self):
283    # Always operate relative to the base directory for better relative-path
284    # handling. This script can be used in any v8 checkout.
285    workdir = self._find_work_dir(os.getcwd())
286    if workdir != os.getcwd():
287      self.verbose_print_1('cd ' + workdir)
288      os.chdir(workdir)
289
290    # Initialize MB as a library.
291    self._mbw = mb.MetaBuildWrapper()
292
293    # TODO(machenbach): Factor out common methods independent of mb arguments.
294    self._mbw.ParseArgs(['lookup', '-f', CONFIG])
295    self._mbw.ReadConfigFile()
296
297    if not self._options.master in self._mbw.builder_groups:
298      print('%s not found in %s\n' % (self._options.master, CONFIG))
299      print('Choose one of:\n%s\n' % (
300          '\n'.join(sorted(self._mbw.builder_groups.keys()))))
301      return 1
302
303    return self._options.func()
304
305
306if __name__ == "__main__":
307  gen = GenerateGnArgs(sys.argv[1:])
308  try:
309    sys.exit(gen.main())
310  except Exception:
311    if not gen._options.verbosity or gen._options.verbosity < 2:
312      print ('\nHint: You can raise verbosity (-vv) to see the output of '
313             'failed commands.\n')
314    raise
315