• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (C) 2019 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15# A collection of utilities for extracting build rule information from GN
16# projects.
17
18from __future__ import print_function
19import errno
20import filecmp
21import json
22import os
23import re
24import shutil
25import subprocess
26import sys
27from compat import iteritems
28
29BUILDFLAGS_TARGET = '//gn:gen_buildflags'
30TARGET_TOOLCHAIN = '//gn/standalone/toolchain:gcc_like_host'
31HOST_TOOLCHAIN = '//gn/standalone/toolchain:gcc_like_host'
32LINKER_UNIT_TYPES = ('executable', 'shared_library', 'static_library')
33
34
35def _check_command_output(cmd, cwd):
36  try:
37    output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, cwd=cwd)
38  except subprocess.CalledProcessError as e:
39    print(
40        'Command "{}" failed in {}:'.format(' '.join(cmd), cwd),
41        file=sys.stderr)
42    print(e.output.decode(), file=sys.stderr)
43    sys.exit(1)
44  else:
45    return output.decode()
46
47
48def repo_root():
49  """Returns an absolute path to the repository root."""
50  return os.path.join(
51      os.path.realpath(os.path.dirname(__file__)), os.path.pardir)
52
53
54def _tool_path(name):
55  return os.path.join(repo_root(), 'tools', name)
56
57
58def prepare_out_directory(gn_args, name, root=repo_root()):
59  """Creates the JSON build description by running GN.
60
61    Returns (path, desc) where |path| is the location of the output directory
62    and |desc| is the JSON build description.
63    """
64  out = os.path.join(root, 'out', name)
65  try:
66    os.makedirs(out)
67  except OSError as e:
68    if e.errno != errno.EEXIST:
69      raise
70  _check_command_output([_tool_path('gn'), 'gen', out,
71                         '--args=%s' % gn_args],
72                        cwd=repo_root())
73  return out
74
75
76def load_build_description(out):
77  """Creates the JSON build description by running GN."""
78  desc = _check_command_output([
79      _tool_path('gn'), 'desc', out, '--format=json', '--all-toolchains', '//*'
80  ],
81                               cwd=repo_root())
82  return json.loads(desc)
83
84
85def create_build_description(gn_args, root=repo_root()):
86  """Prepares a GN out directory and loads the build description from it.
87
88    The temporary out directory is automatically deleted.
89    """
90  out = prepare_out_directory(gn_args, 'tmp.gn_utils', root=root)
91  try:
92    return load_build_description(out)
93  finally:
94    shutil.rmtree(out)
95
96
97def build_targets(out, targets, quiet=False):
98  """Runs ninja to build a list of GN targets in the given out directory.
99
100    Compiling these targets is required so that we can include any generated
101    source files in the amalgamated result.
102    """
103  targets = [t.replace('//', '') for t in targets]
104  with open(os.devnull, 'w') as devnull:
105    stdout = devnull if quiet else None
106    subprocess.check_call(
107        [_tool_path('ninja')] + targets, cwd=out, stdout=stdout)
108
109
110def compute_source_dependencies(out):
111  """For each source file, computes a set of headers it depends on."""
112  ninja_deps = _check_command_output([_tool_path('ninja'), '-t', 'deps'],
113                                     cwd=out)
114  deps = {}
115  current_source = None
116  for line in ninja_deps.split('\n'):
117    filename = os.path.relpath(os.path.join(out, line.strip()), repo_root())
118    if not line or line[0] != ' ':
119      current_source = None
120      continue
121    elif not current_source:
122      # We're assuming the source file is always listed before the
123      # headers.
124      assert os.path.splitext(line)[1] in ['.c', '.cc', '.cpp', '.S']
125      current_source = filename
126      deps[current_source] = []
127    else:
128      assert current_source
129      deps[current_source].append(filename)
130  return deps
131
132
133def label_to_path(label):
134  """Turn a GN output label (e.g., //some_dir/file.cc) into a path."""
135  assert label.startswith('//')
136  return label[2:]
137
138
139def label_without_toolchain(label):
140  """Strips the toolchain from a GN label.
141
142    Return a GN label (e.g //buildtools:protobuf(//gn/standalone/toolchain:
143    gcc_like_host) without the parenthesised toolchain part.
144    """
145  return label.split('(')[0]
146
147
148def label_to_target_name_with_path(label):
149  """
150  Turn a GN label into a target name involving the full path.
151  e.g., //src/perfetto:tests -> src_perfetto_tests
152  """
153  name = re.sub(r'^//:?', '', label)
154  name = re.sub(r'[^a-zA-Z0-9_]', '_', name)
155  return name
156
157
158def gen_buildflags(gn_args, target_file):
159  """Generates the perfetto_build_flags.h for the given config.
160
161    target_file: the path, relative to the repo root, where the generated
162        buildflag header will be copied into.
163    """
164  tmp_out = prepare_out_directory(gn_args, 'tmp.gen_buildflags')
165  build_targets(tmp_out, [BUILDFLAGS_TARGET], quiet=True)
166  src = os.path.join(tmp_out, 'gen', 'build_config', 'perfetto_build_flags.h')
167  shutil.copy(src, os.path.join(repo_root(), target_file))
168  shutil.rmtree(tmp_out)
169
170
171def check_or_commit_generated_files(tmp_files, check):
172  """Checks that gen files are unchanged or renames them to the final location
173
174    Takes in input a list of 'xxx.swp' files that have been written.
175    If check == False, it renames xxx.swp -> xxx.
176    If check == True, it just checks that the contents of 'xxx.swp' == 'xxx'.
177    Returns 0 if no diff was detected, 1 otherwise (to be used as exit code).
178    """
179  res = 0
180  for tmp_file in tmp_files:
181    assert (tmp_file.endswith('.swp'))
182    target_file = os.path.relpath(tmp_file[:-4])
183    if check:
184      if not filecmp.cmp(tmp_file, target_file):
185        sys.stderr.write('%s needs to be regenerated\n' % target_file)
186        res = 1
187      os.unlink(tmp_file)
188    else:
189      os.rename(tmp_file, target_file)
190  return res
191
192
193class GnParser(object):
194  """A parser with some cleverness for GN json desc files
195
196    The main goals of this parser are:
197    1) Deal with the fact that other build systems don't have an equivalent
198       notion to GN's source_set. Conversely to Bazel's and Soong's filegroups,
199       GN source_sets expect that dependencies, cflags and other source_set
200       properties propagate up to the linker unit (static_library, executable or
201       shared_library). This parser simulates the same behavior: when a
202       source_set is encountered, some of its variables (cflags and such) are
203       copied up to the dependent targets. This is to allow gen_xxx to create
204       one filegroup for each source_set and then squash all the other flags
205       onto the linker unit.
206    2) Detect and special-case protobuf targets, figuring out the protoc-plugin
207       being used.
208    """
209
210  class Target(object):
211    """Reperesents A GN target.
212
213        Maked properties are propagated up the dependency chain when a
214        source_set dependency is encountered.
215        """
216
217    def __init__(self, name, type):
218      self.name = name  # e.g. //src/ipc:ipc
219
220      VALID_TYPES = ('static_library', 'shared_library', 'executable', 'group',
221                     'action', 'source_set', 'proto_library')
222      assert (type in VALID_TYPES)
223      self.type = type
224      self.testonly = False
225      self.toolchain = None
226
227      # Only set when type == proto_library.
228      # This is typically: 'proto', 'protozero', 'ipc'.
229      self.proto_plugin = None
230
231      self.sources = set()
232
233      # These are valid only for type == 'action'
234      self.inputs = set()
235      self.outputs = set()
236      self.script = None
237      self.args = []
238
239      # These variables are propagated up when encountering a dependency
240      # on a source_set target.
241      self.cflags = set()
242      self.defines = set()
243      self.deps = set()
244      self.libs = set()
245      self.include_dirs = set()
246      self.ldflags = set()
247      self.source_set_deps = set()  # Transitive set of source_set deps.
248      self.proto_deps = set()  # Transitive set of protobuf deps.
249
250      # Deps on //gn:xxx have this flag set to True. These dependencies
251      # are special because they pull third_party code from buildtools/.
252      # We don't want to keep recursing into //buildtools in generators,
253      # this flag is used to stop the recursion and create an empty
254      # placeholder target once we hit //gn:protoc or similar.
255      self.is_third_party_dep_ = False
256
257    def __lt__(self, other):
258      if isinstance(other, self.__class__):
259        return self.name < other.name
260      raise TypeError(
261          '\'<\' not supported between instances of \'%s\' and \'%s\'' %
262          (type(self).__name__, type(other).__name__))
263
264    def __repr__(self):
265      return json.dumps({
266          k: (list(sorted(v)) if isinstance(v, set) else v)
267          for (k, v) in iteritems(self.__dict__)
268      },
269                        indent=4,
270                        sort_keys=True)
271
272    def update(self, other):
273      for key in ('cflags', 'defines', 'deps', 'include_dirs', 'ldflags',
274                  'source_set_deps', 'proto_deps', 'libs'):
275        self.__dict__[key].update(other.__dict__.get(key, []))
276
277  def __init__(self, gn_desc):
278    self.gn_desc_ = gn_desc
279    self.all_targets = {}
280    self.linker_units = {}  # Executables, shared or static libraries.
281    self.source_sets = {}
282    self.actions = {}
283    self.proto_libs = {}
284
285  def get_target(self, gn_target_name):
286    """Returns a Target object from the fully qualified GN target name.
287
288        It bubbles up variables from source_set dependencies as described in the
289        class-level comments.
290        """
291    target = self.all_targets.get(gn_target_name)
292    if target is not None:
293      return target  # Target already processed.
294
295    desc = self.gn_desc_[gn_target_name]
296    target = GnParser.Target(gn_target_name, desc['type'])
297    target.testonly = desc.get('testonly', False)
298    target.toolchain = desc.get('toolchain', None)
299    self.all_targets[gn_target_name] = target
300
301    # We should never have GN targets directly depend on buidtools. They
302    # should hop via //gn:xxx, so we can give generators an opportunity to
303    # override them.
304    assert (not gn_target_name.startswith('//buildtools'))
305
306    # Don't descend further into third_party targets. Genrators are supposed
307    # to either ignore them or route to other externally-provided targets.
308    if gn_target_name.startswith('//gn'):
309      target.is_third_party_dep_ = True
310      return target
311
312    proto_target_type, proto_desc = self.get_proto_target_type_(target)
313    if proto_target_type is not None:
314      self.proto_libs[target.name] = target
315      target.type = 'proto_library'
316      target.proto_plugin = proto_target_type
317      target.sources.update(proto_desc.get('sources', []))
318      assert (all(x.endswith('.proto') for x in target.sources))
319    elif target.type == 'source_set':
320      self.source_sets[gn_target_name] = target
321      target.sources.update(desc.get('sources', []))
322    elif target.type in LINKER_UNIT_TYPES:
323      self.linker_units[gn_target_name] = target
324      target.sources.update(desc.get('sources', []))
325    elif target.type == 'action':
326      self.actions[gn_target_name] = target
327      target.inputs.update(desc['inputs'])
328      outs = [re.sub('^//out/.+?/gen/', '', x) for x in desc['outputs']]
329      target.outputs.update(outs)
330      target.script = desc['script']
331      # Args are typically relative to the root build dir (../../xxx)
332      # because root build dir is typically out/xxx/).
333      target.args = [re.sub('^../../', '//', x) for x in desc['args']]
334
335    target.cflags.update(desc.get('cflags', []) + desc.get('cflags_cc', []))
336    target.libs.update(desc.get('libs', []))
337    target.ldflags.update(desc.get('ldflags', []))
338    target.defines.update(desc.get('defines', []))
339    target.include_dirs.update(desc.get('include_dirs', []))
340
341    # Recurse in dependencies.
342    for dep_name in desc.get('deps', []):
343      dep = self.get_target(dep_name)
344      if dep.is_third_party_dep_:
345        target.deps.add(dep_name)
346      elif dep.type == 'proto_library':
347        target.proto_deps.add(dep_name)
348        target.proto_deps.update(dep.proto_deps)  # Bubble up deps.
349      elif dep.type == 'source_set':
350        target.source_set_deps.add(dep_name)
351        target.update(dep)  # Bubble up source set's cflags/ldflags etc.
352      elif dep.type == 'group':
353        target.update(dep)  # Bubble up groups's cflags/ldflags etc.
354      elif dep.type == 'action':
355        if proto_target_type is None:
356          target.deps.add(dep_name)
357      elif dep.type in LINKER_UNIT_TYPES:
358        target.deps.add(dep_name)
359
360    return target
361
362  def get_proto_target_type_(self, target):
363    """ Checks if the target is a proto library and return the plugin.
364
365        Returns:
366            (None, None): if the target is not a proto library.
367            (plugin, gen_desc) where |plugin| is 'proto' in the default (lite)
368            case or 'protozero' or 'ipc'; |gen_desc| is the GN json descriptor
369            of the _gen target (the one with .proto sources).
370        """
371    parts = target.name.split('(', 1)
372    name = parts[0]
373    toolchain = '(' + parts[1] if len(parts) > 1 else ''
374    gen_desc = self.gn_desc_.get('%s_gen%s' % (name, toolchain))
375    if gen_desc is None or gen_desc['type'] != 'action':
376      return None, None
377    args = gen_desc.get('args', [])
378    if '/protoc' not in args[0]:
379      return None, None
380    plugin = 'proto'
381    for arg in (arg for arg in args if arg.startswith('--plugin=')):
382      # |arg| at this point looks like:
383      #  --plugin=protoc-gen-plugin=gcc_like_host/protozero_plugin
384      # or
385      #  --plugin=protoc-gen-plugin=protozero_plugin
386      plugin = arg.split('=')[-1].split('/')[-1].replace('_plugin', '')
387    return plugin, gen_desc
388