• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (C) 2019 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15# A collection of utilities for extracting build rule information from GN
16# projects.
17
18from __future__ import print_function
19import collections
20import errno
21import filecmp
22import json
23import os
24import re
25import shutil
26import subprocess
27import sys
28from compat import iteritems
29
30BUILDFLAGS_TARGET = '//gn:gen_buildflags'
31GEN_VERSION_TARGET = '//src/base:version_gen_h'
32TARGET_TOOLCHAIN = '//gn/standalone/toolchain:gcc_like_host'
33HOST_TOOLCHAIN = '//gn/standalone/toolchain:gcc_like_host'
34LINKER_UNIT_TYPES = ('executable', 'shared_library', 'static_library')
35
36# TODO(primiano): investigate these, they require further componentization.
37ODR_VIOLATION_IGNORE_TARGETS = {
38    '//test/cts:perfetto_cts_deps',
39    '//:perfetto_integrationtests',
40}
41
42
43def _check_command_output(cmd, cwd):
44  try:
45    output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, cwd=cwd)
46  except subprocess.CalledProcessError as e:
47    print(
48        'Command "{}" failed in {}:'.format(' '.join(cmd), cwd),
49        file=sys.stderr)
50    print(e.output.decode(), file=sys.stderr)
51    sys.exit(1)
52  else:
53    return output.decode()
54
55
56def repo_root():
57  """Returns an absolute path to the repository root."""
58  return os.path.join(
59      os.path.realpath(os.path.dirname(__file__)), os.path.pardir)
60
61
62def _tool_path(name):
63  wrapper = os.path.abspath(
64      os.path.join(repo_root(), 'tools', 'run_buildtools_binary.py'))
65  return ['python3', wrapper, name]
66
67
68def prepare_out_directory(gn_args, name, root=repo_root()):
69  """Creates the JSON build description by running GN.
70
71    Returns (path, desc) where |path| is the location of the output directory
72    and |desc| is the JSON build description.
73    """
74  out = os.path.join(root, 'out', name)
75  try:
76    os.makedirs(out)
77  except OSError as e:
78    if e.errno != errno.EEXIST:
79      raise
80  _check_command_output(
81      _tool_path('gn') + ['gen', out, '--args=%s' % gn_args], cwd=repo_root())
82  return out
83
84
85def load_build_description(out):
86  """Creates the JSON build description by running GN."""
87  desc = _check_command_output(
88      _tool_path('gn') +
89      ['desc', out, '--format=json', '--all-toolchains', '//*'],
90      cwd=repo_root())
91  return json.loads(desc)
92
93
94def create_build_description(gn_args, root=repo_root()):
95  """Prepares a GN out directory and loads the build description from it.
96
97    The temporary out directory is automatically deleted.
98    """
99  out = prepare_out_directory(gn_args, 'tmp.gn_utils', root=root)
100  try:
101    return load_build_description(out)
102  finally:
103    shutil.rmtree(out)
104
105
106def build_targets(out, targets, quiet=False):
107  """Runs ninja to build a list of GN targets in the given out directory.
108
109    Compiling these targets is required so that we can include any generated
110    source files in the amalgamated result.
111    """
112  targets = [t.replace('//', '') for t in targets]
113  with open(os.devnull, 'w') as devnull:
114    stdout = devnull if quiet else None
115    cmd = _tool_path('ninja') + targets
116    subprocess.check_call(cmd, cwd=os.path.abspath(out), stdout=stdout)
117
118
119def compute_source_dependencies(out):
120  """For each source file, computes a set of headers it depends on."""
121  ninja_deps = _check_command_output(
122      _tool_path('ninja') + ['-t', 'deps'], cwd=out)
123  deps = {}
124  current_source = None
125  for line in ninja_deps.split('\n'):
126    filename = os.path.relpath(os.path.join(out, line.strip()), repo_root())
127    if not line or line[0] != ' ':
128      current_source = None
129      continue
130    elif not current_source:
131      # We're assuming the source file is always listed before the
132      # headers.
133      assert os.path.splitext(line)[1] in ['.c', '.cc', '.cpp', '.S']
134      current_source = filename
135      deps[current_source] = []
136    else:
137      assert current_source
138      deps[current_source].append(filename)
139  return deps
140
141
142def label_to_path(label):
143  """Turn a GN output label (e.g., //some_dir/file.cc) into a path."""
144  assert label.startswith('//')
145  return label[2:]
146
147
148def label_without_toolchain(label):
149  """Strips the toolchain from a GN label.
150
151    Return a GN label (e.g //buildtools:protobuf(//gn/standalone/toolchain:
152    gcc_like_host) without the parenthesised toolchain part.
153    """
154  return label.split('(')[0]
155
156
157def label_to_target_name_with_path(label):
158  """
159  Turn a GN label into a target name involving the full path.
160  e.g., //src/perfetto:tests -> src_perfetto_tests
161  """
162  name = re.sub(r'^//:?', '', label)
163  name = re.sub(r'[^a-zA-Z0-9_]', '_', name)
164  return name
165
166
167def gen_buildflags(gn_args, target_file):
168  """Generates the perfetto_build_flags.h for the given config.
169
170    target_file: the path, relative to the repo root, where the generated
171        buildflag header will be copied into.
172    """
173  tmp_out = prepare_out_directory(gn_args, 'tmp.gen_buildflags')
174  build_targets(tmp_out, [BUILDFLAGS_TARGET], quiet=True)
175  src = os.path.join(tmp_out, 'gen', 'build_config', 'perfetto_build_flags.h')
176  shutil.copy(src, os.path.join(repo_root(), target_file))
177  shutil.rmtree(tmp_out)
178
179
180def check_or_commit_generated_files(tmp_files, check):
181  """Checks that gen files are unchanged or renames them to the final location
182
183    Takes in input a list of 'xxx.swp' files that have been written.
184    If check == False, it renames xxx.swp -> xxx.
185    If check == True, it just checks that the contents of 'xxx.swp' == 'xxx'.
186    Returns 0 if no diff was detected, 1 otherwise (to be used as exit code).
187    """
188  res = 0
189  for tmp_file in tmp_files:
190    assert (tmp_file.endswith('.swp'))
191    target_file = os.path.relpath(tmp_file[:-4])
192    if check:
193      if not filecmp.cmp(tmp_file, target_file):
194        sys.stderr.write('%s needs to be regenerated\n' % target_file)
195        res = 1
196      os.unlink(tmp_file)
197    else:
198      os.rename(tmp_file, target_file)
199  return res
200
201
202class ODRChecker(object):
203  """Detects ODR violations in linker units
204
205  When we turn GN source sets into Soong & Bazel file groups, there is the risk
206  to create ODR violations by including the same file group into different
207  linker unit (this is because other build systems don't have a concept
208  equivalent to GN's source_set). This class navigates the transitive
209  dependencies (mostly static libraries) of a target and detects if multiple
210  paths end up including the same file group. This is to avoid situations like:
211
212  traced.exe -> base(file group)
213  traced.exe -> libperfetto(static lib) -> base(file group)
214  """
215
216  def __init__(self, gn, target_name):
217    self.gn = gn
218    self.root = gn.get_target(target_name)
219    self.source_sets = collections.defaultdict(set)
220    self.deps_visited = set()
221    self.source_set_hdr_only = {}
222
223    self._visit(target_name)
224    num_violations = 0
225    if target_name in ODR_VIOLATION_IGNORE_TARGETS:
226      return
227    for sset, paths in self.source_sets.items():
228      if self.is_header_only(sset):
229        continue
230      if len(paths) != 1:
231        num_violations += 1
232        print(
233            'ODR violation in target %s, multiple paths include %s:\n  %s' %
234            (target_name, sset, '\n  '.join(paths)),
235            file=sys.stderr)
236    if num_violations > 0:
237      raise Exception('%d ODR violations detected. Build generation aborted' %
238                      num_violations)
239
240  def _visit(self, target_name, parent_path=''):
241    target = self.gn.get_target(target_name)
242    path = ((parent_path + ' > ') if parent_path else '') + target_name
243    if not target:
244      raise Exception('Cannot find target %s' % target_name)
245    for ssdep in target.source_set_deps:
246      name_and_path = '%s (via %s)' % (target_name, path)
247      self.source_sets[ssdep].add(name_and_path)
248    deps = set(target.deps).union(
249        target.transitive_proto_deps) - self.deps_visited
250    for dep_name in deps:
251      dep = self.gn.get_target(dep_name)
252      if dep.type == 'executable':
253        continue  # Execs are strong boundaries and don't cause ODR violations.
254      # static_library dependencies should reset the path. It doesn't matter if
255      # we get to a source file via:
256      # source_set1 > static_lib > source.cc OR
257      # source_set1 > source_set2 > static_lib > source.cc
258      # This is NOT an ODR violation because source.cc is linked from the same
259      # static library
260      next_parent_path = path if dep.type != 'static_library' else ''
261      self.deps_visited.add(dep_name)
262      self._visit(dep_name, next_parent_path)
263
264  def is_header_only(self, source_set_name):
265    cached = self.source_set_hdr_only.get(source_set_name)
266    if cached is not None:
267      return cached
268    target = self.gn.get_target(source_set_name)
269    if target.type != 'source_set':
270      raise TypeError('%s is not a source_set' % source_set_name)
271    res = all(src.endswith('.h') for src in target.sources)
272    self.source_set_hdr_only[source_set_name] = res
273    return res
274
275
276class GnParser(object):
277  """A parser with some cleverness for GN json desc files
278
279    The main goals of this parser are:
280    1) Deal with the fact that other build systems don't have an equivalent
281       notion to GN's source_set. Conversely to Bazel's and Soong's filegroups,
282       GN source_sets expect that dependencies, cflags and other source_set
283       properties propagate up to the linker unit (static_library, executable or
284       shared_library). This parser simulates the same behavior: when a
285       source_set is encountered, some of its variables (cflags and such) are
286       copied up to the dependent targets. This is to allow gen_xxx to create
287       one filegroup for each source_set and then squash all the other flags
288       onto the linker unit.
289    2) Detect and special-case protobuf targets, figuring out the protoc-plugin
290       being used.
291    """
292
293  class Target(object):
294    """Reperesents A GN target.
295
296        Maked properties are propagated up the dependency chain when a
297        source_set dependency is encountered.
298        """
299
300    def __init__(self, name, type):
301      self.name = name  # e.g. //src/ipc:ipc
302
303      VALID_TYPES = ('static_library', 'shared_library', 'executable', 'group',
304                     'action', 'source_set', 'proto_library')
305      assert (type in VALID_TYPES)
306      self.type = type
307      self.testonly = False
308      self.toolchain = None
309
310      # These are valid only for type == proto_library.
311      # This is typically: 'proto', 'protozero', 'ipc'.
312      self.proto_plugin = None
313      self.proto_paths = set()
314      self.proto_exports = set()
315
316      self.sources = set()
317      # TODO(primiano): consider whether the public section should be part of
318      # bubbled-up sources.
319      self.public_headers = set()  # 'public'
320
321      # These are valid only for type == 'action'
322      self.inputs = set()
323      self.outputs = set()
324      self.script = None
325      self.args = []
326
327      # These variables are propagated up when encountering a dependency
328      # on a source_set target.
329      self.cflags = set()
330      self.defines = set()
331      self.deps = set()
332      self.libs = set()
333      self.include_dirs = set()
334      self.ldflags = set()
335      self.source_set_deps = set()  # Transitive set of source_set deps.
336      self.proto_deps = set()
337      self.transitive_proto_deps = set()
338
339      # Deps on //gn:xxx have this flag set to True. These dependencies
340      # are special because they pull third_party code from buildtools/.
341      # We don't want to keep recursing into //buildtools in generators,
342      # this flag is used to stop the recursion and create an empty
343      # placeholder target once we hit //gn:protoc or similar.
344      self.is_third_party_dep_ = False
345
346    def __lt__(self, other):
347      if isinstance(other, self.__class__):
348        return self.name < other.name
349      raise TypeError(
350          '\'<\' not supported between instances of \'%s\' and \'%s\'' %
351          (type(self).__name__, type(other).__name__))
352
353    def __repr__(self):
354      return json.dumps({
355          k: (list(sorted(v)) if isinstance(v, set) else v)
356          for (k, v) in iteritems(self.__dict__)
357      },
358                        indent=4,
359                        sort_keys=True)
360
361    def update(self, other):
362      for key in ('cflags', 'defines', 'deps', 'include_dirs', 'ldflags',
363                  'source_set_deps', 'proto_deps', 'transitive_proto_deps',
364                  'libs', 'proto_paths'):
365        self.__dict__[key].update(other.__dict__.get(key, []))
366
367  def __init__(self, gn_desc):
368    self.gn_desc_ = gn_desc
369    self.all_targets = {}
370    self.linker_units = {}  # Executables, shared or static libraries.
371    self.source_sets = {}
372    self.actions = {}
373    self.proto_libs = {}
374
375  def get_target(self, gn_target_name):
376    """Returns a Target object from the fully qualified GN target name.
377
378        It bubbles up variables from source_set dependencies as described in the
379        class-level comments.
380        """
381    target = self.all_targets.get(gn_target_name)
382    if target is not None:
383      return target  # Target already processed.
384
385    desc = self.gn_desc_[gn_target_name]
386    target = GnParser.Target(gn_target_name, desc['type'])
387    target.testonly = desc.get('testonly', False)
388    target.toolchain = desc.get('toolchain', None)
389    self.all_targets[gn_target_name] = target
390
391    # We should never have GN targets directly depend on buidtools. They
392    # should hop via //gn:xxx, so we can give generators an opportunity to
393    # override them.
394    assert (not gn_target_name.startswith('//buildtools'))
395
396    # Don't descend further into third_party targets. Genrators are supposed
397    # to either ignore them or route to other externally-provided targets.
398    if gn_target_name.startswith('//gn'):
399      target.is_third_party_dep_ = True
400      return target
401
402    proto_target_type, proto_desc = self.get_proto_target_type(target)
403    if proto_target_type is not None:
404      self.proto_libs[target.name] = target
405      target.type = 'proto_library'
406      target.proto_plugin = proto_target_type
407      target.proto_paths.update(self.get_proto_paths(proto_desc))
408      target.proto_exports.update(self.get_proto_exports(proto_desc))
409      target.sources.update(proto_desc.get('sources', []))
410      assert (all(x.endswith('.proto') for x in target.sources))
411    elif target.type == 'source_set':
412      self.source_sets[gn_target_name] = target
413      target.sources.update(desc.get('sources', []))
414    elif target.type in LINKER_UNIT_TYPES:
415      self.linker_units[gn_target_name] = target
416      target.sources.update(desc.get('sources', []))
417    elif target.type == 'action':
418      self.actions[gn_target_name] = target
419      target.inputs.update(desc.get('inputs', []))
420      target.sources.update(desc.get('sources', []))
421      outs = [re.sub('^//out/.+?/gen/', '', x) for x in desc['outputs']]
422      target.outputs.update(outs)
423      target.script = desc['script']
424      # Args are typically relative to the root build dir (../../xxx)
425      # because root build dir is typically out/xxx/).
426      target.args = [re.sub('^../../', '//', x) for x in desc['args']]
427
428    # Default for 'public' is //* - all headers in 'sources' are public.
429    # TODO(primiano): if a 'public' section is specified (even if empty), then
430    # the rest of 'sources' is considered inaccessible by gn. Consider
431    # emulating that, so that generated build files don't end up with overly
432    # accessible headers.
433    public_headers = [x for x in desc.get('public', []) if x != '*']
434    target.public_headers.update(public_headers)
435
436    target.cflags.update(desc.get('cflags', []) + desc.get('cflags_cc', []))
437    target.libs.update(desc.get('libs', []))
438    target.ldflags.update(desc.get('ldflags', []))
439    target.defines.update(desc.get('defines', []))
440    target.include_dirs.update(desc.get('include_dirs', []))
441
442    # Recurse in dependencies.
443    for dep_name in desc.get('deps', []):
444      dep = self.get_target(dep_name)
445      if dep.is_third_party_dep_:
446        target.deps.add(dep_name)
447      elif dep.type == 'proto_library':
448        target.proto_deps.add(dep_name)
449        target.transitive_proto_deps.add(dep_name)
450        target.proto_paths.update(dep.proto_paths)
451        target.transitive_proto_deps.update(dep.transitive_proto_deps)
452      elif dep.type == 'source_set':
453        target.source_set_deps.add(dep_name)
454        target.update(dep)  # Bubble up source set's cflags/ldflags etc.
455      elif dep.type == 'group':
456        target.update(dep)  # Bubble up groups's cflags/ldflags etc.
457      elif dep.type == 'action':
458        if proto_target_type is None:
459          target.deps.add(dep_name)
460      elif dep.type in LINKER_UNIT_TYPES:
461        target.deps.add(dep_name)
462
463    return target
464
465  def get_proto_exports(self, proto_desc):
466    # exports in metadata will be available for source_set targets.
467    metadata = proto_desc.get('metadata', {})
468    return metadata.get('exports', [])
469
470  def get_proto_paths(self, proto_desc):
471    # import_dirs in metadata will be available for source_set targets.
472    metadata = proto_desc.get('metadata', {})
473    return metadata.get('import_dirs', [])
474
475  def get_proto_target_type(self, target):
476    """ Checks if the target is a proto library and return the plugin.
477
478        Returns:
479            (None, None): if the target is not a proto library.
480            (plugin, proto_desc) where |plugin| is 'proto' in the default (lite)
481            case or 'protozero' or 'ipc' or 'descriptor'; |proto_desc| is the GN
482            json desc of the target with the .proto sources (_gen target for
483            non-descriptor types or the target itself for descriptor type).
484        """
485    parts = target.name.split('(', 1)
486    name = parts[0]
487    toolchain = '(' + parts[1] if len(parts) > 1 else ''
488
489    # Descriptor targets don't have a _gen target; instead we look for the
490    # characteristic flag in the args of the target itself.
491    desc = self.gn_desc_.get(target.name)
492    if '--descriptor_set_out' in desc.get('args', []):
493      return 'descriptor', desc
494
495    # Source set proto targets have a non-empty proto_library_sources in the
496    # metadata of the description.
497    metadata = desc.get('metadata', {})
498    if 'proto_library_sources' in metadata:
499      return 'source_set', desc
500
501    # In all other cases, we want to look at the _gen target as that has the
502    # important information.
503    gen_desc = self.gn_desc_.get('%s_gen%s' % (name, toolchain))
504    if gen_desc is None or gen_desc['type'] != 'action':
505      return None, None
506    args = gen_desc.get('args', [])
507    if '/protoc' not in args[0]:
508      return None, None
509    plugin = 'proto'
510    for arg in (arg for arg in args if arg.startswith('--plugin=')):
511      # |arg| at this point looks like:
512      #  --plugin=protoc-gen-plugin=gcc_like_host/protozero_plugin
513      # or
514      #  --plugin=protoc-gen-plugin=protozero_plugin
515      plugin = arg.split('=')[-1].split('/')[-1].replace('_plugin', '')
516    return plugin, gen_desc
517