• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright (C) 2017 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16# This tool translates a collection of BUILD.gn files into a mostly equivalent
17# Android.bp file for the Android Soong build system. The input to the tool is a
18# JSON description of the GN build definition generated with the following
19# command:
20#
21#   gn desc out --format=json --all-toolchains "//*" > desc.json
22#
23# The tool is then given a list of GN labels for which to generate Android.bp
24# build rules. The dependencies for the GN labels are squashed to the generated
25# Android.bp target, except for actions which get their own genrule. Some
26# libraries are also mapped to their Android equivalents -- see |builtin_deps|.
27
28import argparse
29import errno
30import json
31import os
32import re
33import shutil
34import subprocess
35import sys
36
37# Default targets to translate to the blueprint file.
38default_targets = [
39    '//:libtraced_shared',
40    '//:perfetto_integrationtests',
41    '//:perfetto_trace_protos',
42    '//:perfetto_unittests',
43    '//:perfetto',
44    '//:traced',
45    '//:traced_probes',
46    '//:trace_to_text',
47]
48
49# Defines a custom init_rc argument to be applied to the corresponding output
50# blueprint target.
51target_initrc = {
52    '//:traced': 'perfetto.rc',
53}
54
55target_host_supported = [
56    '//:perfetto_trace_protos',
57]
58
59target_host_only = [
60    '//:trace_to_text',
61]
62
63# Arguments for the GN output directory.
64gn_args = 'target_os="android" target_cpu="arm" is_debug=false build_with_android=true'
65
66# All module names are prefixed with this string to avoid collisions.
67module_prefix = 'perfetto_'
68
69# Shared libraries which are directly translated to Android system equivalents.
70library_whitelist = [
71    'android',
72    'binder',
73    'log',
74    'services',
75    'utils',
76]
77
78# Name of the module which settings such as compiler flags for all other
79# modules.
80defaults_module = module_prefix + 'defaults'
81
82# Location of the project in the Android source tree.
83tree_path = 'external/perfetto'
84
85# Compiler flags which are passed through to the blueprint.
86cflag_whitelist = r'^-DPERFETTO.*$'
87
88# Compiler defines which are passed through to the blueprint.
89define_whitelist = r'^GOOGLE_PROTO.*$'
90
91# Shared libraries which are not in PDK.
92library_not_in_pdk = {
93    'libandroid',
94    'libservices',
95}
96
97
98def enable_gmock(module):
99    module.static_libs.append('libgmock')
100
101
102def enable_gtest_prod(module):
103    module.static_libs.append('libgtest_prod')
104
105
106def enable_gtest(module):
107    assert module.type == 'cc_test'
108
109
110def enable_protobuf_full(module):
111    module.shared_libs.append('libprotobuf-cpp-full')
112
113
114def enable_protobuf_lite(module):
115    module.shared_libs.append('libprotobuf-cpp-lite')
116
117
118def enable_protoc_lib(module):
119    module.shared_libs.append('libprotoc')
120
121
122def enable_libunwind(module):
123    # libunwind is disabled on Darwin so we cannot depend on it.
124    pass
125
126
127# Android equivalents for third-party libraries that the upstream project
128# depends on.
129builtin_deps = {
130    '//buildtools:gmock': enable_gmock,
131    '//buildtools:gtest': enable_gtest,
132    '//gn:gtest_prod_config': enable_gtest_prod,
133    '//buildtools:gtest_main': enable_gtest,
134    '//buildtools:libunwind': enable_libunwind,
135    '//buildtools:protobuf_full': enable_protobuf_full,
136    '//buildtools:protobuf_lite': enable_protobuf_lite,
137    '//buildtools:protoc_lib': enable_protoc_lib,
138}
139
140# ----------------------------------------------------------------------------
141# End of configuration.
142# ----------------------------------------------------------------------------
143
144
145class Error(Exception):
146    pass
147
148
149class ThrowingArgumentParser(argparse.ArgumentParser):
150    def __init__(self, context):
151        super(ThrowingArgumentParser, self).__init__()
152        self.context = context
153
154    def error(self, message):
155        raise Error('%s: %s' % (self.context, message))
156
157
158class Module(object):
159    """A single module (e.g., cc_binary, cc_test) in a blueprint."""
160
161    def __init__(self, mod_type, name):
162        self.type = mod_type
163        self.name = name
164        self.srcs = []
165        self.comment = None
166        self.shared_libs = []
167        self.static_libs = []
168        self.tools = []
169        self.cmd = None
170        self.host_supported = False
171        self.init_rc = []
172        self.out = []
173        self.export_include_dirs = []
174        self.generated_headers = []
175        self.export_generated_headers = []
176        self.defaults = []
177        self.cflags = set()
178        self.local_include_dirs = []
179        self.user_debug_flag = False
180
181    def to_string(self, output):
182        if self.comment:
183            output.append('// %s' % self.comment)
184        output.append('%s {' % self.type)
185        self._output_field(output, 'name')
186        self._output_field(output, 'srcs')
187        self._output_field(output, 'shared_libs')
188        self._output_field(output, 'static_libs')
189        self._output_field(output, 'tools')
190        self._output_field(output, 'cmd', sort=False)
191        self._output_field(output, 'host_supported')
192        self._output_field(output, 'init_rc')
193        self._output_field(output, 'out')
194        self._output_field(output, 'export_include_dirs')
195        self._output_field(output, 'generated_headers')
196        self._output_field(output, 'export_generated_headers')
197        self._output_field(output, 'defaults')
198        self._output_field(output, 'cflags')
199        self._output_field(output, 'local_include_dirs')
200
201        disable_pdk = any(name in library_not_in_pdk for name in self.shared_libs)
202        if self.user_debug_flag or disable_pdk:
203            output.append('  product_variables: {')
204            if disable_pdk:
205                output.append('    pdk: {')
206                output.append('      enabled: false,')
207                output.append('    },')
208            if self.user_debug_flag:
209                output.append('    debuggable: {')
210                output.append('      cflags: ["-DPERFETTO_BUILD_WITH_ANDROID_USERDEBUG"],')
211                output.append('    },')
212            output.append('  },')
213        output.append('}')
214        output.append('')
215
216    def _output_field(self, output, name, sort=True):
217        value = getattr(self, name)
218        if not value:
219            return
220        if isinstance(value, set):
221            value = sorted(value)
222        if isinstance(value, list):
223            output.append('  %s: [' % name)
224            for item in sorted(value) if sort else value:
225                output.append('    "%s",' % item)
226            output.append('  ],')
227            return
228        if isinstance(value, bool):
229           output.append('  %s: true,' % name)
230           return
231        output.append('  %s: "%s",' % (name, value))
232
233
234class Blueprint(object):
235    """In-memory representation of an Android.bp file."""
236
237    def __init__(self):
238        self.modules = {}
239
240    def add_module(self, module):
241        """Adds a new module to the blueprint, replacing any existing module
242        with the same name.
243
244        Args:
245            module: Module instance.
246        """
247        self.modules[module.name] = module
248
249    def to_string(self, output):
250        for m in sorted(self.modules.itervalues(), key=lambda m: m.name):
251            m.to_string(output)
252
253
254def label_to_path(label):
255    """Turn a GN output label (e.g., //some_dir/file.cc) into a path."""
256    assert label.startswith('//')
257    return label[2:]
258
259
260def label_to_module_name(label):
261    """Turn a GN label (e.g., //:perfetto_tests) into a module name."""
262    module = re.sub(r'^//:?', '', label)
263    module = re.sub(r'[^a-zA-Z0-9_]', '_', module)
264    if not module.startswith(module_prefix) and label not in default_targets:
265        return module_prefix + module
266    return module
267
268
269def label_without_toolchain(label):
270    """Strips the toolchain from a GN label.
271
272    Return a GN label (e.g //buildtools:protobuf(//gn/standalone/toolchain:
273    gcc_like_host) without the parenthesised toolchain part.
274    """
275    return label.split('(')[0]
276
277
278def is_supported_source_file(name):
279    """Returns True if |name| can appear in a 'srcs' list."""
280    return os.path.splitext(name)[1] in ['.c', '.cc', '.proto']
281
282
283def is_generated_by_action(desc, label):
284    """Checks if a label is generated by an action.
285
286    Returns True if a GN output label |label| is an output for any action,
287    i.e., the file is generated dynamically.
288    """
289    for target in desc.itervalues():
290        if target['type'] == 'action' and label in target['outputs']:
291            return True
292    return False
293
294
295def apply_module_dependency(blueprint, desc, module, dep_name):
296    """Recursively collect dependencies for a given module.
297
298    Walk the transitive dependencies for a GN target and apply them to a given
299    module. This effectively flattens the dependency tree so that |module|
300    directly contains all the sources, libraries, etc. in the corresponding GN
301    dependency tree.
302
303    Args:
304        blueprint: Blueprint instance which is being generated.
305        desc: JSON GN description.
306        module: Module to which dependencies should be added.
307        dep_name: GN target of the dependency.
308    """
309    # If the dependency refers to a library which we can replace with an Android
310    # equivalent, stop recursing and patch the dependency in.
311    if label_without_toolchain(dep_name) in builtin_deps:
312        builtin_deps[label_without_toolchain(dep_name)](module)
313        return
314
315    # Similarly some shared libraries are directly mapped to Android
316    # equivalents.
317    target = desc[dep_name]
318    for lib in target.get('libs', []):
319        android_lib = 'lib' + lib
320        if lib in library_whitelist and not android_lib in module.shared_libs:
321            module.shared_libs.append(android_lib)
322
323    type = target['type']
324    if type == 'action':
325        create_modules_from_target(blueprint, desc, dep_name)
326        # Depend both on the generated sources and headers -- see
327        # make_genrules_for_action.
328        module.srcs.append(':' + label_to_module_name(dep_name))
329        module.generated_headers.append(
330            label_to_module_name(dep_name) + '_headers')
331    elif type == 'static_library' and label_to_module_name(
332            dep_name) != module.name:
333        create_modules_from_target(blueprint, desc, dep_name)
334        module.static_libs.append(label_to_module_name(dep_name))
335    elif type == 'shared_library' and label_to_module_name(
336            dep_name) != module.name:
337        module.shared_libs.append(label_to_module_name(dep_name))
338    elif type in ['group', 'source_set', 'executable', 'static_library'
339                  ] and 'sources' in target:
340        # Ignore source files that are generated by actions since they will be
341        # implicitly added by the genrule dependencies.
342        module.srcs.extend(
343            label_to_path(src) for src in target['sources']
344            if is_supported_source_file(src)
345            and not is_generated_by_action(desc, src))
346    module.cflags |= _get_cflags(target)
347
348
349def make_genrules_for_action(blueprint, desc, target_name):
350    """Generate genrules for a GN action.
351
352    GN actions are used to dynamically generate files during the build. The
353    Soong equivalent is a genrule. This function turns a specific kind of
354    genrule which turns .proto files into source and header files into a pair
355    equivalent genrules.
356
357    Args:
358        blueprint: Blueprint instance which is being generated.
359        desc: JSON GN description.
360        target_name: GN target for genrule generation.
361
362    Returns:
363        A (source_genrule, header_genrule) module tuple.
364    """
365    target = desc[target_name]
366
367    # We only support genrules which call protoc (with or without a plugin) to
368    # turn .proto files into header and source files.
369    args = target['args']
370    if not args[0].endswith('/protoc'):
371        raise Error('Unsupported action in target %s: %s' % (target_name,
372                                                             target['args']))
373    parser = ThrowingArgumentParser('Action in target %s (%s)' %
374                                    (target_name, ' '.join(target['args'])))
375    parser.add_argument('--proto_path')
376    parser.add_argument('--cpp_out')
377    parser.add_argument('--plugin')
378    parser.add_argument('--plugin_out')
379    parser.add_argument('protos', nargs=argparse.REMAINDER)
380    args = parser.parse_args(args[1:])
381
382    # Depending on whether we are using the default protoc C++ generator or the
383    # protozero plugin, the output dir is passed as:
384    # --cpp_out=gen/xxx or
385    # --plugin_out=:gen/xxx or
386    # --plugin_out=wrapper_namespace=pbzero:gen/xxx
387    gen_dir = args.cpp_out if args.cpp_out else args.plugin_out.split(':')[1]
388    assert gen_dir.startswith('gen/')
389    gen_dir = gen_dir[4:]
390    cpp_out_dir = ('$(genDir)/%s/%s' % (tree_path, gen_dir)).rstrip('/')
391
392    # TODO(skyostil): Is there a way to avoid hardcoding the tree path here?
393    # TODO(skyostil): Find a way to avoid creating the directory.
394    cmd = [
395        'mkdir -p %s &&' % cpp_out_dir,
396        '$(location aprotoc)',
397        '--cpp_out=%s' % cpp_out_dir
398    ]
399
400    # We create two genrules for each action: one for the protobuf headers and
401    # another for the sources. This is because the module that depends on the
402    # generated files needs to declare two different types of dependencies --
403    # source files in 'srcs' and headers in 'generated_headers' -- and it's not
404    # valid to generate .h files from a source dependency and vice versa.
405    source_module = Module('genrule', label_to_module_name(target_name))
406    source_module.srcs.extend(label_to_path(src) for src in target['sources'])
407    source_module.tools = ['aprotoc']
408
409    header_module = Module('genrule',
410                           label_to_module_name(target_name) + '_headers')
411    header_module.srcs = source_module.srcs[:]
412    header_module.tools = source_module.tools[:]
413    header_module.export_include_dirs = [gen_dir or '.']
414
415    # In GN builds the proto path is always relative to the output directory
416    # (out/tmp.xxx).
417    assert args.proto_path.startswith('../../')
418    cmd += [ '--proto_path=%s/%s' % (tree_path, args.proto_path[6:])]
419
420    namespaces = ['pb']
421    if args.plugin:
422        _, plugin = os.path.split(args.plugin)
423        # TODO(skyostil): Can we detect this some other way?
424        if plugin == 'ipc_plugin':
425            namespaces.append('ipc')
426        elif plugin == 'protoc_plugin':
427            namespaces = ['pbzero']
428        for dep in target['deps']:
429            if desc[dep]['type'] != 'executable':
430                continue
431            _, executable = os.path.split(desc[dep]['outputs'][0])
432            if executable == plugin:
433                cmd += [
434                    '--plugin=protoc-gen-plugin=$(location %s)' %
435                    label_to_module_name(dep)
436                ]
437                source_module.tools.append(label_to_module_name(dep))
438                # Also make sure the module for the tool is generated.
439                create_modules_from_target(blueprint, desc, dep)
440                break
441        else:
442            raise Error('Unrecognized protoc plugin in target %s: %s' %
443                        (target_name, args[i]))
444    if args.plugin_out:
445        plugin_args = args.plugin_out.split(':')[0]
446        cmd += ['--plugin_out=%s:%s' % (plugin_args, cpp_out_dir)]
447
448    cmd += ['$(in)']
449    source_module.cmd = ' '.join(cmd)
450    header_module.cmd = source_module.cmd
451    header_module.tools = source_module.tools[:]
452
453    for ns in namespaces:
454        source_module.out += [
455            '%s/%s' % (tree_path, src.replace('.proto', '.%s.cc' % ns))
456            for src in source_module.srcs
457        ]
458        header_module.out += [
459            '%s/%s' % (tree_path, src.replace('.proto', '.%s.h' % ns))
460            for src in header_module.srcs
461        ]
462    return source_module, header_module
463
464
465def _get_cflags(target):
466    cflags = set(flag for flag in target.get('cflags', [])
467        if re.match(cflag_whitelist, flag))
468    cflags |= set("-D%s" % define for define in target.get('defines', [])
469                  if re.match(define_whitelist, define))
470    return cflags
471
472
473def create_modules_from_target(blueprint, desc, target_name):
474    """Generate module(s) for a given GN target.
475
476    Given a GN target name, generate one or more corresponding modules into a
477    blueprint.
478
479    Args:
480        blueprint: Blueprint instance which is being generated.
481        desc: JSON GN description.
482        target_name: GN target for module generation.
483    """
484    target = desc[target_name]
485    if target['type'] == 'executable':
486        if 'host' in target['toolchain'] or target_name in target_host_only:
487            module_type = 'cc_binary_host'
488        elif target.get('testonly'):
489            module_type = 'cc_test'
490        else:
491            module_type = 'cc_binary'
492        modules = [Module(module_type, label_to_module_name(target_name))]
493    elif target['type'] == 'action':
494        modules = make_genrules_for_action(blueprint, desc, target_name)
495    elif target['type'] == 'static_library':
496        module = Module('cc_library_static', label_to_module_name(target_name))
497        module.export_include_dirs = ['include']
498        modules = [module]
499    elif target['type'] == 'shared_library':
500        modules = [
501            Module('cc_library_shared', label_to_module_name(target_name))
502        ]
503    else:
504        raise Error('Unknown target type: %s' % target['type'])
505
506    for module in modules:
507        module.comment = 'GN target: %s' % target_name
508        if target_name in target_initrc:
509          module.init_rc = [target_initrc[target_name]]
510        if target_name in target_host_supported:
511          module.host_supported = True
512
513        # Don't try to inject library/source dependencies into genrules because
514        # they are not compiled in the traditional sense.
515        if module.type != 'genrule':
516            module.defaults = [defaults_module]
517            apply_module_dependency(blueprint, desc, module, target_name)
518            for dep in resolve_dependencies(desc, target_name):
519                apply_module_dependency(blueprint, desc, module, dep)
520
521        # If the module is a static library, export all the generated headers.
522        if module.type == 'cc_library_static':
523            module.export_generated_headers = module.generated_headers
524
525        blueprint.add_module(module)
526
527
528def resolve_dependencies(desc, target_name):
529    """Return the transitive set of dependent-on targets for a GN target.
530
531    Args:
532        blueprint: Blueprint instance which is being generated.
533        desc: JSON GN description.
534
535    Returns:
536        A set of transitive dependencies in the form of GN targets.
537    """
538
539    if label_without_toolchain(target_name) in builtin_deps:
540        return set()
541    target = desc[target_name]
542    resolved_deps = set()
543    for dep in target.get('deps', []):
544        resolved_deps.add(dep)
545        # Ignore the transitive dependencies of actions because they are
546        # explicitly converted to genrules.
547        if desc[dep]['type'] == 'action':
548            continue
549        # Dependencies on shared libraries shouldn't propagate any transitive
550        # dependencies but only depend on the shared library target
551        if desc[dep]['type'] == 'shared_library':
552            continue
553        resolved_deps.update(resolve_dependencies(desc, dep))
554    return resolved_deps
555
556
557def create_blueprint_for_targets(desc, targets):
558    """Generate a blueprint for a list of GN targets."""
559    blueprint = Blueprint()
560
561    # Default settings used by all modules.
562    defaults = Module('cc_defaults', defaults_module)
563    defaults.local_include_dirs = ['include']
564    defaults.cflags = [
565        '-Wno-error=return-type',
566        '-Wno-sign-compare',
567        '-Wno-sign-promo',
568        '-Wno-unused-parameter',
569        '-fvisibility=hidden',
570        '-Oz',
571    ]
572    defaults.user_debug_flag = True
573
574    blueprint.add_module(defaults)
575    for target in targets:
576        create_modules_from_target(blueprint, desc, target)
577    return blueprint
578
579
580def repo_root():
581    """Returns an absolute path to the repository root."""
582
583    return os.path.join(
584        os.path.realpath(os.path.dirname(__file__)), os.path.pardir)
585
586
587def create_build_description():
588    """Creates the JSON build description by running GN."""
589
590    out = os.path.join(repo_root(), 'out', 'tmp.gen_android_bp')
591    try:
592        try:
593            os.makedirs(out)
594        except OSError as e:
595            if e.errno != errno.EEXIST:
596                raise
597        subprocess.check_output(
598            ['gn', 'gen', out, '--args=%s' % gn_args], cwd=repo_root())
599        desc = subprocess.check_output(
600            ['gn', 'desc', out, '--format=json', '--all-toolchains', '//*'],
601            cwd=repo_root())
602        return json.loads(desc)
603    finally:
604        shutil.rmtree(out)
605
606
607def main():
608    parser = argparse.ArgumentParser(
609        description='Generate Android.bp from a GN description.')
610    parser.add_argument(
611        '--desc',
612        help=
613        'GN description (e.g., gn desc out --format=json --all-toolchains "//*"'
614    )
615    parser.add_argument(
616        '--extras',
617        help='Extra targets to include at the end of the Blueprint file',
618        default=os.path.join(repo_root(), 'Android.bp.extras'),
619    )
620    parser.add_argument(
621        '--output',
622        help='Blueprint file to create',
623        default=os.path.join(repo_root(), 'Android.bp'),
624    )
625    parser.add_argument(
626        'targets',
627        nargs=argparse.REMAINDER,
628        help='Targets to include in the blueprint (e.g., "//:perfetto_tests")')
629    args = parser.parse_args()
630
631    if args.desc:
632        with open(args.desc) as f:
633            desc = json.load(f)
634    else:
635        desc = create_build_description()
636
637    blueprint = create_blueprint_for_targets(desc, args.targets
638                                             or default_targets)
639    output = [
640        """// Copyright (C) 2017 The Android Open Source Project
641//
642// Licensed under the Apache License, Version 2.0 (the "License");
643// you may not use this file except in compliance with the License.
644// You may obtain a copy of the License at
645//
646//      http://www.apache.org/licenses/LICENSE-2.0
647//
648// Unless required by applicable law or agreed to in writing, software
649// distributed under the License is distributed on an "AS IS" BASIS,
650// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
651// See the License for the specific language governing permissions and
652// limitations under the License.
653//
654// This file is automatically generated by %s. Do not edit.
655""" % (__file__)
656    ]
657    blueprint.to_string(output)
658    with open(args.extras, 'r') as r:
659        for line in r:
660            output.append(line.rstrip("\n\r"))
661    with open(args.output, 'w') as f:
662        f.write('\n'.join(output))
663
664
665if __name__ == '__main__':
666    sys.exit(main())
667