#!/usr/bin/env python # Copyright (C) 2018 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # This tool translates a collection of BUILD.gn files into a mostly equivalent # BUILD file for the Bazel build system. The input to the tool is a # JSON description of the GN build definition generated with the following # command: # # gn desc out --format=json --all-toolchains "//*" > desc.json # # The tool is then given a list of GN labels for which to generate Bazel # build rules. from __future__ import print_function import argparse import errno import functools import json import os import re import shutil import subprocess import sys import textwrap # Copyright header for generated code. header = """# Copyright (C) 2019 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # This file is automatically generated by {}. Do not edit. """.format(__file__) # Arguments for the GN output directory. # host_os="linux" is to generate the right build files from Mac OS. gn_args = 'target_os="linux" is_debug=false host_os="linux"' # Default targets to translate to the blueprint file. default_targets = [ '//src/protozero:libprotozero', '//src/trace_processor:trace_processor', '//src/trace_processor:trace_processor_shell_host(//gn/standalone/toolchain:gcc_like_host)', '//tools/trace_to_text:trace_to_text_host(//gn/standalone/toolchain:gcc_like_host)', '//protos/perfetto/config:merged_config_gen', '//protos/perfetto/trace:merged_trace_gen', ] # Aliases to add to the BUILD file alias_targets = { '//src/protozero:libprotozero': 'libprotozero', '//src/trace_processor:trace_processor': 'trace_processor', '//src/trace_processor:trace_processor_shell_host': 'trace_processor_shell', '//tools/trace_to_text:trace_to_text_host': 'trace_to_text', } def enable_sqlite(module): module.deps.add(Label('//third_party/sqlite')) module.deps.add(Label('//third_party/sqlite:sqlite_ext_percentile')) def enable_jsoncpp(module): module.deps.add(Label('//third_party/perfetto/google:jsoncpp')) def enable_linenoise(module): module.deps.add(Label('//third_party/perfetto/google:linenoise')) def enable_gtest_prod(module): module.deps.add(Label('//third_party/perfetto/google:gtest_prod')) def enable_protobuf_full(module): module.deps.add(Label('//third_party/protobuf:libprotoc')) module.deps.add(Label('//third_party/protobuf')) def enable_perfetto_version(module): module.deps.add(Label('//third_party/perfetto/google:perfetto_version')) def disable_module(module): pass # Internal equivalents for third-party libraries that the upstream project # depends on. builtin_deps = { '//gn:jsoncpp_deps': enable_jsoncpp, '//buildtools:linenoise': enable_linenoise, '//buildtools:protobuf_lite': disable_module, '//buildtools:protobuf_full': enable_protobuf_full, '//buildtools:protoc': disable_module, '//buildtools:sqlite': enable_sqlite, '//gn:default_deps': disable_module, '//gn:gtest_prod_config': enable_gtest_prod, '//gn:protoc_lib_deps': enable_protobuf_full, '//gn/standalone:gen_git_revision': enable_perfetto_version, } # ---------------------------------------------------------------------------- # End of configuration. # ---------------------------------------------------------------------------- def check_output(cmd, cwd): try: output = subprocess.check_output( cmd, stderr=subprocess.STDOUT, cwd=cwd) except subprocess.CalledProcessError as e: print('Cmd "{}" failed in {}:'.format( ' '.join(cmd), cwd), file=sys.stderr) print(e.output) exit(1) else: return output class Error(Exception): pass def repo_root(): """Returns an absolute path to the repository root.""" return os.path.join( os.path.realpath(os.path.dirname(__file__)), os.path.pardir) def create_build_description(repo_root): """Creates the JSON build description by running GN.""" out = os.path.join(repo_root, 'out', 'tmp.gen_build') try: try: os.makedirs(out) except OSError as e: if e.errno != errno.EEXIST: raise check_output( ['gn', 'gen', out, '--args=%s' % gn_args], repo_root) desc = check_output( ['gn', 'desc', out, '--format=json', '--all-toolchains', '//*'], repo_root) return json.loads(desc) finally: shutil.rmtree(out) def label_to_path(label): """Turn a GN output label (e.g., //some_dir/file.cc) into a path.""" assert label.startswith('//') return label[2:] def label_to_target_name_with_path(label): """ Turn a GN label into a target name involving the full path. e.g., //src/perfetto:tests -> src_perfetto_tests """ name = re.sub(r'^//:?', '', label) name = re.sub(r'[^a-zA-Z0-9_]', '_', name) return name def label_without_toolchain(label): """Strips the toolchain from a GN label. Return a GN label (e.g //buildtools:protobuf(//gn/standalone/toolchain: gcc_like_host) without the parenthesised toolchain part. """ return label.split('(')[0] def is_public_header(label): """ Returns if this is a c++ header file that is part of the API. Args: label: Label to evaluate """ return label.endswith('.h') and label.startswith('//include/perfetto/') @functools.total_ordering class Label(object): """Represents a label in BUILD file terminology. This class wraps a string label to allow for correct comparision of labels for sorting. Args: label: The string rerepsentation of the label. """ def __init__(self, label): self.label = label def is_absolute(self): return self.label.startswith('//') def dirname(self): return self.label.split(':')[0] if ':' in self.label else self.label def basename(self): return self.label.split(':')[1] if ':' in self.label else '' def __eq__(self, other): return self.label == other.label def __lt__(self, other): return ( self.is_absolute(), self.dirname(), self.basename() ) < ( other.is_absolute(), other.dirname(), other.basename() ) def __str__(self): return self.label def __hash__(self): return hash(self.label) class Writer(object): def __init__(self, output, width=79): self.output = output self.width = width def comment(self, text): for line in textwrap.wrap(text, self.width - 2, break_long_words=False, break_on_hyphens=False): self.output.write('# {}\n'.format(line)) def newline(self): self.output.write('\n') def line(self, s, indent=0): self.output.write(' ' * indent + s + '\n') def variable(self, key, value, sort=True): if value is None: return if isinstance(value, set) or isinstance(value, list): if len(value) == 0: return self.line('{} = ['.format(key), indent=1) for v in sorted(list(value)) if sort else value: self.line('"{}",'.format(v), indent=2) self.line('],', indent=1) elif isinstance(value, basestring): self.line('{} = "{}",'.format(key, value), indent=1) else: self.line('{} = {},'.format(key, value), indent=1) def header(self): self.output.write(header) class Target(object): """In-memory representation of a BUILD target.""" def __init__(self, type, name, gn_name=None): assert type in ('cc_binary', 'cc_library', 'cc_proto_library', 'proto_library', 'filegroup', 'alias', 'pbzero_cc_proto_library', 'genrule', ) self.type = type self.name = name self.srcs = set() self.hdrs = set() self.deps = set() self.visibility = set() self.gn_name = gn_name self.is_pbzero = False self.src_proto_library = None self.outs = set() self.cmd = None self.tools = set() def write(self, writer): if self.gn_name: writer.comment('GN target: {}'.format(self.gn_name)) writer.line('{}('.format(self.type)) writer.variable('name', self.name) writer.variable('srcs', self.srcs) writer.variable('hdrs', self.hdrs) if self.type == 'proto_library' and not self.is_pbzero: if self.srcs: writer.variable('has_services', 1) writer.variable('cc_api_version', 2) if self.srcs: writer.variable('cc_generic_services', 1) writer.variable('src_proto_library', self.src_proto_library) writer.variable('outs', self.outs) writer.variable('cmd', self.cmd) writer.variable('tools', self.tools) # Keep visibility and deps last. writer.variable('visibility', self.visibility) if type != 'filegroup': writer.variable('deps', self.deps) writer.line(')') class Build(object): """In-memory representation of a BUILD file.""" def __init__(self, public, header_lines=[]): self.targets = {} self.public = public self.header_lines = header_lines def add_target(self, target): self.targets[target.name] = target def write(self, writer): writer.header() writer.newline() for line in self.header_lines: writer.line(line) if self.header_lines: writer.newline() if self.public: writer.line( 'package(default_visibility = ["//visibility:public"])') else: writer.line( 'package(default_visibility = ["//third_party/perfetto:__subpackages__"])') writer.newline() writer.line('licenses(["notice"]) # Apache 2.0') writer.newline() writer.line('exports_files(["LICENSE"])') writer.newline() sorted_targets = sorted( self.targets.itervalues(), key=lambda m: m.name) for target in sorted_targets[:-1]: target.write(writer) writer.newline() # BUILD files shouldn't have a trailing new line. sorted_targets[-1].write(writer) class BuildGenerator(object): def __init__(self, desc): self.desc = desc self.action_generated_files = set() for target in self.desc.itervalues(): if target['type'] == 'action': self.action_generated_files.update(target['outputs']) def create_build_for_targets(self, targets): """Generate a BUILD for a list of GN targets and aliases.""" self.build = Build(public=True) proto_cc_import = 'load("//tools/build_defs/proto/cpp:cc_proto_library.bzl", "cc_proto_library")' pbzero_cc_import = 'load("//third_party/perfetto/google:build_defs.bzl", "pbzero_cc_proto_library")' self.proto_build = Build(public=False, header_lines=[ proto_cc_import, pbzero_cc_import]) for target in targets: self.create_target(target) return (self.build, self.proto_build) def resolve_dependencies(self, target_name): """Return the set of direct dependent-on targets for a GN target. Args: desc: JSON GN description. target_name: Name of target Returns: A set of transitive dependencies in the form of GN targets. """ if label_without_toolchain(target_name) in builtin_deps: return set() target = self.desc[target_name] resolved_deps = set() for dep in target.get('deps', []): resolved_deps.add(dep) return resolved_deps def apply_module_sources_to_target(self, target, module_desc): """ Args: target: Module to which dependencies should be added. module_desc: JSON GN description of the module. visibility: Whether the module is visible with respect to the target. """ for src in module_desc['sources']: label = Label(label_to_path(src)) if target.type == 'cc_library' and is_public_header(src): target.hdrs.add(label) else: target.srcs.add(label) def apply_module_dependency(self, target, dep_name): """ Args: build: BUILD instance which is being generated. proto_build: BUILD instance which is being generated to hold protos. desc: JSON GN description. target: Module to which dependencies should be added. dep_name: GN target of the dependency. """ # If the dependency refers to a library which we can replace with an internal # equivalent, stop recursing and patch the dependency in. dep_name_no_toolchain = label_without_toolchain(dep_name) if dep_name_no_toolchain in builtin_deps: builtin_deps[dep_name_no_toolchain](target) return dep_desc = self.desc[dep_name] if dep_desc['type'] == 'source_set': for inner_name in self.resolve_dependencies(dep_name): self.apply_module_dependency(target, inner_name) # Any source set which has a source generated by an action doesn't need # to be depended on as we will depend on the action directly. if any(src in self.action_generated_files for src in dep_desc['sources']): return self.apply_module_sources_to_target(target, dep_desc) elif dep_desc['type'] == 'action': args = dep_desc['args'] if "gen_merged_sql_metrics" in dep_name: dep_target = self.create_merged_sql_metrics_target(dep_name) target.deps.add(Label("//third_party/perfetto:" + dep_target.name)) if target.type == 'cc_library' or target.type == 'cc_binary': target.srcs.update(dep_target.outs) elif args[0].endswith('/protoc'): (proto_target, cc_target) = self.create_proto_target(dep_name) if target.type == 'proto_library': dep_target_name = proto_target.name else: dep_target_name = cc_target.name target.deps.add( Label("//third_party/perfetto/protos:" + dep_target_name)) else: raise Error('Unsupported action in target %s: %s' % (dep_target_name, args)) elif dep_desc['type'] == 'static_library': dep_target = self.create_target(dep_name) target.deps.add(Label("//third_party/perfetto:" + dep_target.name)) elif dep_desc['type'] == 'group': for inner_name in self.resolve_dependencies(dep_name): self.apply_module_dependency(target, inner_name) elif dep_desc['type'] == 'executable': # Just create the dep target but don't add it as a dep because it's an # executable. self.create_target(dep_name) else: raise Error('Unknown target name %s with type: %s' % (dep_name, dep_desc['type'])) def create_merged_sql_metrics_target(self, gn_target_name): target_desc = self.desc[gn_target_name] gn_target_name_no_toolchain = label_without_toolchain(gn_target_name) target = Target( 'genrule', 'gen_merged_sql_metrics', gn_name=gn_target_name_no_toolchain, ) target.outs.update( Label(src[src.index('gen/') + len('gen/'):]) for src in target_desc.get('outputs', []) ) target.cmd = '$(location gen_merged_sql_metrics_py) --cpp_out=$@ $(SRCS)' target.tools.update([ 'gen_merged_sql_metrics_py', ]) target.srcs.update( Label(label_to_path(src)) for src in target_desc.get('inputs', []) if src not in self.action_generated_files ) self.build.add_target(target) return target def create_proto_target(self, gn_target_name): target_desc = self.desc[gn_target_name] args = target_desc['args'] gn_target_name_no_toolchain = label_without_toolchain(gn_target_name) stripped_path = gn_target_name_no_toolchain.replace("protos/perfetto/", "") pretty_target_name = label_to_target_name_with_path(stripped_path) pretty_target_name = pretty_target_name.replace("_lite_gen", "") pretty_target_name = pretty_target_name.replace("_zero_gen", "_zero") proto_target = Target( 'proto_library', pretty_target_name, gn_name=gn_target_name_no_toolchain ) proto_target.is_pbzero = any("pbzero" in arg for arg in args) proto_target.srcs.update([ Label(label_to_path(src).replace('protos/', '')) for src in target_desc.get('sources', []) ]) if not proto_target.is_pbzero: proto_target.visibility.add("//visibility:public") self.proto_build.add_target(proto_target) for dep_name in self.resolve_dependencies(gn_target_name): self.apply_module_dependency(proto_target, dep_name) if proto_target.is_pbzero: # Remove all the protozero srcs from the proto_library. proto_target.srcs.difference_update( [src for src in proto_target.srcs if not src.label.endswith('.proto')]) # Remove all the non-proto deps from the proto_library and add to the cc # library. cc_deps = [ dep for dep in proto_target.deps if not dep.label.startswith('//third_party/perfetto/protos') ] proto_target.deps.difference_update(cc_deps) cc_target_name = proto_target.name + "_cc_proto" cc_target = Target('pbzero_cc_proto_library', cc_target_name, gn_name=gn_target_name_no_toolchain) cc_target.deps.add(Label('//third_party/perfetto:libprotozero')) cc_target.deps.update(cc_deps) # Add the proto_library to the cc_target. cc_target.src_proto_library = \ "//third_party/perfetto/protos:" + proto_target.name self.proto_build.add_target(cc_target) else: cc_target_name = proto_target.name + "_cc_proto" cc_target = Target('cc_proto_library', cc_target_name, gn_name=gn_target_name_no_toolchain) cc_target.visibility.add("//visibility:public") cc_target.deps.add( Label("//third_party/perfetto/protos:" + proto_target.name)) self.proto_build.add_target(cc_target) return (proto_target, cc_target) def create_target(self, gn_target_name): """Generate module(s) for a given GN target. Given a GN target name, generate one or more corresponding modules into a build file. Args: build: Build instance which is being generated. desc: JSON GN description. gn_target_name: GN target name for module generation. """ target_desc = self.desc[gn_target_name] if target_desc['type'] == 'action': args = target_desc['args'] if args[0].endswith('/protoc'): return self.create_proto_target(gn_target_name) else: raise Error('Unsupported action in target %s: %s' % (gn_target_name, args)) elif target_desc['type'] == 'executable': target_type = 'cc_binary' elif target_desc['type'] == 'static_library': target_type = 'cc_library' elif target_desc['type'] == 'source_set': target_type = 'filegroup' else: raise Error('Unknown target type: %s' % target_desc['type']) label_no_toolchain = label_without_toolchain(gn_target_name) target_name_path = label_to_target_name_with_path(label_no_toolchain) target_name = alias_targets.get(label_no_toolchain, target_name_path) target = Target(target_type, target_name, gn_name=label_no_toolchain) target.srcs.update( Label(label_to_path(src)) for src in target_desc.get('sources', []) if src not in self.action_generated_files ) for dep_name in self.resolve_dependencies(gn_target_name): self.apply_module_dependency(target, dep_name) self.build.add_target(target) return target def main(): parser = argparse.ArgumentParser( description='Generate BUILD from a GN description.') parser.add_argument( '--desc', help='GN description (e.g., gn desc out --format=json --all-toolchains "//*"' ) parser.add_argument( '--repo-root', help='Standalone Perfetto repository to generate a GN description', default=repo_root(), ) parser.add_argument( '--extras', help='Extra targets to include at the end of the BUILD file', default=os.path.join(repo_root(), 'BUILD.extras'), ) parser.add_argument( '--output', help='BUILD file to create', default=os.path.join(repo_root(), 'BUILD'), ) parser.add_argument( '--output-proto', help='Proto BUILD file to create', default=os.path.join(repo_root(), 'protos', 'BUILD'), ) parser.add_argument( 'targets', nargs=argparse.REMAINDER, help='Targets to include in the BUILD file (e.g., "//:perfetto_tests")') args = parser.parse_args() if args.desc: with open(args.desc) as f: desc = json.load(f) else: desc = create_build_description(args.repo_root) build_generator = BuildGenerator(desc) build, proto_build = build_generator.create_build_for_targets( args.targets or default_targets) with open(args.output, 'w') as f: writer = Writer(f) build.write(writer) writer.newline() with open(args.extras, 'r') as r: for line in r: writer.line(line.rstrip("\n\r")) with open(args.output_proto, 'w') as f: proto_build.write(Writer(f)) return 0 if __name__ == '__main__': sys.exit(main())