1#!/usr/bin/env python3 2# Copyright (C) 2019 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16# This tool uses a collection of BUILD.gn files and build targets to generate 17# an "amalgamated" C++ header and source file pair which compiles to an 18# equivalent program. The tool also outputs the necessary compiler and linker 19# flags needed to compile the resulting source code. 20 21from __future__ import print_function 22import argparse 23import os 24import re 25import shutil 26import subprocess 27import sys 28import tempfile 29 30import gn_utils 31 32# Default targets to include in the result. 33# TODO(primiano): change this script to recurse into target deps when generating 34# headers, but only for proto targets. .pbzero.h files don't include each other 35# and we need to list targets here individually, which is unmaintainable. 36default_targets = [ 37 '//:libperfetto_client_experimental', 38 '//include/perfetto/protozero:protozero', 39 '//protos/perfetto/config:zero', 40 '//protos/perfetto/trace:zero', 41] 42 43# Arguments for the GN output directory (unless overridden from the command 44# line). 45gn_args = ' '.join([ 46 'enable_perfetto_ipc=true', 47 'is_debug=false', 48 'is_perfetto_build_generator=true', 49 'is_perfetto_embedder=true', 50 'perfetto_enable_git_rev_version_header=true', 51 'use_custom_libcxx=false', 52]) 53 54# By default, the amalgamated .h only recurses in #includes but not in the 55# target deps. In the case of protos we want to follow deps even in lieu of 56# direct #includes. This is because, by design, protozero headers don't 57# include each other but rely on forward declarations. The alternative would 58# be adding each proto sub-target individually (e.g. //proto/trace/gpu:zero), 59# but doing that is unmaintainable. We also do this for cpp bindings since some 60# tracing SDK functions depend on them (and the system tracing IPC mechanism 61# does so too). 62recurse_in_header_deps = '^//protos/.*(cpp|zero)$' 63 64# Compiler flags which aren't filtered out. 65cflag_allowlist = r'^-(W.*|fno-exceptions|fPIC|std.*|fvisibility.*)$' 66 67# Linker flags which aren't filtered out. 68ldflag_allowlist = r'^-()$' 69 70# Libraries which are filtered out. 71lib_denylist = r'^(c|gcc_eh)$' 72 73# Macros which aren't filtered out. 74define_allowlist = r'^(PERFETTO.*|GOOGLE_PROTOBUF.*)$' 75 76# Includes which will be removed from the generated source. 77includes_to_remove = r'^(gtest).*$' 78 79default_cflags = [ 80 # Since we're expanding header files into the generated source file, some 81 # constant may remain unused. 82 '-Wno-unused-const-variable' 83] 84 85# Build flags to satisfy a protobuf (lite or full) dependency. 86protobuf_cflags = [ 87 # Note that these point to the local copy of protobuf in buildtools. In 88 # reality the user of the amalgamated result will have to provide a path to 89 # an installed copy of the exact same version of protobuf which was used to 90 # generate the amalgamated build. 91 '-isystembuildtools/protobuf/src', 92 '-Lbuildtools/protobuf/src/.libs', 93 # We also need to disable some warnings for protobuf. 94 '-Wno-missing-prototypes', 95 '-Wno-missing-variable-declarations', 96 '-Wno-sign-conversion', 97 '-Wno-unknown-pragmas', 98 '-Wno-unused-macros', 99] 100 101# A mapping of dependencies to system libraries. Libraries in this map will not 102# be built statically but instead added as dependencies of the amalgamated 103# project. 104system_library_map = { 105 '//buildtools:protobuf_full': { 106 'libs': ['protobuf'], 107 'cflags': protobuf_cflags, 108 }, 109 '//buildtools:protobuf_lite': { 110 'libs': ['protobuf-lite'], 111 'cflags': protobuf_cflags, 112 }, 113 '//buildtools:protoc_lib': { 114 'libs': ['protoc'] 115 }, 116} 117 118# ---------------------------------------------------------------------------- 119# End of configuration. 120# ---------------------------------------------------------------------------- 121 122tool_name = os.path.basename(__file__) 123project_root = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) 124preamble = """// Copyright (C) 2019 The Android Open Source Project 125// 126// Licensed under the Apache License, Version 2.0 (the "License"); 127// you may not use this file except in compliance with the License. 128// You may obtain a copy of the License at 129// 130// http://www.apache.org/licenses/LICENSE-2.0 131// 132// Unless required by applicable law or agreed to in writing, software 133// distributed under the License is distributed on an "AS IS" BASIS, 134// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 135// See the License for the specific language governing permissions and 136// limitations under the License. 137// 138// This file is automatically generated by %s. Do not edit. 139""" % tool_name 140 141 142def apply_denylist(denylist, items): 143 return [item for item in items if not re.match(denylist, item)] 144 145 146def apply_allowlist(allowlist, items): 147 return [item for item in items if re.match(allowlist, item)] 148 149 150def normalize_path(path): 151 path = os.path.relpath(path, project_root) 152 path = re.sub(r'^out/[^/]+/', '', path) 153 return path 154 155 156class Error(Exception): 157 pass 158 159 160class DependencyNode(object): 161 """A target in a GN build description along with its dependencies.""" 162 163 def __init__(self, target_name): 164 self.target_name = target_name 165 self.dependencies = set() 166 167 def add_dependency(self, target_node): 168 if target_node in self.dependencies: 169 return 170 self.dependencies.add(target_node) 171 172 def iterate_depth_first(self): 173 for node in sorted(self.dependencies, key=lambda n: n.target_name): 174 for node in node.iterate_depth_first(): 175 yield node 176 if self.target_name: 177 yield self 178 179 180class DependencyTree(object): 181 """A tree of GN build target dependencies.""" 182 183 def __init__(self): 184 self.target_to_node_map = {} 185 self.root = self._get_or_create_node(None) 186 187 def _get_or_create_node(self, target_name): 188 if target_name in self.target_to_node_map: 189 return self.target_to_node_map[target_name] 190 node = DependencyNode(target_name) 191 self.target_to_node_map[target_name] = node 192 return node 193 194 def add_dependency(self, from_target, to_target): 195 from_node = self._get_or_create_node(from_target) 196 to_node = self._get_or_create_node(to_target) 197 assert from_node is not to_node 198 from_node.add_dependency(to_node) 199 200 def iterate_depth_first(self): 201 for node in self.root.iterate_depth_first(): 202 yield node 203 204 205class AmalgamatedProject(object): 206 """In-memory representation of an amalgamated source/header pair.""" 207 208 def __init__(self, desc, source_deps, compute_deps_only=False): 209 """Constructor. 210 211 Args: 212 desc: JSON build description. 213 source_deps: A map of (source file, [dependency header]) which is 214 to detect which header files are included by each source file. 215 compute_deps_only: If True, the project will only be used to compute 216 dependency information. Use |get_source_files()| to retrieve 217 the result. 218 """ 219 self.desc = desc 220 self.source_deps = source_deps 221 self.header = [] 222 self.source = [] 223 self.source_defines = [] 224 # Note that we don't support multi-arg flags. 225 self.cflags = set(default_cflags) 226 self.ldflags = set() 227 self.defines = set() 228 self.libs = set() 229 self._dependency_tree = DependencyTree() 230 self._processed_sources = set() 231 self._processed_headers = set() 232 self._processed_header_deps = set() 233 self._processed_source_headers = set() # Header files included from .cc 234 self._include_re = re.compile(r'#include "(.*)"') 235 self._compute_deps_only = compute_deps_only 236 237 def add_target(self, target_name): 238 """Include |target_name| in the amalgamated result.""" 239 self._dependency_tree.add_dependency(None, target_name) 240 self._add_target_dependencies(target_name) 241 self._add_target_flags(target_name) 242 self._add_target_headers(target_name) 243 244 # Recurse into target deps, but only for protos. This generates headers 245 # for all the .{pbzero,gen}.h files, even if they don't #include each other. 246 for _, dep in self._iterate_dep_edges(target_name): 247 if (dep not in self._processed_header_deps and 248 re.match(recurse_in_header_deps, dep)): 249 self._processed_header_deps.add(dep) 250 self.add_target(dep) 251 252 def _iterate_dep_edges(self, target_name): 253 target = self.desc[target_name] 254 for dep in target.get('deps', []): 255 # Ignore system libraries since they will be added as build-time 256 # dependencies. 257 if dep in system_library_map: 258 continue 259 # Don't descend into build action dependencies. 260 if self.desc[dep]['type'] == 'action': 261 continue 262 for sub_target, sub_dep in self._iterate_dep_edges(dep): 263 yield sub_target, sub_dep 264 yield target_name, dep 265 266 def _iterate_target_and_deps(self, target_name): 267 yield target_name 268 for _, dep in self._iterate_dep_edges(target_name): 269 yield dep 270 271 def _add_target_dependencies(self, target_name): 272 for target, dep in self._iterate_dep_edges(target_name): 273 self._dependency_tree.add_dependency(target, dep) 274 275 def process_dep(dep): 276 if dep in system_library_map: 277 self.libs.update(system_library_map[dep].get('libs', [])) 278 self.cflags.update(system_library_map[dep].get('cflags', [])) 279 self.defines.update(system_library_map[dep].get('defines', [])) 280 return True 281 282 def walk_all_deps(target_name): 283 target = self.desc[target_name] 284 for dep in target.get('deps', []): 285 if process_dep(dep): 286 return 287 walk_all_deps(dep) 288 289 walk_all_deps(target_name) 290 291 def _filter_cflags(self, cflags): 292 # Since we want to deduplicate flags, combine two-part switches (e.g., 293 # "-foo bar") into one value ("-foobar") so we can store the result as 294 # a set. 295 result = [] 296 for flag in cflags: 297 if flag.startswith('-'): 298 result.append(flag) 299 else: 300 result[-1] += flag 301 return apply_allowlist(cflag_allowlist, result) 302 303 def _add_target_flags(self, target_name): 304 for target_name in self._iterate_target_and_deps(target_name): 305 target = self.desc[target_name] 306 self.cflags.update(self._filter_cflags(target.get('cflags', []))) 307 self.cflags.update(self._filter_cflags(target.get('cflags_cc', []))) 308 self.ldflags.update( 309 apply_allowlist(ldflag_allowlist, target.get('ldflags', []))) 310 self.libs.update(apply_denylist(lib_denylist, target.get('libs', []))) 311 self.defines.update( 312 apply_allowlist(define_allowlist, target.get('defines', []))) 313 314 def _add_target_headers(self, target_name): 315 target = self.desc[target_name] 316 if not 'sources' in target: 317 return 318 headers = [ 319 gn_utils.label_to_path(s) for s in target['sources'] if s.endswith('.h') 320 ] 321 for header in headers: 322 self._add_header(target_name, header) 323 324 def _get_include_dirs(self, target_name): 325 include_dirs = set() 326 for target_name in self._iterate_target_and_deps(target_name): 327 target = self.desc[target_name] 328 if 'include_dirs' in target: 329 include_dirs.update( 330 [gn_utils.label_to_path(d) for d in target['include_dirs']]) 331 return include_dirs 332 333 def _add_source_included_header(self, include_dirs, allowed_files, 334 header_name): 335 if header_name in self._processed_headers: 336 return 337 if header_name in self._processed_source_headers: 338 return 339 self._processed_source_headers.add(header_name) 340 for include_dir in include_dirs: 341 rel_path = os.path.join(include_dir, header_name) 342 full_path = os.path.join(gn_utils.repo_root(), rel_path) 343 if os.path.exists(full_path): 344 if not rel_path in allowed_files: 345 return 346 with open(full_path) as f: 347 self.source.append( 348 '// %s begin header: %s' % (tool_name, normalize_path(full_path))) 349 self.source.extend( 350 self._process_source_includes(include_dirs, allowed_files, f)) 351 return 352 if self._compute_deps_only: 353 return 354 msg = 'Looked in %s' % ', '.join('"%s"' % d for d in include_dirs) 355 raise Error('Header file %s not found. %s' % (header_name, msg)) 356 357 def _add_source(self, target_name, source_name): 358 if source_name in self._processed_sources: 359 return 360 self._processed_sources.add(source_name) 361 include_dirs = self._get_include_dirs(target_name) 362 deps = self.source_deps[source_name] 363 full_path = os.path.join(gn_utils.repo_root(), source_name) 364 if not os.path.exists(full_path): 365 raise Error('Source file %s not found' % source_name) 366 with open(full_path) as f: 367 self.source.append( 368 '// %s begin source: %s' % (tool_name, normalize_path(full_path))) 369 try: 370 self.source.extend( 371 self._patch_source( 372 source_name, self._process_source_includes( 373 include_dirs, deps, f))) 374 except Error as e: 375 raise Error('Failed adding source %s: %s' % (source_name, e.message)) 376 377 def _add_header_included_header(self, include_dirs, header_name): 378 if header_name in self._processed_headers: 379 return 380 self._processed_headers.add(header_name) 381 for include_dir in include_dirs: 382 full_path = os.path.join(gn_utils.repo_root(), include_dir, header_name) 383 if os.path.exists(full_path): 384 with open(full_path) as f: 385 self.header.append( 386 '// %s begin header: %s' % (tool_name, normalize_path(full_path))) 387 self.header.extend(self._process_header_includes(include_dirs, f)) 388 return 389 if self._compute_deps_only: 390 return 391 msg = 'Looked in %s' % ', '.join('"%s"' % d for d in include_dirs) 392 raise Error('Header file %s not found. %s' % (header_name, msg)) 393 394 def _add_header(self, target_name, header_name): 395 if header_name in self._processed_headers: 396 return 397 self._processed_headers.add(header_name) 398 include_dirs = self._get_include_dirs(target_name) 399 full_path = os.path.join(gn_utils.repo_root(), header_name) 400 if not os.path.exists(full_path): 401 if self._compute_deps_only: 402 return 403 raise Error('Header file %s not found' % header_name) 404 with open(full_path) as f: 405 self.header.append( 406 '// %s begin header: %s' % (tool_name, normalize_path(full_path))) 407 try: 408 self.header.extend(self._process_header_includes(include_dirs, f)) 409 except Error as e: 410 raise Error('Failed adding header %s: %s' % (header_name, e.message)) 411 412 def _patch_source(self, source_name, lines): 413 result = [] 414 namespace = re.sub(r'[^a-z]', '_', 415 os.path.splitext(os.path.basename(source_name))[0]) 416 for line in lines: 417 # Protobuf generates an identical anonymous function into each 418 # message description. Rename all but the first occurrence to avoid 419 # duplicate symbol definitions. 420 line = line.replace('MergeFromFail', '%s_MergeFromFail' % namespace) 421 result.append(line) 422 return result 423 424 def _process_source_includes(self, include_dirs, allowed_files, file): 425 result = [] 426 for line in file: 427 line = line.rstrip('\n') 428 m = self._include_re.match(line) 429 if not m: 430 result.append(line) 431 continue 432 elif re.match(includes_to_remove, m.group(1)): 433 result.append('// %s removed: %s' % (tool_name, line)) 434 else: 435 result.append('// %s expanded: %s' % (tool_name, line)) 436 self._add_source_included_header(include_dirs, allowed_files, 437 m.group(1)) 438 return result 439 440 def _process_header_includes(self, include_dirs, file): 441 result = [] 442 for line in file: 443 line = line.rstrip('\n') 444 m = self._include_re.match(line) 445 if not m: 446 result.append(line) 447 continue 448 elif re.match(includes_to_remove, m.group(1)): 449 result.append('// %s removed: %s' % (tool_name, line)) 450 else: 451 result.append('// %s expanded: %s' % (tool_name, line)) 452 self._add_header_included_header(include_dirs, m.group(1)) 453 return result 454 455 def generate(self): 456 """Prepares the output for this amalgamated project. 457 458 Call save() to persist the result. 459 """ 460 assert not self._compute_deps_only 461 self.source_defines.append('// %s: predefined macros' % tool_name) 462 463 def add_define(name): 464 # Valued macros aren't supported for now. 465 assert '=' not in name 466 self.source_defines.append('#if !defined(%s)' % name) 467 self.source_defines.append('#define %s' % name) 468 self.source_defines.append('#endif') 469 470 for name in self.defines: 471 add_define(name) 472 for target_name, source_name in self.get_source_files(): 473 self._add_source(target_name, source_name) 474 475 def get_source_files(self): 476 """Return a list of (target, [source file]) that describes the source 477 files pulled in by each target which is a dependency of this project. 478 """ 479 source_files = [] 480 for node in self._dependency_tree.iterate_depth_first(): 481 target = self.desc[node.target_name] 482 if not 'sources' in target: 483 continue 484 sources = [(node.target_name, gn_utils.label_to_path(s)) 485 for s in target['sources'] 486 if s.endswith('.cc')] 487 source_files.extend(sources) 488 return source_files 489 490 def _get_nice_path(self, prefix, format): 491 basename = os.path.basename(prefix) 492 return os.path.join( 493 os.path.relpath(os.path.dirname(prefix)), format % basename) 494 495 def _make_directories(self, directory): 496 if not os.path.isdir(directory): 497 os.makedirs(directory) 498 499 def save(self, output_prefix): 500 """Save the generated header and source file pair. 501 502 Returns a message describing the output with build instructions. 503 """ 504 header_file = self._get_nice_path(output_prefix, '%s.h') 505 source_file = self._get_nice_path(output_prefix, '%s.cc') 506 self._make_directories(os.path.dirname(header_file)) 507 self._make_directories(os.path.dirname(source_file)) 508 with open(header_file, 'w') as f: 509 f.write('\n'.join([preamble] + self.header + ['\n'])) 510 with open(source_file, 'w') as f: 511 include_stmt = '#include "%s"' % os.path.basename(header_file) 512 f.write('\n'.join([preamble] + self.source_defines + [include_stmt] + 513 self.source + ['\n'])) 514 build_cmd = self.get_build_command(output_prefix) 515 return """Amalgamated project written to %s and %s. 516 517Build settings: 518 - cflags: %s 519 - ldflags: %s 520 - libs: %s 521 522Example build command: 523 524%s 525""" % (header_file, source_file, ' '.join(self.cflags), ' '.join(self.ldflags), 526 ' '.join(self.libs), ' '.join(build_cmd)) 527 528 def get_build_command(self, output_prefix): 529 """Returns an example command line for building the output source.""" 530 source = self._get_nice_path(output_prefix, '%s.cc') 531 library = self._get_nice_path(output_prefix, 'lib%s.so') 532 533 if sys.platform.startswith('linux'): 534 llvm_script = os.path.join(gn_utils.repo_root(), 'gn', 535 'standalone', 'toolchain', 536 'linux_find_llvm.py') 537 cxx = subprocess.check_output([llvm_script]).splitlines()[2].decode() 538 else: 539 cxx = 'clang++' 540 541 build_cmd = [cxx, source, '-o', library, '-shared'] + \ 542 sorted(self.cflags) + sorted(self.ldflags) 543 for lib in sorted(self.libs): 544 build_cmd.append('-l%s' % lib) 545 return build_cmd 546 547 548def main(): 549 parser = argparse.ArgumentParser( 550 description='Generate an amalgamated header/source pair from a GN ' 551 'build description.') 552 parser.add_argument( 553 '--out', 554 help='The name of the temporary build folder in \'out\'', 555 default='tmp.gen_amalgamated.%u' % os.getpid()) 556 parser.add_argument( 557 '--output', 558 help='Base name of files to create. A .cc/.h extension will be added', 559 default=os.path.join(gn_utils.repo_root(), 'out/amalgamated/perfetto')) 560 parser.add_argument( 561 '--gn_args', 562 help='GN arguments used to prepare the output directory', 563 default=gn_args) 564 parser.add_argument( 565 '--keep', 566 help='Don\'t delete the GN output directory at exit', 567 action='store_true') 568 parser.add_argument( 569 '--build', help='Also compile the generated files', action='store_true') 570 parser.add_argument( 571 '--check', help='Don\'t keep the generated files', action='store_true') 572 parser.add_argument('--quiet', help='Only report errors', action='store_true') 573 parser.add_argument( 574 '--dump-deps', 575 help='List all source files that the amalgamated output depends on', 576 action='store_true') 577 parser.add_argument( 578 'targets', 579 nargs=argparse.REMAINDER, 580 help='Targets to include in the output (e.g., "//:libperfetto")') 581 args = parser.parse_args() 582 targets = args.targets or default_targets 583 584 # The CHANGELOG mtime triggers the perfetto_version.gen.h genrule. This is 585 # to avoid emitting a stale version information in the remote case of somebody 586 # running gen_amalgamated incrementally after having moved to another commit. 587 changelog_path = os.path.join(project_root, 'CHANGELOG') 588 assert(os.path.exists(changelog_path)) 589 subprocess.check_call(['touch', '-c', changelog_path]) 590 591 output = args.output 592 if args.check: 593 output = os.path.join(tempfile.mkdtemp(), 'perfetto_amalgamated') 594 595 out = gn_utils.prepare_out_directory(args.gn_args, args.out) 596 if not args.quiet: 597 print('Building project...') 598 try: 599 desc = gn_utils.load_build_description(out) 600 601 # We need to build everything first so that the necessary header 602 # dependencies get generated. However if we are just dumping dependency 603 # information this can be skipped, allowing cross-platform operation. 604 if not args.dump_deps: 605 gn_utils.build_targets(out, targets) 606 source_deps = gn_utils.compute_source_dependencies(out) 607 project = AmalgamatedProject( 608 desc, source_deps, compute_deps_only=args.dump_deps) 609 610 for target in targets: 611 project.add_target(target) 612 613 if args.dump_deps: 614 source_files = [ 615 source_file for _, source_file in project.get_source_files() 616 ] 617 print('\n'.join(sorted(set(source_files)))) 618 return 619 620 project.generate() 621 result = project.save(output) 622 if not args.quiet: 623 print(result) 624 if args.build: 625 if not args.quiet: 626 sys.stdout.write('Building amalgamated project...') 627 sys.stdout.flush() 628 subprocess.check_call(project.get_build_command(output)) 629 if not args.quiet: 630 print('done') 631 finally: 632 if not args.keep: 633 shutil.rmtree(out) 634 if args.check: 635 shutil.rmtree(os.path.dirname(output)) 636 637 638if __name__ == '__main__': 639 sys.exit(main()) 640