1#!/usr/bin/env python3 2# Copyright (C) 2019 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16# This tool uses a collection of BUILD.gn files and build targets to generate 17# an "amalgamated" C++ header and source file pair which compiles to an 18# equivalent program. The tool also outputs the necessary compiler and linker 19# flags needed to compile the resulting source code. 20 21from __future__ import print_function 22import argparse 23import os 24import re 25import shutil 26import subprocess 27import sys 28import tempfile 29 30import gn_utils 31 32# Default targets to include in the result. 33# TODO(primiano): change this script to recurse into target deps when generating 34# headers, but only for proto targets. .pbzero.h files don't include each other 35# and we need to list targets here individually, which is unmaintainable. 36default_targets = [ 37 '//:libperfetto_client_experimental', 38 '//include/perfetto/protozero:protozero', 39 '//protos/perfetto/config:zero', 40 '//protos/perfetto/trace:zero', 41] 42 43# Arguments for the GN output directory (unless overridden from the command 44# line). 45gn_args = ' '.join([ 46 'enable_perfetto_ipc=true', 47 'enable_perfetto_zlib=false', 48 'is_debug=false', 49 'is_perfetto_build_generator=true', 50 'is_perfetto_embedder=true', 51 'perfetto_enable_git_rev_version_header=true', 52 'use_custom_libcxx=false', 53]) 54 55# By default, the amalgamated .h only recurses in #includes but not in the 56# target deps. In the case of protos we want to follow deps even in lieu of 57# direct #includes. This is because, by design, protozero headers don't 58# include each other but rely on forward declarations. The alternative would 59# be adding each proto sub-target individually (e.g. //proto/trace/gpu:zero), 60# but doing that is unmaintainable. We also do this for cpp bindings since some 61# tracing SDK functions depend on them (and the system tracing IPC mechanism 62# does so too). 63recurse_in_header_deps = '^//protos/.*(cpp|zero)$' 64 65# Compiler flags which aren't filtered out. 66cflag_allowlist = r'^-(W.*|fno-exceptions|fPIC|std.*|fvisibility.*)$' 67 68# Linker flags which aren't filtered out. 69ldflag_allowlist = r'^-()$' 70 71# Libraries which are filtered out. 72lib_denylist = r'^(c|gcc_eh)$' 73 74# Macros which aren't filtered out. 75define_allowlist = r'^(PERFETTO.*|GOOGLE_PROTOBUF.*)$' 76 77# Includes which will be removed from the generated source. 78includes_to_remove = r'^(gtest).*$' 79 80default_cflags = [ 81 # Since we're expanding header files into the generated source file, some 82 # constant may remain unused. 83 '-Wno-unused-const-variable' 84] 85 86# Build flags to satisfy a protobuf (lite or full) dependency. 87protobuf_cflags = [ 88 # Note that these point to the local copy of protobuf in buildtools. In 89 # reality the user of the amalgamated result will have to provide a path to 90 # an installed copy of the exact same version of protobuf which was used to 91 # generate the amalgamated build. 92 '-isystembuildtools/protobuf/src', 93 '-Lbuildtools/protobuf/src/.libs', 94 # We also need to disable some warnings for protobuf. 95 '-Wno-missing-prototypes', 96 '-Wno-missing-variable-declarations', 97 '-Wno-sign-conversion', 98 '-Wno-unknown-pragmas', 99 '-Wno-unused-macros', 100] 101 102# A mapping of dependencies to system libraries. Libraries in this map will not 103# be built statically but instead added as dependencies of the amalgamated 104# project. 105system_library_map = { 106 '//buildtools:protobuf_full': { 107 'libs': ['protobuf'], 108 'cflags': protobuf_cflags, 109 }, 110 '//buildtools:protobuf_lite': { 111 'libs': ['protobuf-lite'], 112 'cflags': protobuf_cflags, 113 }, 114 '//buildtools:protoc_lib': { 115 'libs': ['protoc'] 116 }, 117} 118 119# ---------------------------------------------------------------------------- 120# End of configuration. 121# ---------------------------------------------------------------------------- 122 123tool_name = os.path.basename(__file__) 124project_root = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) 125preamble = """// Copyright (C) 2019 The Android Open Source Project 126// 127// Licensed under the Apache License, Version 2.0 (the "License"); 128// you may not use this file except in compliance with the License. 129// You may obtain a copy of the License at 130// 131// http://www.apache.org/licenses/LICENSE-2.0 132// 133// Unless required by applicable law or agreed to in writing, software 134// distributed under the License is distributed on an "AS IS" BASIS, 135// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 136// See the License for the specific language governing permissions and 137// limitations under the License. 138// 139// This file is automatically generated by %s. Do not edit. 140""" % tool_name 141 142 143def apply_denylist(denylist, items): 144 return [item for item in items if not re.match(denylist, item)] 145 146 147def apply_allowlist(allowlist, items): 148 return [item for item in items if re.match(allowlist, item)] 149 150 151def normalize_path(path): 152 path = os.path.relpath(path, project_root) 153 path = re.sub(r'^out/[^/]+/', '', path) 154 return path 155 156 157class Error(Exception): 158 pass 159 160 161class DependencyNode(object): 162 """A target in a GN build description along with its dependencies.""" 163 164 def __init__(self, target_name): 165 self.target_name = target_name 166 self.dependencies = set() 167 168 def add_dependency(self, target_node): 169 if target_node in self.dependencies: 170 return 171 self.dependencies.add(target_node) 172 173 def iterate_depth_first(self): 174 for node in sorted(self.dependencies, key=lambda n: n.target_name): 175 for node in node.iterate_depth_first(): 176 yield node 177 if self.target_name: 178 yield self 179 180 181class DependencyTree(object): 182 """A tree of GN build target dependencies.""" 183 184 def __init__(self): 185 self.target_to_node_map = {} 186 self.root = self._get_or_create_node(None) 187 188 def _get_or_create_node(self, target_name): 189 if target_name in self.target_to_node_map: 190 return self.target_to_node_map[target_name] 191 node = DependencyNode(target_name) 192 self.target_to_node_map[target_name] = node 193 return node 194 195 def add_dependency(self, from_target, to_target): 196 from_node = self._get_or_create_node(from_target) 197 to_node = self._get_or_create_node(to_target) 198 assert from_node is not to_node 199 from_node.add_dependency(to_node) 200 201 def iterate_depth_first(self): 202 for node in self.root.iterate_depth_first(): 203 yield node 204 205 206class AmalgamatedProject(object): 207 """In-memory representation of an amalgamated source/header pair.""" 208 209 def __init__(self, desc, source_deps, compute_deps_only=False): 210 """Constructor. 211 212 Args: 213 desc: JSON build description. 214 source_deps: A map of (source file, [dependency header]) which is 215 to detect which header files are included by each source file. 216 compute_deps_only: If True, the project will only be used to compute 217 dependency information. Use |get_source_files()| to retrieve 218 the result. 219 """ 220 self.desc = desc 221 self.source_deps = source_deps 222 self.header = [] 223 self.source = [] 224 self.source_defines = [] 225 # Note that we don't support multi-arg flags. 226 self.cflags = set(default_cflags) 227 self.ldflags = set() 228 self.defines = set() 229 self.libs = set() 230 self._dependency_tree = DependencyTree() 231 self._processed_sources = set() 232 self._processed_headers = set() 233 self._processed_header_deps = set() 234 self._processed_source_headers = set() # Header files included from .cc 235 self._include_re = re.compile(r'#include "(.*)"') 236 self._compute_deps_only = compute_deps_only 237 238 def add_target(self, target_name): 239 """Include |target_name| in the amalgamated result.""" 240 self._dependency_tree.add_dependency(None, target_name) 241 self._add_target_dependencies(target_name) 242 self._add_target_flags(target_name) 243 self._add_target_headers(target_name) 244 245 # Recurse into target deps, but only for protos. This generates headers 246 # for all the .{pbzero,gen}.h files, even if they don't #include each other. 247 for _, dep in self._iterate_dep_edges(target_name): 248 if (dep not in self._processed_header_deps and 249 re.match(recurse_in_header_deps, dep)): 250 self._processed_header_deps.add(dep) 251 self.add_target(dep) 252 253 def _iterate_dep_edges(self, target_name): 254 target = self.desc[target_name] 255 for dep in target.get('deps', []): 256 # Ignore system libraries since they will be added as build-time 257 # dependencies. 258 if dep in system_library_map: 259 continue 260 # Don't descend into build action dependencies. 261 if self.desc[dep]['type'] == 'action': 262 continue 263 for sub_target, sub_dep in self._iterate_dep_edges(dep): 264 yield sub_target, sub_dep 265 yield target_name, dep 266 267 def _iterate_target_and_deps(self, target_name): 268 yield target_name 269 for _, dep in self._iterate_dep_edges(target_name): 270 yield dep 271 272 def _add_target_dependencies(self, target_name): 273 for target, dep in self._iterate_dep_edges(target_name): 274 self._dependency_tree.add_dependency(target, dep) 275 276 def process_dep(dep): 277 if dep in system_library_map: 278 self.libs.update(system_library_map[dep].get('libs', [])) 279 self.cflags.update(system_library_map[dep].get('cflags', [])) 280 self.defines.update(system_library_map[dep].get('defines', [])) 281 return True 282 283 def walk_all_deps(target_name): 284 target = self.desc[target_name] 285 for dep in target.get('deps', []): 286 if process_dep(dep): 287 return 288 walk_all_deps(dep) 289 290 walk_all_deps(target_name) 291 292 def _filter_cflags(self, cflags): 293 # Since we want to deduplicate flags, combine two-part switches (e.g., 294 # "-foo bar") into one value ("-foobar") so we can store the result as 295 # a set. 296 result = [] 297 for flag in cflags: 298 if flag.startswith('-'): 299 result.append(flag) 300 else: 301 result[-1] += flag 302 return apply_allowlist(cflag_allowlist, result) 303 304 def _add_target_flags(self, target_name): 305 for target_name in self._iterate_target_and_deps(target_name): 306 target = self.desc[target_name] 307 self.cflags.update(self._filter_cflags(target.get('cflags', []))) 308 self.cflags.update(self._filter_cflags(target.get('cflags_cc', []))) 309 self.ldflags.update( 310 apply_allowlist(ldflag_allowlist, target.get('ldflags', []))) 311 self.libs.update(apply_denylist(lib_denylist, target.get('libs', []))) 312 self.defines.update( 313 apply_allowlist(define_allowlist, target.get('defines', []))) 314 315 def _add_target_headers(self, target_name): 316 target = self.desc[target_name] 317 if not 'sources' in target: 318 return 319 headers = [ 320 gn_utils.label_to_path(s) for s in target['sources'] if s.endswith('.h') 321 ] 322 for header in headers: 323 self._add_header(target_name, header) 324 325 def _get_include_dirs(self, target_name): 326 include_dirs = set() 327 for target_name in self._iterate_target_and_deps(target_name): 328 target = self.desc[target_name] 329 if 'include_dirs' in target: 330 include_dirs.update( 331 [gn_utils.label_to_path(d) for d in target['include_dirs']]) 332 return include_dirs 333 334 def _add_source_included_header(self, include_dirs, allowed_files, 335 header_name): 336 if header_name in self._processed_headers: 337 return 338 if header_name in self._processed_source_headers: 339 return 340 self._processed_source_headers.add(header_name) 341 for include_dir in include_dirs: 342 rel_path = os.path.join(include_dir, header_name) 343 full_path = os.path.join(gn_utils.repo_root(), rel_path) 344 if os.path.exists(full_path): 345 if not rel_path in allowed_files: 346 return 347 with open(full_path) as f: 348 self.source.append('// %s begin header: %s' % 349 (tool_name, normalize_path(full_path))) 350 self.source.extend( 351 self._process_source_includes(include_dirs, allowed_files, f)) 352 return 353 if self._compute_deps_only: 354 return 355 msg = 'Looked in %s' % ', '.join('"%s"' % d for d in include_dirs) 356 raise Error('Header file %s not found. %s' % (header_name, msg)) 357 358 def _add_source(self, target_name, source_name): 359 if source_name in self._processed_sources: 360 return 361 self._processed_sources.add(source_name) 362 include_dirs = self._get_include_dirs(target_name) 363 deps = self.source_deps[source_name] 364 full_path = os.path.join(gn_utils.repo_root(), source_name) 365 if not os.path.exists(full_path): 366 raise Error('Source file %s not found' % source_name) 367 with open(full_path) as f: 368 self.source.append('// %s begin source: %s' % 369 (tool_name, normalize_path(full_path))) 370 try: 371 self.source.extend( 372 self._patch_source( 373 source_name, 374 self._process_source_includes(include_dirs, deps, f))) 375 except Error as e: 376 raise Error('Failed adding source %s: %s' % (source_name, e.message)) 377 378 def _add_header_included_header(self, include_dirs, header_name): 379 if header_name in self._processed_headers: 380 return 381 self._processed_headers.add(header_name) 382 for include_dir in include_dirs: 383 full_path = os.path.join(gn_utils.repo_root(), include_dir, header_name) 384 if os.path.exists(full_path): 385 with open(full_path) as f: 386 self.header.append('// %s begin header: %s' % 387 (tool_name, normalize_path(full_path))) 388 self.header.extend(self._process_header_includes(include_dirs, f)) 389 return 390 if self._compute_deps_only: 391 return 392 msg = 'Looked in %s' % ', '.join('"%s"' % d for d in include_dirs) 393 raise Error('Header file %s not found. %s' % (header_name, msg)) 394 395 def _add_header(self, target_name, header_name): 396 if header_name in self._processed_headers: 397 return 398 self._processed_headers.add(header_name) 399 include_dirs = self._get_include_dirs(target_name) 400 full_path = os.path.join(gn_utils.repo_root(), header_name) 401 if not os.path.exists(full_path): 402 if self._compute_deps_only: 403 return 404 raise Error('Header file %s not found' % header_name) 405 with open(full_path) as f: 406 self.header.append('// %s begin header: %s' % 407 (tool_name, normalize_path(full_path))) 408 try: 409 self.header.extend(self._process_header_includes(include_dirs, f)) 410 except Error as e: 411 raise Error('Failed adding header %s: %s' % (header_name, e.message)) 412 413 def _patch_source(self, source_name, lines): 414 result = [] 415 namespace = re.sub(r'[^a-z]', '_', 416 os.path.splitext(os.path.basename(source_name))[0]) 417 for line in lines: 418 # Protobuf generates an identical anonymous function into each 419 # message description. Rename all but the first occurrence to avoid 420 # duplicate symbol definitions. 421 line = line.replace('MergeFromFail', '%s_MergeFromFail' % namespace) 422 result.append(line) 423 return result 424 425 def _process_source_includes(self, include_dirs, allowed_files, file): 426 result = [] 427 for line in file: 428 line = line.rstrip('\n') 429 m = self._include_re.match(line) 430 if not m: 431 result.append(line) 432 continue 433 elif re.match(includes_to_remove, m.group(1)): 434 result.append('// %s removed: %s' % (tool_name, line)) 435 else: 436 result.append('// %s expanded: %s' % (tool_name, line)) 437 self._add_source_included_header(include_dirs, allowed_files, 438 m.group(1)) 439 return result 440 441 def _process_header_includes(self, include_dirs, file): 442 result = [] 443 for line in file: 444 line = line.rstrip('\n') 445 m = self._include_re.match(line) 446 if not m: 447 result.append(line) 448 continue 449 elif re.match(includes_to_remove, m.group(1)): 450 result.append('// %s removed: %s' % (tool_name, line)) 451 else: 452 result.append('// %s expanded: %s' % (tool_name, line)) 453 self._add_header_included_header(include_dirs, m.group(1)) 454 return result 455 456 def generate(self): 457 """Prepares the output for this amalgamated project. 458 459 Call save() to persist the result. 460 """ 461 assert not self._compute_deps_only 462 self.source_defines.append('// %s: predefined macros' % tool_name) 463 464 def add_define(name): 465 # Valued macros aren't supported for now. 466 assert '=' not in name 467 self.source_defines.append('#if !defined(%s)' % name) 468 self.source_defines.append('#define %s' % name) 469 self.source_defines.append('#endif') 470 471 for name in self.defines: 472 add_define(name) 473 for target_name, source_name in self.get_source_files(): 474 self._add_source(target_name, source_name) 475 476 def get_source_files(self): 477 """Return a list of (target, [source file]) that describes the source 478 files pulled in by each target which is a dependency of this project. 479 """ 480 source_files = [] 481 for node in self._dependency_tree.iterate_depth_first(): 482 target = self.desc[node.target_name] 483 if not 'sources' in target: 484 continue 485 sources = [(node.target_name, gn_utils.label_to_path(s)) 486 for s in target['sources'] 487 if s.endswith('.cc')] 488 source_files.extend(sources) 489 return source_files 490 491 def _get_nice_path(self, prefix, format): 492 basename = os.path.basename(prefix) 493 return os.path.join( 494 os.path.relpath(os.path.dirname(prefix)), format % basename) 495 496 def _make_directories(self, directory): 497 if not os.path.isdir(directory): 498 os.makedirs(directory) 499 500 def save(self, output_prefix, system_buildtools=False): 501 """Save the generated header and source file pair. 502 503 Returns a message describing the output with build instructions. 504 """ 505 header_file = self._get_nice_path(output_prefix, '%s.h') 506 source_file = self._get_nice_path(output_prefix, '%s.cc') 507 self._make_directories(os.path.dirname(header_file)) 508 self._make_directories(os.path.dirname(source_file)) 509 with open(header_file, 'w') as f: 510 f.write('\n'.join([preamble] + self.header + ['\n'])) 511 with open(source_file, 'w') as f: 512 include_stmt = '#include "%s"' % os.path.basename(header_file) 513 f.write('\n'.join([preamble] + self.source_defines + [include_stmt] + 514 self.source + ['\n'])) 515 build_cmd = self.get_build_command(output_prefix, system_buildtools) 516 return """Amalgamated project written to %s and %s. 517 518Build settings: 519 - cflags: %s 520 - ldflags: %s 521 - libs: %s 522 523Example build command: 524 525%s 526""" % (header_file, source_file, ' '.join(self.cflags), ' '.join( 527 self.ldflags), ' '.join(self.libs), ' '.join(build_cmd)) 528 529 def get_build_command(self, output_prefix, system_buildtools=False): 530 """Returns an example command line for building the output source.""" 531 source = self._get_nice_path(output_prefix, '%s.cc') 532 library = self._get_nice_path(output_prefix, 'lib%s.so') 533 534 if sys.platform.startswith('linux') and not system_buildtools: 535 llvm_script = os.path.join(gn_utils.repo_root(), 'gn', 'standalone', 536 'toolchain', 'linux_find_llvm.py') 537 cxx = subprocess.check_output([llvm_script]).splitlines()[2].decode() 538 else: 539 cxx = 'clang++' 540 541 build_cmd = [cxx, source, '-o', library, '-shared'] + \ 542 sorted(self.cflags) + sorted(self.ldflags) 543 for lib in sorted(self.libs): 544 build_cmd.append('-l%s' % lib) 545 return build_cmd 546 547 548def main(): 549 parser = argparse.ArgumentParser( 550 description='Generate an amalgamated header/source pair from a GN ' 551 'build description.') 552 parser.add_argument( 553 '--out', 554 help='The name of the temporary build folder in \'out\'', 555 default='tmp.gen_amalgamated.%u' % os.getpid()) 556 parser.add_argument( 557 '--output', 558 help='Base name of files to create. A .cc/.h extension will be added', 559 default=os.path.join(gn_utils.repo_root(), 'out/amalgamated/perfetto')) 560 parser.add_argument( 561 '--gn_args', 562 help='GN arguments used to prepare the output directory', 563 default=gn_args) 564 parser.add_argument( 565 '--keep', 566 help='Don\'t delete the GN output directory at exit', 567 action='store_true') 568 parser.add_argument( 569 '--build', help='Also compile the generated files', action='store_true') 570 parser.add_argument( 571 '--check', help='Don\'t keep the generated files', action='store_true') 572 parser.add_argument('--quiet', help='Only report errors', action='store_true') 573 parser.add_argument( 574 '--dump-deps', 575 help='List all source files that the amalgamated output depends on', 576 action='store_true') 577 parser.add_argument( 578 '--system_buildtools', 579 help='Use the buildtools (e.g. gn) preinstalled in the system instead ' 580 'of the hermetic ones', 581 action='store_true') 582 parser.add_argument( 583 'targets', 584 nargs=argparse.REMAINDER, 585 help='Targets to include in the output (e.g., "//:libperfetto")') 586 args = parser.parse_args() 587 targets = args.targets or default_targets 588 589 # The CHANGELOG mtime triggers the perfetto_version.gen.h genrule. This is 590 # to avoid emitting a stale version information in the remote case of somebody 591 # running gen_amalgamated incrementally after having moved to another commit. 592 changelog_path = os.path.join(project_root, 'CHANGELOG') 593 assert (os.path.exists(changelog_path)) 594 subprocess.check_call(['touch', '-c', changelog_path]) 595 596 output = args.output 597 if args.check: 598 output = os.path.join(tempfile.mkdtemp(), 'perfetto_amalgamated') 599 600 out = gn_utils.prepare_out_directory(args.gn_args, 601 args.out, 602 system_buildtools=args.system_buildtools) 603 if not args.quiet: 604 print('Building project...') 605 try: 606 desc = gn_utils.load_build_description(out, args.system_buildtools) 607 608 # We need to build everything first so that the necessary header 609 # dependencies get generated. However if we are just dumping dependency 610 # information this can be skipped, allowing cross-platform operation. 611 if not args.dump_deps: 612 gn_utils.build_targets(out, targets, 613 system_buildtools=args.system_buildtools) 614 source_deps = gn_utils.compute_source_dependencies(out, 615 args.system_buildtools) 616 project = AmalgamatedProject( 617 desc, source_deps, compute_deps_only=args.dump_deps) 618 619 for target in targets: 620 project.add_target(target) 621 622 if args.dump_deps: 623 source_files = [ 624 source_file for _, source_file in project.get_source_files() 625 ] 626 print('\n'.join(sorted(set(source_files)))) 627 return 628 629 project.generate() 630 result = project.save(output, args.system_buildtools) 631 if not args.quiet: 632 print(result) 633 if args.build: 634 if not args.quiet: 635 sys.stdout.write('Building amalgamated project...') 636 sys.stdout.flush() 637 subprocess.check_call(project.get_build_command(output, 638 args.system_buildtools)) 639 if not args.quiet: 640 print('done') 641 finally: 642 if not args.keep: 643 shutil.rmtree(out) 644 if args.check: 645 shutil.rmtree(os.path.dirname(output)) 646 647 648if __name__ == '__main__': 649 sys.exit(main()) 650