1# Copyright (C) 2019 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15# A collection of utilities for extracting build rule information from GN 16# projects. 17 18from __future__ import print_function 19import collections 20import errno 21import filecmp 22import json 23import os 24import re 25import shutil 26import subprocess 27import sys 28from compat import iteritems 29 30BUILDFLAGS_TARGET = '//gn:gen_buildflags' 31GEN_VERSION_TARGET = '//src/base:version_gen_h' 32TARGET_TOOLCHAIN = '//gn/standalone/toolchain:gcc_like_host' 33HOST_TOOLCHAIN = '//gn/standalone/toolchain:gcc_like_host' 34LINKER_UNIT_TYPES = ('executable', 'shared_library', 'static_library') 35 36# TODO(primiano): investigate these, they require further componentization. 37ODR_VIOLATION_IGNORE_TARGETS = { 38 '//test/cts:perfetto_cts_deps', 39 '//:perfetto_integrationtests', 40} 41 42 43def _check_command_output(cmd, cwd): 44 try: 45 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, cwd=cwd) 46 except subprocess.CalledProcessError as e: 47 print( 48 'Command "{}" failed in {}:'.format(' '.join(cmd), cwd), 49 file=sys.stderr) 50 print(e.output.decode(), file=sys.stderr) 51 sys.exit(1) 52 else: 53 return output.decode() 54 55 56def repo_root(): 57 """Returns an absolute path to the repository root.""" 58 return os.path.join( 59 os.path.realpath(os.path.dirname(__file__)), os.path.pardir) 60 61 62def _tool_path(name): 63 return os.path.join(repo_root(), 'tools', name) 64 65 66def prepare_out_directory(gn_args, name, root=repo_root()): 67 """Creates the JSON build description by running GN. 68 69 Returns (path, desc) where |path| is the location of the output directory 70 and |desc| is the JSON build description. 71 """ 72 out = os.path.join(root, 'out', name) 73 try: 74 os.makedirs(out) 75 except OSError as e: 76 if e.errno != errno.EEXIST: 77 raise 78 _check_command_output([_tool_path('gn'), 'gen', out, 79 '--args=%s' % gn_args], 80 cwd=repo_root()) 81 return out 82 83 84def load_build_description(out): 85 """Creates the JSON build description by running GN.""" 86 desc = _check_command_output([ 87 _tool_path('gn'), 'desc', out, '--format=json', '--all-toolchains', '//*' 88 ], 89 cwd=repo_root()) 90 return json.loads(desc) 91 92 93def create_build_description(gn_args, root=repo_root()): 94 """Prepares a GN out directory and loads the build description from it. 95 96 The temporary out directory is automatically deleted. 97 """ 98 out = prepare_out_directory(gn_args, 'tmp.gn_utils', root=root) 99 try: 100 return load_build_description(out) 101 finally: 102 shutil.rmtree(out) 103 104 105def build_targets(out, targets, quiet=False): 106 """Runs ninja to build a list of GN targets in the given out directory. 107 108 Compiling these targets is required so that we can include any generated 109 source files in the amalgamated result. 110 """ 111 targets = [t.replace('//', '') for t in targets] 112 with open(os.devnull, 'w') as devnull: 113 stdout = devnull if quiet else None 114 subprocess.check_call( 115 [_tool_path('ninja')] + targets, cwd=out, stdout=stdout) 116 117 118def compute_source_dependencies(out): 119 """For each source file, computes a set of headers it depends on.""" 120 ninja_deps = _check_command_output([_tool_path('ninja'), '-t', 'deps'], 121 cwd=out) 122 deps = {} 123 current_source = None 124 for line in ninja_deps.split('\n'): 125 filename = os.path.relpath(os.path.join(out, line.strip()), repo_root()) 126 if not line or line[0] != ' ': 127 current_source = None 128 continue 129 elif not current_source: 130 # We're assuming the source file is always listed before the 131 # headers. 132 assert os.path.splitext(line)[1] in ['.c', '.cc', '.cpp', '.S'] 133 current_source = filename 134 deps[current_source] = [] 135 else: 136 assert current_source 137 deps[current_source].append(filename) 138 return deps 139 140 141def label_to_path(label): 142 """Turn a GN output label (e.g., //some_dir/file.cc) into a path.""" 143 assert label.startswith('//') 144 return label[2:] 145 146 147def label_without_toolchain(label): 148 """Strips the toolchain from a GN label. 149 150 Return a GN label (e.g //buildtools:protobuf(//gn/standalone/toolchain: 151 gcc_like_host) without the parenthesised toolchain part. 152 """ 153 return label.split('(')[0] 154 155 156def label_to_target_name_with_path(label): 157 """ 158 Turn a GN label into a target name involving the full path. 159 e.g., //src/perfetto:tests -> src_perfetto_tests 160 """ 161 name = re.sub(r'^//:?', '', label) 162 name = re.sub(r'[^a-zA-Z0-9_]', '_', name) 163 return name 164 165 166def gen_buildflags(gn_args, target_file): 167 """Generates the perfetto_build_flags.h for the given config. 168 169 target_file: the path, relative to the repo root, where the generated 170 buildflag header will be copied into. 171 """ 172 tmp_out = prepare_out_directory(gn_args, 'tmp.gen_buildflags') 173 build_targets(tmp_out, [BUILDFLAGS_TARGET], quiet=True) 174 src = os.path.join(tmp_out, 'gen', 'build_config', 'perfetto_build_flags.h') 175 shutil.copy(src, os.path.join(repo_root(), target_file)) 176 shutil.rmtree(tmp_out) 177 178 179def check_or_commit_generated_files(tmp_files, check): 180 """Checks that gen files are unchanged or renames them to the final location 181 182 Takes in input a list of 'xxx.swp' files that have been written. 183 If check == False, it renames xxx.swp -> xxx. 184 If check == True, it just checks that the contents of 'xxx.swp' == 'xxx'. 185 Returns 0 if no diff was detected, 1 otherwise (to be used as exit code). 186 """ 187 res = 0 188 for tmp_file in tmp_files: 189 assert (tmp_file.endswith('.swp')) 190 target_file = os.path.relpath(tmp_file[:-4]) 191 if check: 192 if not filecmp.cmp(tmp_file, target_file): 193 sys.stderr.write('%s needs to be regenerated\n' % target_file) 194 res = 1 195 os.unlink(tmp_file) 196 else: 197 os.rename(tmp_file, target_file) 198 return res 199 200 201class ODRChecker(object): 202 """Detects ODR violations in linker units 203 204 When we turn GN source sets into Soong & Bazel file groups, there is the risk 205 to create ODR violations by including the same file group into different 206 linker unit (this is because other build systems don't have a concept 207 equivalent to GN's source_set). This class navigates the transitive 208 dependencies (mostly static libraries) of a target and detects if multiple 209 paths end up including the same file group. This is to avoid situations like: 210 211 traced.exe -> base(file group) 212 traced.exe -> libperfetto(static lib) -> base(file group) 213 """ 214 215 def __init__(self, gn, target_name): 216 self.gn = gn 217 self.root = gn.get_target(target_name) 218 self.source_sets = collections.defaultdict(set) 219 self.deps_visited = set() 220 self.source_set_hdr_only = {} 221 222 self._visit(target_name) 223 num_violations = 0 224 if target_name in ODR_VIOLATION_IGNORE_TARGETS: 225 return 226 for sset, paths in self.source_sets.items(): 227 if self.is_header_only(sset): 228 continue 229 if len(paths) != 1: 230 num_violations += 1 231 print( 232 'ODR violation in target %s, multiple paths include %s:\n %s' % 233 (target_name, sset, '\n '.join(paths)), 234 file=sys.stderr) 235 if num_violations > 0: 236 raise Exception('%d ODR violations detected. Build generation aborted' % 237 num_violations) 238 239 def _visit(self, target_name, parent_path=''): 240 target = self.gn.get_target(target_name) 241 path = ((parent_path + ' > ') if parent_path else '') + target_name 242 if not target: 243 raise Exception('Cannot find target %s' % target_name) 244 for ssdep in target.source_set_deps: 245 name_and_path = '%s (via %s)' % (target_name, path) 246 self.source_sets[ssdep].add(name_and_path) 247 deps = set(target.deps).union(target.proto_deps) - self.deps_visited 248 for dep_name in deps: 249 dep = self.gn.get_target(dep_name) 250 if dep.type == 'executable': 251 continue # Execs are strong boundaries and don't cause ODR violations. 252 # static_library dependencies should reset the path. It doesn't matter if 253 # we get to a source file via: 254 # source_set1 > static_lib > source.cc OR 255 # source_set1 > source_set2 > static_lib > source.cc 256 # This is NOT an ODR violation because source.cc is linked from the same 257 # static library 258 next_parent_path = path if dep.type != 'static_library' else '' 259 self.deps_visited.add(dep_name) 260 self._visit(dep_name, next_parent_path) 261 262 def is_header_only(self, source_set_name): 263 cached = self.source_set_hdr_only.get(source_set_name) 264 if cached is not None: 265 return cached 266 target = self.gn.get_target(source_set_name) 267 if target.type != 'source_set': 268 raise TypeError('%s is not a source_set' % source_set_name) 269 res = all(src.endswith('.h') for src in target.sources) 270 self.source_set_hdr_only[source_set_name] = res 271 return res 272 273 274class GnParser(object): 275 """A parser with some cleverness for GN json desc files 276 277 The main goals of this parser are: 278 1) Deal with the fact that other build systems don't have an equivalent 279 notion to GN's source_set. Conversely to Bazel's and Soong's filegroups, 280 GN source_sets expect that dependencies, cflags and other source_set 281 properties propagate up to the linker unit (static_library, executable or 282 shared_library). This parser simulates the same behavior: when a 283 source_set is encountered, some of its variables (cflags and such) are 284 copied up to the dependent targets. This is to allow gen_xxx to create 285 one filegroup for each source_set and then squash all the other flags 286 onto the linker unit. 287 2) Detect and special-case protobuf targets, figuring out the protoc-plugin 288 being used. 289 """ 290 291 class Target(object): 292 """Reperesents A GN target. 293 294 Maked properties are propagated up the dependency chain when a 295 source_set dependency is encountered. 296 """ 297 298 def __init__(self, name, type): 299 self.name = name # e.g. //src/ipc:ipc 300 301 VALID_TYPES = ('static_library', 'shared_library', 'executable', 'group', 302 'action', 'source_set', 'proto_library') 303 assert (type in VALID_TYPES) 304 self.type = type 305 self.testonly = False 306 self.toolchain = None 307 308 # These are valid only for type == proto_library. 309 # This is typically: 'proto', 'protozero', 'ipc'. 310 self.proto_plugin = None 311 self.proto_paths = set() 312 313 self.sources = set() 314 # TODO(primiano): consider whether the public section should be part of 315 # bubbled-up sources. 316 self.public_headers = set() # 'public' 317 318 # These are valid only for type == 'action' 319 self.inputs = set() 320 self.outputs = set() 321 self.script = None 322 self.args = [] 323 324 # These variables are propagated up when encountering a dependency 325 # on a source_set target. 326 self.cflags = set() 327 self.defines = set() 328 self.deps = set() 329 self.libs = set() 330 self.include_dirs = set() 331 self.ldflags = set() 332 self.source_set_deps = set() # Transitive set of source_set deps. 333 self.proto_deps = set() # Transitive set of protobuf deps. 334 335 # Deps on //gn:xxx have this flag set to True. These dependencies 336 # are special because they pull third_party code from buildtools/. 337 # We don't want to keep recursing into //buildtools in generators, 338 # this flag is used to stop the recursion and create an empty 339 # placeholder target once we hit //gn:protoc or similar. 340 self.is_third_party_dep_ = False 341 342 def __lt__(self, other): 343 if isinstance(other, self.__class__): 344 return self.name < other.name 345 raise TypeError( 346 '\'<\' not supported between instances of \'%s\' and \'%s\'' % 347 (type(self).__name__, type(other).__name__)) 348 349 def __repr__(self): 350 return json.dumps({ 351 k: (list(sorted(v)) if isinstance(v, set) else v) 352 for (k, v) in iteritems(self.__dict__) 353 }, 354 indent=4, 355 sort_keys=True) 356 357 def update(self, other): 358 for key in ('cflags', 'defines', 'deps', 'include_dirs', 'ldflags', 359 'source_set_deps', 'proto_deps', 'libs', 'proto_paths'): 360 self.__dict__[key].update(other.__dict__.get(key, [])) 361 362 def __init__(self, gn_desc): 363 self.gn_desc_ = gn_desc 364 self.all_targets = {} 365 self.linker_units = {} # Executables, shared or static libraries. 366 self.source_sets = {} 367 self.actions = {} 368 self.proto_libs = {} 369 370 def get_target(self, gn_target_name): 371 """Returns a Target object from the fully qualified GN target name. 372 373 It bubbles up variables from source_set dependencies as described in the 374 class-level comments. 375 """ 376 target = self.all_targets.get(gn_target_name) 377 if target is not None: 378 return target # Target already processed. 379 380 desc = self.gn_desc_[gn_target_name] 381 target = GnParser.Target(gn_target_name, desc['type']) 382 target.testonly = desc.get('testonly', False) 383 target.toolchain = desc.get('toolchain', None) 384 self.all_targets[gn_target_name] = target 385 386 # We should never have GN targets directly depend on buidtools. They 387 # should hop via //gn:xxx, so we can give generators an opportunity to 388 # override them. 389 assert (not gn_target_name.startswith('//buildtools')) 390 391 # Don't descend further into third_party targets. Genrators are supposed 392 # to either ignore them or route to other externally-provided targets. 393 if gn_target_name.startswith('//gn'): 394 target.is_third_party_dep_ = True 395 return target 396 397 proto_target_type, proto_desc = self.get_proto_target_type_(target) 398 if proto_target_type is not None: 399 self.proto_libs[target.name] = target 400 target.type = 'proto_library' 401 target.proto_plugin = proto_target_type 402 target.proto_paths.update(self.get_proto_paths(proto_desc)) 403 target.sources.update(proto_desc.get('sources', [])) 404 assert (all(x.endswith('.proto') for x in target.sources)) 405 elif target.type == 'source_set': 406 self.source_sets[gn_target_name] = target 407 target.sources.update(desc.get('sources', [])) 408 elif target.type in LINKER_UNIT_TYPES: 409 self.linker_units[gn_target_name] = target 410 target.sources.update(desc.get('sources', [])) 411 elif target.type == 'action': 412 self.actions[gn_target_name] = target 413 target.inputs.update(desc.get('inputs', [])) 414 target.sources.update(desc.get('sources', [])) 415 outs = [re.sub('^//out/.+?/gen/', '', x) for x in desc['outputs']] 416 target.outputs.update(outs) 417 target.script = desc['script'] 418 # Args are typically relative to the root build dir (../../xxx) 419 # because root build dir is typically out/xxx/). 420 target.args = [re.sub('^../../', '//', x) for x in desc['args']] 421 422 # Default for 'public' is //* - all headers in 'sources' are public. 423 # TODO(primiano): if a 'public' section is specified (even if empty), then 424 # the rest of 'sources' is considered inaccessible by gn. Consider 425 # emulating that, so that generated build files don't end up with overly 426 # accessible headers. 427 public_headers = [x for x in desc.get('public', []) if x != '*'] 428 target.public_headers.update(public_headers) 429 430 target.cflags.update(desc.get('cflags', []) + desc.get('cflags_cc', [])) 431 target.libs.update(desc.get('libs', [])) 432 target.ldflags.update(desc.get('ldflags', [])) 433 target.defines.update(desc.get('defines', [])) 434 target.include_dirs.update(desc.get('include_dirs', [])) 435 436 # Recurse in dependencies. 437 for dep_name in desc.get('deps', []): 438 dep = self.get_target(dep_name) 439 if dep.is_third_party_dep_: 440 target.deps.add(dep_name) 441 elif dep.type == 'proto_library': 442 target.proto_deps.add(dep_name) 443 target.proto_paths.update(dep.proto_paths) 444 445 # Don't bubble deps for action targets 446 if target.type != 'action': 447 target.proto_deps.update(dep.proto_deps) # Bubble up deps. 448 elif dep.type == 'source_set': 449 target.source_set_deps.add(dep_name) 450 target.update(dep) # Bubble up source set's cflags/ldflags etc. 451 elif dep.type == 'group': 452 target.update(dep) # Bubble up groups's cflags/ldflags etc. 453 elif dep.type == 'action': 454 if proto_target_type is None: 455 target.deps.add(dep_name) 456 elif dep.type in LINKER_UNIT_TYPES: 457 target.deps.add(dep_name) 458 459 return target 460 461 def get_proto_paths(self, proto_desc): 462 # import_dirs in metadata will be available for source_set targets. 463 metadata = proto_desc.get('metadata', {}) 464 import_dirs = metadata.get('import_dirs', []) 465 if import_dirs: 466 return import_dirs 467 468 # For all non-source-set targets, we need to parse the command line 469 # of the protoc invocation. 470 proto_paths = [] 471 args = proto_desc.get('args', []) 472 for i, arg in enumerate(args): 473 if arg != '--proto_path': 474 continue 475 proto_paths.append(re.sub('^../../', '//', args[i + 1])) 476 return proto_paths 477 478 def get_proto_target_type_(self, target): 479 """ Checks if the target is a proto library and return the plugin. 480 481 Returns: 482 (None, None): if the target is not a proto library. 483 (plugin, proto_desc) where |plugin| is 'proto' in the default (lite) 484 case or 'protozero' or 'ipc' or 'descriptor'; |proto_desc| is the GN 485 json desc of the target with the .proto sources (_gen target for 486 non-descriptor types or the target itself for descriptor type). 487 """ 488 parts = target.name.split('(', 1) 489 name = parts[0] 490 toolchain = '(' + parts[1] if len(parts) > 1 else '' 491 492 # Descriptor targets don't have a _gen target; instead we look for the 493 # characteristic flag in the args of the target itself. 494 desc = self.gn_desc_.get(target.name) 495 if '--descriptor_set_out' in desc.get('args', []): 496 return 'descriptor', desc 497 498 # Source set proto targets have a non-empty proto_library_sources in the 499 # metadata of the descirption. 500 metadata = desc.get('metadata', {}) 501 if 'proto_library_sources' in metadata: 502 return 'source_set', desc 503 504 # In all other cases, we want to look at the _gen target as that has the 505 # important information. 506 gen_desc = self.gn_desc_.get('%s_gen%s' % (name, toolchain)) 507 if gen_desc is None or gen_desc['type'] != 'action': 508 return None, None 509 args = gen_desc.get('args', []) 510 if '/protoc' not in args[0]: 511 return None, None 512 plugin = 'proto' 513 for arg in (arg for arg in args if arg.startswith('--plugin=')): 514 # |arg| at this point looks like: 515 # --plugin=protoc-gen-plugin=gcc_like_host/protozero_plugin 516 # or 517 # --plugin=protoc-gen-plugin=protozero_plugin 518 plugin = arg.split('=')[-1].split('/')[-1].replace('_plugin', '') 519 return plugin, gen_desc 520