1# Copyright (C) 2022 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15# A collection of utilities for extracting build rule information from GN 16# projects. 17 18import copy 19import json 20import logging as log 21import os 22import re 23import collections 24 25LINKER_UNIT_TYPES = ('executable', 'shared_library', 'static_library', 'source_set') 26JAVA_BANNED_SCRIPTS = [ 27 "//build/android/gyp/turbine.py", 28 "//build/android/gyp/compile_java.py", 29 "//build/android/gyp/filter_zip.py", 30 "//build/android/gyp/dex.py", 31 "//build/android/gyp/write_build_config.py", 32 "//build/android/gyp/create_r_java.py", 33 "//build/android/gyp/ijar.py", 34 "//build/android/gyp/create_r_java.py", 35 "//build/android/gyp/bytecode_processor.py", 36 "//build/android/gyp/prepare_resources.py", 37 "//build/android/gyp/aar.py", 38 "//build/android/gyp/zip.py", 39] 40RESPONSE_FILE = '{{response_file_name}}' 41TESTING_SUFFIX = "__testing" 42AIDL_INCLUDE_DIRS_REGEX = r'--includes=\[(.*)\]' 43 44def repo_root(): 45 """Returns an absolute path to the repository root.""" 46 return os.path.join( 47 os.path.realpath(os.path.dirname(__file__)), os.path.pardir) 48 49 50def _clean_string(str): 51 return str.replace('\\', '').replace('../../', '').replace('"', '').strip() 52 53def _extract_includes_from_aidl_args(args): 54 for arg in args: 55 is_match = re.match(AIDL_INCLUDE_DIRS_REGEX, arg) 56 if is_match: 57 local_includes = is_match.group(1).split(",") 58 return [_clean_string(local_include) for local_include in local_includes] 59 return [] 60 61 62def label_to_path(label): 63 """Turn a GN output label (e.g., //some_dir/file.cc) into a path.""" 64 assert label.startswith('//') 65 return label[2:] or "./" 66 67def label_without_toolchain(label): 68 """Strips the toolchain from a GN label. 69 70 Return a GN label (e.g //buildtools:protobuf(//gn/standalone/toolchain: 71 gcc_like_host) without the parenthesised toolchain part. 72 """ 73 return label.split('(')[0] 74 75 76def _is_java_source(src): 77 return os.path.splitext(src)[1] == '.java' and not src.startswith("//out/") 78 79 80class GnParser(object): 81 """A parser with some cleverness for GN json desc files 82 83 The main goals of this parser are: 84 1) Deal with the fact that other build systems don't have an equivalent 85 notion to GN's source_set. Conversely to Bazel's and Soong's filegroups, 86 GN source_sets expect that dependencies, cflags and other source_set 87 properties propagate up to the linker unit (static_library, executable or 88 shared_library). This parser simulates the same behavior: when a 89 source_set is encountered, some of its variables (cflags and such) are 90 copied up to the dependent targets. This is to allow gen_xxx to create 91 one filegroup for each source_set and then squash all the other flags 92 onto the linker unit. 93 2) Detect and special-case protobuf targets, figuring out the protoc-plugin 94 being used. 95 """ 96 97 class Target(object): 98 """Reperesents A GN target. 99 100 Maked properties are propagated up the dependency chain when a 101 source_set dependency is encountered. 102 """ 103 class Arch(): 104 """Architecture-dependent properties 105 """ 106 def __init__(self): 107 self.sources = set() 108 self.cflags = set() 109 self.defines = set() 110 self.include_dirs = set() 111 self.deps = set() 112 self.transitive_static_libs_deps = set() 113 self.source_set_deps = set() 114 self.ldflags = set() 115 116 # These are valid only for type == 'action' 117 self.inputs = set() 118 self.outputs = set() 119 self.args = [] 120 self.response_file_contents = '' 121 122 def __init__(self, name, type): 123 self.name = name # e.g. //src/ipc:ipc 124 125 VALID_TYPES = ('static_library', 'shared_library', 'executable', 'group', 126 'action', 'source_set', 'proto_library', 'copy', 'action_foreach') 127 assert (type in VALID_TYPES) 128 self.type = type 129 self.testonly = False 130 self.toolchain = None 131 132 # These are valid only for type == proto_library. 133 # This is typically: 'proto', 'protozero', 'ipc'. 134 self.proto_plugin = None 135 self.proto_paths = set() 136 self.proto_exports = set() 137 self.proto_in_dir = "" 138 139 # TODO(primiano): consider whether the public section should be part of 140 # bubbled-up sources. 141 self.public_headers = set() # 'public' 142 143 # These are valid only for type == 'action' 144 self.script = '' 145 146 # These variables are propagated up when encountering a dependency 147 # on a source_set target. 148 self.libs = set() 149 self.proto_deps = set() 150 self.transitive_proto_deps = set() 151 self.rtti = False 152 153 # TODO: come up with a better way to only run this once. 154 # is_finalized tracks whether finalize() was called on this target. 155 self.is_finalized = False 156 # 'common' is a pseudo-architecture used to store common architecture dependent properties (to 157 # make handling of common vs architecture-specific arguments more consistent). 158 self.arch = {'common': self.Arch()} 159 160 # This is used to get the name/version of libcronet 161 self.output_name = None 162 163 # Properties to forward access to common arch. 164 # TODO: delete these after the transition has been completed. 165 @property 166 def sources(self): 167 return self.arch['common'].sources 168 169 @sources.setter 170 def sources(self, val): 171 self.arch['common'].sources = val 172 173 @property 174 def inputs(self): 175 return self.arch['common'].inputs 176 177 @inputs.setter 178 def inputs(self, val): 179 self.arch['common'].inputs = val 180 181 @property 182 def outputs(self): 183 return self.arch['common'].outputs 184 185 @outputs.setter 186 def outputs(self, val): 187 self.arch['common'].outputs = val 188 189 @property 190 def args(self): 191 return self.arch['common'].args 192 193 @args.setter 194 def args(self, val): 195 self.arch['common'].args = val 196 197 @property 198 def response_file_contents(self): 199 return self.arch['common'].response_file_contents 200 201 @response_file_contents.setter 202 def response_file_contents(self, val): 203 self.arch['common'].response_file_contents = val 204 205 @property 206 def cflags(self): 207 return self.arch['common'].cflags 208 209 @property 210 def defines(self): 211 return self.arch['common'].defines 212 213 @property 214 def deps(self): 215 return self.arch['common'].deps 216 217 @property 218 def include_dirs(self): 219 return self.arch['common'].include_dirs 220 221 @property 222 def ldflags(self): 223 return self.arch['common'].ldflags 224 225 @property 226 def source_set_deps(self): 227 return self.arch['common'].source_set_deps 228 229 def host_supported(self): 230 return 'host' in self.arch 231 232 def device_supported(self): 233 return any([name.startswith('android') for name in self.arch.keys()]) 234 235 def is_linker_unit_type(self): 236 return self.type in LINKER_UNIT_TYPES 237 238 def __lt__(self, other): 239 if isinstance(other, self.__class__): 240 return self.name < other.name 241 raise TypeError( 242 '\'<\' not supported between instances of \'%s\' and \'%s\'' % 243 (type(self).__name__, type(other).__name__)) 244 245 def __repr__(self): 246 return json.dumps({ 247 k: (list(sorted(v)) if isinstance(v, set) else v) 248 for (k, v) in self.__dict__.items() 249 }, 250 indent=4, 251 sort_keys=True) 252 253 def update(self, other, arch): 254 for key in ('cflags', 'defines', 'deps', 'include_dirs', 'ldflags', 255 'source_set_deps', 'proto_deps', 'transitive_proto_deps', 256 'libs', 'proto_paths'): 257 getattr(self, key).update(getattr(other, key, [])) 258 259 for key_in_arch in ('cflags', 'defines', 'include_dirs', 'source_set_deps', 'ldflags'): 260 getattr(self.arch[arch], key_in_arch).update(getattr(other.arch[arch], key_in_arch, [])) 261 262 def get_archs(self): 263 """ Returns a dict of archs without the common arch """ 264 return {arch: val for arch, val in self.arch.items() if arch != 'common'} 265 266 def _finalize_set_attribute(self, key): 267 # Target contains the intersection of arch-dependent properties 268 getattr(self, key).update(set.intersection(*[getattr(arch, key) for arch in 269 self.get_archs().values()])) 270 271 # Deduplicate arch-dependent properties 272 for arch in self.get_archs().values(): 273 getattr(arch, key).difference_update(getattr(self, key)) 274 275 def _finalize_non_set_attribute(self, key): 276 # Only when all the arch has the same non empty value, move the value to the target common 277 val = getattr(list(self.get_archs().values())[0], key) 278 if val and all([val == getattr(arch, key) for arch in self.get_archs().values()]): 279 setattr(self, key, copy.deepcopy(val)) 280 281 def _finalize_attribute(self, key): 282 val = getattr(self, key) 283 if isinstance(val, set): 284 self._finalize_set_attribute(key) 285 elif isinstance(val, (list, str)): 286 self._finalize_non_set_attribute(key) 287 else: 288 raise TypeError(f'Unsupported type: {type(val)}') 289 290 def finalize(self): 291 """Move common properties out of arch-dependent subobjects to Target object. 292 293 TODO: find a better name for this function. 294 """ 295 if self.is_finalized: 296 return 297 self.is_finalized = True 298 299 if len(self.arch) == 1: 300 return 301 302 for key in ('sources', 'cflags', 'defines', 'include_dirs', 'deps', 'source_set_deps', 303 'inputs', 'outputs', 'args', 'response_file_contents', 'ldflags'): 304 self._finalize_attribute(key) 305 306 def get_target_name(self): 307 return self.name[self.name.find(":") + 1:] 308 309 310 def __init__(self, builtin_deps): 311 self.builtin_deps = builtin_deps 312 self.all_targets = {} 313 self.java_sources = collections.defaultdict(set) 314 self.aidl_local_include_dirs = set() 315 self.java_actions = collections.defaultdict(set) 316 317 def _get_response_file_contents(self, action_desc): 318 # response_file_contents are formatted as: 319 # ['--flags', '--flag=true && false'] and need to be formatted as: 320 # '--flags --flag=\"true && false\"' 321 flags = action_desc.get('response_file_contents', []) 322 formatted_flags = [] 323 for flag in flags: 324 if '=' in flag: 325 key, val = flag.split('=') 326 formatted_flags.append('%s=\\"%s\\"' % (key, val)) 327 else: 328 formatted_flags.append(flag) 329 330 return ' '.join(formatted_flags) 331 332 def _is_java_group(self, type_, target_name): 333 # Per https://chromium.googlesource.com/chromium/src/build/+/HEAD/android/docs/java_toolchain.md 334 # java target names must end in "_java". 335 # TODO: There are some other possible variations we might need to support. 336 return type_ == 'group' and target_name.endswith('_java') 337 338 def _get_arch(self, toolchain): 339 if toolchain == '//build/toolchain/android:android_clang_x86': 340 return 'android_x86' 341 elif toolchain == '//build/toolchain/android:android_clang_x64': 342 return 'android_x86_64' 343 elif toolchain == '//build/toolchain/android:android_clang_arm': 344 return 'android_arm' 345 elif toolchain == '//build/toolchain/android:android_clang_arm64': 346 return 'android_arm64' 347 else: 348 return 'host' 349 350 def get_target(self, gn_target_name): 351 """Returns a Target object from the fully qualified GN target name. 352 353 get_target() requires that parse_gn_desc() has already been called. 354 """ 355 # Run this every time as parse_gn_desc can be called at any time. 356 for target in self.all_targets.values(): 357 target.finalize() 358 359 return self.all_targets[label_without_toolchain(gn_target_name)] 360 361 def parse_gn_desc(self, gn_desc, gn_target_name, java_group_name=None, is_test_target=False): 362 """Parses a gn desc tree and resolves all target dependencies. 363 364 It bubbles up variables from source_set dependencies as described in the 365 class-level comments. 366 """ 367 # Use name without toolchain for targets to support targets built for 368 # multiple archs. 369 target_name = label_without_toolchain(gn_target_name) 370 desc = gn_desc[gn_target_name] 371 type_ = desc['type'] 372 arch = self._get_arch(desc['toolchain']) 373 374 if self._is_java_group(type_, target_name): 375 java_group_name = target_name 376 377 if is_test_target: 378 target_name += TESTING_SUFFIX 379 380 target = self.all_targets.get(target_name) 381 if target is None: 382 target = GnParser.Target(target_name, type_) 383 self.all_targets[target_name] = target 384 385 if arch not in target.arch: 386 target.arch[arch] = GnParser.Target.Arch() 387 else: 388 return target # Target already processed. 389 390 if target.name in self.builtin_deps: 391 # return early, no need to parse any further as the module is a builtin. 392 return target 393 394 target.testonly = desc.get('testonly', False) 395 396 proto_target_type, proto_desc = self.get_proto_target_type(gn_desc, gn_target_name) 397 if proto_target_type is not None: 398 target.type = 'proto_library' 399 target.proto_plugin = proto_target_type 400 target.proto_paths.update(self.get_proto_paths(proto_desc)) 401 target.proto_exports.update(self.get_proto_exports(proto_desc)) 402 target.proto_in_dir = self.get_proto_in_dir(proto_desc) 403 for gn_proto_deps_name in proto_desc.get('deps', []): 404 dep = self.parse_gn_desc(gn_desc, gn_proto_deps_name) 405 target.deps.add(dep.name) 406 target.arch[arch].sources.update(proto_desc.get('sources', [])) 407 assert (all(x.endswith('.proto') for x in target.arch[arch].sources)) 408 elif target.type == 'source_set': 409 target.arch[arch].sources.update(desc.get('sources', [])) 410 elif target.is_linker_unit_type(): 411 target.arch[arch].sources.update(desc.get('sources', [])) 412 elif (desc.get("script", "") in JAVA_BANNED_SCRIPTS 413 or self._is_java_group(target.type, target.name)): 414 # java_group identifies the group target generated by the android_library 415 # or java_library template. A java_group must not be added as a 416 # dependency, but sources are collected. 417 log.debug('Found java target %s', target.name) 418 if target.type == "action": 419 # Convert java actions into java_group and keep the inputs for collection. 420 target.inputs.update(desc.get('inputs', [])) 421 target.type = 'java_group' 422 elif target.type in ['action', 'action_foreach']: 423 target.arch[arch].inputs.update(desc.get('inputs', [])) 424 target.arch[arch].sources.update(desc.get('sources', [])) 425 outs = [re.sub('^//out/.+?/gen/', '', x) for x in desc['outputs']] 426 target.arch[arch].outputs.update(outs) 427 # While the arguments might differ, an action should always use the same script for every 428 # architecture. (gen_android_bp's get_action_sanitizer actually relies on this fact. 429 target.script = desc['script'] 430 target.arch[arch].args = desc['args'] 431 target.arch[arch].response_file_contents = self._get_response_file_contents(desc) 432 elif target.type == 'copy': 433 # TODO: copy rules are not currently implemented. 434 pass 435 436 # Default for 'public' is //* - all headers in 'sources' are public. 437 # TODO(primiano): if a 'public' section is specified (even if empty), then 438 # the rest of 'sources' is considered inaccessible by gn. Consider 439 # emulating that, so that generated build files don't end up with overly 440 # accessible headers. 441 public_headers = [x for x in desc.get('public', []) if x != '*'] 442 target.public_headers.update(public_headers) 443 444 target.arch[arch].cflags.update(desc.get('cflags', []) + desc.get('cflags_cc', [])) 445 target.libs.update(desc.get('libs', [])) 446 target.arch[arch].ldflags.update(desc.get('ldflags', [])) 447 target.arch[arch].defines.update(desc.get('defines', [])) 448 target.arch[arch].include_dirs.update(desc.get('include_dirs', [])) 449 target.output_name = desc.get('output_name', None) 450 if "-frtti" in target.arch[arch].cflags: 451 target.rtti = True 452 453 # Recurse in dependencies. 454 for gn_dep_name in desc.get('deps', []): 455 dep = self.parse_gn_desc(gn_desc, gn_dep_name, java_group_name, is_test_target) 456 if dep.type == 'proto_library': 457 target.proto_deps.add(dep.name) 458 target.transitive_proto_deps.add(dep.name) 459 target.proto_paths.update(dep.proto_paths) 460 target.transitive_proto_deps.update(dep.transitive_proto_deps) 461 elif dep.type == 'group': 462 target.update(dep, arch) # Bubble up groups's cflags/ldflags etc. 463 elif dep.type in ['action', 'action_foreach', 'copy']: 464 if proto_target_type is None: 465 target.arch[arch].deps.add(dep.name) 466 elif dep.is_linker_unit_type(): 467 target.arch[arch].deps.add(dep.name) 468 elif dep.type == 'java_group': 469 # Explicitly break dependency chain when a java_group is added. 470 # Java sources are collected and eventually compiled as one large 471 # java_library. 472 pass 473 474 if dep.type in ['static_library', 'source_set']: 475 # Bubble up static_libs and source_set. Necessary, since soong does not propagate 476 # static_libs up the build tree. 477 # Source sets are later translated to static_libraries, so it makes sense 478 # to reuse transitive_static_libs_deps. 479 target.arch[arch].transitive_static_libs_deps.add(dep.name) 480 481 if arch in dep.arch: 482 target.arch[arch].transitive_static_libs_deps.update( 483 dep.arch[arch].transitive_static_libs_deps) 484 target.arch[arch].deps.update(target.arch[arch].transitive_static_libs_deps) 485 486 # Collect java sources. Java sources are kept inside the __compile_java target. 487 # This target can be used for both host and target compilation; only add 488 # the sources if they are destined for the target (i.e. they are a 489 # dependency of the __dex target) 490 # Note: this skips prebuilt java dependencies. These will have to be 491 # added manually when building the jar. 492 if target.name.endswith('__dex'): 493 if dep.name.endswith('__compile_java'): 494 log.debug('Adding java sources for %s', dep.name) 495 java_srcs = [src for src in dep.inputs if _is_java_source(src)] 496 if not is_test_target: 497 # TODO(aymanm): Fix collecting sources for testing modules for java. 498 # Don't collect java source files for test targets. 499 # We only need a specific set of java sources which are hardcoded in gen_android_bp 500 self.java_sources[java_group_name].update(java_srcs) 501 if dep.type in ["action"] and target.type == "java_group": 502 # GN uses an action to compile aidl files. However, this is not needed in soong 503 # as soong can directly have .aidl files in srcs. So adding .aidl files to the java_sources. 504 # TODO: Find a better way/place to do this. 505 if not is_test_target: 506 if '_aidl' in dep.name: 507 self.java_sources[java_group_name].update(dep.arch[arch].sources) 508 self.aidl_local_include_dirs.update( 509 _extract_includes_from_aidl_args(dep.arch[arch].args)) 510 else: 511 # TODO(aymanm): Fix collecting actions for testing modules for java. 512 # Don't collect java actions for test targets. 513 self.java_actions[java_group_name].add(dep.name) 514 return target 515 516 def get_proto_exports(self, proto_desc): 517 # exports in metadata will be available for source_set targets. 518 metadata = proto_desc.get('metadata', {}) 519 return metadata.get('exports', []) 520 521 def get_proto_paths(self, proto_desc): 522 # import_dirs in metadata will be available for source_set targets. 523 metadata = proto_desc.get('metadata', {}) 524 return metadata.get('import_dirs', []) 525 526 527 def get_proto_in_dir(self, proto_desc): 528 args = proto_desc.get('args') 529 return re.sub('^\.\./\.\./', '', args[args.index('--proto-in-dir') + 1]) 530 531 def get_proto_target_type(self, gn_desc, gn_target_name): 532 """ Checks if the target is a proto library and return the plugin. 533 534 Returns: 535 (None, None): if the target is not a proto library. 536 (plugin, proto_desc) where |plugin| is 'proto' in the default (lite) 537 case or 'protozero' or 'ipc' or 'descriptor'; |proto_desc| is the GN 538 json desc of the target with the .proto sources (_gen target for 539 non-descriptor types or the target itself for descriptor type). 540 """ 541 parts = gn_target_name.split('(', 1) 542 name = parts[0] 543 toolchain = '(' + parts[1] if len(parts) > 1 else '' 544 545 # Descriptor targets don't have a _gen target; instead we look for the 546 # characteristic flag in the args of the target itself. 547 desc = gn_desc.get(gn_target_name) 548 if '--descriptor_set_out' in desc.get('args', []): 549 return 'descriptor', desc 550 551 # Source set proto targets have a non-empty proto_library_sources in the 552 # metadata of the description. 553 metadata = desc.get('metadata', {}) 554 if 'proto_library_sources' in metadata: 555 return 'source_set', desc 556 557 # In all other cases, we want to look at the _gen target as that has the 558 # important information. 559 gen_desc = gn_desc.get('%s_gen%s' % (name, toolchain)) 560 if gen_desc is None or gen_desc['type'] != 'action': 561 return None, None 562 if gen_desc['script'] != '//tools/protoc_wrapper/protoc_wrapper.py': 563 return None, None 564 plugin = 'proto' 565 args = gen_desc.get('args', []) 566 for arg in (arg for arg in args if arg.startswith('--plugin=')): 567 # |arg| at this point looks like: 568 # --plugin=protoc-gen-plugin=gcc_like_host/protozero_plugin 569 # or 570 # --plugin=protoc-gen-plugin=protozero_plugin 571 plugin = arg.split('=')[-1].split('/')[-1].replace('_plugin', '') 572 return plugin, gen_desc 573