1# Copyright (C) 2022 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15# A collection of utilities for extracting build rule information from GN 16# projects. 17 18import copy 19import json 20import logging as log 21import os 22import re 23import collections 24 25LINKER_UNIT_TYPES = ('executable', 'shared_library', 'static_library', 'source_set') 26# This is a list of java files that should not be collected 27# as they don't exist right now downstream (eg: apihelpers, cronetEngineBuilderTest). 28# This is temporary solution until they are up-streamed. 29JAVA_FILES_TO_IGNORE = ( 30 "//components/cronet/android/api/src/org/chromium/net/apihelpers/ByteArrayCronetCallback.java", 31 "//components/cronet/android/api/src/org/chromium/net/apihelpers/ContentTypeParametersParser.java", 32 "//components/cronet/android/api/src/org/chromium/net/apihelpers/CronetRequestCompletionListener.java", 33 "//components/cronet/android/api/src/org/chromium/net/apihelpers/CronetResponse.java", 34 "//components/cronet/android/api/src/org/chromium/net/apihelpers/ImplicitFlowControlCallback.java", 35 "//components/cronet/android/api/src/org/chromium/net/apihelpers/InMemoryTransformCronetCallback.java", 36 "//components/cronet/android/api/src/org/chromium/net/apihelpers/JsonCronetCallback.java", 37 "//components/cronet/android/api/src/org/chromium/net/apihelpers/RedirectHandler.java", 38 "//components/cronet/android/api/src/org/chromium/net/apihelpers/RedirectHandlers.java", 39 "//components/cronet/android/api/src/org/chromium/net/apihelpers/StringCronetCallback.java", 40 "//components/cronet/android/api/src/org/chromium/net/apihelpers/UrlRequestCallbacks.java", 41 "//components/cronet/android/test/javatests/src/org/chromium/net/CronetEngineBuilderTest.java", 42 # Api helpers does not exist downstream, hence the tests shouldn't be collected. 43 "//components/cronet/android/test/javatests/src/org/chromium/net/apihelpers/ContentTypeParametersParserTest.java", 44 # androidx-multidex is disabled on unbundled branches. 45 "//base/test/android/java/src/org/chromium/base/multidex/ChromiumMultiDexInstaller.java", 46) 47RESPONSE_FILE = '{{response_file_name}}' 48TESTING_SUFFIX = "__testing" 49AIDL_INCLUDE_DIRS_REGEX = r'--includes=\[(.*)\]' 50AIDL_IMPORT_DIRS_REGEX = r'--imports=\[(.*)\]' 51PROTO_IMPORT_DIRS_REGEX = r'--import-dir=(.*)' 52 53def repo_root(): 54 """Returns an absolute path to the repository root.""" 55 return os.path.join( 56 os.path.realpath(os.path.dirname(__file__)), os.path.pardir) 57 58 59def _clean_string(str): 60 return str.replace('\\', '').replace('../../', '').replace('"', '').strip() 61 62def _clean_aidl_import(orig_str): 63 str = _clean_string(orig_str) 64 src_idx = str.find("src/") 65 if src_idx == -1: 66 raise ValueError(f"Unable to clean aidl import {orig_str}") 67 return str[:src_idx + len("src")] 68 69def _extract_includes_from_aidl_args(args): 70 ret = [] 71 for arg in args: 72 is_match = re.match(AIDL_INCLUDE_DIRS_REGEX, arg) 73 if is_match: 74 local_includes = is_match.group(1).split(",") 75 ret += [_clean_string(local_include) for local_include in local_includes] 76 # Treat imports like include for aidl by removing the package suffix. 77 is_match = re.match(AIDL_IMPORT_DIRS_REGEX, arg) 78 if is_match: 79 local_imports = is_match.group(1).split(",") 80 # Skip "third_party/android_sdk/public/platforms/android-34/framework.aidl" because Soong 81 # already links against the AIDL framework implicitly. 82 ret += [_clean_aidl_import(local_import) for local_import in local_imports 83 if "framework.aidl" not in local_import] 84 return ret 85 86def contains_aidl(sources): 87 return any([src.endswith(".aidl") for src in sources]) 88 89def _get_jni_registration_deps(gn_target_name, gn_desc): 90 # the dependencies are stored within another target with the same name 91 # and a __java_sources suffix, see 92 # https://source.chromium.org/chromium/chromium/src/+/main:third_party/jni_zero/jni_zero.gni;l=117;drc=78e8e27142ed3fddf04fbcd122507517a87cb9ad 93 # for the auto-generated target name. 94 jni_registration_java_target = f'{gn_target_name}__java_sources' 95 if jni_registration_java_target in gn_desc.keys(): 96 return gn_desc[jni_registration_java_target]["deps"] 97 return set() 98 99def label_to_path(label): 100 """Turn a GN output label (e.g., //some_dir/file.cc) into a path.""" 101 assert label.startswith('//') 102 return label[2:] or "./" 103 104def label_without_toolchain(label): 105 """Strips the toolchain from a GN label. 106 107 Return a GN label (e.g //buildtools:protobuf(//gn/standalone/toolchain: 108 gcc_like_host) without the parenthesised toolchain part. 109 """ 110 return label.split('(')[0] 111 112 113def _is_java_source(src): 114 return os.path.splitext(src)[1] == '.java' and not src.startswith("//out/") 115 116 117class GnParser(object): 118 """A parser with some cleverness for GN json desc files 119 120 The main goals of this parser are: 121 1) Deal with the fact that other build systems don't have an equivalent 122 notion to GN's source_set. Conversely to Bazel's and Soong's filegroups, 123 GN source_sets expect that dependencies, cflags and other source_set 124 properties propagate up to the linker unit (static_library, executable or 125 shared_library). This parser simulates the same behavior: when a 126 source_set is encountered, some of its variables (cflags and such) are 127 copied up to the dependent targets. This is to allow gen_xxx to create 128 one filegroup for each source_set and then squash all the other flags 129 onto the linker unit. 130 2) Detect and special-case protobuf targets, figuring out the protoc-plugin 131 being used. 132 """ 133 134 class Target(object): 135 """Reperesents A GN target. 136 137 Maked properties are propagated up the dependency chain when a 138 source_set dependency is encountered. 139 """ 140 class Arch(): 141 """Architecture-dependent properties 142 """ 143 def __init__(self): 144 self.sources = set() 145 self.cflags = set() 146 self.defines = set() 147 self.include_dirs = set() 148 self.deps = set() 149 self.transitive_static_libs_deps = set() 150 self.ldflags = set() 151 152 # These are valid only for type == 'action' 153 self.inputs = set() 154 self.outputs = set() 155 self.args = [] 156 self.response_file_contents = '' 157 158 def __init__(self, name, type): 159 self.name = name # e.g. //src/ipc:ipc 160 161 VALID_TYPES = ('static_library', 'shared_library', 'executable', 'group', 162 'action', 'source_set', 'proto_library', 'copy', 'action_foreach') 163 assert (type in VALID_TYPES) 164 self.type = type 165 self.testonly = False 166 self.toolchain = None 167 168 # These are valid only for type == proto_library. 169 # This is typically: 'proto', 'protozero', 'ipc'. 170 self.proto_plugin = None 171 self.proto_paths = set() 172 self.proto_exports = set() 173 self.proto_in_dir = "" 174 175 # TODO(primiano): consider whether the public section should be part of 176 # bubbled-up sources. 177 self.public_headers = set() # 'public' 178 179 # These are valid only for type == 'action' 180 self.script = '' 181 182 # These variables are propagated up when encountering a dependency 183 # on a source_set target. 184 self.libs = set() 185 self.proto_deps = set() 186 self.rtti = False 187 188 # TODO: come up with a better way to only run this once. 189 # is_finalized tracks whether finalize() was called on this target. 190 self.is_finalized = False 191 # 'common' is a pseudo-architecture used to store common architecture dependent properties (to 192 # make handling of common vs architecture-specific arguments more consistent). 193 self.arch = {'common': self.Arch()} 194 195 # This is used to get the name/version of libcronet 196 self.output_name = None 197 # Local Includes used for AIDL 198 self.local_aidl_includes = set() 199 # Each java_target will contain the transitive java sources found 200 # in generate_jni type target. 201 self.transitive_jni_java_sources = set() 202 # Deps for JNI Registration. Those are not added to deps so that 203 # the generated module would not depend on those deps. 204 self.jni_registration_java_deps = set() 205 # Path to the java jar path. This is used if the java library is 206 # an import of a JAR like `android_java_prebuilt` targets in GN 207 self.jar_path = "" 208 self.sdk_version = "" 209 210 # Properties to forward access to common arch. 211 # TODO: delete these after the transition has been completed. 212 @property 213 def sources(self): 214 return self.arch['common'].sources 215 216 @sources.setter 217 def sources(self, val): 218 self.arch['common'].sources = val 219 220 @property 221 def inputs(self): 222 return self.arch['common'].inputs 223 224 @inputs.setter 225 def inputs(self, val): 226 self.arch['common'].inputs = val 227 228 @property 229 def outputs(self): 230 return self.arch['common'].outputs 231 232 @outputs.setter 233 def outputs(self, val): 234 self.arch['common'].outputs = val 235 236 @property 237 def args(self): 238 return self.arch['common'].args 239 240 @args.setter 241 def args(self, val): 242 self.arch['common'].args = val 243 244 @property 245 def response_file_contents(self): 246 return self.arch['common'].response_file_contents 247 248 @response_file_contents.setter 249 def response_file_contents(self, val): 250 self.arch['common'].response_file_contents = val 251 252 @property 253 def cflags(self): 254 return self.arch['common'].cflags 255 256 @property 257 def defines(self): 258 return self.arch['common'].defines 259 260 @property 261 def deps(self): 262 return self.arch['common'].deps 263 264 @deps.setter 265 def deps(self, val): 266 self.arch['common'].deps = val 267 268 269 @property 270 def include_dirs(self): 271 return self.arch['common'].include_dirs 272 273 @property 274 def ldflags(self): 275 return self.arch['common'].ldflags 276 277 def host_supported(self): 278 return 'host' in self.arch 279 280 def device_supported(self): 281 return any([name.startswith('android') for name in self.arch.keys()]) 282 283 def is_linker_unit_type(self): 284 return self.type in LINKER_UNIT_TYPES 285 286 def __lt__(self, other): 287 if isinstance(other, self.__class__): 288 return self.name < other.name 289 raise TypeError( 290 '\'<\' not supported between instances of \'%s\' and \'%s\'' % 291 (type(self).__name__, type(other).__name__)) 292 293 def __repr__(self): 294 return json.dumps({ 295 k: (list(sorted(v)) if isinstance(v, set) else v) 296 for (k, v) in self.__dict__.items() 297 }, 298 indent=4, 299 sort_keys=True) 300 301 def update(self, other, arch): 302 for key in ('cflags', 'defines', 'deps', 'include_dirs', 'ldflags', 303 'proto_deps', 'libs', 'proto_paths'): 304 getattr(self, key).update(getattr(other, key, [])) 305 306 for key_in_arch in ('cflags', 'defines', 'include_dirs', 'deps', 'ldflags'): 307 getattr(self.arch[arch], key_in_arch).update(getattr(other.arch[arch], key_in_arch, [])) 308 309 def get_archs(self): 310 """ Returns a dict of archs without the common arch """ 311 return {arch: val for arch, val in self.arch.items() if arch != 'common'} 312 313 def _finalize_set_attribute(self, key): 314 # Target contains the intersection of arch-dependent properties 315 getattr(self, key).update(set.intersection(*[getattr(arch, key) for arch in 316 self.get_archs().values()])) 317 318 # Deduplicate arch-dependent properties 319 for arch in self.get_archs().values(): 320 getattr(arch, key).difference_update(getattr(self, key)) 321 322 def _finalize_non_set_attribute(self, key): 323 # Only when all the arch has the same non empty value, move the value to the target common 324 val = getattr(list(self.get_archs().values())[0], key) 325 if val and all([val == getattr(arch, key) for arch in self.get_archs().values()]): 326 setattr(self, key, copy.deepcopy(val)) 327 328 def _finalize_attribute(self, key): 329 val = getattr(self, key) 330 if isinstance(val, set): 331 self._finalize_set_attribute(key) 332 elif isinstance(val, (list, str)): 333 self._finalize_non_set_attribute(key) 334 else: 335 raise TypeError(f'Unsupported type: {type(val)}') 336 337 def finalize(self): 338 """Move common properties out of arch-dependent subobjects to Target object. 339 340 TODO: find a better name for this function. 341 """ 342 if self.is_finalized: 343 return 344 self.is_finalized = True 345 346 if len(self.arch) == 1: 347 return 348 349 for key in ('sources', 'cflags', 'defines', 'include_dirs', 'deps', 350 'inputs', 'outputs', 'args', 'response_file_contents', 'ldflags'): 351 self._finalize_attribute(key) 352 353 def get_target_name(self): 354 return self.name[self.name.find(":") + 1:] 355 356 357 def __init__(self, builtin_deps): 358 self.builtin_deps = builtin_deps 359 self.all_targets = {} 360 self.jni_java_sources = set() 361 362 363 def _get_response_file_contents(self, action_desc): 364 # response_file_contents are formatted as: 365 # ['--flags', '--flag=true && false'] and need to be formatted as: 366 # '--flags --flag=\"true && false\"' 367 flags = action_desc.get('response_file_contents', []) 368 formatted_flags = [] 369 for flag in flags: 370 if '=' in flag: 371 key, val = flag.split('=') 372 formatted_flags.append('%s=\\"%s\\"' % (key, val)) 373 else: 374 formatted_flags.append(flag) 375 376 return ' '.join(formatted_flags) 377 378 def _is_java_group(self, type_, target_name): 379 # Per https://chromium.googlesource.com/chromium/src/build/+/HEAD/android/docs/java_toolchain.md 380 # java target names must end in "_java". 381 # TODO: There are some other possible variations we might need to support. 382 return type_ == 'group' and target_name.endswith('_java') 383 384 def _get_arch(self, toolchain): 385 if toolchain == '//build/toolchain/android:android_clang_x86': 386 return 'android_x86' 387 elif toolchain == '//build/toolchain/android:android_clang_x64': 388 return 'android_x86_64' 389 elif toolchain == '//build/toolchain/android:android_clang_arm': 390 return 'android_arm' 391 elif toolchain == '//build/toolchain/android:android_clang_arm64': 392 return 'android_arm64' 393 elif toolchain == '//build/toolchain/android:android_clang_riscv64': 394 return 'android_riscv64' 395 else: 396 return 'host' 397 398 def get_target(self, gn_target_name): 399 """Returns a Target object from the fully qualified GN target name. 400 401 get_target() requires that parse_gn_desc() has already been called. 402 """ 403 # Run this every time as parse_gn_desc can be called at any time. 404 for target in self.all_targets.values(): 405 target.finalize() 406 407 return self.all_targets[label_without_toolchain(gn_target_name)] 408 409 def parse_gn_desc(self, gn_desc, gn_target_name, java_group_name=None, is_test_target=False): 410 """Parses a gn desc tree and resolves all target dependencies. 411 412 It bubbles up variables from source_set dependencies as described in the 413 class-level comments. 414 """ 415 # Use name without toolchain for targets to support targets built for 416 # multiple archs. 417 target_name = label_without_toolchain(gn_target_name) 418 desc = gn_desc[gn_target_name] 419 type_ = desc['type'] 420 arch = self._get_arch(desc['toolchain']) 421 metadata = desc.get("metadata", {}) 422 423 if is_test_target: 424 target_name += TESTING_SUFFIX 425 426 target = self.all_targets.get(target_name) 427 if target is None: 428 target = GnParser.Target(target_name, type_) 429 self.all_targets[target_name] = target 430 431 if arch not in target.arch: 432 target.arch[arch] = GnParser.Target.Arch() 433 else: 434 return target # Target already processed. 435 436 if 'target_type' in metadata.keys() and metadata["target_type"][0] == 'java_library': 437 target.type = 'java_library' 438 439 if target.name in self.builtin_deps: 440 # return early, no need to parse any further as the module is a builtin. 441 return target 442 443 target.testonly = desc.get('testonly', False) 444 445 deps = desc.get("deps", {}) 446 if desc.get("script", "") == "//tools/protoc_wrapper/protoc_wrapper.py": 447 target.type = 'proto_library' 448 target.proto_plugin = "proto" 449 target.proto_paths.update(self.get_proto_paths(desc)) 450 target.proto_exports.update(self.get_proto_exports(desc)) 451 target.proto_in_dir = self.get_proto_in_dir(desc) 452 target.arch[arch].sources.update(desc.get('sources', [])) 453 target.arch[arch].inputs.update(desc.get('inputs', [])) 454 elif target.type == 'source_set': 455 target.arch[arch].sources.update(source for source in desc.get('sources', []) if not source.startswith("//out")) 456 elif target.is_linker_unit_type(): 457 target.arch[arch].sources.update(source for source in desc.get('sources', []) if not source.startswith("//out")) 458 elif target.type == 'java_library': 459 sources = set() 460 for java_source in metadata.get("source_files", []): 461 if not java_source.startswith("//out") and java_source not in JAVA_FILES_TO_IGNORE: 462 sources.add(java_source) 463 target.sources.update(sources) 464 # Metadata attributes must be list, for jar_path, it is always a list 465 # of size one, the first element is an empty string if `jar_path` is not 466 # defined otherwise it is a path. 467 if metadata.get("jar_path", [""])[0]: 468 target.jar_path = label_to_path(metadata["jar_path"][0]) 469 target.sdk_version = metadata.get('sdk_version', ['current'])[0] 470 deps = metadata.get("all_deps", {}) 471 log.info('Found Java Target %s', target.name) 472 elif target.script == "//build/android/gyp/aidl.py": 473 target.type = "java_library" 474 target.sources.update(desc.get('sources', {})) 475 target.local_aidl_includes = _extract_includes_from_aidl_args(desc.get('args', '')) 476 elif target.type in ['action', 'action_foreach']: 477 target.arch[arch].inputs.update(desc.get('inputs', [])) 478 target.arch[arch].sources.update(desc.get('sources', [])) 479 outs = [re.sub('^//out/.+?/gen/', '', x) for x in desc['outputs']] 480 target.arch[arch].outputs.update(outs) 481 # While the arguments might differ, an action should always use the same script for every 482 # architecture. (gen_android_bp's get_action_sanitizer actually relies on this fact. 483 target.script = desc['script'] 484 target.arch[arch].args = desc['args'] 485 target.arch[arch].response_file_contents = self._get_response_file_contents(desc) 486 # _get_jni_registration_deps will return the dependencies of a target if 487 # the target is of type `generate_jni_registration` otherwise it will 488 # return an empty set. 489 target.jni_registration_java_deps.update(_get_jni_registration_deps(gn_target_name, gn_desc)) 490 # JNI java sources are embedded as metadata inside `jni_headers` targets. 491 # See https://source.chromium.org/chromium/chromium/src/+/main:third_party/jni_zero/jni_zero.gni;l=421;drc=78e8e27142ed3fddf04fbcd122507517a87cb9ad 492 # for more details 493 target.transitive_jni_java_sources.update(metadata.get("jni_source_files_abs", set())) 494 self.jni_java_sources.update(metadata.get("jni_source_files_abs", set())) 495 elif target.type == 'copy': 496 # TODO: copy rules are not currently implemented. 497 pass 498 elif target.type == 'group': 499 # Groups are bubbled upward without creating an equivalent GN target. 500 pass 501 else: 502 raise Exception(f"Encountered GN target with unknown type\nCulprit target: {gn_target_name}\ntype: {type_}") 503 504 # Default for 'public' is //* - all headers in 'sources' are public. 505 # TODO(primiano): if a 'public' section is specified (even if empty), then 506 # the rest of 'sources' is considered inaccessible by gn. Consider 507 # emulating that, so that generated build files don't end up with overly 508 # accessible headers. 509 public_headers = [x for x in desc.get('public', []) if x != '*'] 510 target.public_headers.update(public_headers) 511 512 target.arch[arch].cflags.update(desc.get('cflags', []) + desc.get('cflags_cc', [])) 513 target.libs.update(desc.get('libs', [])) 514 target.arch[arch].ldflags.update(desc.get('ldflags', [])) 515 target.arch[arch].defines.update(desc.get('defines', [])) 516 target.arch[arch].include_dirs.update(desc.get('include_dirs', [])) 517 target.output_name = desc.get('output_name', None) 518 if "-frtti" in target.arch[arch].cflags: 519 target.rtti = True 520 521 for gn_dep_name in set(target.jni_registration_java_deps): 522 dep = self.parse_gn_desc(gn_desc, gn_dep_name, java_group_name, is_test_target) 523 target.transitive_jni_java_sources.update(dep.transitive_jni_java_sources) 524 525 # Recurse in dependencies. 526 for gn_dep_name in set(deps): 527 dep = self.parse_gn_desc(gn_desc, gn_dep_name, java_group_name, is_test_target) 528 529 if dep.type == 'proto_library': 530 target.proto_deps.add(dep.name) 531 elif dep.type == 'group': 532 target.update(dep, arch) # Bubble up groups's cflags/ldflags etc. 533 target.transitive_jni_java_sources.update(dep.transitive_jni_java_sources) 534 elif dep.type in ['action', 'action_foreach', 'copy']: 535 target.arch[arch].deps.add(dep.name) 536 target.transitive_jni_java_sources.update(dep.transitive_jni_java_sources) 537 elif dep.is_linker_unit_type(): 538 target.arch[arch].deps.add(dep.name) 539 elif dep.type == 'java_library': 540 target.deps.add(dep.name) 541 target.transitive_jni_java_sources.update(dep.transitive_jni_java_sources) 542 543 if dep.type in ['static_library', 'source_set']: 544 # Bubble up static_libs and source_set. Necessary, since soong does not propagate 545 # static_libs up the build tree. 546 # Source sets are later translated to static_libraries, so it makes sense 547 # to reuse transitive_static_libs_deps. 548 target.arch[arch].transitive_static_libs_deps.add(dep.name) 549 550 if arch in dep.arch: 551 target.arch[arch].transitive_static_libs_deps.update( 552 dep.arch[arch].transitive_static_libs_deps) 553 target.arch[arch].deps.update(target.arch[arch].transitive_static_libs_deps) 554 return target 555 556 def get_proto_exports(self, proto_desc): 557 # exports in metadata will be available for source_set targets. 558 metadata = proto_desc.get('metadata', {}) 559 return metadata.get('exports', []) 560 561 def get_proto_paths(self, proto_desc): 562 args = proto_desc.get('args') 563 proto_paths = set() 564 for arg in args: 565 is_match = re.match(PROTO_IMPORT_DIRS_REGEX, arg) 566 if is_match: 567 proto_paths.add(re.sub('^\.\./\.\./', '', is_match.group(1))) 568 return proto_paths 569 570 571 def get_proto_in_dir(self, proto_desc): 572 args = proto_desc.get('args') 573 return re.sub('^\.\./\.\./', '', args[args.index('--proto-in-dir') + 1]) 574