1#!/usr/bin/env python3 2# 3# Copyright (C) 2021 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16"""Builds SDK snapshots. 17 18If the environment variable TARGET_BUILD_APPS is nonempty then only the SDKs for 19the APEXes in it are built, otherwise all configured SDKs are built. 20""" 21import argparse 22import dataclasses 23import datetime 24import enum 25import functools 26import io 27import json 28import os 29from pathlib import Path 30import re 31import shutil 32import subprocess 33import sys 34import tempfile 35import typing 36from collections import defaultdict 37from typing import Callable, List 38import zipfile 39 40COPYRIGHT_BOILERPLATE = """ 41// 42// Copyright (C) 2020 The Android Open Source Project 43// 44// Licensed under the Apache License, Version 2.0 (the "License"); 45// you may not use this file except in compliance with the License. 46// You may obtain a copy of the License at 47// 48// http://www.apache.org/licenses/LICENSE-2.0 49// 50// Unless required by applicable law or agreed to in writing, software 51// distributed under the License is distributed on an "AS IS" BASIS, 52// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 53// See the License for the specific language governing permissions and 54// limitations under the License. 55// 56""".lstrip() 57 58 59@dataclasses.dataclass(frozen=True) 60class ConfigVar: 61 """Represents a Soong configuration variable""" 62 # The config variable namespace, e.g. ANDROID. 63 namespace: str 64 65 # The name of the variable within the namespace. 66 name: str 67 68 69@dataclasses.dataclass(frozen=True) 70class FileTransformation: 71 """Performs a transformation on a file within an SDK snapshot zip file.""" 72 73 # The path of the file within the SDK snapshot zip file. 74 path: str 75 76 def apply(self, producer, path, build_release): 77 """Apply the transformation to the path; changing it in place.""" 78 with open(path, "r+", encoding="utf8") as file: 79 self._apply_transformation(producer, file, build_release) 80 81 def _apply_transformation(self, producer, file, build_release): 82 """Apply the transformation to the file. 83 84 The file has been opened in read/write mode so the implementation of 85 this must read the contents and then reset the file to the beginning 86 and write the altered contents. 87 """ 88 raise NotImplementedError 89 90 91@dataclasses.dataclass(frozen=True) 92class SoongConfigVarTransformation(FileTransformation): 93 94 # The configuration variable that will control the prefer setting. 95 configVar: ConfigVar 96 97 # The line containing the prefer property. 98 PREFER_LINE = " prefer: false," 99 100 def _apply_transformation(self, producer, file, build_release): 101 raise NotImplementedError 102 103 104@dataclasses.dataclass(frozen=True) 105class SoongConfigBoilerplateInserter(SoongConfigVarTransformation): 106 """Transforms an Android.bp file to add soong config boilerplate. 107 108 The boilerplate allows the prefer setting of the modules to be controlled 109 through a Soong configuration variable. 110 """ 111 112 # The configuration variable that will control the prefer setting. 113 configVar: ConfigVar 114 115 # The prefix to use for the soong config module types. 116 configModuleTypePrefix: str 117 118 def config_module_type(self, module_type): 119 return self.configModuleTypePrefix + module_type 120 121 def _apply_transformation(self, producer, file, build_release): 122 # TODO(b/174997203): Remove this when we have a proper way to control 123 # prefer flags in Mainline modules. 124 125 header_lines = [] 126 for line in file: 127 line = line.rstrip("\n") 128 if not line.startswith("//"): 129 break 130 header_lines.append(line) 131 132 config_module_types = set() 133 134 content_lines = [] 135 for line in file: 136 line = line.rstrip("\n") 137 138 # Check to see whether the line is the start of a new module type, 139 # e.g. <module-type> { 140 module_header = re.match("([a-z0-9_]+) +{$", line) 141 if not module_header: 142 # It is not so just add the line to the output and skip to the 143 # next line. 144 content_lines.append(line) 145 continue 146 147 module_type = module_header.group(1) 148 module_content = [] 149 150 # Iterate over the Soong module contents 151 for module_line in file: 152 module_line = module_line.rstrip("\n") 153 154 # When the end of the module has been reached then exit. 155 if module_line == "}": 156 break 157 158 # Check to see if the module is an unversioned module, i.e. 159 # without @<version>. If it is then it needs to have the soong 160 # config boilerplate added to control the setting of the prefer 161 # property. Versioned modules do not need that because they are 162 # never preferred. 163 # At the moment this differentiation between versioned and 164 # unversioned relies on the fact that the unversioned modules 165 # set "prefer: false", while the versioned modules do not. That 166 # is a little bit fragile so may require some additional checks. 167 if module_line != self.PREFER_LINE: 168 # The line does not indicate that the module needs the 169 # soong config boilerplate so add the line and skip to the 170 # next one. 171 module_content.append(module_line) 172 continue 173 174 # Add the soong config boilerplate instead of the line: 175 # prefer: false, 176 namespace = self.configVar.namespace 177 name = self.configVar.name 178 module_content.append(f"""\ 179 // Do not prefer prebuilt if the Soong config variable "{name}" in namespace "{namespace}" is true. 180 prefer: true, 181 soong_config_variables: {{ 182 {name}: {{ 183 prefer: false, 184 }}, 185 }},""") 186 187 # Add the module type to the list of module types that need to 188 # have corresponding config module types. 189 config_module_types.add(module_type) 190 191 # Change the module type to the corresponding soong config 192 # module type by adding the prefix. 193 module_type = self.config_module_type(module_type) 194 195 # Generate the module, possibly with the new module type and 196 # containing the soong config variables entry. 197 content_lines.append(module_type + " {") 198 content_lines.extend(module_content) 199 content_lines.append("}") 200 201 # Add the soong_config_module_type module definitions to the header 202 # lines so that they appear before any uses. 203 header_lines.append("") 204 for module_type in sorted(config_module_types): 205 # Create the corresponding soong config module type name by adding 206 # the prefix. 207 config_module_type = self.configModuleTypePrefix + module_type 208 header_lines.append(f""" 209// Soong config variable module type added by {producer.script}. 210soong_config_module_type {{ 211 name: "{config_module_type}", 212 module_type: "{module_type}", 213 config_namespace: "{self.configVar.namespace}", 214 bool_variables: ["{self.configVar.name}"], 215 properties: ["prefer"], 216}} 217""".lstrip()) 218 219 # Overwrite the file with the updated contents. 220 file.seek(0) 221 file.truncate() 222 file.write("\n".join(header_lines + content_lines) + "\n") 223 224 225@dataclasses.dataclass(frozen=True) 226class UseSourceConfigVarTransformation(SoongConfigVarTransformation): 227 228 def _apply_transformation(self, producer, file, build_release): 229 lines = [] 230 for line in file: 231 line = line.rstrip("\n") 232 if line != self.PREFER_LINE: 233 lines.append(line) 234 continue 235 236 # Replace "prefer: false" with "use_source_config_var {...}". 237 namespace = self.configVar.namespace 238 name = self.configVar.name 239 lines.append(f"""\ 240 // Do not prefer prebuilt if the Soong config variable "{name}" in namespace "{namespace}" is true. 241 use_source_config_var: {{ 242 config_namespace: "{namespace}", 243 var_name: "{name}", 244 }},""") 245 246 # Overwrite the file with the updated contents. 247 file.seek(0) 248 file.truncate() 249 file.write("\n".join(lines) + "\n") 250 251# Removes any lines containing prefer 252@dataclasses.dataclass(frozen=True) 253class UseNoPreferPropertyTransformation(SoongConfigVarTransformation): 254 255 def _apply_transformation(self, producer, file, build_release): 256 lines = [] 257 for line in file: 258 line = line.rstrip("\n") 259 if line != self.PREFER_LINE: 260 lines.append(line) 261 continue 262 263 # Overwrite the file with the updated contents. 264 file.seek(0) 265 file.truncate() 266 file.write("\n".join(lines) + "\n") 267 268@dataclasses.dataclass() 269class SubprocessRunner: 270 """Runs subprocesses""" 271 272 # Destination for stdout from subprocesses. 273 # 274 # This (and the following stderr) are needed to allow the tests to be run 275 # in Intellij. This ensures that the tests are run with stdout/stderr 276 # objects that work when passed to subprocess.run(stdout/stderr). Without it 277 # the tests are run with a FlushingStringIO object that has no fileno 278 # attribute - https://youtrack.jetbrains.com/issue/PY-27883. 279 stdout: io.TextIOBase = sys.stdout 280 281 # Destination for stderr from subprocesses. 282 stderr: io.TextIOBase = sys.stderr 283 284 def run(self, *args, **kwargs): 285 return subprocess.run( 286 *args, check=True, stdout=self.stdout, stderr=self.stderr, **kwargs) 287 288 289def sdk_snapshot_zip_file(snapshots_dir, sdk_name): 290 """Get the path to the sdk snapshot zip file.""" 291 return os.path.join(snapshots_dir, f"{sdk_name}-{SDK_VERSION}.zip") 292 293 294def sdk_snapshot_info_file(snapshots_dir, sdk_name): 295 """Get the path to the sdk snapshot info file.""" 296 return os.path.join(snapshots_dir, f"{sdk_name}-{SDK_VERSION}.info") 297 298 299def sdk_snapshot_api_diff_file(snapshots_dir, sdk_name): 300 """Get the path to the sdk snapshot api diff file.""" 301 return os.path.join(snapshots_dir, f"{sdk_name}-{SDK_VERSION}-api-diff.txt") 302 303 304def sdk_snapshot_gantry_metadata_json_file(snapshots_dir, sdk_name): 305 """Get the path to the sdk snapshot gantry metadata json file.""" 306 return os.path.join(snapshots_dir, 307 f"{sdk_name}-{SDK_VERSION}-gantry-metadata.json") 308 309 310# The default time to use in zip entries. Ideally, this should be the same as is 311# used by soong_zip and ziptime but there is no strict need for that to be the 312# case. What matters is this is a fixed time so that the contents of zip files 313# created by this script do not depend on when it is run, only the inputs. 314default_zip_time = datetime.datetime(2008, 1, 1, 0, 0, 0, 0, 315 datetime.timezone.utc) 316 317 318# set the timestamps of the paths to the default_zip_time. 319def set_default_timestamp(base_dir, paths): 320 for path in paths: 321 timestamp = default_zip_time.timestamp() 322 p = os.path.join(base_dir, path) 323 os.utime(p, (timestamp, timestamp)) 324 325 326# Find the git project path of the module_sdk for given module. 327def module_sdk_project_for_module(module, root_dir): 328 module = module.rsplit(".", 1)[1] 329 # git_master-art and aosp-master-art branches does not contain project for 330 # art, hence adding special case for art. 331 if module == "art": 332 return "prebuilts/module_sdk/art" 333 if module == "bt": 334 return "prebuilts/module_sdk/Bluetooth" 335 if module == "media": 336 return "prebuilts/module_sdk/Media" 337 if module == "nfcservices": 338 return "prebuilts/module_sdk/Nfc" 339 if module == "rkpd": 340 return "prebuilts/module_sdk/RemoteKeyProvisioning" 341 if module == "tethering": 342 return "prebuilts/module_sdk/Connectivity" 343 344 target_dir = "" 345 for dir in os.listdir(os.path.join(root_dir, "prebuilts/module_sdk/")): 346 if module.lower() in dir.lower(): 347 if target_dir: 348 print( 349 'Multiple target dirs matched "%s": %s' 350 % (module, (target_dir, dir)) 351 ) 352 sys.exit(1) 353 target_dir = dir 354 if not target_dir: 355 print("Could not find a target dir for %s" % module) 356 sys.exit(1) 357 358 return "prebuilts/module_sdk/%s" % target_dir 359 360 361@dataclasses.dataclass() 362class SnapshotBuilder: 363 """Builds sdk snapshots""" 364 365 # The path to this tool. 366 tool_path: str 367 368 # Used to run subprocesses for building snapshots. 369 subprocess_runner: SubprocessRunner 370 371 # The OUT_DIR environment variable. 372 out_dir: str 373 374 # The out/soong/mainline-sdks directory. 375 mainline_sdks_dir: str = "" 376 377 # True if apex-allowed-deps-check is to be skipped. 378 skip_allowed_deps_check: bool = False 379 380 def __post_init__(self): 381 self.mainline_sdks_dir = os.path.join(self.out_dir, 382 "soong/mainline-sdks") 383 384 def get_sdk_path(self, sdk_name): 385 """Get the path to the sdk snapshot zip file produced by soong""" 386 return os.path.join(self.mainline_sdks_dir, 387 f"{sdk_name}-{SDK_VERSION}.zip") 388 389 def build_target_paths(self, build_release, target_paths): 390 # Extra environment variables to pass to the build process. 391 extraEnv = { 392 # TODO(ngeoffray): remove SOONG_ALLOW_MISSING_DEPENDENCIES, but 393 # we currently break without it. 394 "SOONG_ALLOW_MISSING_DEPENDENCIES": "true", 395 # Set SOONG_SDK_SNAPSHOT_USE_SRCJAR to generate .srcjars inside 396 # sdk zip files as expected by prebuilt drop. 397 "SOONG_SDK_SNAPSHOT_USE_SRCJAR": "true", 398 } 399 extraEnv.update(build_release.soong_env) 400 401 # Unless explicitly specified in the calling environment set 402 # TARGET_BUILD_VARIANT=user. 403 # This MUST be identical to the TARGET_BUILD_VARIANT used to build 404 # the corresponding APEXes otherwise it could result in different 405 # hidden API flags, see http://b/202398851#comment29 for more info. 406 target_build_variant = os.environ.get("TARGET_BUILD_VARIANT", "user") 407 cmd = [ 408 "build/soong/soong_ui.bash", 409 "--make-mode", 410 "--soong-only", 411 f"TARGET_BUILD_VARIANT={target_build_variant}", 412 "TARGET_PRODUCT=mainline_sdk", 413 "MODULE_BUILD_FROM_SOURCE=true", 414 ] + target_paths 415 if not self.skip_allowed_deps_check: 416 cmd += ["apex-allowed-deps-check"] 417 print_command(extraEnv, cmd) 418 env = os.environ.copy() 419 env.update(extraEnv) 420 self.subprocess_runner.run(cmd, env=env) 421 422 def build_snapshots(self, build_release, modules): 423 # Compute the paths to all the Soong generated sdk snapshot files 424 # required by this script. 425 paths = [ 426 sdk_snapshot_zip_file(self.mainline_sdks_dir, sdk) 427 for module in modules 428 for sdk in module.sdks 429 ] 430 431 if paths: 432 self.build_target_paths(build_release, paths) 433 return self.mainline_sdks_dir 434 435 def build_snapshots_for_build_r(self, build_release, modules): 436 # Build the snapshots as standard. 437 snapshot_dir = self.build_snapshots(build_release, modules) 438 439 # Each module will extract needed files from the original snapshot zip 440 # file and then use that to create a replacement zip file. 441 r_snapshot_dir = os.path.join(snapshot_dir, "for-R-build") 442 shutil.rmtree(r_snapshot_dir, ignore_errors=True) 443 444 build_number_file = os.path.join(self.out_dir, "soong/build_number.txt") 445 446 for module in modules: 447 apex = module.apex 448 dest_dir = os.path.join(r_snapshot_dir, apex) 449 os.makedirs(dest_dir, exist_ok=True) 450 451 # Write the bp file in the sdk_library sub-directory rather than the 452 # root of the zip file as it will be unpacked in a directory that 453 # already contains an Android.bp file that defines the corresponding 454 # apex_set. 455 bp_file = os.path.join(dest_dir, "sdk_library/Android.bp") 456 os.makedirs(os.path.dirname(bp_file), exist_ok=True) 457 458 # The first sdk in the list is the name to use. 459 sdk_name = module.sdks[0] 460 461 with open(bp_file, "w", encoding="utf8") as bp: 462 bp.write("// DO NOT EDIT. Auto-generated by the following:\n") 463 bp.write(f"// {self.tool_path}\n") 464 bp.write(COPYRIGHT_BOILERPLATE) 465 aosp_apex = google_to_aosp_name(apex) 466 467 for library in module.for_r_build.sdk_libraries: 468 module_name = library.name 469 shared_library = str(library.shared_library).lower() 470 sdk_file = sdk_snapshot_zip_file(snapshot_dir, sdk_name) 471 extract_matching_files_from_zip( 472 sdk_file, dest_dir, 473 sdk_library_files_pattern( 474 scope_pattern=r"(public|system|module-lib)", 475 name_pattern=fr"({module_name}(-removed|-stubs)?)")) 476 477 available_apexes = [f'"{aosp_apex}"'] 478 if aosp_apex != "com.android.tethering": 479 available_apexes.append(f'"test_{aosp_apex}"') 480 apex_available = ",\n ".join(available_apexes) 481 482 bp.write(f""" 483java_sdk_library_import {{ 484 name: "{module_name}", 485 owner: "google", 486 prefer: true, 487 shared_library: {shared_library}, 488 apex_available: [ 489 {apex_available}, 490 ], 491 public: {{ 492 jars: ["public/{module_name}-stubs.jar"], 493 current_api: "public/{module_name}.txt", 494 removed_api: "public/{module_name}-removed.txt", 495 sdk_version: "module_current", 496 }}, 497 system: {{ 498 jars: ["system/{module_name}-stubs.jar"], 499 current_api: "system/{module_name}.txt", 500 removed_api: "system/{module_name}-removed.txt", 501 sdk_version: "module_current", 502 }}, 503 module_lib: {{ 504 jars: ["module-lib/{module_name}-stubs.jar"], 505 current_api: "module-lib/{module_name}.txt", 506 removed_api: "module-lib/{module_name}-removed.txt", 507 sdk_version: "module_current", 508 }}, 509}} 510""") 511 512 # Copy the build_number.txt file into the snapshot. 513 snapshot_build_number_file = os.path.join( 514 dest_dir, "snapshot-creation-build-number.txt") 515 shutil.copy(build_number_file, snapshot_build_number_file) 516 517 # Make sure that all the paths being added to the zip file have a 518 # fixed timestamp so that the contents of the zip file do not depend 519 # on when this script is run, only the inputs. 520 for root, dirs, files in os.walk(dest_dir): 521 set_default_timestamp(root, dirs) 522 set_default_timestamp(root, files) 523 524 # Now zip up the files into a snapshot zip file. 525 base_file = os.path.join(r_snapshot_dir, sdk_name + "-current") 526 shutil.make_archive(base_file, "zip", dest_dir) 527 528 return r_snapshot_dir 529 530 @staticmethod 531 def does_sdk_library_support_latest_api(sdk_library): 532 if sdk_library == "conscrypt.module.platform.api" or \ 533 sdk_library == "conscrypt.module.intra.core.api": 534 return False 535 return True 536 537 def latest_api_file_targets(self, sdk_info_file): 538 # Read the sdk info file and fetch the latest scope targets. 539 with open(sdk_info_file, "r", encoding="utf8") as sdk_info_file_object: 540 sdk_info_file_json = json.loads(sdk_info_file_object.read()) 541 542 target_paths = [] 543 target_dict = {} 544 for jsonItem in sdk_info_file_json: 545 if not jsonItem["@type"] == "java_sdk_library": 546 continue 547 548 sdk_library = jsonItem["@name"] 549 if not self.does_sdk_library_support_latest_api(sdk_library): 550 continue 551 552 target_dict[sdk_library] = {} 553 for scope in jsonItem["scopes"]: 554 scope_json = jsonItem["scopes"][scope] 555 target_dict[sdk_library][scope] = {} 556 target_list = [ 557 "current_api", "latest_api", "removed_api", 558 "latest_removed_api" 559 ] 560 for target in target_list: 561 target_dict[sdk_library][scope][target] = scope_json[target] 562 target_paths.append(scope_json["latest_api"]) 563 target_paths.append(scope_json["latest_removed_api"]) 564 target_paths.append(scope_json["latest_api"] 565 .replace(".latest", ".latest.extension_version")) 566 target_paths.append(scope_json["latest_removed_api"] 567 .replace(".latest", ".latest.extension_version")) 568 569 return target_paths, target_dict 570 571 def build_sdk_scope_targets(self, build_release, modules): 572 # Build the latest scope targets for each module sdk 573 # Compute the paths to all the latest scope targets for each module sdk. 574 target_paths = [] 575 target_dict = {} 576 for module in modules: 577 for sdk in module.sdks: 578 sdk_type = sdk_type_from_name(sdk) 579 if not sdk_type.providesApis: 580 continue 581 582 sdk_info_file = sdk_snapshot_info_file(self.mainline_sdks_dir, 583 sdk) 584 paths, dict_item = self.latest_api_file_targets(sdk_info_file) 585 target_paths.extend(paths) 586 target_dict[sdk_info_file] = dict_item 587 if target_paths: 588 self.build_target_paths(build_release, target_paths) 589 return target_dict 590 591 def appendDiffToFile(self, file_object, sdk_zip_file, current_api, 592 latest_api, snapshots_dir): 593 """Extract current api and find its diff with the latest api.""" 594 with zipfile.ZipFile(sdk_zip_file, "r") as zipObj: 595 extracted_current_api = zipObj.extract( 596 member=current_api, path=snapshots_dir) 597 # The diff tool has an exit code of 0, 1 or 2 depending on whether 598 # it find no differences, some differences or an error (like missing 599 # file). As 0 or 1 are both valid results this cannot use check=True 600 # so disable the pylint check. 601 # pylint: disable=subprocess-run-check 602 diff = subprocess.run([ 603 "diff", "-u0", latest_api, extracted_current_api, "--label", 604 latest_api, "--label", extracted_current_api 605 ], 606 capture_output=True).stdout.decode("utf-8") 607 file_object.write(diff) 608 609 def create_snapshot_gantry_metadata_and_api_diff(self, sdk, target_dict, 610 snapshots_dir, 611 module_extension_version): 612 """Creates gantry metadata and api diff files for each module sdk. 613 614 For each module sdk, the scope targets are obtained for each java sdk 615 library and the api diff files are generated by performing a diff 616 operation between the current api file vs the latest api file. 617 """ 618 sdk_info_file = sdk_snapshot_info_file(snapshots_dir, sdk) 619 sdk_zip_file = sdk_snapshot_zip_file(snapshots_dir, sdk) 620 sdk_api_diff_file = sdk_snapshot_api_diff_file(snapshots_dir, sdk) 621 622 gantry_metadata_dict = {} 623 with open( 624 sdk_api_diff_file, "w", 625 encoding="utf8") as sdk_api_diff_file_object: 626 last_finalized_version_set = set() 627 for sdk_library in target_dict[sdk_info_file]: 628 for scope in target_dict[sdk_info_file][sdk_library]: 629 scope_json = target_dict[sdk_info_file][sdk_library][scope] 630 current_api = scope_json["current_api"] 631 latest_api = scope_json["latest_api"] 632 self.appendDiffToFile(sdk_api_diff_file_object, 633 sdk_zip_file, current_api, latest_api, 634 snapshots_dir) 635 636 removed_api = scope_json["removed_api"] 637 latest_removed_api = scope_json["latest_removed_api"] 638 self.appendDiffToFile(sdk_api_diff_file_object, 639 sdk_zip_file, removed_api, 640 latest_removed_api, snapshots_dir) 641 642 def read_extension_version(target): 643 extension_target = target.replace( 644 ".latest", ".latest.extension_version") 645 with open( 646 extension_target, "r", encoding="utf8") as file: 647 version = int(file.read()) 648 # version equal to -1 means "not an extension version". 649 if version != -1: 650 last_finalized_version_set.add(version) 651 652 read_extension_version(scope_json["latest_api"]) 653 read_extension_version(scope_json["latest_removed_api"]) 654 655 if len(last_finalized_version_set) == 0: 656 # Either there is no java sdk library or all java sdk libraries 657 # have not been finalized in sdk extensions yet and hence have 658 # last finalized version set as -1. 659 gantry_metadata_dict["last_finalized_version"] = -1 660 elif len(last_finalized_version_set) == 1: 661 # All java sdk library extension version match. 662 gantry_metadata_dict["last_finalized_version"] =\ 663 last_finalized_version_set.pop() 664 else: 665 # Fail the build 666 raise ValueError( 667 "Not all sdk libraries finalized with the same version.\n") 668 669 gantry_metadata_dict["api_diff_file"] = sdk_api_diff_file.rsplit( 670 "/", 1)[-1] 671 gantry_metadata_dict["api_diff_file_size"] = os.path.getsize( 672 sdk_api_diff_file) 673 gantry_metadata_dict[ 674 "module_extension_version"] = module_extension_version 675 sdk_metadata_json_file = sdk_snapshot_gantry_metadata_json_file( 676 snapshots_dir, sdk) 677 678 gantry_metadata_json_object = json.dumps(gantry_metadata_dict, indent=4) 679 with open(sdk_metadata_json_file, 680 "w") as gantry_metadata_json_file_object: 681 gantry_metadata_json_file_object.write(gantry_metadata_json_object) 682 683 if os.path.getsize(sdk_metadata_json_file) > 1048576: # 1 MB 684 raise ValueError("Metadata file size should not exceed 1 MB.\n") 685 686 def get_module_extension_version(self): 687 return int( 688 subprocess.run([ 689 "build/soong/soong_ui.bash", "--dumpvar-mode", 690 "PLATFORM_SDK_EXTENSION_VERSION" 691 ], 692 capture_output=True).stdout.decode("utf-8").strip()) 693 694 def build_snapshot_gantry_metadata_and_api_diff(self, modules, target_dict, 695 snapshots_dir): 696 """For each module sdk, create the metadata and api diff file.""" 697 module_extension_version = self.get_module_extension_version() 698 for module in modules: 699 for sdk in module.sdks: 700 sdk_type = sdk_type_from_name(sdk) 701 if not sdk_type.providesApis: 702 continue 703 self.create_snapshot_gantry_metadata_and_api_diff( 704 sdk, target_dict, snapshots_dir, module_extension_version) 705 706 707# The sdk version to build 708# 709# This is legacy from the time when this could generate versioned sdk snapshots. 710SDK_VERSION = "current" 711 712# The initially empty list of build releases. Every BuildRelease that is created 713# automatically appends itself to this list. 714ALL_BUILD_RELEASES = [] 715 716 717class PreferHandling(enum.Enum): 718 """Enumeration of the various ways of handling prefer properties""" 719 720 # No special prefer property handling is required. 721 NONE = enum.auto() 722 723 # Apply the SoongConfigBoilerplateInserter transformation. 724 SOONG_CONFIG = enum.auto() 725 726 # Use the use_source_config_var property added in T. 727 USE_SOURCE_CONFIG_VAR_PROPERTY = enum.auto() 728 729 # No prefer in Android.bp file 730 # Starting with V, prebuilts will be enabled using apex_contributions flags. 731 USE_NO_PREFER_PROPERTY = enum.auto() 732 733 734@dataclasses.dataclass(frozen=True) 735@functools.total_ordering 736class BuildRelease: 737 """Represents a build release""" 738 739 # The name of the build release, e.g. Q, R, S, T, etc. 740 name: str 741 742 # The function to call to create the snapshot in the dist, that covers 743 # building and copying the snapshot into the dist. 744 creator: Callable[ 745 ["BuildRelease", "SdkDistProducer", List["MainlineModule"]], None] 746 747 # The sub-directory of dist/mainline-sdks into which the build release 748 # specific snapshots will be copied. 749 # 750 # Defaults to for-<name>-build. 751 sub_dir: str = None 752 753 # Additional environment variables to pass to Soong when building the 754 # snapshots for this build release. 755 # 756 # Defaults to { 757 # "SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE": <name>, 758 # } 759 soong_env: typing.Dict[str, str] = None 760 761 # The position of this instance within the BUILD_RELEASES list. 762 ordinal: int = dataclasses.field(default=-1, init=False) 763 764 # Whether this build release supports the Soong config boilerplate that is 765 # used to control the prefer setting of modules via a Soong config variable. 766 preferHandling: PreferHandling = \ 767 PreferHandling.USE_SOURCE_CONFIG_VAR_PROPERTY 768 769 # Whether the generated snapshots should include flagged APIs. Defaults to 770 # false because flagged APIs are not suitable for use outside Android. 771 include_flagged_apis: bool = False 772 773 # Whether the build release should generate Gantry metadata and API diff. 774 generate_gantry_metadata_and_api_diff: bool = False 775 776 def __post_init__(self): 777 # The following use object.__setattr__ as this object is frozen and 778 # attempting to set the fields directly would cause an exception to be 779 # thrown. 780 object.__setattr__(self, "ordinal", len(ALL_BUILD_RELEASES)) 781 # Add this to the end of the list of all build releases. 782 ALL_BUILD_RELEASES.append(self) 783 # If no sub_dir was specified then set the default. 784 if self.sub_dir is None: 785 object.__setattr__(self, "sub_dir", f"for-{self.name}-build") 786 # If no soong_env was specified then set the default. 787 if self.soong_env is None: 788 object.__setattr__( 789 self, 790 "soong_env", 791 { 792 # Set SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE to generate a 793 # snapshot suitable for a specific target build release. 794 "SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE": self.name, 795 }) 796 797 def __eq__(self, other): 798 return self.ordinal == other.ordinal 799 800 def __le__(self, other): 801 return self.ordinal <= other.ordinal 802 803 804def create_no_dist_snapshot(_: BuildRelease, __: "SdkDistProducer", 805 modules: List["MainlineModule"]): 806 """A place holder dist snapshot creation function that does nothing.""" 807 print(f"create_no_dist_snapshot for modules {[m.apex for m in modules]}") 808 809 810def create_dist_snapshot_for_r(build_release: BuildRelease, 811 producer: "SdkDistProducer", 812 modules: List["MainlineModule"]): 813 """Generate a snapshot suitable for use in an R build.""" 814 producer.product_dist_for_build_r(build_release, modules) 815 816 817def create_sdk_snapshots_in_soong(build_release: BuildRelease, 818 producer: "SdkDistProducer", 819 modules: List["MainlineModule"]): 820 """Builds sdks and populates the dist for unbundled modules.""" 821 producer.produce_unbundled_dist_for_build_release(build_release, modules) 822 823 824def create_latest_sdk_snapshots(build_release: BuildRelease, 825 producer: "SdkDistProducer", 826 modules: List["MainlineModule"]): 827 """Builds and populates the latest release, including bundled modules.""" 828 producer.produce_unbundled_dist_for_build_release(build_release, modules) 829 producer.produce_bundled_dist_for_build_release(build_release, modules) 830 831 832Q = BuildRelease( 833 name="Q", 834 # At the moment we do not generate a snapshot for Q. 835 creator=create_no_dist_snapshot, 836 # This does not support or need any special prefer property handling. 837 preferHandling=PreferHandling.NONE, 838) 839R = BuildRelease( 840 name="R", 841 # Generate a simple snapshot for R. 842 creator=create_dist_snapshot_for_r, 843 # By default a BuildRelease creates an environment to pass to Soong that 844 # creates a release specific snapshot. However, Soong does not yet (and is 845 # unlikely to) support building an sdk snapshot for R so create an empty 846 # environment to pass to Soong instead. 847 soong_env={}, 848 # This does not support or need any special prefer property handling. 849 preferHandling=PreferHandling.NONE, 850) 851S = BuildRelease( 852 name="S", 853 # Generate a snapshot for this build release using Soong. 854 creator=create_sdk_snapshots_in_soong, 855 # This requires the SoongConfigBoilerplateInserter transformation to be 856 # applied. 857 preferHandling=PreferHandling.SOONG_CONFIG, 858) 859Tiramisu = BuildRelease( 860 name="Tiramisu", 861 # Generate a snapshot for this build release using Soong. 862 creator=create_sdk_snapshots_in_soong, 863 # This build release supports the use_source_config_var property. 864 preferHandling=PreferHandling.USE_SOURCE_CONFIG_VAR_PROPERTY, 865) 866UpsideDownCake = BuildRelease( 867 name="UpsideDownCake", 868 # Generate a snapshot for this build release using Soong. 869 creator=create_sdk_snapshots_in_soong, 870 # This build release supports the use_source_config_var property. 871 preferHandling=PreferHandling.USE_SOURCE_CONFIG_VAR_PROPERTY, 872) 873VanillaIceCream = BuildRelease( 874 name="VanillaIceCream", 875 # Generate a snapshot for this build release using Soong. 876 creator=create_sdk_snapshots_in_soong, 877 # Starting with V, setting `prefer|use_source_config_var` on soong modules 878 # in prebuilts/module_sdk is not necessary. 879 # prebuilts will be enabled using apex_contributions release build flags. 880 preferHandling=PreferHandling.USE_NO_PREFER_PROPERTY, 881) 882Baklava = BuildRelease( 883 name="Baklava", 884 # Generate a snapshot for this build release using Soong. 885 creator=create_sdk_snapshots_in_soong, 886 # There are no build release specific environment variables to pass to 887 # Soong. 888 soong_env={}, 889 # Starting with V, setting `prefer|use_source_config_var` on soong modules 890 # in prebuilts/module_sdk is not necessary. 891 # prebuilts will be enabled using apex_contributions release build flags. 892 preferHandling=PreferHandling.USE_NO_PREFER_PROPERTY, 893) 894 895# Insert additional BuildRelease definitions for following releases here, 896# before LATEST. 897 898# A build release for the latest build excluding flagged apis. 899NEXT = BuildRelease( 900 name="next", 901 creator=create_latest_sdk_snapshots, 902 # There are no build release specific environment variables to pass to 903 # Soong. 904 soong_env={}, 905 generate_gantry_metadata_and_api_diff=True, 906 # Starting with V, setting `prefer|use_source_config_var` on soong modules 907 # in prebuilts/module_sdk is not necessary. 908 # prebuilts will be enabled using apex_contributions release build flags. 909 preferHandling=PreferHandling.USE_NO_PREFER_PROPERTY, 910) 911 912# The build release for the latest build supported by this build, i.e. the 913# current build. This must be the last BuildRelease defined in this script. 914LATEST = BuildRelease( 915 name="latest", 916 creator=create_latest_sdk_snapshots, 917 # There are no build release specific environment variables to pass to 918 # Soong. 919 soong_env={}, 920 # Latest must include flagged APIs because it may be dropped into the main 921 # Android branches. 922 include_flagged_apis=True, 923 generate_gantry_metadata_and_api_diff=True, 924 # Starting with V, setting `prefer|use_source_config_var` on soong modules 925 # in prebuilts/module_sdk is not necessary. 926 # prebuilts will be enabled using apex_contributions release build flags. 927 preferHandling=PreferHandling.USE_NO_PREFER_PROPERTY, 928) 929 930 931@dataclasses.dataclass(frozen=True) 932class SdkLibrary: 933 """Information about a java_sdk_library.""" 934 935 # The name of java_sdk_library module. 936 name: str 937 938 # True if the sdk_library module is a shared library. 939 shared_library: bool = False 940 941 942@dataclasses.dataclass(frozen=True) 943class ForRBuild: 944 """Data structure needed for generating a snapshot for an R build.""" 945 946 # The java_sdk_library modules to export to the r snapshot. 947 sdk_libraries: typing.List[SdkLibrary] = dataclasses.field( 948 default_factory=list) 949 950 951@dataclasses.dataclass(frozen=True) 952class MainlineModule: 953 """Represents an unbundled mainline module. 954 955 This is a module that is distributed as a prebuilt and intended to be 956 updated with Mainline trains. 957 """ 958 # The name of the apex. 959 apex: str 960 961 # The names of the sdk and module_exports. 962 sdks: list[str] 963 964 # The first build release in which the SDK snapshot for this module is 965 # needed. 966 # 967 # Note: This is not necessarily the same build release in which the SDK 968 # source was first included. So, a module that was added in build T 969 # could potentially be used in an S release and so its SDK will need 970 # to be made available for S builds. 971 first_release: BuildRelease 972 973 # The configuration variable, defaults to ANDROID:module_build_from_source 974 configVar: ConfigVar = ConfigVar( 975 namespace="ANDROID", 976 name="module_build_from_source", 977 ) 978 979 for_r_build: typing.Optional[ForRBuild] = None 980 981 # The last release on which this module was optional. 982 # 983 # Some modules are optional when they are first released, usually because 984 # some vendors of Android devices have their own customizations of the 985 # module that they would like to preserve and which cannot yet be achieved 986 # through the existing APIs. Once those issues have been resolved then they 987 # will become mandatory. 988 # 989 # This field records the last build release in which they are optional. It 990 # defaults to None which indicates that the module was never optional. 991 # 992 # TODO(b/238203992): remove the following warning once all modules can be 993 # treated as optional at build time. 994 # 995 # DO NOT use this attr for anything other than controlling whether the 996 # generated snapshot uses its own Soong config variable or the common one. 997 # That is because this is being temporarily used to force Permission to have 998 # its own Soong config variable even though Permission is not actually 999 # optional at runtime on a GMS capable device. 1000 # 1001 # b/238203992 will make all modules have their own Soong config variable by 1002 # default at which point this will no longer be needed on Permission and so 1003 # it can be used to indicate that a module is optional at runtime. 1004 last_optional_release: typing.Optional[BuildRelease] = None 1005 1006 # The short name for the module. 1007 # 1008 # Defaults to the last part of the apex name. 1009 short_name: str = "" 1010 1011 # Additional transformations 1012 additional_transformations: list[FileTransformation] = None 1013 1014 # The module key of SdkModule Enum defined in 1015 # packages/modules/common/proto/sdk.proto. 1016 module_proto_key: str = "" 1017 1018 def __post_init__(self): 1019 # If short_name is not set then set it to the last component of the apex 1020 # name. 1021 if not self.short_name: 1022 short_name = self.apex.rsplit(".", 1)[-1] 1023 object.__setattr__(self, "short_name", short_name) 1024 1025 def is_bundled(self): 1026 """Returns true for bundled modules. See BundledMainlineModule.""" 1027 return False 1028 1029 def transformations(self, build_release, sdk_type): 1030 """Returns the transformations to apply to this module's snapshot(s).""" 1031 transformations = [] 1032 1033 config_var = self.configVar 1034 1035 # If the module is optional then it needs its own Soong config 1036 # variable to allow it to be managed separately from other modules. 1037 if self.last_optional_release: 1038 config_var = ConfigVar( 1039 namespace=f"{self.short_name}_module", 1040 name="source_build", 1041 ) 1042 1043 prefer_handling = build_release.preferHandling 1044 if prefer_handling == PreferHandling.SOONG_CONFIG: 1045 sdk_type_prefix = sdk_type.configModuleTypePrefix 1046 config_module_type_prefix = \ 1047 f"{self.short_name}{sdk_type_prefix}_prebuilt_" 1048 inserter = SoongConfigBoilerplateInserter( 1049 "Android.bp", 1050 configVar=config_var, 1051 configModuleTypePrefix=config_module_type_prefix) 1052 transformations.append(inserter) 1053 elif prefer_handling == PreferHandling.USE_SOURCE_CONFIG_VAR_PROPERTY: 1054 transformation = UseSourceConfigVarTransformation( 1055 "Android.bp", configVar=config_var) 1056 transformations.append(transformation) 1057 elif prefer_handling == PreferHandling.USE_NO_PREFER_PROPERTY: 1058 transformation = UseNoPreferPropertyTransformation( 1059 "Android.bp", configVar=config_var 1060 ) 1061 transformations.append(transformation) 1062 1063 if self.additional_transformations and build_release > R: 1064 transformations.extend(self.additional_transformations) 1065 1066 return transformations 1067 1068 def is_required_for(self, target_build_release): 1069 """True if this module is required for the target build release.""" 1070 return self.first_release <= target_build_release 1071 1072 1073@dataclasses.dataclass(frozen=True) 1074class BundledMainlineModule(MainlineModule): 1075 """Represents a bundled Mainline module or a platform SDK for module use. 1076 1077 A bundled module is always preloaded into the platform images. 1078 """ 1079 1080 # Defaults to the latest build, i.e. the build on which this script is run 1081 # as bundled modules are, by definition, only needed in this build. 1082 first_release: BuildRelease = LATEST 1083 1084 def is_bundled(self): 1085 return True 1086 1087 def transformations(self, build_release, sdk_type): 1088 # Bundled modules are only used on thin branches where the corresponding 1089 # sources are absent, so skip transformations and keep the default 1090 # `prefer: false`. 1091 return [] 1092 1093 1094# List of mainline modules. 1095MAINLINE_MODULES = [ 1096 MainlineModule( 1097 apex="com.android.adservices", 1098 sdks=["adservices-module-sdk"], 1099 first_release=Tiramisu, 1100 last_optional_release=LATEST, 1101 module_proto_key="AD_SERVICES", 1102 ), 1103 MainlineModule( 1104 apex="com.android.appsearch", 1105 sdks=["appsearch-sdk"], 1106 first_release=Tiramisu, 1107 last_optional_release=LATEST, 1108 module_proto_key="APPSEARCH", 1109 ), 1110 MainlineModule( 1111 apex="com.android.art", 1112 sdks=[ 1113 "art-module-sdk", 1114 "art-module-test-exports", 1115 "art-module-host-exports", 1116 ], 1117 first_release=S, 1118 # Override the config... fields. 1119 configVar=ConfigVar( 1120 namespace="art_module", 1121 name="source_build", 1122 ), 1123 module_proto_key="ART", 1124 ), 1125 MainlineModule( 1126 apex="com.android.bt", 1127 sdks=["bt-module-sdk"], 1128 first_release=Baklava, 1129 # Bluetooth is optional. 1130 last_optional_release=LATEST, 1131 module_proto_key="", 1132 ), 1133 MainlineModule( 1134 apex="com.android.configinfrastructure", 1135 sdks=["configinfrastructure-sdk"], 1136 first_release=UpsideDownCake, 1137 last_optional_release=LATEST, 1138 module_proto_key="CONFIG_INFRASTRUCTURE", 1139 ), 1140 MainlineModule( 1141 apex="com.android.conscrypt", 1142 sdks=[ 1143 "conscrypt-module-sdk", 1144 "conscrypt-module-test-exports", 1145 "conscrypt-module-host-exports", 1146 ], 1147 first_release=Q, 1148 # No conscrypt java_sdk_library modules are exported to the R snapshot. 1149 # Conscrypt was updatable in R but the generate_ml_bundle.sh does not 1150 # appear to generate a snapshot for it. 1151 for_r_build=None, 1152 last_optional_release=LATEST, 1153 module_proto_key="CONSCRYPT", 1154 ), 1155 MainlineModule( 1156 apex="com.android.crashrecovery", 1157 sdks=["crashrecovery-sdk"], 1158 first_release=Baklava, 1159 last_optional_release=LATEST, 1160 module_proto_key="", 1161 ), 1162 MainlineModule( 1163 apex="com.android.devicelock", 1164 sdks=["devicelock-module-sdk"], 1165 first_release=UpsideDownCake, 1166 # Treat DeviceLock as optional at build time 1167 # TODO(b/238203992): remove once all modules are optional at build time. 1168 last_optional_release=LATEST, 1169 module_proto_key="", 1170 ), 1171 MainlineModule( 1172 apex="com.android.healthfitness", 1173 sdks=["healthfitness-module-sdk"], 1174 first_release=UpsideDownCake, 1175 last_optional_release=LATEST, 1176 module_proto_key="HEALTH_FITNESS", 1177 ), 1178 MainlineModule( 1179 apex="com.android.ipsec", 1180 sdks=["ipsec-module-sdk"], 1181 first_release=R, 1182 for_r_build=ForRBuild(sdk_libraries=[ 1183 SdkLibrary( 1184 name="android.net.ipsec.ike", 1185 shared_library=True, 1186 ), 1187 ]), 1188 last_optional_release=LATEST, 1189 module_proto_key="IPSEC", 1190 ), 1191 MainlineModule( 1192 apex="com.android.media", 1193 sdks=["media-module-sdk"], 1194 first_release=R, 1195 for_r_build=ForRBuild(sdk_libraries=[ 1196 SdkLibrary(name="framework-media"), 1197 ]), 1198 last_optional_release=LATEST, 1199 module_proto_key="MEDIA", 1200 ), 1201 MainlineModule( 1202 apex="com.android.mediaprovider", 1203 sdks=["mediaprovider-module-sdk"], 1204 first_release=R, 1205 for_r_build=ForRBuild(sdk_libraries=[ 1206 SdkLibrary(name="framework-mediaprovider"), 1207 ]), 1208 # MP is a mandatory mainline module but in some cases (b/294190883) this 1209 # needs to be optional for Android Go on T. GTS tests might be needed to 1210 # to check the specific condition mentioned in the bug. 1211 last_optional_release=LATEST, 1212 module_proto_key="MEDIA_PROVIDER", 1213 ), 1214 MainlineModule( 1215 apex="com.android.nfcservices", 1216 sdks=["nfcservices-module-sdk"], 1217 first_release=Baklava, 1218 # NFC is optional. 1219 last_optional_release=LATEST, 1220 module_proto_key="", 1221 ), 1222 MainlineModule( 1223 apex="com.android.ondevicepersonalization", 1224 sdks=["ondevicepersonalization-module-sdk"], 1225 first_release=Tiramisu, 1226 last_optional_release=LATEST, 1227 module_proto_key="ON_DEVICE_PERSONALIZATION", 1228 ), 1229 MainlineModule( 1230 apex="com.android.permission", 1231 sdks=["permission-module-sdk"], 1232 first_release=R, 1233 for_r_build=ForRBuild(sdk_libraries=[ 1234 SdkLibrary(name="framework-permission"), 1235 # framework-permission-s is not needed on R as it contains classes 1236 # that are provided in R by non-updatable parts of the 1237 # bootclasspath. 1238 ]), 1239 # Although Permission is not, and has never been, optional for GMS 1240 # capable devices it does need to be treated as optional at build time 1241 # when building non-GMS devices. 1242 # TODO(b/238203992): remove once all modules are optional at build time. 1243 last_optional_release=LATEST, 1244 module_proto_key="PERMISSIONS", 1245 ), 1246 MainlineModule( 1247 apex="com.android.profiling", 1248 sdks=["profiling-module-sdk"], 1249 first_release=Baklava, 1250 # Profiling is optional. 1251 last_optional_release=LATEST, 1252 module_proto_key="", 1253 ), 1254 MainlineModule( 1255 apex="com.android.rkpd", 1256 sdks=["rkpd-sdk"], 1257 first_release=UpsideDownCake, 1258 # Rkpd has always been and is still optional. 1259 last_optional_release=LATEST, 1260 module_proto_key="", 1261 ), 1262 MainlineModule( 1263 apex="com.android.scheduling", 1264 sdks=["scheduling-sdk"], 1265 first_release=S, 1266 last_optional_release=LATEST, 1267 module_proto_key="SCHEDULING", 1268 ), 1269 MainlineModule( 1270 apex="com.android.sdkext", 1271 sdks=[ 1272 "sdkextensions-sdk", 1273 "sdkextensions-host-exports", 1274 ], 1275 first_release=R, 1276 for_r_build=ForRBuild(sdk_libraries=[ 1277 SdkLibrary(name="framework-sdkextensions"), 1278 ]), 1279 last_optional_release=LATEST, 1280 module_proto_key="SDK_EXTENSIONS", 1281 ), 1282 MainlineModule( 1283 apex="com.android.os.statsd", 1284 sdks=["statsd-module-sdk"], 1285 first_release=R, 1286 for_r_build=ForRBuild(sdk_libraries=[ 1287 SdkLibrary(name="framework-statsd"), 1288 ]), 1289 last_optional_release=LATEST, 1290 module_proto_key="STATSD", 1291 ), 1292 MainlineModule( 1293 apex="com.android.tethering", 1294 sdks=["tethering-module-sdk"], 1295 first_release=R, 1296 for_r_build=ForRBuild(sdk_libraries=[ 1297 SdkLibrary(name="framework-tethering"), 1298 ]), 1299 last_optional_release=LATEST, 1300 module_proto_key="TETHERING", 1301 ), 1302 MainlineModule( 1303 apex="com.android.uwb", 1304 sdks=["uwb-module-sdk"], 1305 first_release=Tiramisu, 1306 # Uwb has always been and is still optional. 1307 last_optional_release=LATEST, 1308 module_proto_key="", 1309 ), 1310 MainlineModule( 1311 apex="com.android.wifi", 1312 sdks=["wifi-module-sdk"], 1313 first_release=R, 1314 for_r_build=ForRBuild(sdk_libraries=[ 1315 SdkLibrary(name="framework-wifi"), 1316 ]), 1317 # Wifi has always been and is still optional. 1318 last_optional_release=LATEST, 1319 module_proto_key="", 1320 ), 1321] 1322 1323# List of Mainline modules that currently are never built unbundled. They must 1324# not specify first_release, and they don't have com.google.android 1325# counterparts. 1326BUNDLED_MAINLINE_MODULES = [ 1327 BundledMainlineModule( 1328 apex="com.android.i18n", 1329 sdks=[ 1330 "i18n-module-sdk", 1331 "i18n-module-test-exports", 1332 "i18n-module-host-exports", 1333 ], 1334 ), 1335 BundledMainlineModule( 1336 apex="com.android.runtime", 1337 sdks=[ 1338 "runtime-module-host-exports", 1339 "runtime-module-sdk", 1340 ], 1341 ), 1342 BundledMainlineModule( 1343 apex="com.android.tzdata", 1344 sdks=["tzdata-module-test-exports"], 1345 ), 1346] 1347 1348# List of platform SDKs for Mainline module use. 1349PLATFORM_SDKS_FOR_MAINLINE = [ 1350 BundledMainlineModule( 1351 apex="platform-mainline", 1352 sdks=[ 1353 "platform-mainline-sdk", 1354 "platform-mainline-test-exports", 1355 ], 1356 ), 1357] 1358 1359 1360@dataclasses.dataclass 1361class SdkDistProducer: 1362 """Produces the DIST_DIR/mainline-sdks and DIST_DIR/stubs directories. 1363 1364 Builds SDK snapshots for mainline modules and then copies them into the 1365 DIST_DIR/mainline-sdks directory. Also extracts the sdk_library txt, jar and 1366 srcjar files from each SDK snapshot and copies them into the DIST_DIR/stubs 1367 directory. 1368 """ 1369 1370 # Used to run subprocesses for this. 1371 subprocess_runner: SubprocessRunner 1372 1373 # Builds sdk snapshots 1374 snapshot_builder: SnapshotBuilder 1375 1376 # The DIST_DIR environment variable. 1377 dist_dir: str = "uninitialized-dist" 1378 1379 # The path to this script. It may be inserted into files that are 1380 # transformed to document where the changes came from. 1381 script: str = sys.argv[0] 1382 1383 # The path to the mainline-sdks dist directory for unbundled modules. 1384 # 1385 # Initialized in __post_init__(). 1386 mainline_sdks_dir: str = dataclasses.field(init=False) 1387 1388 # The path to the mainline-sdks dist directory for bundled modules and 1389 # platform SDKs. 1390 # 1391 # Initialized in __post_init__(). 1392 bundled_mainline_sdks_dir: str = dataclasses.field(init=False) 1393 1394 def __post_init__(self): 1395 self.mainline_sdks_dir = os.path.join(self.dist_dir, "mainline-sdks") 1396 self.bundled_mainline_sdks_dir = os.path.join(self.dist_dir, 1397 "bundled-mainline-sdks") 1398 1399 def prepare(self): 1400 pass 1401 1402 def produce_dist(self, modules, build_releases): 1403 # Prepare the dist directory for the sdks. 1404 self.prepare() 1405 1406 # Group build releases so that those with the same Soong environment are 1407 # run consecutively to avoid having to regenerate ninja files. 1408 grouped_by_env = defaultdict(list) 1409 for build_release in build_releases: 1410 grouped_by_env[str(build_release.soong_env)].append(build_release) 1411 ordered = [br for _, group in grouped_by_env.items() for br in group] 1412 1413 for build_release in ordered: 1414 # Only build modules that are required for this build release. 1415 filtered_modules = [ 1416 m for m in modules if m.is_required_for(build_release) 1417 ] 1418 if filtered_modules: 1419 print(f"Building SDK snapshots for {build_release.name}" 1420 f" build release") 1421 build_release.creator(build_release, self, filtered_modules) 1422 1423 def product_dist_for_build_r(self, build_release, modules): 1424 # Although we only need a subset of the files that a java_sdk_library 1425 # adds to an sdk snapshot generating the whole snapshot is the simplest 1426 # way to ensure that all the necessary files are produced. 1427 1428 # Filter out any modules that do not provide sdk for R. 1429 modules = [m for m in modules if m.for_r_build] 1430 1431 snapshot_dir = self.snapshot_builder.build_snapshots_for_build_r( 1432 build_release, modules) 1433 self.populate_unbundled_dist(build_release, modules, snapshot_dir) 1434 1435 def produce_unbundled_dist_for_build_release(self, build_release, modules): 1436 modules = [m for m in modules if not m.is_bundled()] 1437 snapshots_dir = self.snapshot_builder.build_snapshots( 1438 build_release, modules) 1439 if build_release.generate_gantry_metadata_and_api_diff: 1440 target_dict = self.snapshot_builder.build_sdk_scope_targets( 1441 build_release, modules) 1442 self.snapshot_builder.build_snapshot_gantry_metadata_and_api_diff( 1443 modules, target_dict, snapshots_dir) 1444 self.populate_unbundled_dist(build_release, modules, snapshots_dir) 1445 return snapshots_dir 1446 1447 def produce_bundled_dist_for_build_release(self, build_release, modules): 1448 modules = [m for m in modules if m.is_bundled()] 1449 if modules: 1450 snapshots_dir = self.snapshot_builder.build_snapshots( 1451 build_release, modules) 1452 self.populate_bundled_dist(build_release, modules, snapshots_dir) 1453 1454 def dist_sdk_snapshot_gantry_metadata_and_api_diff(self, sdk_dist_dir, sdk, 1455 module, snapshots_dir): 1456 """Copy the sdk snapshot api diff file to a dist directory.""" 1457 sdk_type = sdk_type_from_name(sdk) 1458 if not sdk_type.providesApis: 1459 return 1460 1461 sdk_dist_module_subdir = os.path.join(sdk_dist_dir, module.apex) 1462 sdk_dist_subdir = os.path.join(sdk_dist_module_subdir, "sdk") 1463 os.makedirs(sdk_dist_subdir, exist_ok=True) 1464 sdk_api_diff_path = sdk_snapshot_api_diff_file(snapshots_dir, sdk) 1465 shutil.copy(sdk_api_diff_path, sdk_dist_subdir) 1466 1467 sdk_gantry_metadata_json_path = sdk_snapshot_gantry_metadata_json_file( 1468 snapshots_dir, sdk) 1469 sdk_dist_gantry_metadata_json_path = os.path.join( 1470 sdk_dist_module_subdir, "gantry-metadata.json") 1471 shutil.copy(sdk_gantry_metadata_json_path, 1472 sdk_dist_gantry_metadata_json_path) 1473 1474 def dist_generate_sdk_supported_modules_file(self, modules): 1475 sdk_modules_file = os.path.join(self.dist_dir, "sdk-modules.txt") 1476 os.makedirs(os.path.dirname(sdk_modules_file), exist_ok=True) 1477 with open(sdk_modules_file, "w", encoding="utf8") as file: 1478 for module in modules: 1479 if module in MAINLINE_MODULES: 1480 file.write(aosp_to_google_name(module.apex) + "\n") 1481 1482 def generate_mainline_modules_info_file(self, modules, root_dir): 1483 mainline_modules_info_file = os.path.join( 1484 self.dist_dir, "mainline-modules-info.json" 1485 ) 1486 os.makedirs(os.path.dirname(mainline_modules_info_file), exist_ok=True) 1487 mainline_modules_info_dict = {} 1488 for module in modules: 1489 if module not in MAINLINE_MODULES: 1490 continue 1491 module_name = aosp_to_google_name(module.apex) 1492 mainline_modules_info_dict[module_name] = dict() 1493 mainline_modules_info_dict[module_name]["module_sdk_project"] = ( 1494 module_sdk_project_for_module(module_name, root_dir) 1495 ) 1496 mainline_modules_info_dict[module_name][ 1497 "module_proto_key" 1498 ] = module.module_proto_key 1499 # The first sdk in the list is the name to use. 1500 mainline_modules_info_dict[module_name]["sdk_name"] = module.sdks[0] 1501 1502 with open(mainline_modules_info_file, "w", encoding="utf8") as file: 1503 json.dump(mainline_modules_info_dict, file, indent=4) 1504 1505 def populate_unbundled_dist(self, build_release, modules, snapshots_dir): 1506 build_release_dist_dir = os.path.join(self.mainline_sdks_dir, 1507 build_release.sub_dir) 1508 for module in modules: 1509 for sdk in module.sdks: 1510 sdk_dist_dir = os.path.join(build_release_dist_dir, SDK_VERSION) 1511 if build_release.generate_gantry_metadata_and_api_diff: 1512 self.dist_sdk_snapshot_gantry_metadata_and_api_diff( 1513 sdk_dist_dir, sdk, module, snapshots_dir) 1514 self.populate_dist_snapshot(build_release, module, sdk, 1515 sdk_dist_dir, snapshots_dir) 1516 1517 def populate_bundled_dist(self, build_release, modules, snapshots_dir): 1518 sdk_dist_dir = self.bundled_mainline_sdks_dir 1519 for module in modules: 1520 for sdk in module.sdks: 1521 self.populate_dist_snapshot(build_release, module, sdk, 1522 sdk_dist_dir, snapshots_dir) 1523 1524 def populate_dist_snapshot(self, build_release, module, sdk, sdk_dist_dir, 1525 snapshots_dir): 1526 sdk_type = sdk_type_from_name(sdk) 1527 subdir = sdk_type.name 1528 1529 # HostExports are not needed for R. 1530 if build_release == R and sdk_type == HostExports: 1531 return 1532 1533 sdk_dist_subdir = os.path.join(sdk_dist_dir, module.apex, subdir) 1534 sdk_path = sdk_snapshot_zip_file(snapshots_dir, sdk) 1535 transformations = module.transformations(build_release, sdk_type) 1536 self.dist_sdk_snapshot_zip( 1537 build_release, sdk_path, sdk_dist_subdir, transformations) 1538 1539 def dist_sdk_snapshot_zip( 1540 self, build_release, src_sdk_zip, sdk_dist_dir, transformations): 1541 """Copy the sdk snapshot zip file to a dist directory. 1542 1543 If no transformations are provided then this simply copies the show sdk 1544 snapshot zip file to the dist dir. However, if transformations are 1545 provided then the files to be transformed are extracted from the 1546 snapshot zip file, they are transformed to files in a separate directory 1547 and then a new zip file is created in the dist directory with the 1548 original files replaced by the newly transformed files. build_release is 1549 provided for transformations if it is needed. 1550 """ 1551 os.makedirs(sdk_dist_dir, exist_ok=True) 1552 dest_sdk_zip = os.path.join(sdk_dist_dir, os.path.basename(src_sdk_zip)) 1553 print(f"Copying sdk snapshot {src_sdk_zip} to {dest_sdk_zip}") 1554 1555 # If no transformations are provided then just copy the zip file 1556 # directly. 1557 if len(transformations) == 0: 1558 shutil.copy(src_sdk_zip, sdk_dist_dir) 1559 return 1560 1561 with tempfile.TemporaryDirectory() as tmp_dir: 1562 # Create a single pattern that will match any of the paths provided 1563 # in the transformations. 1564 pattern = "|".join( 1565 [f"({re.escape(t.path)})" for t in transformations]) 1566 1567 # Extract the matching files from the zip into the temporary 1568 # directory. 1569 extract_matching_files_from_zip(src_sdk_zip, tmp_dir, pattern) 1570 1571 # Apply the transformations to the extracted files in situ. 1572 apply_transformations(self, tmp_dir, transformations, build_release) 1573 1574 # Replace the original entries in the zip with the transformed 1575 # files. 1576 paths = [transformation.path for transformation in transformations] 1577 copy_zip_and_replace(self, src_sdk_zip, dest_sdk_zip, tmp_dir, 1578 paths) 1579 1580 1581def print_command(env, cmd): 1582 print(" ".join([f"{name}={value}" for name, value in env.items()] + cmd)) 1583 1584 1585def sdk_library_files_pattern(*, scope_pattern=r"[^/]+", name_pattern=r"[^/]+"): 1586 """Return a pattern to match sdk_library related files in an sdk snapshot""" 1587 return rf"sdk_library/{scope_pattern}/{name_pattern}\.(txt|jar|srcjar)" 1588 1589 1590def extract_matching_files_from_zip(zip_path, dest_dir, pattern): 1591 """Extracts files from a zip file into a destination directory. 1592 1593 The extracted files are those that match the specified regular expression 1594 pattern. 1595 """ 1596 os.makedirs(dest_dir, exist_ok=True) 1597 with zipfile.ZipFile(zip_path) as zip_file: 1598 for filename in zip_file.namelist(): 1599 if re.match(pattern, filename): 1600 print(f" extracting {filename}") 1601 zip_file.extract(filename, dest_dir) 1602 1603 1604def copy_zip_and_replace(producer, src_zip_path, dest_zip_path, src_dir, paths): 1605 """Copies a zip replacing some of its contents in the process. 1606 1607 The files to replace are specified by the paths parameter and are relative 1608 to the src_dir. 1609 """ 1610 # Get the absolute paths of the source and dest zip files so that they are 1611 # not affected by a change of directory. 1612 abs_src_zip_path = os.path.abspath(src_zip_path) 1613 abs_dest_zip_path = os.path.abspath(dest_zip_path) 1614 1615 # Make sure that all the paths being added to the zip file have a fixed 1616 # timestamp so that the contents of the zip file do not depend on when this 1617 # script is run, only the inputs. 1618 set_default_timestamp(src_dir, paths) 1619 1620 producer.subprocess_runner.run( 1621 ["zip", "-q", abs_src_zip_path, "--out", abs_dest_zip_path] + paths, 1622 # Change into the source directory before running zip. 1623 cwd=src_dir) 1624 1625 1626def apply_transformations(producer, tmp_dir, transformations, build_release): 1627 for transformation in transformations: 1628 path = os.path.join(tmp_dir, transformation.path) 1629 1630 # Record the timestamp of the file. 1631 modified = os.path.getmtime(path) 1632 1633 # Transform the file. 1634 transformation.apply(producer, path, build_release) 1635 1636 # Reset the timestamp of the file to the original timestamp before the 1637 # transformation was applied. 1638 os.utime(path, (modified, modified)) 1639 1640 1641def create_producer(tool_path, skip_allowed_deps_check): 1642 # Variables initialized from environment variables that are set by the 1643 # calling mainline_modules_sdks.sh. 1644 out_dir = os.environ["OUT_DIR"] 1645 dist_dir = os.environ["DIST_DIR"] 1646 1647 top_dir = os.environ["ANDROID_BUILD_TOP"] 1648 tool_path = os.path.relpath(tool_path, top_dir) 1649 tool_path = tool_path.replace(".py", ".sh") 1650 1651 subprocess_runner = SubprocessRunner() 1652 snapshot_builder = SnapshotBuilder( 1653 tool_path=tool_path, 1654 subprocess_runner=subprocess_runner, 1655 out_dir=out_dir, 1656 skip_allowed_deps_check=skip_allowed_deps_check, 1657 ) 1658 return SdkDistProducer( 1659 subprocess_runner=subprocess_runner, 1660 snapshot_builder=snapshot_builder, 1661 dist_dir=dist_dir, 1662 ) 1663 1664 1665def aosp_to_google(module): 1666 """Transform an AOSP module into a Google module""" 1667 new_apex = aosp_to_google_name(module.apex) 1668 # Create a copy of the AOSP module with the internal specific APEX name. 1669 return dataclasses.replace(module, apex=new_apex) 1670 1671 1672def aosp_to_google_name(name): 1673 """Transform an AOSP module name into a Google module name""" 1674 return name.replace("com.android.", "com.google.android.") 1675 1676 1677def google_to_aosp_name(name): 1678 """Transform a Google module name into an AOSP module name""" 1679 return name.replace("com.google.android.", "com.android.") 1680 1681 1682@dataclasses.dataclass(frozen=True) 1683class SdkType: 1684 name: str 1685 1686 configModuleTypePrefix: str 1687 1688 providesApis: bool = False 1689 1690 1691Sdk = SdkType( 1692 name="sdk", 1693 configModuleTypePrefix="", 1694 providesApis=True, 1695) 1696HostExports = SdkType( 1697 name="host-exports", 1698 configModuleTypePrefix="_host_exports", 1699) 1700TestExports = SdkType( 1701 name="test-exports", 1702 configModuleTypePrefix="_test_exports", 1703) 1704 1705 1706def sdk_type_from_name(name): 1707 if name.endswith("-sdk"): 1708 return Sdk 1709 if name.endswith("-host-exports"): 1710 return HostExports 1711 if name.endswith("-test-exports"): 1712 return TestExports 1713 1714 raise Exception(f"{name} is not a valid sdk name, expected it to end" 1715 f" with -(sdk|host-exports|test-exports)") 1716 1717 1718def filter_modules(modules, target_build_apps): 1719 if target_build_apps: 1720 target_build_apps = target_build_apps.split() 1721 return [m for m in modules if m.apex in target_build_apps] 1722 return modules 1723 1724 1725def main(args): 1726 """Program entry point.""" 1727 if not os.path.exists("build/make/core/Makefile"): 1728 sys.exit("This script must be run from the top of the tree.") 1729 1730 args_parser = argparse.ArgumentParser( 1731 description="Build snapshot zips for consumption by Gantry.") 1732 args_parser.add_argument( 1733 "--tool-path", 1734 help="The path to this tool.", 1735 default="unspecified", 1736 ) 1737 args_parser.add_argument( 1738 "--build-release", 1739 action="append", 1740 choices=[br.name for br in ALL_BUILD_RELEASES], 1741 help="A target build for which snapshots are required. " 1742 "If it is \"latest\" then Mainline module SDKs from platform and " 1743 "bundled modules are included.", 1744 ) 1745 args_parser.add_argument( 1746 "--build-platform-sdks-for-mainline", 1747 action="store_true", 1748 help="Also build the platform SDKs for Mainline modules. " 1749 "Defaults to true when TARGET_BUILD_APPS is not set. " 1750 "Applicable only if the \"latest\" build release is built.", 1751 ) 1752 args_parser.add_argument( 1753 "--skip-allowed-deps-check", 1754 action="store_true", 1755 help="Skip apex-allowed-deps-check.", 1756 ) 1757 args = args_parser.parse_args(args) 1758 1759 build_releases = ALL_BUILD_RELEASES 1760 if args.build_release: 1761 selected_build_releases = {b.lower() for b in args.build_release} 1762 build_releases = [ 1763 b for b in build_releases 1764 if b.name.lower() in selected_build_releases 1765 ] 1766 1767 target_build_apps = os.environ.get("TARGET_BUILD_APPS") 1768 modules = filter_modules(MAINLINE_MODULES + BUNDLED_MAINLINE_MODULES, 1769 target_build_apps) 1770 1771 # Also build the platform Mainline SDKs either if no specific modules are 1772 # requested or if --build-platform-sdks-for-mainline is given. 1773 if not target_build_apps or args.build_platform_sdks_for_mainline: 1774 modules += PLATFORM_SDKS_FOR_MAINLINE 1775 1776 producer = create_producer(args.tool_path, args.skip_allowed_deps_check) 1777 producer.dist_generate_sdk_supported_modules_file(modules) 1778 producer.generate_mainline_modules_info_file( 1779 modules, os.environ["ANDROID_BUILD_TOP"] 1780 ) 1781 producer.produce_dist(modules, build_releases) 1782 1783 1784if __name__ == "__main__": 1785 main(sys.argv[1:]) 1786