1#!/usr/bin/env python3 2# 3# Copyright (C) 2021 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16"""Builds SDK snapshots. 17 18If the environment variable TARGET_BUILD_APPS is nonempty then only the SDKs for 19the APEXes in it are built, otherwise all configured SDKs are built. 20""" 21import argparse 22import dataclasses 23import datetime 24import enum 25import functools 26import io 27import json 28import os 29import re 30import shutil 31import subprocess 32import sys 33import tempfile 34import typing 35from collections import defaultdict 36from typing import Callable, List 37import zipfile 38 39COPYRIGHT_BOILERPLATE = """ 40// 41// Copyright (C) 2020 The Android Open Source Project 42// 43// Licensed under the Apache License, Version 2.0 (the "License"); 44// you may not use this file except in compliance with the License. 45// You may obtain a copy of the License at 46// 47// http://www.apache.org/licenses/LICENSE-2.0 48// 49// Unless required by applicable law or agreed to in writing, software 50// distributed under the License is distributed on an "AS IS" BASIS, 51// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 52// See the License for the specific language governing permissions and 53// limitations under the License. 54// 55""".lstrip() 56 57 58@dataclasses.dataclass(frozen=True) 59class ConfigVar: 60 """Represents a Soong configuration variable""" 61 # The config variable namespace, e.g. ANDROID. 62 namespace: str 63 64 # The name of the variable within the namespace. 65 name: str 66 67 68@dataclasses.dataclass(frozen=True) 69class FileTransformation: 70 """Performs a transformation on a file within an SDK snapshot zip file.""" 71 72 # The path of the file within the SDK snapshot zip file. 73 path: str 74 75 def apply(self, producer, path, build_release): 76 """Apply the transformation to the path; changing it in place.""" 77 with open(path, "r+", encoding="utf8") as file: 78 self._apply_transformation(producer, file, build_release) 79 80 def _apply_transformation(self, producer, file, build_release): 81 """Apply the transformation to the file. 82 83 The file has been opened in read/write mode so the implementation of 84 this must read the contents and then reset the file to the beginning 85 and write the altered contents. 86 """ 87 raise NotImplementedError 88 89 90@dataclasses.dataclass(frozen=True) 91class SoongConfigVarTransformation(FileTransformation): 92 93 # The configuration variable that will control the prefer setting. 94 configVar: ConfigVar 95 96 # The line containing the prefer property. 97 PREFER_LINE = " prefer: false," 98 99 def _apply_transformation(self, producer, file, build_release): 100 raise NotImplementedError 101 102 103@dataclasses.dataclass(frozen=True) 104class SoongConfigBoilerplateInserter(SoongConfigVarTransformation): 105 """Transforms an Android.bp file to add soong config boilerplate. 106 107 The boilerplate allows the prefer setting of the modules to be controlled 108 through a Soong configuration variable. 109 """ 110 111 # The configuration variable that will control the prefer setting. 112 configVar: ConfigVar 113 114 # The prefix to use for the soong config module types. 115 configModuleTypePrefix: str 116 117 def config_module_type(self, module_type): 118 return self.configModuleTypePrefix + module_type 119 120 def _apply_transformation(self, producer, file, build_release): 121 # TODO(b/174997203): Remove this when we have a proper way to control 122 # prefer flags in Mainline modules. 123 124 header_lines = [] 125 for line in file: 126 line = line.rstrip("\n") 127 if not line.startswith("//"): 128 break 129 header_lines.append(line) 130 131 config_module_types = set() 132 133 content_lines = [] 134 for line in file: 135 line = line.rstrip("\n") 136 137 # Check to see whether the line is the start of a new module type, 138 # e.g. <module-type> { 139 module_header = re.match("([a-z0-9_]+) +{$", line) 140 if not module_header: 141 # It is not so just add the line to the output and skip to the 142 # next line. 143 content_lines.append(line) 144 continue 145 146 module_type = module_header.group(1) 147 module_content = [] 148 149 # Iterate over the Soong module contents 150 for module_line in file: 151 module_line = module_line.rstrip("\n") 152 153 # When the end of the module has been reached then exit. 154 if module_line == "}": 155 break 156 157 # Check to see if the module is an unversioned module, i.e. 158 # without @<version>. If it is then it needs to have the soong 159 # config boilerplate added to control the setting of the prefer 160 # property. Versioned modules do not need that because they are 161 # never preferred. 162 # At the moment this differentiation between versioned and 163 # unversioned relies on the fact that the unversioned modules 164 # set "prefer: false", while the versioned modules do not. That 165 # is a little bit fragile so may require some additional checks. 166 if module_line != self.PREFER_LINE: 167 # The line does not indicate that the module needs the 168 # soong config boilerplate so add the line and skip to the 169 # next one. 170 module_content.append(module_line) 171 continue 172 173 # Add the soong config boilerplate instead of the line: 174 # prefer: false, 175 namespace = self.configVar.namespace 176 name = self.configVar.name 177 module_content.append(f"""\ 178 // Do not prefer prebuilt if the Soong config variable "{name}" in namespace "{namespace}" is true. 179 prefer: true, 180 soong_config_variables: {{ 181 {name}: {{ 182 prefer: false, 183 }}, 184 }},""") 185 186 # Add the module type to the list of module types that need to 187 # have corresponding config module types. 188 config_module_types.add(module_type) 189 190 # Change the module type to the corresponding soong config 191 # module type by adding the prefix. 192 module_type = self.config_module_type(module_type) 193 194 # Generate the module, possibly with the new module type and 195 # containing the soong config variables entry. 196 content_lines.append(module_type + " {") 197 content_lines.extend(module_content) 198 content_lines.append("}") 199 200 # Add the soong_config_module_type module definitions to the header 201 # lines so that they appear before any uses. 202 header_lines.append("") 203 for module_type in sorted(config_module_types): 204 # Create the corresponding soong config module type name by adding 205 # the prefix. 206 config_module_type = self.configModuleTypePrefix + module_type 207 header_lines.append(f""" 208// Soong config variable module type added by {producer.script}. 209soong_config_module_type {{ 210 name: "{config_module_type}", 211 module_type: "{module_type}", 212 config_namespace: "{self.configVar.namespace}", 213 bool_variables: ["{self.configVar.name}"], 214 properties: ["prefer"], 215}} 216""".lstrip()) 217 218 # Overwrite the file with the updated contents. 219 file.seek(0) 220 file.truncate() 221 file.write("\n".join(header_lines + content_lines) + "\n") 222 223 224@dataclasses.dataclass(frozen=True) 225class UseSourceConfigVarTransformation(SoongConfigVarTransformation): 226 227 def _apply_transformation(self, producer, file, build_release): 228 lines = [] 229 for line in file: 230 line = line.rstrip("\n") 231 if line != self.PREFER_LINE: 232 lines.append(line) 233 continue 234 235 # Replace "prefer: false" with "use_source_config_var {...}". 236 namespace = self.configVar.namespace 237 name = self.configVar.name 238 lines.append(f"""\ 239 // Do not prefer prebuilt if the Soong config variable "{name}" in namespace "{namespace}" is true. 240 use_source_config_var: {{ 241 config_namespace: "{namespace}", 242 var_name: "{name}", 243 }},""") 244 245 # Overwrite the file with the updated contents. 246 file.seek(0) 247 file.truncate() 248 file.write("\n".join(lines) + "\n") 249 250 251@dataclasses.dataclass() 252class SubprocessRunner: 253 """Runs subprocesses""" 254 255 # Destination for stdout from subprocesses. 256 # 257 # This (and the following stderr) are needed to allow the tests to be run 258 # in Intellij. This ensures that the tests are run with stdout/stderr 259 # objects that work when passed to subprocess.run(stdout/stderr). Without it 260 # the tests are run with a FlushingStringIO object that has no fileno 261 # attribute - https://youtrack.jetbrains.com/issue/PY-27883. 262 stdout: io.TextIOBase = sys.stdout 263 264 # Destination for stderr from subprocesses. 265 stderr: io.TextIOBase = sys.stderr 266 267 def run(self, *args, **kwargs): 268 return subprocess.run( 269 *args, check=True, stdout=self.stdout, stderr=self.stderr, **kwargs) 270 271 272def sdk_snapshot_zip_file(snapshots_dir, sdk_name): 273 """Get the path to the sdk snapshot zip file.""" 274 return os.path.join(snapshots_dir, f"{sdk_name}-{SDK_VERSION}.zip") 275 276 277def sdk_snapshot_info_file(snapshots_dir, sdk_name): 278 """Get the path to the sdk snapshot info file.""" 279 return os.path.join(snapshots_dir, f"{sdk_name}-{SDK_VERSION}.info") 280 281 282def sdk_snapshot_api_diff_file(snapshots_dir, sdk_name): 283 """Get the path to the sdk snapshot api diff file.""" 284 return os.path.join(snapshots_dir, f"{sdk_name}-{SDK_VERSION}-api-diff.txt") 285 286 287def sdk_snapshot_gantry_metadata_json_file(snapshots_dir, sdk_name): 288 """Get the path to the sdk snapshot gantry metadata json file.""" 289 return os.path.join(snapshots_dir, 290 f"{sdk_name}-{SDK_VERSION}-gantry-metadata.json") 291 292 293# The default time to use in zip entries. Ideally, this should be the same as is 294# used by soong_zip and ziptime but there is no strict need for that to be the 295# case. What matters is this is a fixed time so that the contents of zip files 296# created by this script do not depend on when it is run, only the inputs. 297default_zip_time = datetime.datetime(2008, 1, 1, 0, 0, 0, 0, 298 datetime.timezone.utc) 299 300 301# set the timestamps of the paths to the default_zip_time. 302def set_default_timestamp(base_dir, paths): 303 for path in paths: 304 timestamp = default_zip_time.timestamp() 305 p = os.path.join(base_dir, path) 306 os.utime(p, (timestamp, timestamp)) 307 308 309@dataclasses.dataclass() 310class SnapshotBuilder: 311 """Builds sdk snapshots""" 312 313 # The path to this tool. 314 tool_path: str 315 316 # Used to run subprocesses for building snapshots. 317 subprocess_runner: SubprocessRunner 318 319 # The OUT_DIR environment variable. 320 out_dir: str 321 322 # The out/soong/mainline-sdks directory. 323 mainline_sdks_dir: str = "" 324 325 def __post_init__(self): 326 self.mainline_sdks_dir = os.path.join(self.out_dir, 327 "soong/mainline-sdks") 328 329 def get_sdk_path(self, sdk_name): 330 """Get the path to the sdk snapshot zip file produced by soong""" 331 return os.path.join(self.mainline_sdks_dir, 332 f"{sdk_name}-{SDK_VERSION}.zip") 333 334 def build_target_paths(self, build_release, target_paths): 335 # Extra environment variables to pass to the build process. 336 extraEnv = { 337 # TODO(ngeoffray): remove SOONG_ALLOW_MISSING_DEPENDENCIES, but 338 # we currently break without it. 339 "SOONG_ALLOW_MISSING_DEPENDENCIES": "true", 340 # Set SOONG_SDK_SNAPSHOT_USE_SRCJAR to generate .srcjars inside 341 # sdk zip files as expected by prebuilt drop. 342 "SOONG_SDK_SNAPSHOT_USE_SRCJAR": "true", 343 } 344 extraEnv.update(build_release.soong_env) 345 346 # Unless explicitly specified in the calling environment set 347 # TARGET_BUILD_VARIANT=user. 348 # This MUST be identical to the TARGET_BUILD_VARIANT used to build 349 # the corresponding APEXes otherwise it could result in different 350 # hidden API flags, see http://b/202398851#comment29 for more info. 351 target_build_variant = os.environ.get("TARGET_BUILD_VARIANT", "user") 352 cmd = [ 353 "build/soong/soong_ui.bash", 354 "--make-mode", 355 "--soong-only", 356 f"TARGET_BUILD_VARIANT={target_build_variant}", 357 "TARGET_PRODUCT=mainline_sdk", 358 "MODULE_BUILD_FROM_SOURCE=true", 359 "out/soong/apex/depsinfo/new-allowed-deps.txt.check", 360 ] + target_paths 361 print_command(extraEnv, cmd) 362 env = os.environ.copy() 363 env.update(extraEnv) 364 self.subprocess_runner.run(cmd, env=env) 365 366 def build_snapshots(self, build_release, modules): 367 # Compute the paths to all the Soong generated sdk snapshot files 368 # required by this script. 369 paths = [ 370 sdk_snapshot_zip_file(self.mainline_sdks_dir, sdk) 371 for module in modules 372 for sdk in module.sdks 373 ] 374 375 self.build_target_paths(build_release, paths) 376 return self.mainline_sdks_dir 377 378 def build_snapshots_for_build_r(self, build_release, modules): 379 # Build the snapshots as standard. 380 snapshot_dir = self.build_snapshots(build_release, modules) 381 382 # Each module will extract needed files from the original snapshot zip 383 # file and then use that to create a replacement zip file. 384 r_snapshot_dir = os.path.join(snapshot_dir, "for-R-build") 385 shutil.rmtree(r_snapshot_dir, ignore_errors=True) 386 387 build_number_file = os.path.join(self.out_dir, "soong/build_number.txt") 388 389 for module in modules: 390 apex = module.apex 391 dest_dir = os.path.join(r_snapshot_dir, apex) 392 os.makedirs(dest_dir, exist_ok=True) 393 394 # Write the bp file in the sdk_library sub-directory rather than the 395 # root of the zip file as it will be unpacked in a directory that 396 # already contains an Android.bp file that defines the corresponding 397 # apex_set. 398 bp_file = os.path.join(dest_dir, "sdk_library/Android.bp") 399 os.makedirs(os.path.dirname(bp_file), exist_ok=True) 400 401 # The first sdk in the list is the name to use. 402 sdk_name = module.sdks[0] 403 404 with open(bp_file, "w", encoding="utf8") as bp: 405 bp.write("// DO NOT EDIT. Auto-generated by the following:\n") 406 bp.write(f"// {self.tool_path}\n") 407 bp.write(COPYRIGHT_BOILERPLATE) 408 aosp_apex = google_to_aosp_name(apex) 409 410 for library in module.for_r_build.sdk_libraries: 411 module_name = library.name 412 shared_library = str(library.shared_library).lower() 413 sdk_file = sdk_snapshot_zip_file(snapshot_dir, sdk_name) 414 extract_matching_files_from_zip( 415 sdk_file, dest_dir, 416 sdk_library_files_pattern( 417 scope_pattern=r"(public|system|module-lib)", 418 name_pattern=fr"({module_name}(-removed|-stubs)?)")) 419 420 available_apexes = [f'"{aosp_apex}"'] 421 if aosp_apex != "com.android.tethering": 422 available_apexes.append(f'"test_{aosp_apex}"') 423 apex_available = ",\n ".join(available_apexes) 424 425 bp.write(f""" 426java_sdk_library_import {{ 427 name: "{module_name}", 428 owner: "google", 429 prefer: true, 430 shared_library: {shared_library}, 431 apex_available: [ 432 {apex_available}, 433 ], 434 public: {{ 435 jars: ["public/{module_name}-stubs.jar"], 436 current_api: "public/{module_name}.txt", 437 removed_api: "public/{module_name}-removed.txt", 438 sdk_version: "module_current", 439 }}, 440 system: {{ 441 jars: ["system/{module_name}-stubs.jar"], 442 current_api: "system/{module_name}.txt", 443 removed_api: "system/{module_name}-removed.txt", 444 sdk_version: "module_current", 445 }}, 446 module_lib: {{ 447 jars: ["module-lib/{module_name}-stubs.jar"], 448 current_api: "module-lib/{module_name}.txt", 449 removed_api: "module-lib/{module_name}-removed.txt", 450 sdk_version: "module_current", 451 }}, 452}} 453""") 454 455 # Copy the build_number.txt file into the snapshot. 456 snapshot_build_number_file = os.path.join( 457 dest_dir, "snapshot-creation-build-number.txt") 458 shutil.copy(build_number_file, snapshot_build_number_file) 459 460 # Make sure that all the paths being added to the zip file have a 461 # fixed timestamp so that the contents of the zip file do not depend 462 # on when this script is run, only the inputs. 463 for root, dirs, files in os.walk(dest_dir): 464 set_default_timestamp(root, dirs) 465 set_default_timestamp(root, files) 466 467 # Now zip up the files into a snapshot zip file. 468 base_file = os.path.join(r_snapshot_dir, sdk_name + "-current") 469 shutil.make_archive(base_file, "zip", dest_dir) 470 471 return r_snapshot_dir 472 473 @staticmethod 474 def does_sdk_library_support_latest_api(sdk_library): 475 if sdk_library == "conscrypt.module.platform.api": 476 return False 477 return True 478 479 def latest_api_file_targets(self, sdk_info_file): 480 # Read the sdk info file and fetch the latest scope targets. 481 with open(sdk_info_file, "r", encoding="utf8") as sdk_info_file_object: 482 sdk_info_file_json = json.loads(sdk_info_file_object.read()) 483 484 target_paths = [] 485 target_dict = {} 486 for jsonItem in sdk_info_file_json: 487 if not jsonItem["@type"] == "java_sdk_library": 488 continue 489 490 sdk_library = jsonItem["@name"] 491 if not self.does_sdk_library_support_latest_api(sdk_library): 492 continue 493 494 target_dict[sdk_library] = {} 495 for scope in jsonItem["scopes"]: 496 scope_json = jsonItem["scopes"][scope] 497 target_dict[sdk_library][scope] = {} 498 target_list = [ 499 "current_api", "latest_api", "removed_api", 500 "latest_removed_api" 501 ] 502 for target in target_list: 503 target_dict[sdk_library][scope][target] = scope_json[target] 504 target_paths.append(scope_json["latest_api"]) 505 target_paths.append(scope_json["latest_removed_api"]) 506 507 return target_paths, target_dict 508 509 def build_sdk_scope_targets(self, build_release, modules): 510 # Build the latest scope targets for each module sdk 511 # Compute the paths to all the latest scope targets for each module sdk. 512 target_paths = [] 513 target_dict = {} 514 for module in modules: 515 for sdk in module.sdks: 516 sdk_type = sdk_type_from_name(sdk) 517 if not sdk_type.providesApis: 518 continue 519 520 sdk_info_file = sdk_snapshot_info_file(self.mainline_sdks_dir, 521 sdk) 522 paths, dict_item = self.latest_api_file_targets(sdk_info_file) 523 target_paths.extend(paths) 524 target_dict[sdk_info_file] = dict_item 525 self.build_target_paths(build_release, target_paths) 526 return target_dict 527 528 def appendDiffToFile(self, file_object, sdk_zip_file, current_api, 529 latest_api, snapshots_dir): 530 """Extract current api and find its diff with the latest api.""" 531 with zipfile.ZipFile(sdk_zip_file, "r") as zipObj: 532 extracted_current_api = zipObj.extract( 533 member=current_api, path=snapshots_dir) 534 # The diff tool has an exit code of 0, 1 or 2 depending on whether 535 # it find no differences, some differences or an error (like missing 536 # file). As 0 or 1 are both valid results this cannot use check=True 537 # so disable the pylint check. 538 # pylint: disable=subprocess-run-check 539 diff = subprocess.run([ 540 "diff", "-u0", latest_api, extracted_current_api, "--label", 541 latest_api, "--label", extracted_current_api 542 ], 543 capture_output=True).stdout.decode("utf-8") 544 file_object.write(diff) 545 546 def create_snapshot_gantry_metadata_and_api_diff(self, sdk, target_dict, 547 snapshots_dir, 548 module_extension_version): 549 """Creates gantry metadata and api diff files for each module sdk. 550 551 For each module sdk, the scope targets are obtained for each java sdk 552 library and the api diff files are generated by performing a diff 553 operation between the current api file vs the latest api file. 554 """ 555 sdk_info_file = sdk_snapshot_info_file(snapshots_dir, sdk) 556 sdk_zip_file = sdk_snapshot_zip_file(snapshots_dir, sdk) 557 sdk_api_diff_file = sdk_snapshot_api_diff_file(snapshots_dir, sdk) 558 559 gantry_metadata_dict = {} 560 with open( 561 sdk_api_diff_file, "w", 562 encoding="utf8") as sdk_api_diff_file_object: 563 for sdk_library in target_dict[sdk_info_file]: 564 for scope in target_dict[sdk_info_file][sdk_library]: 565 scope_json = target_dict[sdk_info_file][sdk_library][scope] 566 current_api = scope_json["current_api"] 567 latest_api = scope_json["latest_api"] 568 self.appendDiffToFile(sdk_api_diff_file_object, 569 sdk_zip_file, current_api, latest_api, 570 snapshots_dir) 571 572 removed_api = scope_json["removed_api"] 573 latest_removed_api = scope_json["latest_removed_api"] 574 self.appendDiffToFile(sdk_api_diff_file_object, 575 sdk_zip_file, removed_api, 576 latest_removed_api, snapshots_dir) 577 578 gantry_metadata_dict["api_diff_file"] = sdk_api_diff_file.rsplit( 579 "/", 1)[-1] 580 gantry_metadata_dict["api_diff_file_size"] = os.path.getsize( 581 sdk_api_diff_file) 582 gantry_metadata_dict[ 583 "module_extension_version"] = module_extension_version 584 sdk_metadata_json_file = sdk_snapshot_gantry_metadata_json_file( 585 snapshots_dir, sdk) 586 587 gantry_metadata_json_object = json.dumps(gantry_metadata_dict, indent=4) 588 with open(sdk_metadata_json_file, 589 "w") as gantry_metadata_json_file_object: 590 gantry_metadata_json_file_object.write(gantry_metadata_json_object) 591 592 def get_module_extension_version(self): 593 return int( 594 subprocess.run([ 595 "build/soong/soong_ui.bash", "--dumpvar-mode", 596 "PLATFORM_SDK_EXTENSION_VERSION" 597 ], 598 capture_output=True).stdout.decode("utf-8").strip()) 599 600 def build_snapshot_gantry_metadata_and_api_diff(self, modules, target_dict, 601 snapshots_dir): 602 """For each module sdk, create the metadata and api diff file.""" 603 module_extension_version = self.get_module_extension_version() 604 for module in modules: 605 for sdk in module.sdks: 606 sdk_type = sdk_type_from_name(sdk) 607 if not sdk_type.providesApis: 608 continue 609 self.create_snapshot_gantry_metadata_and_api_diff( 610 sdk, target_dict, snapshots_dir, module_extension_version) 611 612 613# The sdk version to build 614# 615# This is legacy from the time when this could generate versioned sdk snapshots. 616SDK_VERSION = "current" 617 618# The initially empty list of build releases. Every BuildRelease that is created 619# automatically appends itself to this list. 620ALL_BUILD_RELEASES = [] 621 622 623class PreferHandling(enum.Enum): 624 """Enumeration of the various ways of handling prefer properties""" 625 626 # No special prefer property handling is required. 627 NONE = enum.auto() 628 629 # Apply the SoongConfigBoilerplateInserter transformation. 630 SOONG_CONFIG = enum.auto() 631 632 # Use the use_source_config_var property added in T. 633 USE_SOURCE_CONFIG_VAR_PROPERTY = enum.auto() 634 635 636@dataclasses.dataclass(frozen=True) 637@functools.total_ordering 638class BuildRelease: 639 """Represents a build release""" 640 641 # The name of the build release, e.g. Q, R, S, T, etc. 642 name: str 643 644 # The function to call to create the snapshot in the dist, that covers 645 # building and copying the snapshot into the dist. 646 creator: Callable[ 647 ["BuildRelease", "SdkDistProducer", List["MainlineModule"]], None] 648 649 # The sub-directory of dist/mainline-sdks into which the build release 650 # specific snapshots will be copied. 651 # 652 # Defaults to for-<name>-build. 653 sub_dir: str = None 654 655 # Additional environment variables to pass to Soong when building the 656 # snapshots for this build release. 657 # 658 # Defaults to { 659 # "SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE": <name>, 660 # } 661 soong_env: typing.Dict[str, str] = None 662 663 # The position of this instance within the BUILD_RELEASES list. 664 ordinal: int = dataclasses.field(default=-1, init=False) 665 666 # Whether this build release supports the Soong config boilerplate that is 667 # used to control the prefer setting of modules via a Soong config variable. 668 preferHandling: PreferHandling = \ 669 PreferHandling.USE_SOURCE_CONFIG_VAR_PROPERTY 670 671 def __post_init__(self): 672 # The following use object.__setattr__ as this object is frozen and 673 # attempting to set the fields directly would cause an exception to be 674 # thrown. 675 object.__setattr__(self, "ordinal", len(ALL_BUILD_RELEASES)) 676 # Add this to the end of the list of all build releases. 677 ALL_BUILD_RELEASES.append(self) 678 # If no sub_dir was specified then set the default. 679 if self.sub_dir is None: 680 object.__setattr__(self, "sub_dir", f"for-{self.name}-build") 681 # If no soong_env was specified then set the default. 682 if self.soong_env is None: 683 object.__setattr__( 684 self, 685 "soong_env", 686 { 687 # Set SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE to generate a 688 # snapshot suitable for a specific target build release. 689 "SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE": self.name, 690 }) 691 692 def __eq__(self, other): 693 return self.ordinal == other.ordinal 694 695 def __le__(self, other): 696 return self.ordinal <= other.ordinal 697 698 699def create_no_dist_snapshot(_: BuildRelease, __: "SdkDistProducer", 700 modules: List["MainlineModule"]): 701 """A place holder dist snapshot creation function that does nothing.""" 702 print(f"create_no_dist_snapshot for modules {[m.apex for m in modules]}") 703 704 705def create_dist_snapshot_for_r(build_release: BuildRelease, 706 producer: "SdkDistProducer", 707 modules: List["MainlineModule"]): 708 """Generate a snapshot suitable for use in an R build.""" 709 producer.product_dist_for_build_r(build_release, modules) 710 711 712def create_sdk_snapshots_in_soong(build_release: BuildRelease, 713 producer: "SdkDistProducer", 714 modules: List["MainlineModule"]): 715 """Builds sdks and populates the dist for unbundled modules.""" 716 producer.produce_unbundled_dist_for_build_release(build_release, modules) 717 718 719def create_latest_sdk_snapshots(build_release: BuildRelease, 720 producer: "SdkDistProducer", 721 modules: List["MainlineModule"]): 722 """Builds and populates the latest release, including bundled modules.""" 723 producer.produce_unbundled_dist_for_build_release(build_release, modules) 724 producer.produce_bundled_dist_for_build_release(build_release, modules) 725 726 727Q = BuildRelease( 728 name="Q", 729 # At the moment we do not generate a snapshot for Q. 730 creator=create_no_dist_snapshot, 731 # This does not support or need any special prefer property handling. 732 preferHandling=PreferHandling.NONE, 733) 734R = BuildRelease( 735 name="R", 736 # Generate a simple snapshot for R. 737 creator=create_dist_snapshot_for_r, 738 # By default a BuildRelease creates an environment to pass to Soong that 739 # creates a release specific snapshot. However, Soong does not yet (and is 740 # unlikely to) support building an sdk snapshot for R so create an empty 741 # environment to pass to Soong instead. 742 soong_env={}, 743 # This does not support or need any special prefer property handling. 744 preferHandling=PreferHandling.NONE, 745) 746S = BuildRelease( 747 name="S", 748 # Generate a snapshot for this build release using Soong. 749 creator=create_sdk_snapshots_in_soong, 750 # This requires the SoongConfigBoilerplateInserter transformation to be 751 # applied. 752 preferHandling=PreferHandling.SOONG_CONFIG, 753) 754Tiramisu = BuildRelease( 755 name="Tiramisu", 756 # Generate a snapshot for this build release using Soong. 757 creator=create_sdk_snapshots_in_soong, 758 # This build release supports the use_source_config_var property. 759 preferHandling=PreferHandling.USE_SOURCE_CONFIG_VAR_PROPERTY, 760) 761UpsideDownCake = BuildRelease( 762 name="UpsideDownCake", 763 # Generate a snapshot for this build release using Soong. 764 creator=create_sdk_snapshots_in_soong, 765 # This build release supports the use_source_config_var property. 766 preferHandling=PreferHandling.USE_SOURCE_CONFIG_VAR_PROPERTY, 767) 768 769# Insert additional BuildRelease definitions for following releases here, 770# before LATEST. 771 772# The build release for the latest build supported by this build, i.e. the 773# current build. This must be the last BuildRelease defined in this script. 774LATEST = BuildRelease( 775 name="latest", 776 creator=create_latest_sdk_snapshots, 777 # There are no build release specific environment variables to pass to 778 # Soong. 779 soong_env={}, 780) 781 782 783@dataclasses.dataclass(frozen=True) 784class SdkLibrary: 785 """Information about a java_sdk_library.""" 786 787 # The name of java_sdk_library module. 788 name: str 789 790 # True if the sdk_library module is a shared library. 791 shared_library: bool = False 792 793 794@dataclasses.dataclass(frozen=True) 795class ForRBuild: 796 """Data structure needed for generating a snapshot for an R build.""" 797 798 # The java_sdk_library modules to export to the r snapshot. 799 sdk_libraries: typing.List[SdkLibrary] = dataclasses.field( 800 default_factory=list) 801 802 803@dataclasses.dataclass(frozen=True) 804class MainlineModule: 805 """Represents an unbundled mainline module. 806 807 This is a module that is distributed as a prebuilt and intended to be 808 updated with Mainline trains. 809 """ 810 # The name of the apex. 811 apex: str 812 813 # The names of the sdk and module_exports. 814 sdks: list[str] 815 816 # The first build release in which the SDK snapshot for this module is 817 # needed. 818 # 819 # Note: This is not necessarily the same build release in which the SDK 820 # source was first included. So, a module that was added in build T 821 # could potentially be used in an S release and so its SDK will need 822 # to be made available for S builds. 823 first_release: BuildRelease 824 825 # The configuration variable, defaults to ANDROID:module_build_from_source 826 configVar: ConfigVar = ConfigVar( 827 namespace="ANDROID", 828 name="module_build_from_source", 829 ) 830 831 for_r_build: typing.Optional[ForRBuild] = None 832 833 # The last release on which this module was optional. 834 # 835 # Some modules are optional when they are first released, usually because 836 # some vendors of Android devices have their own customizations of the 837 # module that they would like to preserve and which cannot yet be achieved 838 # through the existing APIs. Once those issues have been resolved then they 839 # will become mandatory. 840 # 841 # This field records the last build release in which they are optional. It 842 # defaults to None which indicates that the module was never optional. 843 # 844 # TODO(b/238203992): remove the following warning once all modules can be 845 # treated as optional at build time. 846 # 847 # DO NOT use this attr for anything other than controlling whether the 848 # generated snapshot uses its own Soong config variable or the common one. 849 # That is because this is being temporarily used to force Permission to have 850 # its own Soong config variable even though Permission is not actually 851 # optional at runtime on a GMS capable device. 852 # 853 # b/238203992 will make all modules have their own Soong config variable by 854 # default at which point this will no longer be needed on Permission and so 855 # it can be used to indicate that a module is optional at runtime. 856 last_optional_release: typing.Optional[BuildRelease] = None 857 858 # The short name for the module. 859 # 860 # Defaults to the last part of the apex name. 861 short_name: str = "" 862 863 # Additional transformations 864 additional_transformations: list[FileTransformation] = None 865 866 def __post_init__(self): 867 # If short_name is not set then set it to the last component of the apex 868 # name. 869 if not self.short_name: 870 short_name = self.apex.rsplit(".", 1)[-1] 871 object.__setattr__(self, "short_name", short_name) 872 873 def is_bundled(self): 874 """Returns true for bundled modules. See BundledMainlineModule.""" 875 return False 876 877 def transformations(self, build_release, sdk_type): 878 """Returns the transformations to apply to this module's snapshot(s).""" 879 transformations = [] 880 881 config_var = self.configVar 882 883 # If the module is optional then it needs its own Soong config 884 # variable to allow it to be managed separately from other modules. 885 if (self.last_optional_release and 886 self.last_optional_release > build_release): 887 config_var = ConfigVar( 888 namespace=f"{self.short_name}_module", 889 name="source_build", 890 ) 891 892 prefer_handling = build_release.preferHandling 893 if prefer_handling == PreferHandling.SOONG_CONFIG: 894 sdk_type_prefix = sdk_type.configModuleTypePrefix 895 config_module_type_prefix = \ 896 f"{self.short_name}{sdk_type_prefix}_prebuilt_" 897 inserter = SoongConfigBoilerplateInserter( 898 "Android.bp", 899 configVar=config_var, 900 configModuleTypePrefix=config_module_type_prefix) 901 transformations.append(inserter) 902 elif prefer_handling == PreferHandling.USE_SOURCE_CONFIG_VAR_PROPERTY: 903 transformation = UseSourceConfigVarTransformation( 904 "Android.bp", configVar=config_var) 905 transformations.append(transformation) 906 907 if self.additional_transformations and build_release > R: 908 transformations.extend(self.additional_transformations) 909 910 return transformations 911 912 def is_required_for(self, target_build_release): 913 """True if this module is required for the target build release.""" 914 return self.first_release <= target_build_release 915 916 917@dataclasses.dataclass(frozen=True) 918class BundledMainlineModule(MainlineModule): 919 """Represents a bundled Mainline module or a platform SDK for module use. 920 921 A bundled module is always preloaded into the platform images. 922 """ 923 924 # Defaults to the latest build, i.e. the build on which this script is run 925 # as bundled modules are, by definition, only needed in this build. 926 first_release: BuildRelease = LATEST 927 928 def is_bundled(self): 929 return True 930 931 def transformations(self, build_release, sdk_type): 932 # Bundled modules are only used on thin branches where the corresponding 933 # sources are absent, so skip transformations and keep the default 934 # `prefer: false`. 935 return [] 936 937 938# List of mainline modules. 939MAINLINE_MODULES = [ 940 MainlineModule( 941 apex="com.android.adservices", 942 sdks=["adservices-module-sdk"], 943 first_release=Tiramisu, 944 ), 945 MainlineModule( 946 apex="com.android.appsearch", 947 sdks=["appsearch-sdk"], 948 first_release=Tiramisu, 949 ), 950 MainlineModule( 951 apex="com.android.art", 952 sdks=[ 953 "art-module-sdk", 954 "art-module-test-exports", 955 "art-module-host-exports", 956 ], 957 first_release=S, 958 # Override the config... fields. 959 configVar=ConfigVar( 960 namespace="art_module", 961 name="source_build", 962 ), 963 ), 964 MainlineModule( 965 apex="com.android.btservices", 966 sdks=["btservices-module-sdk"], 967 first_release=UpsideDownCake, 968 # Bluetooth has always been and is still optional. 969 last_optional_release=LATEST, 970 ), 971 MainlineModule( 972 apex="com.android.configinfrastructure", 973 sdks=["configinfrastructure-sdk"], 974 first_release=UpsideDownCake, 975 ), 976 MainlineModule( 977 apex="com.android.conscrypt", 978 sdks=[ 979 "conscrypt-module-sdk", 980 "conscrypt-module-test-exports", 981 "conscrypt-module-host-exports", 982 ], 983 first_release=Q, 984 # No conscrypt java_sdk_library modules are exported to the R snapshot. 985 # Conscrypt was updatable in R but the generate_ml_bundle.sh does not 986 # appear to generate a snapshot for it. 987 for_r_build=None, 988 ), 989 MainlineModule( 990 apex="com.android.healthfitness", 991 sdks=["healthfitness-module-sdk"], 992 first_release=UpsideDownCake, 993 ), 994 MainlineModule( 995 apex="com.android.ipsec", 996 sdks=["ipsec-module-sdk"], 997 first_release=R, 998 for_r_build=ForRBuild(sdk_libraries=[ 999 SdkLibrary( 1000 name="android.net.ipsec.ike", 1001 shared_library=True, 1002 ), 1003 ]), 1004 ), 1005 MainlineModule( 1006 apex="com.android.media", 1007 sdks=["media-module-sdk"], 1008 first_release=R, 1009 for_r_build=ForRBuild(sdk_libraries=[ 1010 SdkLibrary(name="framework-media"), 1011 ]), 1012 ), 1013 MainlineModule( 1014 apex="com.android.mediaprovider", 1015 sdks=["mediaprovider-module-sdk"], 1016 first_release=R, 1017 for_r_build=ForRBuild(sdk_libraries=[ 1018 SdkLibrary(name="framework-mediaprovider"), 1019 ]), 1020 ), 1021 MainlineModule( 1022 apex="com.android.ondevicepersonalization", 1023 sdks=["ondevicepersonalization-module-sdk"], 1024 first_release=Tiramisu, 1025 ), 1026 MainlineModule( 1027 apex="com.android.permission", 1028 sdks=["permission-module-sdk"], 1029 first_release=R, 1030 for_r_build=ForRBuild(sdk_libraries=[ 1031 SdkLibrary(name="framework-permission"), 1032 # framework-permission-s is not needed on R as it contains classes 1033 # that are provided in R by non-updatable parts of the 1034 # bootclasspath. 1035 ]), 1036 # Although Permission is not, and has never been, optional for GMS 1037 # capable devices it does need to be treated as optional at build time 1038 # when building non-GMS devices. 1039 # TODO(b/238203992): remove once all modules are optional at build time. 1040 last_optional_release=LATEST, 1041 ), 1042 MainlineModule( 1043 apex="com.android.rkpd", 1044 sdks=["rkpd-sdk"], 1045 first_release=UpsideDownCake, 1046 # Rkpd has always been and is still optional. 1047 last_optional_release=LATEST, 1048 ), 1049 MainlineModule( 1050 apex="com.android.scheduling", 1051 sdks=["scheduling-sdk"], 1052 first_release=S, 1053 ), 1054 MainlineModule( 1055 apex="com.android.sdkext", 1056 sdks=["sdkextensions-sdk"], 1057 first_release=R, 1058 for_r_build=ForRBuild(sdk_libraries=[ 1059 SdkLibrary(name="framework-sdkextensions"), 1060 ]), 1061 ), 1062 MainlineModule( 1063 apex="com.android.os.statsd", 1064 sdks=["statsd-module-sdk"], 1065 first_release=R, 1066 for_r_build=ForRBuild(sdk_libraries=[ 1067 SdkLibrary(name="framework-statsd"), 1068 ]), 1069 ), 1070 MainlineModule( 1071 apex="com.android.tethering", 1072 sdks=["tethering-module-sdk"], 1073 first_release=R, 1074 for_r_build=ForRBuild(sdk_libraries=[ 1075 SdkLibrary(name="framework-tethering"), 1076 ]), 1077 ), 1078 MainlineModule( 1079 apex="com.android.uwb", 1080 sdks=["uwb-module-sdk"], 1081 first_release=Tiramisu, 1082 # Uwb has always been and is still optional. 1083 last_optional_release=LATEST, 1084 ), 1085 MainlineModule( 1086 apex="com.android.wifi", 1087 sdks=["wifi-module-sdk"], 1088 first_release=R, 1089 for_r_build=ForRBuild(sdk_libraries=[ 1090 SdkLibrary(name="framework-wifi"), 1091 ]), 1092 # Wifi has always been and is still optional. 1093 last_optional_release=LATEST, 1094 ), 1095] 1096 1097# List of Mainline modules that currently are never built unbundled. They must 1098# not specify first_release, and they don't have com.google.android 1099# counterparts. 1100BUNDLED_MAINLINE_MODULES = [ 1101 BundledMainlineModule( 1102 apex="com.android.i18n", 1103 sdks=[ 1104 "i18n-module-sdk", 1105 "i18n-module-test-exports", 1106 "i18n-module-host-exports", 1107 ], 1108 ), 1109 BundledMainlineModule( 1110 apex="com.android.runtime", 1111 sdks=[ 1112 "runtime-module-host-exports", 1113 "runtime-module-sdk", 1114 ], 1115 ), 1116 BundledMainlineModule( 1117 apex="com.android.tzdata", 1118 sdks=["tzdata-module-test-exports"], 1119 ), 1120] 1121 1122# List of platform SDKs for Mainline module use. 1123PLATFORM_SDKS_FOR_MAINLINE = [ 1124 BundledMainlineModule( 1125 apex="platform-mainline", 1126 sdks=[ 1127 "platform-mainline-sdk", 1128 "platform-mainline-test-exports", 1129 ], 1130 ), 1131] 1132 1133 1134@dataclasses.dataclass 1135class SdkDistProducer: 1136 """Produces the DIST_DIR/mainline-sdks and DIST_DIR/stubs directories. 1137 1138 Builds SDK snapshots for mainline modules and then copies them into the 1139 DIST_DIR/mainline-sdks directory. Also extracts the sdk_library txt, jar and 1140 srcjar files from each SDK snapshot and copies them into the DIST_DIR/stubs 1141 directory. 1142 """ 1143 1144 # Used to run subprocesses for this. 1145 subprocess_runner: SubprocessRunner 1146 1147 # Builds sdk snapshots 1148 snapshot_builder: SnapshotBuilder 1149 1150 # The DIST_DIR environment variable. 1151 dist_dir: str = "uninitialized-dist" 1152 1153 # The path to this script. It may be inserted into files that are 1154 # transformed to document where the changes came from. 1155 script: str = sys.argv[0] 1156 1157 # The path to the mainline-sdks dist directory for unbundled modules. 1158 # 1159 # Initialized in __post_init__(). 1160 mainline_sdks_dir: str = dataclasses.field(init=False) 1161 1162 # The path to the mainline-sdks dist directory for bundled modules and 1163 # platform SDKs. 1164 # 1165 # Initialized in __post_init__(). 1166 bundled_mainline_sdks_dir: str = dataclasses.field(init=False) 1167 1168 def __post_init__(self): 1169 self.mainline_sdks_dir = os.path.join(self.dist_dir, "mainline-sdks") 1170 self.bundled_mainline_sdks_dir = os.path.join(self.dist_dir, 1171 "bundled-mainline-sdks") 1172 1173 def prepare(self): 1174 pass 1175 1176 def produce_dist(self, modules, build_releases): 1177 # Prepare the dist directory for the sdks. 1178 self.prepare() 1179 1180 # Group build releases so that those with the same Soong environment are 1181 # run consecutively to avoid having to regenerate ninja files. 1182 grouped_by_env = defaultdict(list) 1183 for build_release in build_releases: 1184 grouped_by_env[str(build_release.soong_env)].append(build_release) 1185 ordered = [br for _, group in grouped_by_env.items() for br in group] 1186 1187 for build_release in ordered: 1188 # Only build modules that are required for this build release. 1189 filtered_modules = [ 1190 m for m in modules if m.is_required_for(build_release) 1191 ] 1192 if filtered_modules: 1193 print(f"Building SDK snapshots for {build_release.name}" 1194 f" build release") 1195 build_release.creator(build_release, self, filtered_modules) 1196 1197 def product_dist_for_build_r(self, build_release, modules): 1198 # Although we only need a subset of the files that a java_sdk_library 1199 # adds to an sdk snapshot generating the whole snapshot is the simplest 1200 # way to ensure that all the necessary files are produced. 1201 1202 # Filter out any modules that do not provide sdk for R. 1203 modules = [m for m in modules if m.for_r_build] 1204 1205 snapshot_dir = self.snapshot_builder.build_snapshots_for_build_r( 1206 build_release, modules) 1207 self.populate_unbundled_dist(build_release, modules, snapshot_dir) 1208 1209 def produce_unbundled_dist_for_build_release(self, build_release, modules): 1210 modules = [m for m in modules if not m.is_bundled()] 1211 snapshots_dir = self.snapshot_builder.build_snapshots( 1212 build_release, modules) 1213 if build_release == LATEST: 1214 target_dict = self.snapshot_builder.build_sdk_scope_targets( 1215 build_release, modules) 1216 self.snapshot_builder.build_snapshot_gantry_metadata_and_api_diff( 1217 modules, target_dict, snapshots_dir) 1218 self.populate_unbundled_dist(build_release, modules, snapshots_dir) 1219 return snapshots_dir 1220 1221 def produce_bundled_dist_for_build_release(self, build_release, modules): 1222 modules = [m for m in modules if m.is_bundled()] 1223 if modules: 1224 snapshots_dir = self.snapshot_builder.build_snapshots( 1225 build_release, modules) 1226 self.populate_bundled_dist(build_release, modules, snapshots_dir) 1227 1228 def dist_sdk_snapshot_gantry_metadata_and_api_diff(self, sdk_dist_dir, sdk, 1229 module, snapshots_dir): 1230 """Copy the sdk snapshot api diff file to a dist directory.""" 1231 sdk_type = sdk_type_from_name(sdk) 1232 if not sdk_type.providesApis: 1233 return 1234 1235 sdk_dist_module_subdir = os.path.join(sdk_dist_dir, module.apex) 1236 sdk_dist_subdir = os.path.join(sdk_dist_module_subdir, "sdk") 1237 os.makedirs(sdk_dist_subdir, exist_ok=True) 1238 sdk_api_diff_path = sdk_snapshot_api_diff_file(snapshots_dir, sdk) 1239 shutil.copy(sdk_api_diff_path, sdk_dist_subdir) 1240 1241 sdk_gantry_metadata_json_path = sdk_snapshot_gantry_metadata_json_file( 1242 snapshots_dir, sdk) 1243 sdk_dist_gantry_metadata_json_path = os.path.join( 1244 sdk_dist_module_subdir, "gantry-metadata.json") 1245 shutil.copy(sdk_gantry_metadata_json_path, 1246 sdk_dist_gantry_metadata_json_path) 1247 1248 def populate_unbundled_dist(self, build_release, modules, snapshots_dir): 1249 build_release_dist_dir = os.path.join(self.mainline_sdks_dir, 1250 build_release.sub_dir) 1251 for module in modules: 1252 for sdk in module.sdks: 1253 sdk_dist_dir = os.path.join(build_release_dist_dir, SDK_VERSION) 1254 if build_release == LATEST: 1255 self.dist_sdk_snapshot_gantry_metadata_and_api_diff( 1256 sdk_dist_dir, sdk, module, snapshots_dir) 1257 self.populate_dist_snapshot(build_release, module, sdk, 1258 sdk_dist_dir, snapshots_dir) 1259 1260 def populate_bundled_dist(self, build_release, modules, snapshots_dir): 1261 sdk_dist_dir = self.bundled_mainline_sdks_dir 1262 for module in modules: 1263 for sdk in module.sdks: 1264 self.populate_dist_snapshot(build_release, module, sdk, 1265 sdk_dist_dir, snapshots_dir) 1266 1267 def populate_dist_snapshot(self, build_release, module, sdk, sdk_dist_dir, 1268 snapshots_dir): 1269 sdk_type = sdk_type_from_name(sdk) 1270 subdir = sdk_type.name 1271 1272 sdk_dist_subdir = os.path.join(sdk_dist_dir, module.apex, subdir) 1273 sdk_path = sdk_snapshot_zip_file(snapshots_dir, sdk) 1274 sdk_type = sdk_type_from_name(sdk) 1275 transformations = module.transformations(build_release, sdk_type) 1276 self.dist_sdk_snapshot_zip( 1277 build_release, sdk_path, sdk_dist_subdir, transformations) 1278 1279 def dist_sdk_snapshot_zip( 1280 self, build_release, src_sdk_zip, sdk_dist_dir, transformations): 1281 """Copy the sdk snapshot zip file to a dist directory. 1282 1283 If no transformations are provided then this simply copies the show sdk 1284 snapshot zip file to the dist dir. However, if transformations are 1285 provided then the files to be transformed are extracted from the 1286 snapshot zip file, they are transformed to files in a separate directory 1287 and then a new zip file is created in the dist directory with the 1288 original files replaced by the newly transformed files. build_release is 1289 provided for transformations if it is needed. 1290 """ 1291 os.makedirs(sdk_dist_dir, exist_ok=True) 1292 dest_sdk_zip = os.path.join(sdk_dist_dir, os.path.basename(src_sdk_zip)) 1293 print(f"Copying sdk snapshot {src_sdk_zip} to {dest_sdk_zip}") 1294 1295 # If no transformations are provided then just copy the zip file 1296 # directly. 1297 if len(transformations) == 0: 1298 shutil.copy(src_sdk_zip, sdk_dist_dir) 1299 return 1300 1301 with tempfile.TemporaryDirectory() as tmp_dir: 1302 # Create a single pattern that will match any of the paths provided 1303 # in the transformations. 1304 pattern = "|".join( 1305 [f"({re.escape(t.path)})" for t in transformations]) 1306 1307 # Extract the matching files from the zip into the temporary 1308 # directory. 1309 extract_matching_files_from_zip(src_sdk_zip, tmp_dir, pattern) 1310 1311 # Apply the transformations to the extracted files in situ. 1312 apply_transformations(self, tmp_dir, transformations, build_release) 1313 1314 # Replace the original entries in the zip with the transformed 1315 # files. 1316 paths = [transformation.path for transformation in transformations] 1317 copy_zip_and_replace(self, src_sdk_zip, dest_sdk_zip, tmp_dir, 1318 paths) 1319 1320 1321def print_command(env, cmd): 1322 print(" ".join([f"{name}={value}" for name, value in env.items()] + cmd)) 1323 1324 1325def sdk_library_files_pattern(*, scope_pattern=r"[^/]+", name_pattern=r"[^/]+"): 1326 """Return a pattern to match sdk_library related files in an sdk snapshot""" 1327 return rf"sdk_library/{scope_pattern}/{name_pattern}\.(txt|jar|srcjar)" 1328 1329 1330def extract_matching_files_from_zip(zip_path, dest_dir, pattern): 1331 """Extracts files from a zip file into a destination directory. 1332 1333 The extracted files are those that match the specified regular expression 1334 pattern. 1335 """ 1336 os.makedirs(dest_dir, exist_ok=True) 1337 with zipfile.ZipFile(zip_path) as zip_file: 1338 for filename in zip_file.namelist(): 1339 if re.match(pattern, filename): 1340 print(f" extracting {filename}") 1341 zip_file.extract(filename, dest_dir) 1342 1343 1344def copy_zip_and_replace(producer, src_zip_path, dest_zip_path, src_dir, paths): 1345 """Copies a zip replacing some of its contents in the process. 1346 1347 The files to replace are specified by the paths parameter and are relative 1348 to the src_dir. 1349 """ 1350 # Get the absolute paths of the source and dest zip files so that they are 1351 # not affected by a change of directory. 1352 abs_src_zip_path = os.path.abspath(src_zip_path) 1353 abs_dest_zip_path = os.path.abspath(dest_zip_path) 1354 1355 # Make sure that all the paths being added to the zip file have a fixed 1356 # timestamp so that the contents of the zip file do not depend on when this 1357 # script is run, only the inputs. 1358 set_default_timestamp(src_dir, paths) 1359 1360 producer.subprocess_runner.run( 1361 ["zip", "-q", abs_src_zip_path, "--out", abs_dest_zip_path] + paths, 1362 # Change into the source directory before running zip. 1363 cwd=src_dir) 1364 1365 1366def apply_transformations(producer, tmp_dir, transformations, build_release): 1367 for transformation in transformations: 1368 path = os.path.join(tmp_dir, transformation.path) 1369 1370 # Record the timestamp of the file. 1371 modified = os.path.getmtime(path) 1372 1373 # Transform the file. 1374 transformation.apply(producer, path, build_release) 1375 1376 # Reset the timestamp of the file to the original timestamp before the 1377 # transformation was applied. 1378 os.utime(path, (modified, modified)) 1379 1380 1381def create_producer(tool_path): 1382 # Variables initialized from environment variables that are set by the 1383 # calling mainline_modules_sdks.sh. 1384 out_dir = os.environ["OUT_DIR"] 1385 dist_dir = os.environ["DIST_DIR"] 1386 1387 top_dir = os.environ["ANDROID_BUILD_TOP"] 1388 tool_path = os.path.relpath(tool_path, top_dir) 1389 tool_path = tool_path.replace(".py", ".sh") 1390 1391 subprocess_runner = SubprocessRunner() 1392 snapshot_builder = SnapshotBuilder( 1393 tool_path=tool_path, 1394 subprocess_runner=subprocess_runner, 1395 out_dir=out_dir, 1396 ) 1397 return SdkDistProducer( 1398 subprocess_runner=subprocess_runner, 1399 snapshot_builder=snapshot_builder, 1400 dist_dir=dist_dir, 1401 ) 1402 1403 1404def aosp_to_google(module): 1405 """Transform an AOSP module into a Google module""" 1406 new_apex = aosp_to_google_name(module.apex) 1407 # Create a copy of the AOSP module with the internal specific APEX name. 1408 return dataclasses.replace(module, apex=new_apex) 1409 1410 1411def aosp_to_google_name(name): 1412 """Transform an AOSP module name into a Google module name""" 1413 return name.replace("com.android.", "com.google.android.") 1414 1415 1416def google_to_aosp_name(name): 1417 """Transform a Google module name into an AOSP module name""" 1418 return name.replace("com.google.android.", "com.android.") 1419 1420 1421@dataclasses.dataclass(frozen=True) 1422class SdkType: 1423 name: str 1424 1425 configModuleTypePrefix: str 1426 1427 providesApis: bool = False 1428 1429 1430Sdk = SdkType( 1431 name="sdk", 1432 configModuleTypePrefix="", 1433 providesApis=True, 1434) 1435HostExports = SdkType( 1436 name="host-exports", 1437 configModuleTypePrefix="_host_exports", 1438) 1439TestExports = SdkType( 1440 name="test-exports", 1441 configModuleTypePrefix="_test_exports", 1442) 1443 1444 1445def sdk_type_from_name(name): 1446 if name.endswith("-sdk"): 1447 return Sdk 1448 if name.endswith("-host-exports"): 1449 return HostExports 1450 if name.endswith("-test-exports"): 1451 return TestExports 1452 1453 raise Exception(f"{name} is not a valid sdk name, expected it to end" 1454 f" with -(sdk|host-exports|test-exports)") 1455 1456 1457def filter_modules(modules, target_build_apps): 1458 if target_build_apps: 1459 target_build_apps = target_build_apps.split() 1460 return [m for m in modules if m.apex in target_build_apps] 1461 return modules 1462 1463 1464def main(args): 1465 """Program entry point.""" 1466 if not os.path.exists("build/make/core/Makefile"): 1467 sys.exit("This script must be run from the top of the tree.") 1468 1469 args_parser = argparse.ArgumentParser( 1470 description="Build snapshot zips for consumption by Gantry.") 1471 args_parser.add_argument( 1472 "--tool-path", 1473 help="The path to this tool.", 1474 default="unspecified", 1475 ) 1476 args_parser.add_argument( 1477 "--build-release", 1478 action="append", 1479 choices=[br.name for br in ALL_BUILD_RELEASES], 1480 help="A target build for which snapshots are required. " 1481 "If it is \"latest\" then Mainline module SDKs from platform and " 1482 "bundled modules are included.", 1483 ) 1484 args_parser.add_argument( 1485 "--build-platform-sdks-for-mainline", 1486 action="store_true", 1487 help="Also build the platform SDKs for Mainline modules. " 1488 "Defaults to true when TARGET_BUILD_APPS is not set. " 1489 "Applicable only if the \"latest\" build release is built.", 1490 ) 1491 args = args_parser.parse_args(args) 1492 1493 build_releases = ALL_BUILD_RELEASES 1494 if args.build_release: 1495 selected_build_releases = {b.lower() for b in args.build_release} 1496 build_releases = [ 1497 b for b in build_releases 1498 if b.name.lower() in selected_build_releases 1499 ] 1500 1501 target_build_apps = os.environ.get("TARGET_BUILD_APPS") 1502 modules = filter_modules(MAINLINE_MODULES + BUNDLED_MAINLINE_MODULES, 1503 target_build_apps) 1504 1505 # Also build the platform Mainline SDKs either if no specific modules are 1506 # requested or if --build-platform-sdks-for-mainline is given. 1507 if not target_build_apps or args.build_platform_sdks_for_mainline: 1508 modules += PLATFORM_SDKS_FOR_MAINLINE 1509 1510 producer = create_producer(args.tool_path) 1511 producer.produce_dist(modules, build_releases) 1512 1513 1514if __name__ == "__main__": 1515 main(sys.argv[1:]) 1516