1#!/usr/bin/python3 2 3"""Updates prebuilt libraries used by Android builds. 4 5For details on how to use this script, visit go/update-prebuilts. 6""" 7import os 8import sys 9import zipfile 10import re 11import argparse 12import subprocess 13import shlex 14import glob 15import shutil 16import unittest 17 18# Modules not in Android repo. Ok to ignore if they are not really used. 19try: 20 import six 21except ImportError: 22 six = None 23 24from urllib import request 25from shutil import which 26from distutils.version import LooseVersion 27from pathlib import Path 28from io import StringIO 29from typing import Iterable, Optional 30import xml.etree.ElementTree as ET 31from maven import MavenLibraryInfo, GMavenArtifact, maven_path_for_artifact 32from buildserver import fetch_and_extract, extract_artifact, \ 33 parse_build_id, fetch_artifact as buildserver_fetch_artifact, fetch_artifacts as buildserver_fetch_artifacts 34from utils import print_e, append, cp, mv, rm 35 36 37current_path = 'current' 38framework_sdk_target = 'sdk' 39androidx_dir = os.path.join(current_path, 'androidx') 40androidx_owners = os.path.join(androidx_dir, 'OWNERS') 41java_plugins_bp_path = os.path.join(androidx_dir, 'JavaPlugins.bp') 42test_mapping_file = os.path.join(androidx_dir, 'TEST_MAPPING') 43drop_config_toml = os.path.join(androidx_dir, 'drop_config.toml') 44compose_test_mapping_file = os.path.join(androidx_dir, 'm2repository/androidx/compose/TEST_MAPPING') 45gmaven_dir = os.path.join(current_path, 'gmaven') 46extras_dir = os.path.join(current_path, 'extras') 47buildtools_dir = 'tools' 48jetifier_dir = os.path.join(buildtools_dir, 'jetifier', 'jetifier-standalone') 49repo_root_dir = Path(sys.argv[0]).resolve().parents[3] 50extension_sdk_finalization_cmd = 'prebuilts/build-tools/path/linux-x86/python3 %s -r "{readme}" {local_mode} -b {bug} -f {extension_version} {build_id}' % ( 51 "packages/modules/common/tools/finalize_sdk.py" 52) 53temp_dir = os.path.join(os.getcwd(), 'support_tmp') 54os.chdir(os.path.dirname(os.path.dirname(os.path.realpath(sys.argv[0])))) 55git_dir = os.getcwd() 56 57# Suffixes used by KMP artifacts. If an artifact in maven_to_make ends with one 58# of these, it will replace the anchor artifact. 59kmp_suffixes = ['android','jvm'] 60 61# Leave map blank to automatically populate name and path: 62# - Name format is MAVEN.replaceAll(':','_') 63# - Path format is MAVEN.replaceAll(':','/').replaceAll('.','/') 64maven_to_make = { 65 # AndroidX 66 'androidx.benchmark:benchmark-macro': {}, 67 'androidx.benchmark:benchmark-macro-junit4': {}, 68 'androidx.benchmark:benchmark-common': {}, 69 'androidx.benchmark:benchmark-junit4': {}, 70 'androidx.tracing:tracing': {}, 71 'androidx.tracing:tracing-perfetto': {}, 72 'androidx.tracing:tracing-perfetto-binary': {}, 73 'androidx.tracing:tracing-perfetto-handshake': {}, 74 'androidx.tracing:tracing-perfetto-common': {}, 75 'androidx.tracing:tracing-ktx': {}, 76 'androidx.slice:slice-builders': {}, 77 'androidx.slice:slice-core': {}, 78 'androidx.slice:slice-view': {}, 79 'androidx.remotecallback:remotecallback': {}, 80 'androidx.remotecallback:remotecallback-processor': { 81 'host': True 82 }, 83 'androidx.versionedparcelable:versionedparcelable': {}, 84 'androidx.vectordrawable:vectordrawable-animated': {}, 85 'androidx.activity:activity': {}, 86 'androidx.activity:activity-ktx': {}, 87 'androidx.annotation:annotation-jvm': { 88 'host_and_device': True 89 }, 90 'androidx.annotation:annotation-experimental': {}, 91 'androidx.asynclayoutinflater:asynclayoutinflater': {}, 92 'androidx.camera:camera-viewfinder':{}, 93 'androidx.camera:camera-camera2' :{}, 94 'androidx.camera:camera-core': {}, 95 'androidx.camera:camera-lifecycle': {}, 96 'androidx.camera:camera-extensions': {}, 97 'androidx.collection:collection-ktx': {}, 98 'androidx.collection:collection-jvm': {}, 99 'androidx.concurrent:concurrent-futures': {}, 100 'androidx.concurrent:concurrent-futures-ktx': {}, 101 'androidx.concurrent:concurrent-listenablefuture-callback': {}, 102 'androidx.concurrent:concurrent-listenablefuture': {}, 103 'androidx.core:core': {}, 104 'androidx.core:core-animation': {}, 105 'androidx.core:core-animation-testing': {}, 106 'androidx.core:core-ktx': {}, 107 'androidx.core.uwb:uwb': {}, 108 'androidx.core.uwb:uwb-rxjava3': {}, 109 'androidx.contentpaging:contentpaging': {}, 110 'androidx.coordinatorlayout:coordinatorlayout': {}, 111 'androidx.datastore:datastore-android': {}, 112 'androidx.datastore:datastore-core-okio-jvm': {}, 113 'androidx.datastore:datastore-core-android': {}, 114 'androidx.datastore:datastore-preferences-android': {}, 115 'androidx.datastore:datastore-preferences-core-jvm': {}, 116 'androidx.datastore:datastore-preferences-rxjava2': {}, 117 'androidx.datastore:datastore-rxjava2': {}, 118 'androidx.legacy:legacy-support-core-ui': {}, 119 'androidx.legacy:legacy-support-core-utils': {}, 120 'androidx.cursoradapter:cursoradapter': {}, 121 'androidx.browser:browser': {}, 122 'androidx.customview:customview': {}, 123 'androidx.customview:customview-poolingcontainer': {}, 124 'androidx.credentials:credentials': {}, 125 'androidx.documentfile:documentfile': {}, 126 'androidx.drawerlayout:drawerlayout': {}, 127 'androidx.dynamicanimation:dynamicanimation': {}, 128 'androidx.emoji:emoji': {}, 129 'androidx.emoji:emoji-appcompat': {}, 130 'androidx.emoji:emoji-bundled': {}, 131 'androidx.emoji2:emoji2': {}, 132 'androidx.emoji2:emoji2-views-helper': {}, 133 'androidx.exifinterface:exifinterface': {}, 134 'androidx.fragment:fragment': {}, 135 'androidx.fragment:fragment-ktx': {}, 136 'androidx.fragment:fragment-testing': {}, 137 'androidx.fragment:fragment-testing-manifest': {}, 138 'androidx.heifwriter:heifwriter': {}, 139 'androidx.health:health-services-client': {}, 140 'androidx.interpolator:interpolator': {}, 141 'androidx.loader:loader': {}, 142 'androidx.media:media': {}, 143 'androidx.media2:media2-player': {}, 144 'androidx.media2:media2-session': {}, 145 'androidx.media2:media2-common': {}, 146 'androidx.media2:media2-exoplayer': {}, 147 'androidx.media2:media2-widget': {}, 148 'androidx.navigation:navigation-common': {}, 149 'androidx.navigation:navigation-common-ktx': {}, 150 'androidx.navigation:navigation-fragment': {}, 151 'androidx.navigation:navigation-fragment-ktx': {}, 152 'androidx.navigation:navigation-runtime': {}, 153 'androidx.navigation:navigation-runtime-ktx': {}, 154 'androidx.navigation:navigation-ui': {}, 155 'androidx.navigation:navigation-ui-ktx': {}, 156 'androidx.percentlayout:percentlayout': {}, 157 'androidx.print:print': {}, 158 'androidx.privacysandbox.ads:ads-adservices': {}, 159 'androidx.privacysandbox.ads:ads-adservices-java': {}, 160 'androidx.privacysandbox.ui:ui-client': {}, 161 'androidx.privacysandbox.ui:ui-provider': {}, 162 'androidx.privacysandbox.ui:ui-core': {}, 163 'androidx.privacysandbox.sdkruntime:sdkruntime-client': {}, 164 'androidx.privacysandbox.sdkruntime:sdkruntime-core': {}, 165 'androidx.privacysandbox.tools:tools': { 166 'host': True 167 }, 168 'androidx.privacysandbox.tools:tools-apicompiler': { 169 'host': True 170 }, 171 'androidx.privacysandbox.tools:tools-apigenerator': { 172 'host': True 173 }, 174 'androidx.privacysandbox.tools:tools-apipackager': { 175 'host': True 176 }, 177 'androidx.privacysandbox.tools:tools-core': { 178 'host': True 179 }, 180 'androidx.privacysandbox.ui:ui-tests': {}, 181 'androidx.recommendation:recommendation': {}, 182 'androidx.recyclerview:recyclerview-selection': {}, 183 'androidx.savedstate:savedstate': {}, 184 'androidx.savedstate:savedstate-ktx': {}, 185 'androidx.slidingpanelayout:slidingpanelayout': {}, 186 'androidx.swiperefreshlayout:swiperefreshlayout': {}, 187 'androidx.textclassifier:textclassifier': {}, 188 'androidx.transition:transition': {}, 189 'androidx.transition:transition-ktx': {}, 190 'androidx.tvprovider:tvprovider': {}, 191 'androidx.legacy:legacy-support-v13': {}, 192 'androidx.legacy:legacy-preference-v14': {}, 193 'androidx.leanback:leanback': {}, 194 'androidx.leanback:leanback-grid': {}, 195 'androidx.leanback:leanback-preference': {}, 196 'androidx.legacy:legacy-support-v4': {}, 197 'androidx.appcompat:appcompat': {}, 198 'androidx.appcompat:appcompat-resources': {}, 199 'androidx.cardview:cardview': {}, 200 'androidx.gridlayout:gridlayout': {}, 201 'androidx.mediarouter:mediarouter': {}, 202 'androidx.palette:palette': {}, 203 'androidx.preference:preference': {}, 204 'androidx.recyclerview:recyclerview': {}, 205 'androidx.vectordrawable:vectordrawable': {}, 206 'androidx.viewpager:viewpager': {}, 207 'androidx.viewpager2:viewpager2': {}, 208 'androidx.wear:wear': {}, 209 'androidx.wear:wear-ongoing': {}, 210 'androidx.javascriptengine:javascriptengine': {}, 211 'androidx.webkit:webkit': {}, 212 'androidx.biometric:biometric': {}, 213 'androidx.autofill:autofill': {}, 214 'androidx.appsearch:appsearch': {}, 215 'androidx.appsearch:appsearch-builtin-types': {}, 216 'androidx.appsearch:appsearch-compiler': { 217 'name': 'androidx.appsearch_appsearch-compiler', 218 'host': True 219 }, 220 'androidx.appsearch:appsearch-local-storage': { 221 'name': 'androidx.appsearch_appsearch_local_storage' 222 }, 223 'androidx.appsearch:appsearch-platform-storage': {}, 224 'androidx.car.app:app': {}, 225 'androidx.car.app:app-automotive': {}, 226 'androidx.car.app:app-testing': {}, 227 'androidx.startup:startup-runtime': {}, 228 'androidx.window:window': { 229 'optional-uses-libs': { 230 'androidx.window.extensions', 231 'androidx.window.sidecar' 232 } 233 }, 234 'androidx.window.extensions:extensions': {}, 235 'androidx.window.extensions.core:core': {}, 236 'androidx.window:window-core': {}, 237 'androidx.window:window-java':{}, 238 'androidx.resourceinspection:resourceinspection-annotation': {}, 239 'androidx.profileinstaller:profileinstaller': {}, 240 'androidx.test.uiautomator:uiautomator': {}, 241 242 # AndroidX for Compose 243 'androidx.compose.compiler:compiler-hosted': { 244 'host': True 245 }, 246 'androidx.compose.animation:animation-android': {}, 247 'androidx.compose.animation:animation-core-android': {}, 248 'androidx.compose.animation:animation-graphics-android': {}, 249 'androidx.compose.foundation:foundation-android': {}, 250 'androidx.compose.foundation:foundation-layout-android': {}, 251 'androidx.compose.foundation:foundation-text-android': {}, 252 'androidx.compose.material:material-android': {}, 253 'androidx.compose.material:material-icons-core-android': {}, 254 'androidx.compose.material:material-icons-extended-android': {}, 255 'androidx.compose.material:material-ripple-android': {}, 256 'androidx.compose.material3:material3-android': {}, 257 'androidx.compose.material3:material3-window-size-class-android': {}, 258 'androidx.compose.runtime:runtime-android': {}, 259 'androidx.compose.runtime:runtime-livedata': {}, 260 'androidx.compose.runtime:runtime-saveable-android': {}, 261 'androidx.compose.runtime:runtime-tracing': {}, 262 'androidx.compose.ui:ui-util-android': {}, 263 'androidx.compose.ui:ui-android': {}, 264 'androidx.compose.ui:ui-geometry-android': {}, 265 'androidx.compose.ui:ui-graphics-android': {}, 266 'androidx.compose.ui:ui-test-manifest': {}, 267 'androidx.compose.ui:ui-test-android': {}, 268 'androidx.compose.ui:ui-test-junit4-android': {}, 269 'androidx.compose.ui:ui-text-android': {}, 270 'androidx.compose.ui:ui-tooling-android': {}, 271 'androidx.compose.ui:ui-tooling-data-android': {}, 272 'androidx.compose.ui:ui-tooling-preview-android': {}, 273 'androidx.compose.ui:ui-unit-android': {}, 274 'androidx.activity:activity-compose': {}, 275 'androidx.navigation:navigation-compose': { }, 276 'androidx.lifecycle:lifecycle-viewmodel-compose': { }, 277 278 # Compose for wear 279 'androidx.wear.compose:compose-material-core': {}, 280 'androidx.wear.compose:compose-foundation': {}, 281 'androidx.wear.compose:compose-material': {}, 282 'androidx.wear.compose:compose-navigation': {}, 283 284 # AndroidX for Multidex 285 'androidx.multidex:multidex': {}, 286 'androidx.multidex:multidex-instrumentation': {}, 287 288 # AndroidX for Constraint Layout 289 'androidx.constraintlayout:constraintlayout': { 290 'name': 'androidx-constraintlayout_constraintlayout' 291 }, 292 'androidx.constraintlayout:constraintlayout-solver': { 293 'name': 'androidx-constraintlayout_constraintlayout-solver' 294 }, 295 'androidx.constraintlayout:constraintlayout-core': {}, 296 'androidx.constraintlayout:constraintlayout-compose-android': {}, 297 # AndroidX for Architecture Components 298 'androidx.arch.core:core-common': {}, 299 'androidx.arch.core:core-runtime': {}, 300 'androidx.arch.core:core-testing': {}, 301 'androidx.lifecycle:lifecycle-common': {}, 302 'androidx.lifecycle:lifecycle-common-java8': {}, 303 'androidx.lifecycle:lifecycle-extensions': {}, 304 'androidx.lifecycle:lifecycle-livedata': {}, 305 'androidx.lifecycle:lifecycle-livedata-ktx': {}, 306 'androidx.lifecycle:lifecycle-livedata-core': {}, 307 'androidx.lifecycle:lifecycle-livedata-core-ktx': {}, 308 'androidx.lifecycle:lifecycle-process': {}, 309 'androidx.lifecycle:lifecycle-runtime': {}, 310 'androidx.lifecycle:lifecycle-runtime-ktx': {}, 311 'androidx.lifecycle:lifecycle-runtime-compose': {}, 312 'androidx.lifecycle:lifecycle-runtime-testing': {}, 313 'androidx.lifecycle:lifecycle-service': {}, 314 'androidx.lifecycle:lifecycle-viewmodel': {}, 315 'androidx.lifecycle:lifecycle-viewmodel-ktx': {}, 316 'androidx.lifecycle:lifecycle-viewmodel-savedstate': {}, 317 'androidx.paging:paging-common-jvm': {}, 318 'androidx.paging:paging-common-ktx': {}, 319 'androidx.paging:paging-guava': {}, 320 'androidx.paging:paging-runtime': {}, 321 'androidx.sqlite:sqlite': {}, 322 'androidx.sqlite:sqlite-framework': {}, 323 'androidx.room:room-common-jvm': { 324 'host_and_device': True 325 }, 326 'androidx.room:room-compiler': { 327 'host': True, 328 'extra-static-libs': { 329 'guava' 330 } 331 }, 332 'androidx.room:room-guava': {}, 333 'androidx.room:room-migration': { 334 'host_and_device': True 335 }, 336 'androidx.room:room-ktx': {}, 337 'androidx.room:room-paging': {}, 338 'androidx.room:room-paging-guava': {}, 339 'androidx.room:room-runtime': {}, 340 'androidx.room:room-testing': {}, 341 'androidx.room:room-compiler-processing': { 342 'host': True 343 }, 344 'androidx.work:work-runtime': {}, 345 'androidx.work:work-runtime-ktx': {}, 346 'androidx.work:work-testing': {}, 347 348 # Third-party dependencies 349 'com.google.android:flexbox': { 350 'name': 'flexbox', 351 'path': 'flexbox' 352 }, 353 354 # Androidx Material Design Components 355 'com.google.android.material:material': {}, 356} 357 358# Mapping of POM dependencies to Soong build targets 359deps_rewrite = { 360 'auto-common': 'auto_common', 361 'auto-value-annotations': 'auto_value_annotations', 362 'com.google.auto.value:auto-value': 'libauto_value_plugin', 363 'com.google.protobuf:protobuf-java': 'libprotobuf-java-full', 364 'com.google.protobuf:protobuf-javalite': 'libprotobuf-java-lite', 365 'org.ow2.asm:asm': 'ow2-asm', 366 'org.ow2.asm:asm-commons': 'ow2-asm-commons', 367 'monitor': 'androidx.test.monitor', 368 'rules': 'androidx.test.rules', 369 'runner': 'androidx.test.runner', 370 'androidx.test:core': 'androidx.test.core', 371 'com.squareup:javapoet': 'javapoet', 372 'com.squareup.okio:okio-jvm': 'okio-lib', 373 'com.google.guava:listenablefuture': 'guava-listenablefuture-prebuilt-jar', 374 'sqlite-jdbc': 'xerial-sqlite-jdbc', 375 'com.intellij:annotations': 'jetbrains-annotations', 376 'javax.annotation:javax.annotation-api': 'javax-annotation-api-prebuilt-host-jar', 377 'org.robolectric:robolectric': 'Robolectric_all-target', 378 'org.jetbrains.kotlin:kotlin-stdlib-common': 'kotlin-stdlib', 379 'org.jetbrains.kotlinx:kotlinx-coroutines-core': 'kotlinx_coroutines', 380 'org.jetbrains.kotlinx:kotlinx-coroutines-test-jvm': 'kotlinx_coroutines_test', 381 'org.jetbrains.kotlinx:kotlinx-coroutines-guava': 'kotlinx_coroutines_guava', 382 'org.jetbrains.kotlinx:kotlinx-coroutines-android': 'kotlinx_coroutines_android', 383 'org.jetbrains.kotlinx:kotlinx-coroutines-test':'kotlinx_coroutines_test', 384 'org.jetbrains.kotlinx:kotlinx-coroutines-rx2': 'kotlinx_coroutines_rx2', 385 'org.jetbrains.kotlinx:kotlinx-metadata-jvm': 'kotlinx_metadata_jvm', 386 'androidx.test.espresso:espresso-core':'androidx.test.espresso.core', 387 'androidx.test.espresso:espresso-idling-resource':'androidx.test.espresso.idling-resource', 388 'androidx.datastore:datastore-core-jvm': 'androidx.datastore_datastore-core', 389} 390 391# List of artifacts that will be updated from GMaven 392# Use pattern: `group:library:version:extension` 393# e.g.: 394# androidx.appcompat:appcompat:1.2.0:aar 395# Use `latest` to always fetch the latest version. 396# e.g.: 397# androidx.appcompat:appcompat:latest:aar 398# Also make sure you add `group:library`:{} to maven_to_make as well. 399gmaven_artifacts = {} 400 401# Always remove these files. 402denylist_files = [ 403 'annotations.zip', 404 'public.txt', 405 'R.txt', 406 'AndroidManifest.xml', 407 os.path.join('libs', 'noto-emoji-compat-java.jar') 408] 409 410# Explicitly allow-listed initializers 411enabled_initializers = set([ 412 'androidx.lifecycle.ProcessLifecycleInitializer', 413 'androidx.work.WorkManagerInitializer', 414 # TODO(282947321): update after http://aosp/2600447 lands 415 'androidx.compose.runtime.tracing.TracingInitializer', 416]) 417 418android_manifest_namepaces = { 419 'android': 'http://schemas.android.com/apk/res/android', 420 'tools': 'http://schemas.android.com/tools' 421} 422 423startup_initializer_pattern = re.compile(r'(\s+)android:value="androidx.startup".*') 424 425artifact_pattern = re.compile(r'^(.+?)-(\d+\.\d+\.\d+(?:-\w+\d+)?(?:-[\d.]+)*)\.(jar|aar)$') 426 427 428def name_for_artifact(group_artifact): 429 """Returns the build system target name for a given library's Maven coordinate. 430 431 Args: 432 group_artifact: an unversioned Maven artifact coordinate, ex. androidx.core:core 433 Returns: 434 The build system target name for the artifact, ex. androidx.core_core. 435 """ 436 for kmp_suffix in kmp_suffixes: 437 if group_artifact.endswith("-" + kmp_suffix): 438 loc = group_artifact.rfind("-" + kmp_suffix) 439 group_artifact = group_artifact[0:loc] 440 if group_artifact in maven_to_make: 441 raise ValueError(f'Do not specify KMP anchor artifact in ' 442 f'maven_to_make: {group_artifact}') 443 deps_rewrite[group_artifact] = group_artifact.replace(':', '_') 444 break 445 return group_artifact.replace(':', '_') 446 447 448def path_for_artifact(group_artifact): 449 """Returns the file system path for a given library's Maven coordinate. 450 451 Args: 452 group_artifact: an unversioned Maven artifact coordinate, ex. androidx.core:core 453 Returns: 454 The file system path for the artifact, ex. androidx/core/core. 455 """ 456 return group_artifact.replace('.', '/').replace(':', '/') 457 458 459def populate_maven_to_make(mapping): 460 """Modifies the input mapping by expanding Maven coordinate keys into build target names and 461 paths. 462 463 Args: 464 mapping: a map where the keys are Maven coordinates 465 """ 466 for key in mapping: 467 if 'name' not in mapping[key]: 468 mapping[key]['name'] = name_for_artifact(key) 469 if 'path' not in maven_to_make[key]: 470 mapping[key]['path'] = path_for_artifact(key) 471 472 473def detect_artifacts(maven_repo_dirs): 474 """Parses Maven libraries from the specified directories. 475 476 Args: 477 maven_repo_dirs: a list of maven repository roots 478 Returns: 479 A map of Maven coordinate keys to MavenLibraryInfo objects parsed from POM files. 480 """ 481 maven_lib_info = {} 482 483 # Find the latest revision for each artifact, remove others 484 for repo_dir in maven_repo_dirs: 485 for root, dirs, files in os.walk(repo_dir): 486 for file in files: 487 if file[-4:] == '.pom': 488 # Read the POM (hack hack hack). 489 group_id = '' 490 artifact_id = '' 491 version = '' 492 file = os.path.join(root, file) 493 with open(file) as pom_file: 494 for line in pom_file: 495 if line[:11] == ' <groupId>': 496 group_id = line[11:-11] 497 elif line[:14] == ' <artifactId>': 498 artifact_id = line[14:-14] 499 elif line[:11] == ' <version>': 500 version = line[11:-11] 501 if group_id == '' or artifact_id == '' or version == '': 502 print_e('Failed to find Maven artifact data in ' + file) 503 continue 504 505 # Locate the artifact. 506 artifact_file = file[:-4] 507 if os.path.exists(artifact_file + '.jar'): 508 artifact_file = artifact_file + '.jar' 509 elif os.path.exists(artifact_file + '.aar'): 510 artifact_file = artifact_file + '.aar' 511 else: 512 # This error only occurs for a handful of gradle.plugin artifacts that only 513 # ship POM files, so we probably don't need to log unless we're debugging. 514 # print_e('Failed to find artifact for ' + artifact_file) 515 continue 516 517 # Make relative to root. 518 artifact_file = artifact_file[len(root) + 1:] 519 520 # Find the mapping. 521 group_artifact = group_id + ':' + artifact_id 522 if group_artifact in maven_to_make: 523 key = group_artifact 524 elif artifact_id in maven_to_make: 525 key = artifact_id 526 else: 527 # No mapping entry, skip this library. 528 continue 529 530 # Store the latest version. 531 version = LooseVersion(version) 532 if key not in maven_lib_info \ 533 or version > maven_lib_info[key].version: 534 maven_lib_info[key] = MavenLibraryInfo(key, group_id, artifact_id, version, 535 root, repo_dir, artifact_file) 536 537 return maven_lib_info 538 539 540def find_invalid_spec(artifact_list): 541 """Verifies whether all the artifacts in the list correspond to an entry in maven_to_make. 542 543 Args: 544 artifact_list: list of group IDs or artifact coordinates 545 Returns: 546 The first invalid artifact specification in the list, or None if all specs are valid. 547 """ 548 if artifact_list is None: 549 return None 550 for prefix in artifact_list: 551 has_prefix = False 552 for artifact_id in maven_to_make: 553 if artifact_id.startswith(prefix): 554 has_prefix = True 555 break 556 if not has_prefix: 557 return prefix 558 return None 559 560 561def transform_maven_repos(maven_repo_dirs, transformed_dir, extract_res=True, 562 write_pom2bp_cmd=True, include_static_deps=True, include=None, 563 exclude=None, prepend=None): 564 """Transforms a standard Maven repository to be compatible with the Android build system. 565 566 When using the include argument by itself, all other libraries will be excluded. When using the 567 exclude argument by itself, all other libraries will be included. When using both arguments, the 568 inclusion list will be applied followed by the exclusion list. 569 570 Args: 571 maven_repo_dirs: path to local Maven repository 572 transformed_dir: relative path for output, ex. androidx 573 extract_res: whether to extract Android resources like AndroidManifest.xml from AARs 574 write_pom2bp_cmd: whether pom2bp should write its own invocation arguments to output 575 include_static_deps: whether to pass --static-deps to pom2bp 576 include: list of Maven groupIds or unversioned artifact coordinates to include for 577 updates, ex. androidx.core or androidx.core:core 578 exclude: list of Maven groupIds or unversioned artifact coordinates to exclude from 579 updates, ex. androidx.core or androidx.core:core 580 prepend: Path to a file containing text to be inserted at the beginning of the generated 581 build file 582 Returns: 583 True if successful, false otherwise. 584 """ 585 # If neither include nor exclude is set, fall back to legacy behavior of including everything. 586 include_all = exclude is None and include is None 587 588 if exclude is None: 589 exclude = [] 590 if include is None: 591 include = [] 592 593 cwd = os.getcwd() 594 local_repo = os.path.join(cwd, transformed_dir) 595 working_dir = temp_dir 596 597 # Handle inclusions by stashing the remote artifacts for the inclusions, replacing the entire 598 # remote repo with the local repo, then restoring the stashed artifacts. 599 for remote_repo in maven_repo_dirs: 600 remote_repo = os.path.join(cwd, remote_repo) 601 paths_to_copy = [] 602 603 # If we're including everything, move the entire repo to temp. 604 if include_all: 605 cp(remote_repo, working_dir) 606 else: 607 # Move included artifacts from repo to temp. 608 for group_artifact in include: 609 artifact_path = os.path.join('m2repository', path_for_artifact(group_artifact)) 610 remote_path = os.path.join(remote_repo, artifact_path) 611 working_path = os.path.join(working_dir, artifact_path) 612 if os.path.exists(remote_path): 613 print(f'Included {group_artifact} in update') 614 paths_to_copy.append([remote_path, working_path]) 615 for [remote_path, working_path] in paths_to_copy: 616 mv(remote_path, working_path) 617 618 # Replace all remaining artifacts in remote repo with local repo. 619 cp(local_repo, remote_repo) 620 621 # If we're including everything, restore the entire repo. 622 if include_all: 623 cp(working_dir, remote_repo) 624 else: 625 # Restore included artifacts to remote repo. 626 for [remote_path, working_path] in paths_to_copy: 627 mv(working_path, remote_path) 628 629 # Handle exclusions by replacing the remote artifacts for the exclusions with local artifacts. 630 # This must happen before we parse the artifacts. 631 for remote_repo in maven_repo_dirs: 632 for group_artifact in exclude: 633 artifact_path = os.path.join('m2repository', path_for_artifact(group_artifact)) 634 remote_path = os.path.join(remote_repo, artifact_path) 635 if os.path.exists(remote_path): 636 rm(remote_path) 637 local_path = os.path.join(local_repo, artifact_path) 638 if os.path.exists(local_path): 639 print(f'Excluded {group_artifact} from update, used local artifact') 640 mv(local_path, remote_path) 641 else: 642 print(f'Excluded {group_artifact} from update, no local artifact present') 643 644 # Parse artifacts. 645 maven_lib_info = detect_artifacts(maven_repo_dirs) 646 647 if not maven_lib_info: 648 print_e('Failed to detect artifacts') 649 return False 650 651 # Move libraries into the working directory, performing any necessary transformations. 652 for info in maven_lib_info.values(): 653 transform_maven_lib(working_dir, info, extract_res) 654 655 # Generate a single Android.bp that specifies to use all of the above artifacts. 656 makefile = os.path.join(working_dir, 'Android.bp') 657 with open(makefile, 'w') as f: 658 args = ['pom2bp'] 659 args.extend(['-sdk-version', '31']) 660 args.extend(['-default-min-sdk-version', '24']) 661 if not write_pom2bp_cmd: 662 args.extend(['-write-cmd=false']) 663 if include_static_deps: 664 args.append('-static-deps') 665 if prepend: 666 args.append(f'-prepend={prepend}') 667 rewrite_names = sorted(maven_to_make.keys()) 668 args.extend([f'-rewrite=^{name}$={maven_to_make[name]["name"]}' for name in rewrite_names]) 669 args.extend([f'-rewrite=^{key}$={value}' for key, value in deps_rewrite.items()]) 670 args.extend(["-extra-static-libs=" + maven_to_make[name]['name'] + "=" + ",".join( 671 sorted(maven_to_make[name]['extra-static-libs'])) for name in maven_to_make if 672 'extra-static-libs' in maven_to_make[name]]) 673 args.extend(["-optional-uses-libs=" + maven_to_make[name]['name'] + "=" + ",".join( 674 sorted(maven_to_make[name]['optional-uses-libs'])) for name in maven_to_make if 675 'optional-uses-libs' in maven_to_make[name]]) 676 args.extend([f'-host={name}' for name in maven_to_make 677 if maven_to_make[name].get('host')]) 678 args.extend([f'-host-and-device={name}' for name in maven_to_make 679 if maven_to_make[name].get('host_and_device')]) 680 args.extend(['.']) 681 subprocess.check_call(args, stdout=f, cwd=working_dir) 682 683 # Replace the old directory. 684 local_repo = os.path.join(cwd, transformed_dir) 685 mv(working_dir, local_repo) 686 return True 687 688 689def transform_maven_lib(working_dir, artifact_info, extract_res): 690 """Transforms the specified artifact for use in the Android build system. 691 692 Moves relevant files for the artifact represented by artifact_info of type MavenLibraryInfo into 693 the appropriate path inside working_dir, unpacking files needed by the build system from AARs. 694 695 Args: 696 working_dir: The directory into which the artifact should be moved 697 artifact_info: A MavenLibraryInfo representing the library artifact 698 extract_res: True to extract resources from AARs, false otherwise. 699 """ 700 # Move library into working dir 701 new_dir = os.path.normpath( 702 os.path.join(working_dir, os.path.relpath(artifact_info.dir, artifact_info.repo_dir))) 703 mv(artifact_info.dir, new_dir) 704 705 maven_lib_type = os.path.splitext(artifact_info.file)[1][1:] 706 707 group_artifact = artifact_info.key 708 make_lib_name = maven_to_make[group_artifact]['name'] 709 make_dir_name = maven_to_make[group_artifact]['path'] 710 711 artifact_file = os.path.join(new_dir, artifact_info.file) 712 713 if maven_lib_type == 'aar': 714 if extract_res: 715 target_dir = os.path.join(working_dir, make_dir_name) 716 if not os.path.exists(target_dir): 717 os.makedirs(target_dir) 718 719 process_aar(artifact_file, target_dir) 720 721 with zipfile.ZipFile(artifact_file) as zip_file: 722 manifests_dir = os.path.join(working_dir, 'manifests') 723 lib_path = Path(os.path.join(manifests_dir, make_lib_name)) 724 manifest_path = lib_path / 'AndroidManifest.xml' 725 zip_file.extract('AndroidManifest.xml', lib_path.as_posix()) 726 contents = check_startup_initializers(manifest_path) 727 if contents: 728 manifest_path.write_text(contents) 729 730 731def process_aar(artifact_file, target_dir): 732 """Extracts and cleans up the contents of an AAR file to the specified directory. 733 734 Removes classes.jar, empty directories, and denylisted files. 735 736 Args: 737 artifact_file: path to the AAR to extract 738 target_dir: directory into which the contents should be extracted 739 """ 740 # Extract AAR file to target_dir. 741 with zipfile.ZipFile(artifact_file) as zip_file: 742 zip_file.extractall(target_dir) 743 744 # Remove classes.jar 745 classes_jar = os.path.join(target_dir, 'classes.jar') 746 if os.path.exists(classes_jar): 747 os.remove(classes_jar) 748 749 # Remove empty dirs. 750 for root, dirs, files in os.walk(target_dir, topdown=False): 751 for dir_name in dirs: 752 dir_path = os.path.join(root, dir_name) 753 if not os.listdir(dir_path): 754 os.rmdir(dir_path) 755 756 # Remove top-level cruft. 757 for file in denylist_files: 758 file_path = os.path.join(target_dir, file) 759 if os.path.exists(file_path): 760 os.remove(file_path) 761 762 763def fetch_gmaven_artifact(artifact): 764 """Fetch a GMaven artifact. 765 766 Downloads a GMaven artifact 767 (https://developer.android.com/studio/build/dependencies#gmaven-access) 768 769 Args: 770 artifact: an instance of GMavenArtifact. 771 """ 772 pom_path = maven_path_for_artifact( 773 'gmaven', artifact.group, artifact.library, artifact.version, 'pom') 774 artifact_path = maven_path_for_artifact( 775 'gmaven', artifact.group, artifact.library, artifact.version, artifact.ext) 776 777 download_file_to_disk(artifact.get_pom_file_url(), pom_path) 778 download_file_to_disk(artifact.get_artifact_url(), artifact_path) 779 780 return os.path.dirname(artifact_path) 781 782 783def download_file_to_disk(url, filepath): 784 """Download the file at URL to the location dictated by the path. 785 786 Args: 787 url: Remote URL to download file from. 788 filepath: Filesystem path to write the file to. 789 """ 790 print(f'Downloading URL: {url}') 791 file_data = request.urlopen(url) 792 793 try: 794 os.makedirs(os.path.dirname(filepath)) 795 except os.error: 796 # This is a common situation - os.makedirs fails if dir already exists. 797 pass 798 try: 799 with open(filepath, 'wb') as f: 800 f.write(six.ensure_binary(file_data.read())) 801 except Exception as e: 802 print_e(e.__class__, 'occurred while reading', filepath) 803 os.remove(os.path.dirname(filepath)) 804 raise 805 806 807def check_startup_initializers(manifest_path: Path) -> Optional[str]: 808 try: 809 for prefix in android_manifest_namepaces: 810 ET.register_namespace(prefix, android_manifest_namepaces[prefix]) 811 812 # Use ElementTree to check if we need updates. 813 # That way we avoid false positives. 814 contents = manifest_path.read_text() 815 root = ET.fromstring(contents) 816 needs_changes = _check_node(root) 817 if needs_changes: 818 # Ideally we would use ElementTree here. 819 # Instead, we are using regular expressions here so we can 820 # preserve comments and whitespaces. 821 lines = contents.splitlines() 822 output = StringIO() 823 for line in lines: 824 matcher = startup_initializer_pattern.match(line) 825 if matcher: 826 prefix = matcher.group(1) 827 # Adding an explicit tools:node="remove" so this is still traceable 828 # when looking at the source. 829 output.write(f'{prefix}android:value="androidx.startup"\n') 830 output.write(f'{prefix}tools:node="remove" />') 831 else: 832 output.write(line) 833 output.write('\n') 834 835 output.write('\n') 836 return output.getvalue() 837 except BaseException as exception: 838 print( 839 f'Unable to parse manifest file with path {manifest_path}.\n\n Details ({exception})' 840 ) 841 842def _attribute_name(namespace: str, attribute: str) -> str: 843 if not namespace in android_manifest_namepaces: 844 raise ValueError(f'Unexpected namespace {namespace}') 845 846 return f'{{{android_manifest_namepaces[namespace]}}}{attribute}' 847 848 849def _check_node(node: ET.Element) -> bool: 850 for child in node: 851 # Find the initialization provider 852 is_provider = child.tag == 'provider' 853 provider_name = child.attrib.get(_attribute_name('android', 'name')) 854 is_initialization_provider = provider_name == 'androidx.startup.InitializationProvider' 855 856 if is_provider and is_initialization_provider: 857 metadata_nodes = child.findall('meta-data', namespaces=android_manifest_namepaces) 858 return _needs_disable_initialization(metadata_nodes) 859 860 if len(child) > 0: 861 return _check_node(child) 862 863 return False 864 865 866def _needs_disable_initialization(metadata_nodes: Iterable[ET.Element]) -> bool: 867 needs_update = False 868 for node in metadata_nodes: 869 name = node.attrib.get(_attribute_name('android', 'name')) 870 value = node.attrib.get(_attribute_name('android', 'value')) 871 if value == 'androidx.startup': 872 if name not in enabled_initializers: 873 needs_update = True 874 875 return needs_update 876 877 878def update_gmaven(gmaven_artifacts_list): 879 artifacts = [GMavenArtifact(artifact) for artifact in gmaven_artifacts_list] 880 for artifact in artifacts: 881 if artifact.version == 'latest': 882 artifact.version = artifact.get_latest_version() 883 884 if not transform_maven_repos(['gmaven'], gmaven_dir, extract_res=False): 885 return [] 886 return [artifact.key for artifact in artifacts] 887 888 889def update_jetifier(target, build_id, beyond_corp): 890 """ 891 Fetches and extracts Jetifier tool prebuilts. 892 893 Args: 894 target: Android build server target name 895 build_id: Android build server ID 896 beyond_corp: Whether to use BeyondCorp-compatible artifact fetcher 897 Return: 898 Whether the prebuilt was successfully updated. 899 """ 900 repo_file = 'jetifier-standalone.zip' 901 repo_dir = fetch_and_extract(target, build_id.url_id, repo_file, beyond_corp) 902 if not repo_dir: 903 print_e('Failed to extract Jetifier') 904 return False 905 906 rm(jetifier_dir) 907 mv(os.path.join(repo_dir, 'jetifier-standalone'), jetifier_dir) 908 os.chmod(os.path.join(jetifier_dir, 'bin', 'jetifier-standalone'), 0o755) 909 return True 910 911 912def update_constraint(local_file): 913 """ 914 Extracts ConstraintLayout library prebuilts. 915 916 Args: 917 local_file: local Maven repository ZIP containing library artifacts 918 Return: 919 Whether the prebuilts were successfully updated. 920 """ 921 repo_dir = extract_artifact(local_file) 922 if not repo_dir: 923 print_e('Failed to extract Constraint Layout') 924 return False 925 return transform_maven_repos([repo_dir], os.path.join(extras_dir, 'constraint-layout-x'), 926 extract_res=False) 927 928 929def update_material(local_file): 930 """ 931 Extracts Material Design Components library prebuilts. 932 933 Args: 934 local_file: local Maven repository ZIP containing library artifacts 935 Return: 936 Whether the prebuilts were successfully updated. 937 """ 938 design_dir = extract_artifact(local_file) 939 if not design_dir: 940 print_e('Failed to extract Material Design Components') 941 return False 942 return transform_maven_repos([design_dir], os.path.join(extras_dir, 'material-design-x'), 943 extract_res=False) 944 945 946def fetch_artifact(target, build_id, artifact_path, beyond_corp, local_mode): 947 if not local_mode: 948 return buildserver_fetch_artifact(target, build_id, artifact_path, beyond_corp) 949 950 copy_from = os.path.join(repo_root_dir.resolve(), 'out/dist', artifact_path) 951 copy_to = os.path.join('.', os.path.dirname(artifact_path)) 952 print(f'Copying {copy_from} to {copy_to}...') 953 result_path = None 954 try: 955 if not os.path.exists(copy_to): 956 os.makedirs(copy_to) 957 copied = 0 958 for file in glob.glob(copy_from): 959 result_path = shutil.copy(file, copy_to) 960 copied += 1 961 # Multiple files, return destination folder. 962 if copied > 1: 963 result_path = artifact_path 964 except Exception as e: 965 print(f'Error: {e} occured while copying') 966 raise 967 return result_path 968 969 970def fetch_artifacts(target, build_id, artifact_dict, beyond_corp, local_mode): 971 if not local_mode: 972 return buildserver_fetch_artifacts(target, build_id, artifact_dict, beyond_corp) 973 974 for artifact, target_path in artifact_dict.items(): 975 artifact_path = fetch_artifact(target, build_id.url_id, artifact, beyond_corp, local_mode) 976 if not artifact_path: 977 return False 978 mv(artifact_path, target_path) 979 return True 980 981 982def update_framework(target, build_id, sdk_dir, beyond_corp, local_mode): 983 api_scope_list = ['public', 'system', 'test', 'module-lib', 'system-server'] 984 if sdk_dir == 'current': 985 api_scope_list.append('core') 986 987 for api_scope in api_scope_list: 988 target_dir = os.path.join(sdk_dir, api_scope) 989 if api_scope == 'core': 990 artifact_to_path = {'core.current.stubs.jar': os.path.join(target_dir, 'android.jar')} 991 else: 992 artifact_to_path = { 993 'apistubs/android/' + api_scope + '/*.jar': os.path.join(target_dir, '*'), 994 } 995 if api_scope == 'public' or api_scope == 'module-lib': 996 # Distinct core-for-system-modules.jar files are only provided 997 # for the public and module-lib API surfaces. 998 artifact_to_path[ 999 'system-modules/' + api_scope + '/core-for-system-modules.jar'] = os.path.join( 1000 target_dir, '*') 1001 1002 if not fetch_artifacts(target, build_id, artifact_to_path, beyond_corp, local_mode): 1003 return False 1004 1005 if api_scope == 'public': 1006 # Fetch a few artifacts from the public sdk. 1007 if local_mode: 1008 artifact = 'android-sdk*.zip' 1009 else: 1010 artifact = f'sdk-repo-linux-platforms-{build_id.url_id}.zip' 1011 artifact_path = fetch_artifact(target, build_id.url_id, artifact, beyond_corp, local_mode) 1012 if not artifact_path: 1013 return False 1014 1015 with zipfile.ZipFile(artifact_path) as zipFile: 1016 extra_files = [ 1017 'android.jar', 1018 'framework.aidl', 1019 'uiautomator.jar'] 1020 for filename in extra_files: 1021 matches = list(filter(lambda path: filename in path, zipFile.namelist())) 1022 if len(matches) != 1: 1023 print_e('Expected 1 file named \'%s\' in zip %s, found %d' % 1024 (filename, zipFile.filename, len(matches))) 1025 return False 1026 zip_path = matches[0] 1027 src_path = zipFile.extract(zip_path) 1028 dst_path = os.path.join(target_dir, filename) 1029 mv(src_path, dst_path) 1030 1031 # Fetch the lint api databases 1032 lint_database_artifacts = {} 1033 for api_scope in ['public', 'system', 'module-lib', 'system-server']: 1034 data_folder = 'data' if api_scope == 'public' else api_scope + '-data' 1035 lint_database_artifacts[os.path.join(data_folder, 'api-versions.xml')] = os.path.join(sdk_dir, api_scope, 'data', 'api-versions.xml') 1036 lint_database_artifacts[os.path.join(data_folder, 'annotations.zip')] = os.path.join(sdk_dir, api_scope, 'data', 'annotations.zip') 1037 lint_database_artifacts['finalized-flags.txt'] = os.path.join(sdk_dir, 'finalized-flags.txt') 1038 fetch_artifacts(target, build_id, lint_database_artifacts, beyond_corp, local_mode) 1039 1040 return True 1041 1042 1043def update_makefile(build_id): 1044 template = '"%s",\n\ 1045 "current"' 1046 makefile = os.path.join(git_dir, 'Android.bp') 1047 1048 with open(makefile, 'r+') as f: 1049 contents = f.read().replace('"current"', template % build_id) 1050 f.seek(0) 1051 f.write(contents) 1052 1053 return True 1054 1055 1056def finalize_sdk(target, build_id, sdk_version, beyond_corp, local_mode): 1057 target_finalize_dir = sdk_version 1058 1059 for api_scope in ['public', 'system', 'test', 'module-lib', 'system-server']: 1060 artifact_to_path = {f'apistubs/android/{api_scope}/api/*.txt': os.path.join( 1061 target_finalize_dir, api_scope, 'api', '*')} 1062 1063 if not fetch_artifacts(target, build_id, artifact_to_path, beyond_corp, local_mode): 1064 return False 1065 1066 return update_framework(target, build_id, target_finalize_dir, beyond_corp, local_mode) and update_makefile( 1067 target_finalize_dir) 1068 1069 1070def update_framework_current(target, build_id, beyond_corp, local_mode): 1071 return update_framework(target, build_id, current_path, beyond_corp, local_mode) 1072 1073 1074def update_buildtools(target, arch, build_id, beyond_corp): 1075 artifact_path = fetch_and_extract(target, build_id.url_id, 1076 f'sdk-repo-{arch}-build-tools-{build_id.fs_id}.zip', 1077 beyond_corp) 1078 if not artifact_path: 1079 return False 1080 1081 top_level_dir = os.listdir(artifact_path)[0] 1082 src_path = os.path.join(artifact_path, top_level_dir) 1083 dst_path = os.path.join(buildtools_dir, arch) 1084 1085 # There are a few libraries that have been manually added to the 1086 # build tools, copy them from the destination back to the source 1087 # before the destination is overwritten. 1088 files_to_save = ( 1089 'lib64/libconscrypt_openjdk_jni.dylib', 1090 'lib64/libconscrypt_openjdk_jni.so', 1091 'bin/lib64/libwinpthread-1.dll', 1092 ) 1093 for file in files_to_save: 1094 src_file = os.path.join(dst_path, file) 1095 dst_file = os.path.join(src_path, file) 1096 if os.path.exists(dst_path): 1097 mv(src_file, dst_file) 1098 1099 mv(src_path, dst_path) 1100 1101 # Move all top-level files to /bin and make them executable 1102 bin_path = os.path.join(dst_path, 'bin') 1103 top_level_files = filter(lambda e: os.path.isfile(os.path.join(dst_path, e)), os.listdir(dst_path)) 1104 for file in top_level_files: 1105 src_file = os.path.join(dst_path, file) 1106 dst_file = os.path.join(bin_path, file) 1107 mv(src_file, dst_file) 1108 os.chmod(dst_file, 0o755) 1109 1110 # Make the files under lld-bin executable 1111 lld_bin_files = os.listdir(os.path.join(dst_path, 'lld-bin')) 1112 for file in lld_bin_files: 1113 os.chmod(os.path.join(dst_path, 'lld-bin', file), 0o755) 1114 1115 # Remove renderscript 1116 rm(os.path.join(dst_path, 'renderscript')) 1117 1118 return True 1119 1120 1121def has_uncommitted_changes(): 1122 try: 1123 # Make sure we don't overwrite any pending changes. 1124 diff_command = f'cd {git_dir} && git diff --quiet' 1125 subprocess.check_call(diff_command, shell=True) 1126 subprocess.check_call(f'{diff_command} --cached', shell=True) 1127 return False 1128 except subprocess.CalledProcessError: 1129 return True 1130 1131 1132def check_platform_sdk_version_format(s): 1133 # Verify that the string is using either of these formats: 1134 # 1135 # - A single non-zero integer 1136 # - A non-zero integer followed by a dot followed by another integer 1137 # 1138 # Examples of allowed strings: 1, 1.0, 2.34 1139 # Examples of disallowed strings: 0.1, 1.01 1140 if not re.match(r'[1-9][0-9]*(\.(0|[1-9][0-9]*))?', s): 1141 raise ValueError('bad platform SDK version format') 1142 return s 1143 1144 1145def main(): 1146 parser = argparse.ArgumentParser( 1147 description='Update current prebuilts') 1148 parser.add_argument( 1149 'source', nargs='?', 1150 help='Build server build ID or local Maven ZIP file') 1151 parser.add_argument( 1152 '-m', '--material', action='store_true', 1153 help='If specified, updates only Material Design Components') 1154 parser.add_argument( 1155 '-c', '--constraint', action='store_true', 1156 help='If specified, updates only Constraint Layout') 1157 parser.add_argument( 1158 '-j', '--jetifier', action='store_true', 1159 help='If specified, updates only Jetifier') 1160 parser.add_argument( 1161 '-p', '--platform', action='store_true', 1162 help='If specified, updates only the Android Platform') 1163 parser.add_argument( 1164 '-f', '--finalize_sdk', type=check_platform_sdk_version_format, 1165 help='Finalize the build as the specified SDK version. The version can be either a single integer ' 1166 'like 36, or a major.minor version like 36.1. Must be used together with -e') 1167 parser.add_argument( 1168 '-e', '--finalize_extension', type=int, 1169 help='Finalize the build as the specified extension SDK version. Must be used together with -f') 1170 parser.add_argument('--bug', type=int, help='The bug number to add to the commit message.') 1171 parser.add_argument( 1172 '--sdk_target', 1173 default=framework_sdk_target, 1174 help='If specified, the name of the build target from which to retrieve the SDK when -p or -f ' 1175 'is specified.') 1176 parser.add_argument( 1177 '-b', '--buildtools', action='store_true', 1178 help='If specified, updates only the Build Tools') 1179 parser.add_argument( 1180 '--include', action='append', default=[], 1181 help='If specified with -x, includes the specified Jetpack library Maven group or artifact for ' 1182 'updates. Applied before exclude.') 1183 parser.add_argument( 1184 '--exclude', action='append', default=[], 1185 help='If specified with -x, excludes the specified Jetpack library Maven group or artifact ' 1186 'from updates') 1187 parser.add_argument( 1188 '-g', '--gmaven', action='store_true', 1189 help='If specified, updates only the artifact from GMaven libraries excluding those covered by ' 1190 'other arguments') 1191 parser.add_argument( 1192 '--commit-first', action='store_true', 1193 help='If specified, then if uncommited changes exist, commit before continuing') 1194 parser.add_argument( 1195 '--beyond-corp', action='store_true', 1196 help='If specified, then fetch artifacts with tooling that works on BeyondCorp devices') 1197 parser.add_argument( 1198 '--local_mode', action="store_true", 1199 help='Local mode: use locally built artifacts and don\'t upload the result to Gerrit.') 1200 rm(temp_dir) 1201 1202 args = parser.parse_args() 1203 1204 # Validate combinations of arguments. 1205 if not args.source and (args.platform or args.buildtools or args.jetifier 1206 or args.material or args.finalize_sdk 1207 or args.constraint): 1208 parser.error('You must specify a build ID or local Maven ZIP file') 1209 sys.exit(1) 1210 if not (args.gmaven or args.platform or args.buildtools or args.jetifier 1211 or args.material or args.finalize_sdk 1212 or args.finalize_extension or args.constraint): 1213 parser.error('You must specify at least one target to update') 1214 sys.exit(1) 1215 if args.local_mode and not args.finalize_sdk: 1216 parser.error('Local mode can only be used when finalizing an SDK.') 1217 sys.exit(1) 1218 if (args.finalize_sdk is None) != (args.finalize_extension is None): 1219 parser.error('Either both or neither of -e and -f must be specified.') 1220 sys.exit(1) 1221 if args.finalize_sdk and not args.bug: 1222 parser.error('Specifying a bug ID with --bug is required when finalizing an SDK.') 1223 sys.exit(1) 1224 1225 # Validate the build environment for POM-dependent targets. 1226 if (args.constraint or args.material or args.gmaven) \ 1227 and which('pom2bp') is None: 1228 parser.error('Cannot find pom2bp in path; please run lunch to set up build environment. ' 1229 'You may also need to run \'m pom2bp\' if it hasn\'t been built already.') 1230 sys.exit(1) 1231 1232 # Validate include/exclude arguments. 1233 if args.exclude: 1234 invalid_spec = find_invalid_spec(args.exclude) 1235 if invalid_spec: 1236 parser.error('Unknown artifact specification in exclude: ' + invalid_spec) 1237 sys.exit(1) 1238 if args.include: 1239 invalid_spec = find_invalid_spec(args.include) 1240 if invalid_spec: 1241 parser.error('Unknown artifact specification in include: ' + invalid_spec) 1242 sys.exit(1) 1243 1244 # Validate the git status. 1245 if not args.local_mode and has_uncommitted_changes(): 1246 if args.commit_first: 1247 subprocess.check_call(f'cd {git_dir} && git add -u', shell=True) 1248 subprocess.check_call(f'cd {git_dir} && git commit -m \'save working state\'', 1249 shell=True) 1250 if not args.local_mode and has_uncommitted_changes(): 1251 self_file = os.path.basename(__file__) 1252 print_e(f'FAIL: There are uncommitted changes here. Please commit or stash before ' 1253 f'continuing, because {self_file} will run "git reset --hard" if execution fails') 1254 sys.exit(1) 1255 1256 if args.bug: 1257 commit_msg_suffix = f'\n\nBug: {args.bug}' 1258 else: 1259 commit_msg_suffix = '' 1260 1261 # Are we fetching a build ID or using a local file? 1262 build_id = None 1263 file = None 1264 if args.source: 1265 build_id = parse_build_id(args.source) 1266 if build_id is None: 1267 file = args.source 1268 1269 try: 1270 components = [] 1271 if args.constraint: 1272 if update_constraint(file): 1273 components.append('Constraint Layout') 1274 else: 1275 print_e('Failed to update Constraint Layout, aborting...') 1276 sys.exit(1) 1277 if args.material: 1278 if update_material(file): 1279 components.append('Material Design Components') 1280 else: 1281 print_e('Failed to update Material Design Components, aborting...') 1282 sys.exit(1) 1283 if args.gmaven: 1284 updated_artifacts = update_gmaven(gmaven_artifacts) 1285 if updated_artifacts: 1286 components.append('\n'.join(updated_artifacts)) 1287 else: 1288 print_e('Failed to update GMaven, aborting...') 1289 sys.exit(1) 1290 if args.jetifier: 1291 if update_jetifier('androidx', build_id, args.beyond_corp): 1292 components.append('Jetifier') 1293 else: 1294 print_e('Failed to update Jetifier, aborting...') 1295 sys.exit(1) 1296 if args.platform or args.finalize_sdk: 1297 if update_framework_current(args.sdk_target, build_id, args.beyond_corp, args.local_mode): 1298 components.append('platform SDK') 1299 else: 1300 print_e('Failed to update platform SDK, aborting...') 1301 sys.exit(1) 1302 if args.finalize_sdk: 1303 n = args.finalize_sdk 1304 if not finalize_sdk(args.sdk_target, build_id, n, args.beyond_corp, args.local_mode): 1305 print_e('Failed to finalize SDK %d, aborting...' % n) 1306 sys.exit(1) 1307 1308 if not args.local_mode: 1309 # HACK: extension sdk finalization will create a new branch, hiding this commit. 1310 # Let's create it in advance for now. 1311 # TODO(b/228451704) do a proper fix? 1312 branch_name = 'finalize-%d' % args.finalize_extension 1313 subprocess.check_output(['repo', 'start', branch_name]) 1314 # We commit the finalized dir separately from the current sdk update. 1315 msg = f'Import final sdk version {n} from build {build_id.url_id}{commit_msg_suffix}' 1316 subprocess.check_call(['git', 'add', '%s' % n]) 1317 subprocess.check_call(['git', 'add', 'Android.bp']) 1318 subprocess.check_call(['git', 'commit', '-m', msg]) 1319 1320 # Finalize extension sdk level 1321 readme = (f'Finalized together with ' 1322 f'Android {args.finalize_sdk} (all modules)') 1323 cmd = extension_sdk_finalization_cmd.format( 1324 readme=readme, 1325 bug=args.bug, 1326 extension_version=args.finalize_extension, 1327 build_id=build_id.url_id, 1328 local_mode='--local_mode' if args.local_mode else '') 1329 subprocess.check_call(shlex.split(cmd), cwd=repo_root_dir.resolve()) 1330 if args.buildtools: 1331 if update_buildtools('sdk-sdk_mac', 'darwin', build_id, args.beyond_corp) \ 1332 and update_buildtools('sdk', 'linux', build_id, args.beyond_corp) \ 1333 and update_buildtools('sdk', 'windows', build_id, args.beyond_corp): 1334 components.append('build tools') 1335 else: 1336 print_e('Failed to update build tools, aborting...') 1337 sys.exit(1) 1338 1339 if args.local_mode: 1340 print('Updated prebuilts using locally built artifacts. Don\'t submit or use for anything besides local testing.') 1341 sys.exit(0) 1342 1343 # Build the git commit. 1344 subprocess.check_call(['git', 'add', current_path, buildtools_dir]) 1345 1346 # Build the commit message. 1347 components_msg = ', '.join(components) 1348 argv_msg = ' '.join(sys.argv) 1349 if not args.source and args.gmaven: 1350 src_msg = 'GMaven' 1351 elif not args.source.isnumeric(): 1352 src_msg = 'local Maven ZIP' 1353 else: 1354 src_msg = f'build {build_id.url_id}' 1355 msg = f'Import {components_msg} from {src_msg}\n\n{argv_msg}{commit_msg_suffix}' 1356 1357 # Create the git commit. 1358 subprocess.check_call(['git', 'commit', '-q', '-m', msg]) 1359 1360 if args.finalize_sdk: 1361 print('NOTE: Created three commits:') 1362 subprocess.check_call(['git', 'log', '-3', '--oneline']) 1363 else: 1364 print('Created commit:') 1365 subprocess.check_call(['git', 'log', '-1', '--oneline']) 1366 print('Remember to test this change before uploading it to Gerrit!') 1367 1368 except Exception as e: 1369 print(f'ERROR: {e} occured while updating prebuilts') 1370 raise 1371 finally: 1372 if args.local_mode: 1373 print('No cleaning up in local mode, manual cleanup required.') 1374 else: 1375 # Revert all stray files, including the downloaded zip. 1376 try: 1377 with open(os.devnull, 'w') as bitbucket: 1378 subprocess.check_call(['git', 'add', '-Af', '.'], stdout=bitbucket) 1379 subprocess.check_call( 1380 ['git', 'commit', '-m', 'COMMIT TO REVERT - RESET ME!!!', '--allow-empty'], 1381 stdout=bitbucket) 1382 subprocess.check_call(['git', 'reset', '--hard', 'HEAD~1'], stdout=bitbucket) 1383 except subprocess.CalledProcessError: 1384 print_e('ERROR: Failed cleaning up, manual cleanup required!!!') 1385 1386 1387# Add automatic entries to maven_to_make. 1388populate_maven_to_make(maven_to_make) 1389 1390if __name__ == '__main__': 1391 main() 1392 1393 1394class Foo(unittest.TestCase): 1395 def test_check_platform_sdk_version_format(self): 1396 # valid input (single int) 1397 check_platform_sdk_version_format("1") 1398 check_platform_sdk_version_format("10") 1399 1400 # valid input (major.minor version) 1401 check_platform_sdk_version_format("1.0") 1402 check_platform_sdk_version_format("1.01") 1403 check_platform_sdk_version_format("2.34") 1404 1405 # invalid input 1406 with self.assertRaises(ValueError): 1407 check_platform_sdk_version_format("") 1408 with self.assertRaises(ValueError): 1409 check_platform_sdk_version_format("0") 1410 with self.assertRaises(ValueError): 1411 check_platform_sdk_version_format("foo") 1412