1#!/usr/bin/env python3 2# Copyright (C) 2020 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16# Exports AppSearch Androidx code to Framework 17# 18# NOTE: This will remove and replace all files in the 19# packages/modules/AppSearch path. 20# 21# Example usage (from root dir of androidx workspace): 22# $ ./frameworks/support/appsearch/exportToFramework.py "$HOME/android/master" "<jetpack git sha>" 23 24# Special directives supported by this script: 25# 26# Causes the file where it appears to not be copied at all: 27# @exportToFramework:skipFile() 28# 29# Causes the text appearing between startStrip() and endStrip() to be removed during export: 30# // @exportToFramework:startStrip() ... // @exportToFramework:endStrip() 31# 32# Replaced with @hide: 33# <!--@exportToFramework:hide--> 34# 35# Removes the text appearing between ifJetpack() and else(), and causes the text appearing between 36# else() and --> to become uncommented, to support framework-only Javadocs: 37# <!--@exportToFramework:ifJetpack()--> 38# Jetpack-only Javadoc 39# <!--@exportToFramework:else() 40# Framework-only Javadoc 41# --> 42# Note: Using the above pattern, you can hide a method in Jetpack but unhide it in Framework like 43# this: 44# <!--@exportToFramework:ifJetpack()-->@hide<!--@exportToFramework:else()--> 45 46import os 47import re 48import subprocess 49import sys 50 51# Jetpack paths relative to frameworks/support/appsearch 52JETPACK_API_ROOT = 'appsearch/src/main/java/androidx/appsearch' 53JETPACK_API_TEST_ROOT = 'appsearch/src/androidTest/java/androidx/appsearch' 54JETPACK_IMPL_ROOT = 'appsearch-local-storage/src/main/java/androidx/appsearch' 55JETPACK_IMPL_TEST_ROOT = 'appsearch-local-storage/src/androidTest/java/androidx/appsearch' 56JETPACK_TEST_UTIL_ROOT = 'appsearch-test-util/src/main/java/androidx/appsearch' 57JETPACK_TEST_UTIL_TEST_ROOT = 'appsearch-test-util/src/androidTest/java/androidx/appsearch' 58 59# Framework paths relative to packages/modules/AppSearch 60FRAMEWORK_API_ROOT = 'framework/java/external/android/app/appsearch' 61FRAMEWORK_API_TEST_ROOT = 'testing/coretests/src/android/app/appsearch/external' 62FRAMEWORK_IMPL_ROOT = 'service/java/com/android/server/appsearch/external' 63FRAMEWORK_IMPL_TEST_ROOT = 'testing/servicestests/src/com/android/server/appsearch/external' 64FRAMEWORK_TEST_UTIL_ROOT = ( 65 '../../../cts/tests/appsearch/testutils/src/android/app/appsearch/testutil/external') 66FRAMEWORK_TEST_UTIL_TEST_ROOT = 'testing/servicestests/src/android/app/appsearch/testutil/external' 67FRAMEWORK_CTS_TEST_ROOT = '../../../cts/tests/appsearch/src/com/android/cts/appsearch/external' 68GOOGLE_JAVA_FORMAT = ( 69 '../../../prebuilts/tools/common/google-java-format/google-java-format') 70 71# Miscellaneous constants 72SHA_FILE_NAME = 'synced_jetpack_sha.txt' 73 74class ExportToFramework: 75 def __init__(self, jetpack_appsearch_root, framework_appsearch_root): 76 self._jetpack_appsearch_root = jetpack_appsearch_root 77 self._framework_appsearch_root = framework_appsearch_root 78 self._written_files = [] 79 80 def _PruneDir(self, dir_to_prune): 81 for walk_path, walk_folders, walk_files in os.walk(dir_to_prune): 82 for walk_filename in walk_files: 83 abs_path = os.path.join(walk_path, walk_filename) 84 print('Prune: remove "%s"' % abs_path) 85 os.remove(abs_path) 86 87 def _TransformExportedToCts(self, contents): 88 """ 89 Blanket transforms for files that are being exported to the CTS test repo (platform/cts). 90 File specific transforms will still be applied in _TransformAndCopyFileToPath 91 """ 92 contents = (contents 93 .replace('com.google.android.icing.proto.', 94 'com.android.server.appsearch.icing.proto.') 95 ) 96 return contents 97 98 def _TransformAndCopyFile( 99 self, source_path, default_dest_path, transform_func=None, ignore_skips=False): 100 """ 101 Transforms the file located at 'source_path' and writes it into 'default_dest_path'. 102 103 An @exportToFramework:skip() directive will skip the copy process. 104 An @exportToFramework:copyToPath() directive will override default_dest_path with another 105 path relative to framework_appsearch_root (which is usually packages/modules/AppSearch) 106 """ 107 with open(source_path, 'r') as fh: 108 contents = fh.read() 109 110 if not ignore_skips and '@exportToFramework:skipFile()' in contents: 111 print('Skipping: "%s" -> "%s"' % (source_path, default_dest_path), file=sys.stderr) 112 return 113 114 copy_to_path = re.search(r'@exportToFramework:copyToPath\(([^)]+)\)', contents) 115 if copy_to_path: 116 dest_path = os.path.join(self._framework_appsearch_root, copy_to_path.group(1)) 117 # Check if the file is being exported to the CTS test repo. 118 if "cts/tests/appsearch/" in dest_path: 119 contents = self._TransformExportedToCts(contents) 120 else: 121 dest_path = default_dest_path 122 123 self._TransformAndCopyFileToPath(source_path, dest_path, contents, transform_func) 124 125 def _TransformAndCopyFileToPath(self, source_path, dest_path, contents, transform_func=None): 126 """Transforms the file located at 'source_path' and writes it into 'dest_path'.""" 127 print('Copy: "%s" -> "%s"' % (source_path, dest_path), file=sys.stderr) 128 if transform_func: 129 contents = transform_func(contents) 130 os.makedirs(os.path.dirname(dest_path), exist_ok=True) 131 with open(dest_path, 'w') as fh: 132 fh.write(contents) 133 134 # Save file for future formatting 135 self._written_files.append(dest_path) 136 137 def _TransformCommonCode(self, contents): 138 # Apply stripping 139 contents = re.sub( 140 r'\/\/ @exportToFramework:startStrip\(\).*?\/\/ @exportToFramework:endStrip\(\)', 141 '', 142 contents, 143 flags=re.DOTALL) 144 145 # Apply if/elses in javadocs 146 contents = re.sub( 147 r'<!--@exportToFramework:ifJetpack\(\)-->.*?<!--@exportToFramework:else\(\)(.*?)-->', 148 r'\1', 149 contents, 150 flags=re.DOTALL) 151 152 # Add additional imports if required 153 imports_to_add = [] 154 for import_to_add in imports_to_add: 155 contents = re.sub( 156 r'^(\s*package [^;]+;\s*)$', r'\1\nimport %s;\n' % import_to_add, contents, 157 flags=re.MULTILINE) 158 159 # Remove all imports for stub CREATOR classes imported for SafeParcelable 160 # If there are more use cases in the future we might want to add 161 # imports_to_delete 162 contents = re.sub( 163 r'import androidx\.appsearch\.safeparcel\.stub.*?\;', 164 '', contents, flags=re.MULTILINE) 165 166 # Apply in-place replacements 167 contents = (contents 168 .replace('androidx.appsearch.app', 'android.app.appsearch') 169 .replace( 170 'androidx.appsearch.localstorage.', 171 'com.android.server.appsearch.external.localstorage.') 172 .replace('androidx.appsearch.flags.FlaggedApi', 'android.annotation.FlaggedApi') 173 .replace('androidx.appsearch.flags.Flags', 'com.android.appsearch.flags.Flags') 174 .replace( 175 'androidx.appsearch.annotation.CurrentTimeMillis', 176 'android.annotation.CurrentTimeMillis') 177 .replace( 178 'androidx.appsearch.annotation.SystemApi', 179 'android.annotation.SystemApi') 180 .replace('androidx.appsearch', 'android.app.appsearch') 181 .replace( 182 'androidx.annotation.GuardedBy', 183 'com.android.internal.annotations.GuardedBy') 184 .replace( 185 'androidx.annotation.VisibleForTesting', 186 'com.android.internal.annotations.VisibleForTesting') 187 .replace('androidx.annotation.', 'android.annotation.') 188 .replace('androidx.collection.ArrayMap', 'android.util.ArrayMap') 189 .replace('androidx.collection.ArraySet', 'android.util.ArraySet') 190 .replace( 191 'androidx.core.util.ObjectsCompat', 192 'java.util.Objects') 193 # Preconditions.checkNotNull is replaced with Objects.requireNonNull. We add both 194 # imports and let google-java-format sort out which one is unused. 195 .replace( 196 'import androidx.core.util.Preconditions;', 197 'import java.util.Objects; import com.android.internal.util.Preconditions;') 198 .replace('import androidx.annotation.RestrictTo;', '') 199 .replace('@RestrictTo(RestrictTo.Scope.LIBRARY)', '') 200 .replace('@RestrictTo(RestrictTo.Scope.LIBRARY_GROUP)', '') 201 .replace('Preconditions.checkNotNull(', 'Objects.requireNonNull(') 202 .replace('ObjectsCompat.', 'Objects.') 203 .replace('<!--@exportToFramework:hide-->', '@hide') 204 .replace('@exportToFramework:hide', '@hide') 205 .replace('// @exportToFramework:skipFile()', '') 206 .replace('@ExperimentalAppSearchApi', '') 207 .replace('@OptIn(markerClass = ExperimentalAppSearchApi.class)', '') 208 ) 209 contents = re.sub(r'\/\/ @exportToFramework:copyToPath\([^)]+\)', '', contents) 210 contents = re.sub(r'@RequiresFeature\([^)]*\)', '', contents, flags=re.DOTALL) 211 contents = re.sub(r'@RequiresOptIn\([^)]+\)', '', contents) 212 213 # Jetpack methods have the Async suffix, but framework doesn't. Strip the Async suffix 214 # to allow the same documentation to compile for both. 215 contents = re.sub(r'(#[a-zA-Z0-9_]+)Async}', r'\1}', contents) 216 contents = re.sub( 217 r'(\@see [^#]+#[a-zA-Z0-9_]+)Async$', r'\1', contents, flags=re.MULTILINE) 218 return contents 219 220 def _TransformTestCode(self, contents): 221 contents = (contents 222 .replace( 223 'androidx.appsearch.testutil.flags.CheckFlagsRule', 224 'android.platform.test.flag.junit.CheckFlagsRule') 225 .replace( 226 'androidx.appsearch.testutil.flags.DeviceFlagsValueProvider', 227 'android.platform.test.flag.junit.DeviceFlagsValueProvider') 228 .replace( 229 'androidx.appsearch.testutil.flags.RequiresFlagsEnabled', 230 'android.platform.test.annotations.RequiresFlagsEnabled') 231 .replace( 232 'androidx.appsearch.testutil.flags.RequiresFlagsDisabled', 233 'android.platform.test.annotations.RequiresFlagsDisabled') 234 .replace('androidx.appsearch.testutil.', 'android.app.appsearch.testutil.') 235 .replace( 236 'package androidx.appsearch.testutil;', 237 'package android.app.appsearch.testutil;') 238 .replace( 239 'import androidx.appsearch.localstorage.LocalStorage;', 240 'import android.app.appsearch.AppSearchManager;') 241 .replace('LocalStorage.', 'AppSearchManager.') 242 ) 243 for shim in [ 244 'AppSearchSession', 'GlobalSearchSession', 'EnterpriseGlobalSearchSession', 245 'SearchResults']: 246 contents = re.sub(r"([^a-zA-Z])(%s)([^a-zA-Z0-9])" % shim, r'\1\2Shim\3', contents) 247 return self._TransformCommonCode(contents) 248 249 def _TransformAndCopyFolder(self, source_dir, dest_dir, transform_func=None): 250 for currentpath, folders, files in os.walk(source_dir): 251 dir_rel_to_root = os.path.relpath(currentpath, source_dir) 252 for filename in files: 253 source_abs_path = os.path.join(currentpath, filename) 254 dest_path = os.path.join(dest_dir, dir_rel_to_root, filename) 255 self._TransformAndCopyFile(source_abs_path, dest_path, transform_func) 256 257 def _ExportApiCode(self): 258 # Prod source 259 api_source_dir = os.path.join(self._jetpack_appsearch_root, JETPACK_API_ROOT) 260 api_dest_dir = os.path.join(self._framework_appsearch_root, FRAMEWORK_API_ROOT) 261 262 # Unit tests 263 api_test_source_dir = os.path.join(self._jetpack_appsearch_root, JETPACK_API_TEST_ROOT) 264 api_test_dest_dir = os.path.join(self._framework_appsearch_root, FRAMEWORK_API_TEST_ROOT) 265 266 # CTS tests 267 cts_test_source_dir = os.path.join(api_test_source_dir, 'cts') 268 cts_test_dest_dir = os.path.join(self._framework_appsearch_root, FRAMEWORK_CTS_TEST_ROOT) 269 270 # Test utils 271 test_util_source_dir = os.path.join(self._jetpack_appsearch_root, JETPACK_TEST_UTIL_ROOT) 272 test_util_dest_dir = os.path.join(self._framework_appsearch_root, FRAMEWORK_TEST_UTIL_ROOT) 273 274 # Prune existing files 275 self._PruneDir(api_dest_dir) 276 self._PruneDir(api_test_dest_dir) 277 self._PruneDir(cts_test_dest_dir) 278 self._PruneDir(test_util_dest_dir) 279 280 # Copy api classes. We can't use _TransformAndCopyFolder here because we 281 # need to specially handle the 'app' package. 282 print('~~~ Copying API classes ~~~') 283 def _TransformApiCode(contents): 284 contents = contents.replace( 285 'package androidx.appsearch.app;', 286 'package android.app.appsearch;') 287 return self._TransformCommonCode(contents) 288 for currentpath, folders, files in os.walk(api_source_dir): 289 dir_rel_to_root = os.path.relpath(currentpath, api_source_dir) 290 for filename in files: 291 # Figure out what folder to place them into 292 source_abs_path = os.path.join(currentpath, filename) 293 if dir_rel_to_root == 'app': 294 # Files in the 'app' folder live in the root of the platform tree 295 dest_path = os.path.join(api_dest_dir, filename) 296 else: 297 dest_path = os.path.join(api_dest_dir, dir_rel_to_root, filename) 298 self._TransformAndCopyFile(source_abs_path, dest_path, _TransformApiCode) 299 300 # Copy api unit tests. We can't use _TransformAndCopyFolder here because we need to skip the 301 # 'util' and 'cts' subfolders. 302 print('~~~ Copying API unit tests ~~~') 303 for currentpath, folders, files in os.walk(api_test_source_dir): 304 if (currentpath.startswith(cts_test_source_dir) or 305 currentpath.startswith(test_util_source_dir)): 306 continue 307 dir_rel_to_root = os.path.relpath(currentpath, api_test_source_dir) 308 for filename in files: 309 source_abs_path = os.path.join(currentpath, filename) 310 dest_path = os.path.join(api_test_dest_dir, dir_rel_to_root, filename) 311 self._TransformAndCopyFile(source_abs_path, dest_path, self._TransformTestCode) 312 313 # Copy CTS tests 314 print('~~~ Copying CTS tests ~~~') 315 self._TransformAndCopyFolder( 316 cts_test_source_dir, cts_test_dest_dir, transform_func=self._TransformTestCode) 317 318 # Copy test utils 319 print('~~~ Copying test utils ~~~') 320 self._TransformAndCopyFolder( 321 test_util_source_dir, test_util_dest_dir, transform_func=self._TransformTestCode) 322 for iface_file in ( 323 'AppSearchSession.java', 'GlobalSearchSession.java', 324 'EnterpriseGlobalSearchSession.java', 'SearchResults.java'): 325 dest_file_name = os.path.splitext(iface_file)[0] + 'Shim.java' 326 self._TransformAndCopyFile( 327 os.path.join(api_source_dir, 'app/' + iface_file), 328 os.path.join(test_util_dest_dir, dest_file_name), 329 transform_func=self._TransformTestCode, 330 ignore_skips=True) 331 332 def _ExportImplCode(self): 333 impl_source_dir = os.path.join(self._jetpack_appsearch_root, JETPACK_IMPL_ROOT) 334 impl_test_source_dir = os.path.join(self._jetpack_appsearch_root, JETPACK_IMPL_TEST_ROOT) 335 impl_dest_dir = os.path.join(self._framework_appsearch_root, FRAMEWORK_IMPL_ROOT) 336 impl_test_dest_dir = os.path.join(self._framework_appsearch_root, FRAMEWORK_IMPL_TEST_ROOT) 337 test_util_test_source_dir = os.path.join( 338 self._jetpack_appsearch_root, JETPACK_TEST_UTIL_TEST_ROOT) 339 test_util_test_dest_dir = os.path.join( 340 self._framework_appsearch_root, FRAMEWORK_TEST_UTIL_TEST_ROOT) 341 342 # Prune 343 self._PruneDir(impl_dest_dir) 344 self._PruneDir(impl_test_dest_dir) 345 self._PruneDir(test_util_test_dest_dir) 346 347 # Copy impl classes 348 def _TransformImplCode(contents): 349 contents = (contents 350 .replace('package androidx.appsearch', 351 'package com.android.server.appsearch.external') 352 .replace('com.google.android.icing.protobuf.', 'com.google.protobuf.') 353 ) 354 return self._TransformCommonCode(contents) 355 self._TransformAndCopyFolder( 356 impl_source_dir, impl_dest_dir, transform_func=_TransformImplCode) 357 358 # Copy servicestests 359 def _TransformImplTestCode(contents): 360 contents = (contents 361 .replace('package androidx.appsearch', 362 'package com.android.server.appsearch.external') 363 .replace('com.google.android.icing.proto.', 364 'com.android.server.appsearch.icing.proto.') 365 .replace('com.google.android.appsearch.proto.', 366 'com.android.server.appsearch.appsearch.proto.') 367 .replace('com.google.android.icing.protobuf.', 368 'com.android.server.appsearch.protobuf.') 369 ) 370 return self._TransformTestCode(contents) 371 self._TransformAndCopyFolder( 372 impl_test_source_dir, impl_test_dest_dir, transform_func=_TransformImplTestCode) 373 self._TransformAndCopyFolder( 374 test_util_test_source_dir, 375 test_util_test_dest_dir, 376 transform_func=self._TransformTestCode) 377 378 def _FormatWrittenFiles(self): 379 google_java_format_cmd = [GOOGLE_JAVA_FORMAT, '--aosp', '-i'] + self._written_files 380 print('$ ' + ' '.join(google_java_format_cmd)) 381 subprocess.check_call(google_java_format_cmd, cwd=self._framework_appsearch_root) 382 383 def ExportCode(self): 384 self._ExportApiCode() 385 self._ExportImplCode() 386 self._FormatWrittenFiles() 387 388 def WriteShaFile(self, sha): 389 """Copies the git sha of the most recent public CL into a file on the framework side. 390 391 This file is used for tracking, to determine what framework is synced to. 392 393 You must always provide a sha of a submitted submitted git commit. If you abandon the CL 394 pointed to by this sha, the next person syncing framework will be unable to find what CL it 395 is synced to. 396 397 The previous content of the sha file, if any, is returned. 398 """ 399 file_path = os.path.join(self._framework_appsearch_root, SHA_FILE_NAME) 400 old_sha = None 401 if os.path.isfile(file_path): 402 with open(file_path, 'r') as fh: 403 old_sha = fh.read().rstrip() 404 with open(file_path, 'w') as fh: 405 print(sha, file=fh) 406 print('Wrote "%s"' % file_path) 407 return old_sha 408 409 def FormatCommitMessage(self, old_sha, new_sha): 410 print('\nCommand to diff old version to new version:') 411 print(' git log --pretty=format:"* %h %s" {}..{} -- appsearch/'.format(old_sha, new_sha)) 412 pretty_log = subprocess.check_output([ 413 'git', 414 'log', 415 '--pretty=format:* %h %s', 416 '{}..{}'.format(old_sha, new_sha), 417 '--', 418 'appsearch/' 419 ]).decode("utf-8") 420 bug_output = subprocess.check_output([ 421 '/bin/sh', 422 '-c', 423 'git log {}..{} -- appsearch/ | grep Bug: | sort | uniq'.format(old_sha, new_sha) 424 ]).decode("utf-8") 425 426 print('\n--------------------------------------------------') 427 print('Update Framework from Jetpack.\n') 428 print(pretty_log) 429 print() 430 for line in bug_output.splitlines(): 431 print(line.strip()) 432 print('Test: Presubmit\n') 433 print('--------------------------------------------------\n') 434 435 436if __name__ == '__main__': 437 if len(sys.argv) != 3: 438 print('Usage: %s <path/to/framework/checkout> <git sha of head jetpack commit>' % ( 439 sys.argv[0]), 440 file=sys.stderr) 441 sys.exit(1) 442 if sys.argv[2].startswith('I'): 443 print('Error: Git sha "%s" looks like a changeid. Please provide a git sha instead.' % ( 444 sys.argv[2]), 445 file=sys.stderr) 446 sys.exit(1) 447 448 source_dir = os.path.normpath(os.path.dirname(sys.argv[0])) 449 dest_dir = os.path.normpath(sys.argv[1]) 450 dest_dir = os.path.join(dest_dir, 'packages/modules/AppSearch') 451 if not os.path.isdir(dest_dir): 452 print('Destination path "%s" does not exist or is not a directory' % ( 453 dest_dir), 454 file=sys.stderr) 455 sys.exit(1) 456 exporter = ExportToFramework(source_dir, dest_dir) 457 exporter.ExportCode() 458 459 # Update the sha file 460 new_sha = sys.argv[2] 461 old_sha = exporter.WriteShaFile(new_sha) 462 if old_sha and old_sha != new_sha: 463 exporter.FormatCommitMessage(old_sha, new_sha) 464