1#!/usr/bin/env python3 2# 3# Copyright 2018 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Collect the source paths from dependency information.""" 18 19from __future__ import absolute_import 20 21import glob 22import logging 23import os 24import re 25 26from aidegen import constant 27from aidegen.lib import errors 28from aidegen.lib import common_util 29from aidegen.lib.common_util import COLORED_INFO 30from atest import atest_utils 31from atest import constants 32 33# Parse package name from the package declaration line of a java. 34# Group matches "foo.bar" of line "package foo.bar;" or "package foo.bar" 35_PACKAGE_RE = re.compile(r'\s*package\s+(?P<package>[^(;|\s)]+)\s*', re.I) 36 37_ANDROID_SUPPORT_PATH_KEYWORD = 'prebuilts/sdk/current/' 38_JAR = '.jar' 39_TARGET_LIBS = [_JAR] 40_JARJAR_RULES_FILE = 'jarjar-rules.txt' 41_JAVA = '.java' 42_KOTLIN = '.kt' 43_TARGET_FILES = [_JAVA, _KOTLIN] 44_KEY_INSTALLED = 'installed' 45_KEY_JARJAR_RULES = 'jarjar_rules' 46_KEY_JARS = 'jars' 47_KEY_PATH = 'path' 48_KEY_SRCS = 'srcs' 49_KEY_TESTS = 'tests' 50_SRCJAR = '.srcjar' 51_AAPT2_DIR = 'out/target/common/obj/APPS/%s_intermediates/aapt2' 52_AAPT2_SRCJAR = 'out/target/common/obj/APPS/%s_intermediates/aapt2.srcjar' 53_IGNORE_DIRS = [ 54 # The java files under this directory have to be ignored because it will 55 # cause duplicated classes by libcore/ojluni/src/main/java. 56 'libcore/ojluni/src/lambda/java' 57] 58_DIS_ROBO_BUILD_ENV_VAR = {'DISABLE_ROBO_RUN_TESTS': 'true'} 59_SKIP_BUILD_WARN = ( 60 'You choose "--skip-build". Skip building jar and module might increase ' 61 'the risk of the absence of some jar or R/AIDL/logtags java files and ' 62 'cause the red lines to appear in IDE tool.') 63 64 65def multi_projects_locate_source(projects, verbose, depth, ide_name, 66 skip_build=True): 67 """Locate the paths of dependent source folders and jar files with projects. 68 69 Args: 70 projects: A list of ProjectInfo instances. Information of a project such 71 as project relative path, project real path, project 72 dependencies. 73 verbose: A boolean, if true displays full build output. 74 depth: An integer shows the depth of module dependency referenced by 75 source. Zero means the max module depth. 76 ide_name: A string stands for the IDE name, default is IntelliJ. 77 skip_build: A boolean default to true, if true skip building jar and 78 srcjar files, otherwise build them. 79 """ 80 if skip_build: 81 print('\n{} {}\n'.format(COLORED_INFO('Warning:'), _SKIP_BUILD_WARN)) 82 for project in projects: 83 locate_source(project, verbose, depth, ide_name, build=not skip_build) 84 85 86def locate_source(project, verbose, depth, ide_name, build=True): 87 """Locate the paths of dependent source folders and jar files. 88 89 Try to reference source folder path as dependent module unless the 90 dependent module should be referenced to a jar file, such as modules have 91 jars and jarjar_rules parameter. 92 For example: 93 Module: asm-6.0 94 java_import { 95 name: 'asm-6.0', 96 host_supported: true, 97 jars: ['asm-6.0.jar'], 98 } 99 Module: bouncycastle 100 java_library { 101 name: 'bouncycastle', 102 ... 103 target: { 104 android: { 105 jarjar_rules: 'jarjar-rules.txt', 106 }, 107 }, 108 } 109 110 Args: 111 project: A ProjectInfo instance. Information of a project such as 112 project relative path, project real path, project dependencies. 113 verbose: A boolean, if true displays full build output. 114 depth: An integer shows the depth of module dependency referenced by 115 source. Zero means the max module depth. 116 ide_name: A string stands for the IDE name, default is IntelliJ. 117 build: A boolean default to true, if true skip building jar and srcjar 118 files, otherwise build them. 119 120 Example usage: 121 project.source_path = locate_source(project, verbose, False) 122 E.g. 123 project.source_path = { 124 'source_folder_path': ['path/to/source/folder1', 125 'path/to/source/folder2', ...], 126 'test_folder_path': ['path/to/test/folder', ...], 127 'jar_path': ['path/to/jar/file1', 'path/to/jar/file2', ...] 128 } 129 """ 130 if not hasattr(project, 'dep_modules') or not project.dep_modules: 131 raise errors.EmptyModuleDependencyError( 132 'Dependent modules dictionary is empty.') 133 dependencies = project.source_path 134 rebuild_targets = set() 135 for module_name, module_data in project.dep_modules.items(): 136 module = _generate_moduledata(module_name, module_data, ide_name, 137 project.project_relative_path, depth) 138 module.locate_sources_path() 139 dependencies['source_folder_path'].update(module.src_dirs) 140 dependencies['test_folder_path'].update(module.test_dirs) 141 _append_jars_as_dependencies(dependencies, module) 142 if module.build_targets: 143 rebuild_targets |= module.build_targets 144 if rebuild_targets: 145 if build: 146 _build_dependencies(verbose, rebuild_targets) 147 locate_source(project, verbose, depth, ide_name, build=False) 148 else: 149 logging.warning('Jar files or modules build failed:\n\t%s.', 150 '\n\t'.join(rebuild_targets)) 151 152 153def _build_dependencies(verbose, rebuild_targets): 154 """Build the jar or srcjar files of the modules if it don't exist. 155 156 Args: 157 verbose: A boolean, if true displays full build output. 158 rebuild_targets: A list of jar or srcjar files which do not exist. 159 """ 160 logging.info(('Ready to build the jar or srcjar files.')) 161 targets = ['-k'] 162 targets.extend(list(rebuild_targets)) 163 if not atest_utils.build(targets, verbose, _DIS_ROBO_BUILD_ENV_VAR): 164 message = ('Build failed!\n{}\nAIDEGen will proceed but dependency ' 165 'correctness is not guaranteed if not all targets being ' 166 'built successfully.'.format('\n'.join(targets))) 167 print('\n{} {}\n'.format(COLORED_INFO('Warning:'), message)) 168 169 170def _generate_moduledata(module_name, module_data, ide_name, project_relpath, 171 depth): 172 """Generate a module class to collect dependencies in IntelliJ or Eclipse. 173 174 Args: 175 module_name: Name of the module. 176 module_data: A dictionary holding a module information. 177 ide_name: A string stands for the IDE name. 178 project_relpath: A string stands for the project's relative path. 179 depth: An integer shows the depth of module dependency referenced by 180 source. Zero means the max module depth. 181 182 Returns: 183 A ModuleData class. 184 """ 185 if ide_name == constant.IDE_ECLIPSE: 186 return EclipseModuleData(module_name, module_data, project_relpath) 187 return ModuleData(module_name, module_data, depth) 188 189 190def _append_jars_as_dependencies(dependent_data, module): 191 """Add given module's jar files into dependent_data as dependencies. 192 193 Args: 194 dependent_data: A dictionary contains the dependent source paths and 195 jar files. 196 module: A ModuleData instance. 197 """ 198 if module.jar_files: 199 dependent_data['jar_path'].update(module.jar_files) 200 for jar in list(module.jar_files): 201 dependent_data['jar_module_path'].update({jar: module.module_path}) 202 # Collecting the jar files of default core modules as dependencies. 203 if constant.KEY_DEP in module.module_data: 204 dependent_data['jar_path'].update([ 205 x for x in module.module_data[constant.KEY_DEP] 206 if common_util.is_target(x, _TARGET_LIBS) 207 ]) 208 209 210class ModuleData(): 211 """ModuleData class. 212 213 Attributes: 214 All following relative paths stand for the path relative to the android 215 repo root. 216 217 module_path: A string of the relative path to the module. 218 src_dirs: A set to keep the unique source folder relative paths. 219 test_dirs: A set to keep the unique test folder relative paths. 220 jar_files: A set to keep the unique jar file relative paths. 221 referenced_by_jar: A boolean to check if the module is referenced by a 222 jar file. 223 build_targets: A set to keep the unique build target jar or srcjar file 224 relative paths which are ready to be rebuld. 225 missing_jars: A set to keep the jar file relative paths if it doesn't 226 exist. 227 specific_soong_path: A string of the relative path to the module's 228 intermediates folder under out/. 229 """ 230 231 def __init__(self, module_name, module_data, depth): 232 """Initialize ModuleData. 233 234 Args: 235 module_name: Name of the module. 236 module_data: A dictionary holding a module information. 237 depth: An integer shows the depth of module dependency referenced by 238 source. Zero means the max module depth. 239 For example: 240 { 241 'class': ['APPS'], 242 'path': ['path/to/the/module'], 243 'depth': 0, 244 'dependencies': ['bouncycastle', 'ims-common'], 245 'srcs': [ 246 'path/to/the/module/src/com/android/test.java', 247 'path/to/the/module/src/com/google/test.java', 248 'out/soong/.intermediates/path/to/the/module/test/src/ 249 com/android/test.srcjar' 250 ], 251 'installed': ['out/target/product/generic_x86_64/ 252 system/framework/framework.jar'], 253 'jars': ['settings.jar'], 254 'jarjar_rules': ['jarjar-rules.txt'] 255 } 256 """ 257 assert module_name, 'Module name can\'t be null.' 258 assert module_data, 'Module data of %s can\'t be null.' % module_name 259 self.module_name = module_name 260 self.module_data = module_data 261 self._init_module_path() 262 self._init_module_depth(depth) 263 self.src_dirs = set() 264 self.test_dirs = set() 265 self.jar_files = set() 266 self.referenced_by_jar = False 267 self.build_targets = set() 268 self.missing_jars = set() 269 self.specific_soong_path = os.path.join( 270 'out/soong/.intermediates', self.module_path, self.module_name) 271 272 def _is_app_module(self): 273 """Check if the current module's class is APPS""" 274 return self._check_key('class') and 'APPS' in self.module_data['class'] 275 276 def _is_target_module(self): 277 """Check if the current module is a target module. 278 279 A target module is the target project or a module under the 280 target project and it's module depth is 0. 281 For example: aidegen Settings framework 282 The target projects are Settings and framework so they are also 283 target modules. And the dependent module SettingsUnitTests's path 284 is packages/apps/Settings/tests/unit so it also a target module. 285 """ 286 return self.module_depth == 0 287 288 def _is_module_in_apps(self): 289 """Check if the current module is under packages/apps.""" 290 _apps_path = os.path.join('packages', 'apps') 291 return self.module_path.startswith(_apps_path) 292 293 def _collect_r_srcs_paths(self): 294 """Collect the source folder of R.java. 295 296 For modules under packages/apps, check if exists an intermediates 297 directory which contains R.java. If it does not exist, build the 298 aapt2.srcjar of the module to generate. Build system will finally copy 299 the R.java from a intermediates directory to the central R directory 300 after building successfully. So set the central R directory 301 out/target/common/R as a default source folder in IntelliJ. 302 """ 303 if (self._is_app_module() and self._is_target_module() and 304 self._is_module_in_apps()): 305 # The directory contains R.java for apps in packages/apps. 306 r_src_dir = _AAPT2_DIR % self.module_name 307 if not os.path.exists(common_util.get_abs_path(r_src_dir)): 308 self.build_targets.add(_AAPT2_SRCJAR % self.module_name) 309 # In case the central R folder been deleted, uses the intermediate 310 # folder as the dependency to R.java. 311 self.src_dirs.add(r_src_dir) 312 # Add the central R as a default source folder. 313 self.src_dirs.add('out/target/common/R') 314 315 def _init_module_path(self): 316 """Inintialize self.module_path.""" 317 self.module_path = (self.module_data[_KEY_PATH][0] 318 if _KEY_PATH in self.module_data 319 and self.module_data[_KEY_PATH] else '') 320 321 def _init_module_depth(self, depth): 322 """Inintialize module depth's settings. 323 324 Set the module's depth from module info when user have -d parameter. 325 Set the -d value from user input, default to 0. 326 327 Args: 328 depth: the depth to be set. 329 """ 330 self.module_depth = (int(self.module_data[constant.KEY_DEPTH]) 331 if depth else 0) 332 self.depth_by_source = depth 333 334 def _is_android_supported_module(self): 335 """Determine if this is an Android supported module.""" 336 return self.module_path.startswith(_ANDROID_SUPPORT_PATH_KEYWORD) 337 338 def _check_jarjar_rules_exist(self): 339 """Check if jarjar rules exist.""" 340 return (_KEY_JARJAR_RULES in self.module_data and 341 self.module_data[_KEY_JARJAR_RULES][0] == _JARJAR_RULES_FILE) 342 343 def _check_jars_exist(self): 344 """Check if jars exist.""" 345 return _KEY_JARS in self.module_data and self.module_data[_KEY_JARS] 346 347 def _collect_srcs_paths(self): 348 """Collect source folder paths in src_dirs from module_data['srcs'].""" 349 if self._check_key(_KEY_SRCS): 350 scanned_dirs = set() 351 for src_item in self.module_data[_KEY_SRCS]: 352 src_dir = None 353 src_item = os.path.relpath(src_item) 354 if src_item.endswith(_SRCJAR): 355 self._append_jar_from_installed(self.specific_soong_path) 356 elif common_util.is_target(src_item, _TARGET_FILES): 357 # Only scan one java file in each source directories. 358 src_item_dir = os.path.dirname(src_item) 359 if src_item_dir not in scanned_dirs: 360 scanned_dirs.add(src_item_dir) 361 src_dir = self._get_source_folder(src_item) 362 else: 363 # To record what files except java and srcjar in the srcs. 364 logging.debug('%s is not in parsing scope.', src_item) 365 if src_dir: 366 self._add_to_source_or_test_dirs(src_dir) 367 368 def _check_key(self, key): 369 """Check if key is in self.module_data and not empty. 370 371 Args: 372 key: the key to be checked. 373 """ 374 return key in self.module_data and self.module_data[key] 375 376 def _add_to_source_or_test_dirs(self, src_dir): 377 """Add folder to source or test directories. 378 379 Args: 380 src_dir: the directory to be added. 381 """ 382 if not any(path in src_dir for path in _IGNORE_DIRS): 383 # Build the module if the source path not exists. The java is 384 # normally generated for AIDL or logtags file. 385 if not os.path.exists(common_util.get_abs_path(src_dir)): 386 self.build_targets.add(self.module_name) 387 if self._is_test_module(src_dir): 388 self.test_dirs.add(src_dir) 389 else: 390 self.src_dirs.add(src_dir) 391 392 @staticmethod 393 def _is_test_module(src_dir): 394 """Check if the module path is a test module path. 395 396 Args: 397 src_dir: the directory to be checked. 398 399 Returns: 400 True if module path is a test module path, otherwise False. 401 """ 402 return _KEY_TESTS in src_dir.split(os.sep) 403 404 # pylint: disable=inconsistent-return-statements 405 @staticmethod 406 def _get_source_folder(java_file): 407 """Parsing a java to get the package name to filter out source path. 408 409 There are 3 steps to get the source path from a java. 410 1. Parsing a java to get package name. 411 For example: 412 The java_file is:path/to/the/module/src/main/java/com/android/ 413 first.java 414 The package name of java_file is com.android. 415 2. Transfer package name to package path: 416 For example: 417 The package path of com.android is com/android. 418 3. Remove the package path and file name from the java path. 419 For example: 420 The path after removing package path and file name is 421 path/to/the/module/src/main/java. 422 As a result, path/to/the/module/src/main/java is the source path parsed 423 from path/to/the/module/src/main/java/com/android/first.java. 424 425 Returns: 426 source_folder: A string of path to source folder(e.g. src/main/java) 427 or none when it failed to get package name. 428 """ 429 abs_java_path = common_util.get_abs_path(java_file) 430 if os.path.exists(abs_java_path): 431 with open(abs_java_path) as data: 432 for line in data.read().splitlines(): 433 match = _PACKAGE_RE.match(line) 434 if match: 435 package_name = match.group('package') 436 package_path = package_name.replace(os.extsep, os.sep) 437 source_folder, _, _ = java_file.rpartition(package_path) 438 return source_folder.strip(os.sep) 439 440 def _append_jar_file(self, jar_path): 441 """Append a path to the jar file into self.jar_files if it's exists. 442 443 Args: 444 jar_path: A path supposed to be a jar file. 445 446 Returns: 447 Boolean: True if jar_path is an existing jar file. 448 """ 449 if common_util.is_target(jar_path, _TARGET_LIBS): 450 self.referenced_by_jar = True 451 if os.path.isfile(common_util.get_abs_path(jar_path)): 452 self.jar_files.add(jar_path) 453 else: 454 self.missing_jars.add(jar_path) 455 return True 456 457 def _append_jar_from_installed(self, specific_dir=None): 458 """Append a jar file's path to the list of jar_files with matching 459 path_prefix. 460 461 There might be more than one jar in "installed" parameter and only the 462 first jar file is returned. If specific_dir is set, the jar file must be 463 under the specific directory or its sub-directory. 464 465 Args: 466 specific_dir: A string of path. 467 """ 468 if (_KEY_INSTALLED in self.module_data 469 and self.module_data[_KEY_INSTALLED]): 470 for jar in self.module_data[_KEY_INSTALLED]: 471 if specific_dir and not jar.startswith(specific_dir): 472 continue 473 if self._append_jar_file(jar): 474 break 475 476 def _set_jars_jarfile(self): 477 """Append prebuilt jars of module into self.jar_files. 478 479 Some modules' sources are prebuilt jar files instead of source java 480 files. The jar files can be imported into IntelliJ as a dependency 481 directly. There is only jar file name in self.module_data['jars'], it 482 has to be combined with self.module_data['path'] to append into 483 self.jar_files. 484 For example: 485 'asm-6.0': { 486 'jars': [ 487 'asm-6.0.jar' 488 ], 489 'path': [ 490 'prebuilts/misc/common/asm' 491 ], 492 }, 493 Path to the jar file is prebuilts/misc/common/asm/asm-6.0.jar. 494 """ 495 if _KEY_JARS in self.module_data and self.module_data[_KEY_JARS]: 496 for jar_name in self.module_data[_KEY_JARS]: 497 if self._check_key(_KEY_INSTALLED): 498 self._append_jar_from_installed() 499 else: 500 jar_path = os.path.join(self.module_path, jar_name) 501 jar_abs = common_util.get_abs_path(jar_path) 502 if not os.path.isfile( 503 jar_abs) and jar_name.endswith('prebuilt.jar'): 504 rel_path = self._get_jar_path_from_prebuilts(jar_name) 505 if rel_path: 506 jar_path = rel_path 507 self._append_jar_file(jar_path) 508 509 @staticmethod 510 def _get_jar_path_from_prebuilts(jar_name): 511 """Get prebuilt jar file from prebuilts folder. 512 513 If the prebuilt jar file we get from method _set_jars_jarfile() does not 514 exist, we should search the prebuilt jar file in prebuilts folder. 515 For example: 516 'platformprotos': { 517 'jars': [ 518 'platformprotos-prebuilt.jar' 519 ], 520 'path': [ 521 'frameworks/base' 522 ], 523 }, 524 We get an incorrect path: 'frameworks/base/platformprotos-prebuilt.jar' 525 If the file does not exist, we should search the file name from 526 prebuilts folder. If we can get the correct path from 'prebuilts', we 527 can replace it with the incorrect path. 528 529 Args: 530 jar_name: The prebuilt jar file name. 531 532 Return: 533 A relative prebuilt jar file path if found, otherwise None. 534 """ 535 rel_path = '' 536 search = os.sep.join( 537 [constant.ANDROID_ROOT_PATH, 'prebuilts/**', jar_name]) 538 results = glob.glob(search, recursive=True) 539 if results: 540 jar_abs = results[0] 541 rel_path = os.path.relpath( 542 jar_abs, os.environ.get(constants.ANDROID_BUILD_TOP, os.sep)) 543 return rel_path 544 545 def locate_sources_path(self): 546 """Locate source folders' paths or jar files.""" 547 if self.module_depth > self.depth_by_source: 548 self._append_jar_from_installed(self.specific_soong_path) 549 else: 550 if self._is_android_supported_module(): 551 self._append_jar_from_installed() 552 elif self._check_jarjar_rules_exist(): 553 self._append_jar_from_installed(self.specific_soong_path) 554 elif self._check_jars_exist(): 555 self._set_jars_jarfile() 556 self._collect_srcs_paths() 557 # If there is no source/tests folder of the module, reference the 558 # module by jar. 559 if not self.src_dirs and not self.test_dirs: 560 self._append_jar_from_installed() 561 self._collect_r_srcs_paths() 562 if self.referenced_by_jar and self.missing_jars: 563 self.build_targets |= self.missing_jars 564 565 566class EclipseModuleData(ModuleData): 567 """Deal with modules data for Eclipse 568 569 Only project target modules use source folder type and the other ones use 570 jar as their source. We'll combine both to establish the whole project's 571 dependencies. If the source folder used to build dependency jar file exists 572 in Android, we should provide the jar file path as <linkedResource> item in 573 source data. 574 """ 575 576 def __init__(self, module_name, module_data, project_relpath): 577 """Initialize EclipseModuleData. 578 579 Only project target modules apply source folder type, so set the depth 580 of module referenced by source to 0. 581 582 Args: 583 module_name: String type, name of the module. 584 module_data: A dictionary contains a module information. 585 project_relpath: A string stands for the project's relative path. 586 """ 587 super().__init__(module_name, module_data, depth=0) 588 self.is_project = common_util.is_project_path_relative_module( 589 module_data, project_relpath) 590 591 def locate_sources_path(self): 592 """Locate source folders' paths or jar files. 593 594 Only collect source folders for the project modules and collect jar 595 files for the other dependent modules. 596 """ 597 if self.is_project: 598 self._locate_project_source_path() 599 else: 600 self._locate_jar_path() 601 if self.referenced_by_jar and self.missing_jars: 602 self.build_targets |= self.missing_jars 603 604 def _locate_project_source_path(self): 605 """Locate the source folder paths of the project module. 606 607 A project module is the target modules or paths that users key in 608 aidegen command. Collecting the source folders is necessary for 609 developers to edit code. And also collect the central R folder for the 610 dependency of resources. 611 """ 612 self._collect_srcs_paths() 613 self._collect_r_srcs_paths() 614 615 def _locate_jar_path(self): 616 """Locate the jar path of the module. 617 618 Use jar files for dependency modules for Eclipse. Collect the jar file 619 path with different cases. 620 """ 621 if self._check_jarjar_rules_exist(): 622 self._append_jar_from_installed(self.specific_soong_path) 623 elif self._check_jars_exist(): 624 self._set_jars_jarfile() 625 else: 626 self._append_jar_from_installed() 627