1#!/usr/bin/env python3 2# 3# Copyright 2018 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Project information.""" 18 19from __future__ import absolute_import 20 21import logging 22import os 23import time 24 25from aidegen import constant 26from aidegen.lib import aidegen_metrics 27from aidegen.lib import common_util 28from aidegen.lib import errors 29from aidegen.lib import module_info 30from aidegen.lib import project_config 31from aidegen.lib import source_locator 32from aidegen.idea import iml 33 34from atest import atest_utils 35 36_CONVERT_MK_URL = ('https://android.googlesource.com/platform/build/soong/' 37 '#convert-android_mk-files') 38_ROBOLECTRIC_MODULE = 'Robolectric_all' 39_NOT_TARGET = ('The module %s does not contain any Java or Kotlin file, ' 40 'therefore we skip this module in the project.') 41# The module fake-framework have the same package name with framework but empty 42# content. It will impact the dependency for framework when referencing the 43# package from fake-framework in IntelliJ. 44_EXCLUDE_MODULES = ['fake-framework'] 45# When we use atest_utils.build(), there is a command length limit on 46# soong_ui.bash. We reserve 5000 characters for rewriting the command line 47# in soong_ui.bash. 48_CMD_LENGTH_BUFFER = 5000 49# For each argument, it needs a space to separate following argument. 50_BLANK_SIZE = 1 51_CORE_MODULES = [constant.FRAMEWORK_ALL, constant.CORE_ALL, 52 'org.apache.http.legacy.stubs.system'] 53_BUILD_BP_JSON_ENV_ON = { 54 constant.GEN_JAVA_DEPS: 'true', 55 constant.GEN_CC_DEPS: 'true', 56 constant.GEN_COMPDB: 'true', 57 constant.GEN_RUST: 'true' 58} 59 60 61class ProjectInfo: 62 """Project information. 63 64 Users should call config_project first before starting using ProjectInfo. 65 66 Class attributes: 67 modules_info: An AidegenModuleInfo instance whose name_to_module_info is 68 combining module-info.json with module_bp_java_deps.json. 69 projects: A list of instances of ProjectInfo that are generated in an 70 AIDEGen command. 71 72 Attributes: 73 project_absolute_path: The absolute path of the project. 74 project_relative_path: The relative path of the project to 75 common_util.get_android_root_dir(). 76 project_module_names: A set of module names under project_absolute_path 77 directory or it's subdirectories. 78 dep_modules: A dict has recursively dependent modules of 79 project_module_names. 80 iml_path: The project's iml file path. 81 source_path: A dictionary to keep following data: 82 source_folder_path: A set contains the source folder 83 relative paths. 84 test_folder_path: A set contains the test folder relative 85 paths. 86 jar_path: A set contains the jar file paths. 87 jar_module_path: A dictionary contains the jar file and 88 the module's path mapping, only used in 89 Eclipse. 90 r_java_path: A set contains the relative path to the 91 R.java files, only used in Eclipse. 92 srcjar_path: A source content descriptor only used in 93 IntelliJ. 94 e.g. out/.../aapt2.srcjar!/ 95 The "!/" is a content descriptor for 96 compressed files in IntelliJ. 97 is_main_project: A boolean to verify the project is main project. 98 dependencies: A list of dependency projects' iml file names, e.g. base, 99 framework-all. 100 iml_name: The iml project file name of this project. 101 rel_out_soong_jar_path: A string of relative project path in the 102 'out/soong/.intermediates' directory, e.g., if 103 self.project_relative_path = 'frameworks/base' 104 the rel_out_soong_jar_path should be 105 'out/soong/.intermediates/frameworks/base/'. 106 """ 107 108 modules_info = None 109 projects = [] 110 111 def __init__(self, target=None, is_main_project=False): 112 """ProjectInfo initialize. 113 114 Args: 115 target: Includes target module or project path from user input, when 116 locating the target, project with matching module name of 117 the given target has a higher priority than project path. 118 is_main_project: A boolean, default is False. True if the target is 119 the main project, otherwise False. 120 """ 121 rel_path, abs_path = common_util.get_related_paths( 122 self.modules_info, target) 123 self.module_name = self.get_target_name(target, abs_path) 124 self.is_main_project = is_main_project 125 self.project_module_names = set( 126 self.modules_info.get_module_names(rel_path)) 127 self.project_relative_path = rel_path 128 self.project_absolute_path = abs_path 129 self.iml_path = '' 130 self._set_default_modules() 131 self._init_source_path() 132 if target == constant.FRAMEWORK_ALL: 133 self.dep_modules = self.get_dep_modules([target]) 134 else: 135 self.dep_modules = self.get_dep_modules() 136 self._filter_out_modules() 137 self.dependencies = [] 138 self.iml_name = iml.IMLGenerator.get_unique_iml_name(abs_path) 139 self.rel_out_soong_jar_path = self._get_rel_project_out_soong_jar_path() 140 141 def _set_default_modules(self): 142 """Append default hard-code modules, source paths and jar files. 143 144 1. framework: Framework module is always needed for dependencies, but it 145 might not always be located by module dependency. 146 2. org.apache.http.legacy.stubs.system: The module can't be located 147 through module dependency. Without it, a lot of java files will have 148 error of "cannot resolve symbol" in IntelliJ since they import 149 packages android.Manifest and com.android.internal.R. 150 """ 151 # Set the default modules framework-all and core-all as the core 152 # dependency modules. 153 self.project_module_names.update(_CORE_MODULES) 154 155 def _init_source_path(self): 156 """Initialize source_path dictionary.""" 157 self.source_path = { 158 'source_folder_path': set(), 159 'test_folder_path': set(), 160 'jar_path': set(), 161 'jar_module_path': {}, 162 'r_java_path': set(), 163 'srcjar_path': set() 164 } 165 166 def _search_android_make_files(self): 167 """Search project and dependency modules contain Android.mk files. 168 169 If there is only Android.mk but no Android.bp, we'll show the warning 170 message, otherwise we won't. 171 172 Yields: 173 A string: the relative path of Android.mk. 174 """ 175 if (common_util.exist_android_mk(self.project_absolute_path) and 176 not common_util.exist_android_bp(self.project_absolute_path)): 177 yield '\t' + os.path.join(self.project_relative_path, 178 constant.ANDROID_MK) 179 for mod_name in self.dep_modules: 180 rel_path, abs_path = common_util.get_related_paths( 181 self.modules_info, mod_name) 182 if rel_path and abs_path: 183 if (common_util.exist_android_mk(abs_path) 184 and not common_util.exist_android_bp(abs_path)): 185 yield '\t' + os.path.join(rel_path, constant.ANDROID_MK) 186 187 def _get_modules_under_project_path(self, rel_path): 188 """Find qualified modules under the rel_path. 189 190 Find modules which contain any Java or Kotlin file as a target module. 191 If it's the whole source tree project, add all modules into it. 192 193 Args: 194 rel_path: A string, the project's relative path. 195 196 Returns: 197 A set of module names. 198 """ 199 logging.info('Find modules contain any Java or Kotlin file under %s.', 200 rel_path) 201 if rel_path == '': 202 return self.modules_info.name_to_module_info.keys() 203 modules = set() 204 root_dir = common_util.get_android_root_dir() 205 for name, data in self.modules_info.name_to_module_info.items(): 206 if module_info.AidegenModuleInfo.is_project_path_relative_module( 207 data, rel_path): 208 if common_util.check_java_or_kotlin_file_exists( 209 os.path.join(root_dir, data[constant.KEY_PATH][0])): 210 modules.add(name) 211 else: 212 logging.debug(_NOT_TARGET, name) 213 return modules 214 215 def _get_robolectric_dep_module(self, modules): 216 """Return the robolectric module set as dependency if any module is a 217 robolectric test. 218 219 Args: 220 modules: A set of modules. 221 222 Returns: 223 A set with a robolectric_all module name if one of the modules 224 needs the robolectric test module. Otherwise return empty list. 225 """ 226 for module in modules: 227 if self.modules_info.is_robolectric_test(module): 228 return {_ROBOLECTRIC_MODULE} 229 return set() 230 231 def _filter_out_modules(self): 232 """Filter out unnecessary modules.""" 233 for module in _EXCLUDE_MODULES: 234 self.dep_modules.pop(module, None) 235 236 def get_dep_modules(self, module_names=None, depth=0): 237 """Recursively find dependent modules of the project. 238 239 Find dependent modules by dependencies parameter of each module. 240 For example: 241 The module_names is ['m1']. 242 The modules_info is 243 { 244 'm1': {'dependencies': ['m2'], 'path': ['path_to_m1']}, 245 'm2': {'path': ['path_to_m4']}, 246 'm3': {'path': ['path_to_m1']} 247 'm4': {'path': []} 248 } 249 The result dependent modules are: 250 { 251 'm1': {'dependencies': ['m2'], 'path': ['path_to_m1'] 252 'depth': 0}, 253 'm2': {'path': ['path_to_m4'], 'depth': 1}, 254 'm3': {'path': ['path_to_m1'], 'depth': 0} 255 } 256 Note that: 257 1. m4 is not in the result as it's not among dependent modules. 258 2. m3 is in the result as it has the same path to m1. 259 260 Args: 261 module_names: A set of module names. 262 depth: An integer shows the depth of module dependency referenced by 263 source. Zero means the max module depth. 264 265 Returns: 266 deps: A dict contains all dependent modules data of given modules. 267 """ 268 dep = {} 269 children = set() 270 if not module_names: 271 module_names = self.project_module_names 272 module_names.update( 273 self._get_modules_under_project_path( 274 self.project_relative_path)) 275 module_names.update(self._get_robolectric_dep_module(module_names)) 276 self.project_module_names = set() 277 for name in module_names: 278 if (name in self.modules_info.name_to_module_info 279 and name not in self.project_module_names): 280 dep[name] = self.modules_info.name_to_module_info[name] 281 dep[name][constant.KEY_DEPTH] = depth 282 self.project_module_names.add(name) 283 if (constant.KEY_DEPENDENCIES in dep[name] 284 and dep[name][constant.KEY_DEPENDENCIES]): 285 children.update(dep[name][constant.KEY_DEPENDENCIES]) 286 if children: 287 dep.update(self.get_dep_modules(children, depth + 1)) 288 return dep 289 290 @staticmethod 291 def generate_projects(targets): 292 """Generate a list of projects in one time by a list of module names. 293 294 Args: 295 targets: A list of target modules or project paths from user input, 296 when locating the target, project with matched module name 297 of the target has a higher priority than project path. 298 299 Returns: 300 List: A list of ProjectInfo instances. 301 """ 302 return [ProjectInfo(target, i == 0) for i, target in enumerate(targets)] 303 304 @staticmethod 305 def get_target_name(target, abs_path): 306 """Get target name from target's absolute path. 307 308 If the project is for entire Android source tree, change the target to 309 source tree's root folder name. In this way, we give IDE project file 310 a more specific name. e.g, master.iml. 311 312 Args: 313 target: Includes target module or project path from user input, when 314 locating the target, project with matching module name of 315 the given target has a higher priority than project path. 316 abs_path: A string, target's absolute path. 317 318 Returns: 319 A string, the target name. 320 """ 321 if abs_path == common_util.get_android_root_dir(): 322 return os.path.basename(abs_path) 323 return target 324 325 def locate_source(self, build=True): 326 """Locate the paths of dependent source folders and jar files. 327 328 Try to reference source folder path as dependent module unless the 329 dependent module should be referenced to a jar file, such as modules 330 have jars and jarjar_rules parameter. 331 For example: 332 Module: asm-6.0 333 java_import { 334 name: 'asm-6.0', 335 host_supported: true, 336 jars: ['asm-6.0.jar'], 337 } 338 Module: bouncycastle 339 java_library { 340 name: 'bouncycastle', 341 ... 342 target: { 343 android: { 344 jarjar_rules: 'jarjar-rules.txt', 345 }, 346 }, 347 } 348 349 Args: 350 build: A boolean default to true. If false, skip building jar and 351 srcjar files, otherwise build them. 352 353 Example usage: 354 project.source_path = project.locate_source() 355 E.g. 356 project.source_path = { 357 'source_folder_path': ['path/to/source/folder1', 358 'path/to/source/folder2', ...], 359 'test_folder_path': ['path/to/test/folder', ...], 360 'jar_path': ['path/to/jar/file1', 'path/to/jar/file2', ...] 361 } 362 """ 363 if not hasattr(self, 'dep_modules') or not self.dep_modules: 364 raise errors.EmptyModuleDependencyError( 365 'Dependent modules dictionary is empty.') 366 rebuild_targets = set() 367 for module_name, module_data in self.dep_modules.items(): 368 module = self._generate_moduledata(module_name, module_data) 369 module.locate_sources_path() 370 self.source_path['source_folder_path'].update(set(module.src_dirs)) 371 self.source_path['test_folder_path'].update(set(module.test_dirs)) 372 self.source_path['r_java_path'].update(set(module.r_java_paths)) 373 self.source_path['srcjar_path'].update(set(module.srcjar_paths)) 374 self._append_jars_as_dependencies(module) 375 rebuild_targets.update(module.build_targets) 376 config = project_config.ProjectConfig.get_instance() 377 if config.is_skip_build: 378 return 379 if rebuild_targets: 380 if build: 381 logging.info('\nThe batch_build_dependencies function is ' 382 'called by ProjectInfo\'s locate_source method.') 383 batch_build_dependencies(rebuild_targets) 384 self.locate_source(build=False) 385 else: 386 logging.warning('Jar or srcjar files build skipped:\n\t%s.', 387 '\n\t'.join(rebuild_targets)) 388 389 def _generate_moduledata(self, module_name, module_data): 390 """Generate a module class to collect dependencies in IDE. 391 392 The rules of initialize a module data instance: if ide_object isn't None 393 and its ide_name is 'eclipse', we'll create an EclipseModuleData 394 instance otherwise create a ModuleData instance. 395 396 Args: 397 module_name: Name of the module. 398 module_data: A dictionary holding a module information. 399 400 Returns: 401 A ModuleData class. 402 """ 403 ide_name = project_config.ProjectConfig.get_instance().ide_name 404 if ide_name == constant.IDE_ECLIPSE: 405 return source_locator.EclipseModuleData( 406 module_name, module_data, self.project_relative_path) 407 depth = project_config.ProjectConfig.get_instance().depth 408 return source_locator.ModuleData(module_name, module_data, depth) 409 410 def _append_jars_as_dependencies(self, module): 411 """Add given module's jar files into dependent_data as dependencies. 412 413 Args: 414 module: A ModuleData instance. 415 """ 416 if module.jar_files: 417 self.source_path['jar_path'].update(module.jar_files) 418 for jar in list(module.jar_files): 419 self.source_path['jar_module_path'].update({ 420 jar: 421 module.module_path 422 }) 423 # Collecting the jar files of default core modules as dependencies. 424 if constant.KEY_DEPENDENCIES in module.module_data: 425 self.source_path['jar_path'].update([ 426 x for x in module.module_data[constant.KEY_DEPENDENCIES] 427 if common_util.is_target(x, constant.TARGET_LIBS) 428 ]) 429 430 def _get_rel_project_out_soong_jar_path(self): 431 """Gets the projects' jar path in 'out/soong/.intermediates' folder. 432 433 Gets the relative project's jar path in the 'out/soong/.intermediates' 434 directory. For example, if the self.project_relative_path is 435 'frameworks/base', the returned value should be 436 'out/soong/.intermediates/frameworks/base/'. 437 438 Returns: 439 A string of relative project path in out/soong/.intermediates/ 440 directory, e.g. 'out/soong/.intermediates/frameworks/base/'. 441 """ 442 rdir = os.path.relpath(common_util.get_soong_out_path(), 443 common_util.get_android_root_dir()) 444 return os.sep.join( 445 [rdir, constant.INTERMEDIATES, self.project_relative_path]) + os.sep 446 447 @classmethod 448 def multi_projects_locate_source(cls, projects): 449 """Locate the paths of dependent source folders and jar files. 450 451 Args: 452 projects: A list of ProjectInfo instances. Information of a project 453 such as project relative path, project real path, project 454 dependencies. 455 """ 456 cls.projects = projects 457 for project in projects: 458 project.locate_source() 459 _update_iml_dep_modules(project) 460 461 462class MultiProjectsInfo(ProjectInfo): 463 """Multiple projects info. 464 465 Usage example: 466 if folder_base: 467 project = MultiProjectsInfo(['module_name']) 468 project.collect_all_dep_modules() 469 project.gen_folder_base_dependencies() 470 else: 471 ProjectInfo.generate_projects(['module_name']) 472 473 Attributes: 474 _targets: A list of module names or project paths. 475 path_to_sources: A dictionary of modules' sources, the module's path 476 as key and the sources as value. 477 e.g. 478 { 479 'frameworks/base': { 480 'src_dirs': [], 481 'test_dirs': [], 482 'r_java_paths': [], 483 'srcjar_paths': [], 484 'jar_files': [], 485 'dep_paths': [], 486 } 487 } 488 """ 489 490 def __init__(self, targets=None): 491 """MultiProjectsInfo initialize. 492 493 Args: 494 targets: A list of module names or project paths from user's input. 495 """ 496 super().__init__(targets[0], True) 497 self._targets = targets 498 self.path_to_sources = {} 499 500 @staticmethod 501 def _clear_srcjar_paths(module): 502 """Clears the srcjar_paths. 503 504 Args: 505 module: A ModuleData instance. 506 """ 507 module.srcjar_paths = [] 508 509 def _collect_framework_srcjar_info(self, module): 510 """Clears the framework's srcjars. 511 512 Args: 513 module: A ModuleData instance. 514 """ 515 if module.module_path == constant.FRAMEWORK_PATH: 516 framework_srcjar_path = os.path.join(constant.FRAMEWORK_PATH, 517 constant.FRAMEWORK_SRCJARS) 518 if module.module_name == constant.FRAMEWORK_ALL: 519 self.path_to_sources[framework_srcjar_path] = { 520 'src_dirs': [], 521 'test_dirs': [], 522 'r_java_paths': [], 523 'srcjar_paths': module.srcjar_paths, 524 'jar_files': [], 525 'dep_paths': [constant.FRAMEWORK_PATH], 526 } 527 # In the folder base case, AIDEGen has to ignore all module's srcjar 528 # files under the frameworks/base except the framework-all. Because 529 # there are too many duplicate srcjars of modules under the 530 # frameworks/base. So that AIDEGen keeps the srcjar files only from 531 # the framework-all module. Other modules' srcjar files will be 532 # removed. However, when users choose the module base case, srcjar 533 # files will be collected by the ProjectInfo class, so that the 534 # removing srcjar_paths in this class does not impact the 535 # srcjar_paths collection of modules in the ProjectInfo class. 536 self._clear_srcjar_paths(module) 537 538 def collect_all_dep_modules(self): 539 """Collects all dependency modules for the projects.""" 540 self.project_module_names.clear() 541 module_names = set(_CORE_MODULES) 542 for target in self._targets: 543 relpath, _ = common_util.get_related_paths(self.modules_info, 544 target) 545 module_names.update(self._get_modules_under_project_path(relpath)) 546 module_names.update(self._get_robolectric_dep_module(module_names)) 547 self.dep_modules = self.get_dep_modules(module_names) 548 549 def gen_folder_base_dependencies(self, module): 550 """Generates the folder base dependencies dictionary. 551 552 Args: 553 module: A ModuleData instance. 554 """ 555 mod_path = module.module_path 556 if not mod_path: 557 logging.debug('The %s\'s path is empty.', module.module_name) 558 return 559 self._collect_framework_srcjar_info(module) 560 if mod_path not in self.path_to_sources: 561 self.path_to_sources[mod_path] = { 562 'src_dirs': module.src_dirs, 563 'test_dirs': module.test_dirs, 564 'r_java_paths': module.r_java_paths, 565 'srcjar_paths': module.srcjar_paths, 566 'jar_files': module.jar_files, 567 'dep_paths': module.dep_paths, 568 } 569 else: 570 for key, val in self.path_to_sources[mod_path].items(): 571 val.extend([v for v in getattr(module, key) if v not in val]) 572 573 574def batch_build_dependencies(rebuild_targets): 575 """Batch build the jar or srcjar files of the modules if they don't exist. 576 577 Command line has the max length limit, MAX_ARG_STRLEN, and 578 MAX_ARG_STRLEN = (PAGE_SIZE * 32). 579 If the build command is longer than MAX_ARG_STRLEN, this function will 580 separate the rebuild_targets into chunks with size less or equal to 581 MAX_ARG_STRLEN to make sure it can be built successfully. 582 583 Args: 584 rebuild_targets: A set of jar or srcjar files which do not exist. 585 """ 586 start_time = time.time() 587 logging.info('Ready to build the jar or srcjar files. Files count = %s', 588 str(len(rebuild_targets))) 589 arg_max = os.sysconf('SC_PAGE_SIZE') * 32 - _CMD_LENGTH_BUFFER 590 rebuild_targets = list(rebuild_targets) 591 for start, end in iter(_separate_build_targets(rebuild_targets, arg_max)): 592 _build_target(rebuild_targets[start:end]) 593 duration = time.time() - start_time 594 logging.debug('Build Time, duration = %s', str(duration)) 595 aidegen_metrics.performance_metrics(constant.TYPE_AIDEGEN_BUILD_TIME, 596 duration) 597 598 599def _build_target(targets): 600 """Build the jar or srcjar files. 601 602 Use -k to keep going when some targets can't be built or build failed. 603 Use -j to speed up building. 604 605 Args: 606 targets: A list of jar or srcjar files which need to build. 607 """ 608 build_cmd = ['-k', '-j'] 609 build_cmd.extend(list(targets)) 610 atest_utils.update_build_env(_BUILD_BP_JSON_ENV_ON) 611 if not atest_utils.build(build_cmd): 612 message = ('Build failed!\n{}\nAIDEGen will proceed but dependency ' 613 'correctness is not guaranteed if not all targets being ' 614 'built successfully.'.format('\n'.join(targets))) 615 print('\n{} {}\n'.format(common_util.COLORED_INFO('Warning:'), message)) 616 617 618def _separate_build_targets(build_targets, max_length): 619 """Separate the build_targets by limit the command size to max command 620 length. 621 622 Args: 623 build_targets: A list to be separated. 624 max_length: The max number of each build command length. 625 626 Yields: 627 The start index and end index of build_targets. 628 """ 629 arg_len = 0 630 first_item_index = 0 631 for i, item in enumerate(build_targets): 632 arg_len = arg_len + len(item) + _BLANK_SIZE 633 if arg_len > max_length: 634 yield first_item_index, i 635 first_item_index = i 636 arg_len = len(item) + _BLANK_SIZE 637 if first_item_index < len(build_targets): 638 yield first_item_index, len(build_targets) 639 640 641def _update_iml_dep_modules(project): 642 """Gets the dependent modules in the project's iml file. 643 644 The jar files which have the same source codes as cls.projects' source files 645 should be removed from the dependencies.iml file's jar paths. The codes are 646 written in aidegen.project.project_splitter.py. 647 We should also add the jar project's unique iml name into self.dependencies 648 which later will be written into its own iml project file. If we don't 649 remove these files in dependencies.iml, it will cause the duplicated codes 650 in IDE and raise issues. For example, when users do 'refactor' and rename a 651 class in the IDE, it will search all sources and dependencies' jar paths and 652 lead to the error. 653 """ 654 keys = ('source_folder_path', 'test_folder_path', 'r_java_path', 655 'srcjar_path', 'jar_path') 656 for key in keys: 657 for jar in project.source_path[key]: 658 for prj in ProjectInfo.projects: 659 if prj is project: 660 continue 661 if (prj.rel_out_soong_jar_path in jar and 662 jar.endswith(constant.JAR_EXT)): 663 if prj.iml_name not in project.dependencies: 664 project.dependencies.append(prj.iml_name) 665 break 666