1#!/usr/bin/env python3 2# 3# Copyright 2018 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Project information.""" 18 19from __future__ import absolute_import 20 21import logging 22import os 23import time 24 25from aidegen import constant 26from aidegen.lib import aidegen_metrics 27from aidegen.lib import common_util 28from aidegen.lib import errors 29from aidegen.lib import module_info 30from aidegen.lib import project_config 31from aidegen.lib import source_locator 32from aidegen.idea import iml 33 34from atest import atest_utils 35 36_CONVERT_MK_URL = ('https://android.googlesource.com/platform/build/soong/' 37 '#convert-android_mk-files') 38_ROBOLECTRIC_MODULE = 'Robolectric_all' 39_NOT_TARGET = ('The module %s does not contain any Java or Kotlin file, ' 40 'therefore we skip this module in the project.') 41# The module fake-framework have the same package name with framework but empty 42# content. It will impact the dependency for framework when referencing the 43# package from fake-framework in IntelliJ. 44_EXCLUDE_MODULES = ['fake-framework'] 45# When we use atest_utils.build(), there is a command length limit on 46# soong_ui.bash. We reserve 5000 characters for rewriting the command line 47# in soong_ui.bash. 48_CMD_LENGTH_BUFFER = 5000 49# For each argument, it need a space to separate following argument. 50_BLANK_SIZE = 1 51_CORE_MODULES = [constant.FRAMEWORK_ALL, constant.CORE_ALL, 52 'org.apache.http.legacy.stubs.system'] 53_BUILD_BP_JSON_ENV_ON = { 54 constant.GEN_JAVA_DEPS: 'true', 55 constant.GEN_CC_DEPS: 'true', 56 constant.GEN_COMPDB: 'true', 57 constant.GEN_RUST: 'true' 58} 59 60 61class ProjectInfo: 62 """Project information. 63 64 Users should call config_project first before starting using ProjectInfo. 65 66 Class attributes: 67 modules_info: An AidegenModuleInfo instance whose name_to_module_info is 68 combining module-info.json with module_bp_java_deps.json. 69 projects: A list of instances of ProjectInfo that are generated in an 70 AIDEGen command. 71 72 Attributes: 73 project_absolute_path: The absolute path of the project. 74 project_relative_path: The relative path of the project to 75 common_util.get_android_root_dir(). 76 project_module_names: A set of module names under project_absolute_path 77 directory or it's subdirectories. 78 dep_modules: A dict has recursively dependent modules of 79 project_module_names. 80 iml_path: The project's iml file path. 81 source_path: A dictionary to keep following data: 82 source_folder_path: A set contains the source folder 83 relative paths. 84 test_folder_path: A set contains the test folder relative 85 paths. 86 jar_path: A set contains the jar file paths. 87 jar_module_path: A dictionary contains the jar file and 88 the module's path mapping, only used in 89 Eclipse. 90 r_java_path: A set contains the relative path to the 91 R.java files, only used in Eclipse. 92 srcjar_path: A source content descriptor only used in 93 IntelliJ. 94 e.g. out/.../aapt2.srcjar!/ 95 The "!/" is a content descriptor for 96 compressed files in IntelliJ. 97 is_main_project: A boolean to verify the project is main project. 98 dependencies: A list of dependency projects' iml file names, e.g. base, 99 framework-all. 100 iml_name: The iml project file name of this project. 101 rel_out_soong_jar_path: A string of relative project path in the 102 'out/soong/.intermediates' directory, e.g., if 103 self.project_relative_path = 'frameworks/base' 104 the rel_out_soong_jar_path should be 105 'out/soong/.intermediates/frameworks/base/'. 106 """ 107 108 modules_info = None 109 110 def __init__(self, target=None, is_main_project=False): 111 """ProjectInfo initialize. 112 113 Args: 114 target: Includes target module or project path from user input, when 115 locating the target, project with matching module name of 116 the given target has a higher priority than project path. 117 is_main_project: A boolean, default is False. True if the target is 118 the main project, otherwise False. 119 """ 120 rel_path, abs_path = common_util.get_related_paths( 121 self.modules_info, target) 122 self.module_name = self.get_target_name(target, abs_path) 123 self.is_main_project = is_main_project 124 self.project_module_names = set( 125 self.modules_info.get_module_names(rel_path)) 126 self.project_relative_path = rel_path 127 self.project_absolute_path = abs_path 128 self.iml_path = '' 129 self._set_default_modues() 130 self._init_source_path() 131 if target == constant.FRAMEWORK_ALL: 132 self.dep_modules = self.get_dep_modules([target]) 133 else: 134 self.dep_modules = self.get_dep_modules() 135 self._filter_out_modules() 136 self.dependencies = [] 137 self.iml_name = iml.IMLGenerator.get_unique_iml_name(abs_path) 138 self.rel_out_soong_jar_path = self._get_rel_project_out_soong_jar_path() 139 140 def _set_default_modues(self): 141 """Append default hard-code modules, source paths and jar files. 142 143 1. framework: Framework module is always needed for dependencies but it 144 might not always be located by module dependency. 145 2. org.apache.http.legacy.stubs.system: The module can't be located 146 through module dependency. Without it, a lot of java files will have 147 error of "cannot resolve symbol" in IntelliJ since they import 148 packages android.Manifest and com.android.internal.R. 149 """ 150 # Set the default modules framework-all and core-all as the core 151 # dependency modules. 152 self.project_module_names.update(_CORE_MODULES) 153 154 def _init_source_path(self): 155 """Initialize source_path dictionary.""" 156 self.source_path = { 157 'source_folder_path': set(), 158 'test_folder_path': set(), 159 'jar_path': set(), 160 'jar_module_path': dict(), 161 'r_java_path': set(), 162 'srcjar_path': set() 163 } 164 165 def _search_android_make_files(self): 166 """Search project and dependency modules contain Android.mk files. 167 168 If there is only Android.mk but no Android.bp, we'll show the warning 169 message, otherwise we won't. 170 171 Yields: 172 A string: the relative path of Android.mk. 173 """ 174 if (common_util.exist_android_mk(self.project_absolute_path) and 175 not common_util.exist_android_bp(self.project_absolute_path)): 176 yield '\t' + os.path.join(self.project_relative_path, 177 constant.ANDROID_MK) 178 for mod_name in self.dep_modules: 179 rel_path, abs_path = common_util.get_related_paths( 180 self.modules_info, mod_name) 181 if rel_path and abs_path: 182 if (common_util.exist_android_mk(abs_path) 183 and not common_util.exist_android_bp(abs_path)): 184 yield '\t' + os.path.join(rel_path, constant.ANDROID_MK) 185 186 def _get_modules_under_project_path(self, rel_path): 187 """Find qualified modules under the rel_path. 188 189 Find modules which contain any Java or Kotlin file as a target module. 190 If it's the whole source tree project, add all modules into it. 191 192 Args: 193 rel_path: A string, the project's relative path. 194 195 Returns: 196 A set of module names. 197 """ 198 logging.info('Find modules contain any Java or Kotlin file under %s.', 199 rel_path) 200 if rel_path == '': 201 return self.modules_info.name_to_module_info.keys() 202 modules = set() 203 root_dir = common_util.get_android_root_dir() 204 for name, data in self.modules_info.name_to_module_info.items(): 205 if module_info.AidegenModuleInfo.is_project_path_relative_module( 206 data, rel_path): 207 if common_util.check_java_or_kotlin_file_exists( 208 os.path.join(root_dir, data[constant.KEY_PATH][0])): 209 modules.add(name) 210 else: 211 logging.debug(_NOT_TARGET, name) 212 return modules 213 214 def _get_robolectric_dep_module(self, modules): 215 """Return the robolectric module set as dependency if any module is a 216 robolectric test. 217 218 Args: 219 modules: A set of modules. 220 221 Returns: 222 A set with a robolectric_all module name if one of the modules 223 needs the robolectric test module. Otherwise return empty list. 224 """ 225 for module in modules: 226 if self.modules_info.is_robolectric_test(module): 227 return {_ROBOLECTRIC_MODULE} 228 return set() 229 230 def _filter_out_modules(self): 231 """Filter out unnecessary modules.""" 232 for module in _EXCLUDE_MODULES: 233 self.dep_modules.pop(module, None) 234 235 def get_dep_modules(self, module_names=None, depth=0): 236 """Recursively find dependent modules of the project. 237 238 Find dependent modules by dependencies parameter of each module. 239 For example: 240 The module_names is ['m1']. 241 The modules_info is 242 { 243 'm1': {'dependencies': ['m2'], 'path': ['path_to_m1']}, 244 'm2': {'path': ['path_to_m4']}, 245 'm3': {'path': ['path_to_m1']} 246 'm4': {'path': []} 247 } 248 The result dependent modules are: 249 { 250 'm1': {'dependencies': ['m2'], 'path': ['path_to_m1'] 251 'depth': 0}, 252 'm2': {'path': ['path_to_m4'], 'depth': 1}, 253 'm3': {'path': ['path_to_m1'], 'depth': 0} 254 } 255 Note that: 256 1. m4 is not in the result as it's not among dependent modules. 257 2. m3 is in the result as it has the same path to m1. 258 259 Args: 260 module_names: A set of module names. 261 depth: An integer shows the depth of module dependency referenced by 262 source. Zero means the max module depth. 263 264 Returns: 265 deps: A dict contains all dependent modules data of given modules. 266 """ 267 dep = {} 268 children = set() 269 if not module_names: 270 module_names = self.project_module_names 271 module_names.update( 272 self._get_modules_under_project_path( 273 self.project_relative_path)) 274 module_names.update(self._get_robolectric_dep_module(module_names)) 275 self.project_module_names = set() 276 for name in module_names: 277 if (name in self.modules_info.name_to_module_info 278 and name not in self.project_module_names): 279 dep[name] = self.modules_info.name_to_module_info[name] 280 dep[name][constant.KEY_DEPTH] = depth 281 self.project_module_names.add(name) 282 if (constant.KEY_DEPENDENCIES in dep[name] 283 and dep[name][constant.KEY_DEPENDENCIES]): 284 children.update(dep[name][constant.KEY_DEPENDENCIES]) 285 if children: 286 dep.update(self.get_dep_modules(children, depth + 1)) 287 return dep 288 289 @staticmethod 290 def generate_projects(targets): 291 """Generate a list of projects in one time by a list of module names. 292 293 Args: 294 targets: A list of target modules or project paths from user input, 295 when locating the target, project with matched module name 296 of the target has a higher priority than project path. 297 298 Returns: 299 List: A list of ProjectInfo instances. 300 """ 301 return [ProjectInfo(target, i == 0) for i, target in enumerate(targets)] 302 303 @staticmethod 304 def get_target_name(target, abs_path): 305 """Get target name from target's absolute path. 306 307 If the project is for entire Android source tree, change the target to 308 source tree's root folder name. In this way, we give IDE project file 309 a more specific name. e.g, master.iml. 310 311 Args: 312 target: Includes target module or project path from user input, when 313 locating the target, project with matching module name of 314 the given target has a higher priority than project path. 315 abs_path: A string, target's absolute path. 316 317 Returns: 318 A string, the target name. 319 """ 320 if abs_path == common_util.get_android_root_dir(): 321 return os.path.basename(abs_path) 322 return target 323 324 def locate_source(self, build=True): 325 """Locate the paths of dependent source folders and jar files. 326 327 Try to reference source folder path as dependent module unless the 328 dependent module should be referenced to a jar file, such as modules 329 have jars and jarjar_rules parameter. 330 For example: 331 Module: asm-6.0 332 java_import { 333 name: 'asm-6.0', 334 host_supported: true, 335 jars: ['asm-6.0.jar'], 336 } 337 Module: bouncycastle 338 java_library { 339 name: 'bouncycastle', 340 ... 341 target: { 342 android: { 343 jarjar_rules: 'jarjar-rules.txt', 344 }, 345 }, 346 } 347 348 Args: 349 build: A boolean default to true. If false, skip building jar and 350 srcjar files, otherwise build them. 351 352 Example usage: 353 project.source_path = project.locate_source() 354 E.g. 355 project.source_path = { 356 'source_folder_path': ['path/to/source/folder1', 357 'path/to/source/folder2', ...], 358 'test_folder_path': ['path/to/test/folder', ...], 359 'jar_path': ['path/to/jar/file1', 'path/to/jar/file2', ...] 360 } 361 """ 362 if not hasattr(self, 'dep_modules') or not self.dep_modules: 363 raise errors.EmptyModuleDependencyError( 364 'Dependent modules dictionary is empty.') 365 rebuild_targets = set() 366 for module_name, module_data in self.dep_modules.items(): 367 module = self._generate_moduledata(module_name, module_data) 368 module.locate_sources_path() 369 self.source_path['source_folder_path'].update(set(module.src_dirs)) 370 self.source_path['test_folder_path'].update(set(module.test_dirs)) 371 self.source_path['r_java_path'].update(set(module.r_java_paths)) 372 self.source_path['srcjar_path'].update(set(module.srcjar_paths)) 373 self._append_jars_as_dependencies(module) 374 rebuild_targets.update(module.build_targets) 375 config = project_config.ProjectConfig.get_instance() 376 if config.is_skip_build: 377 return 378 if rebuild_targets: 379 if build: 380 logging.info('\nThe batch_build_dependencies function is ' 381 'called by ProjectInfo\'s locate_source method.') 382 batch_build_dependencies(rebuild_targets) 383 self.locate_source(build=False) 384 else: 385 logging.warning('Jar or srcjar files build skipped:\n\t%s.', 386 '\n\t'.join(rebuild_targets)) 387 388 def _generate_moduledata(self, module_name, module_data): 389 """Generate a module class to collect dependencies in IDE. 390 391 The rules of initialize a module data instance: if ide_object isn't None 392 and its ide_name is 'eclipse', we'll create an EclipseModuleData 393 instance otherwise create a ModuleData instance. 394 395 Args: 396 module_name: Name of the module. 397 module_data: A dictionary holding a module information. 398 399 Returns: 400 A ModuleData class. 401 """ 402 ide_name = project_config.ProjectConfig.get_instance().ide_name 403 if ide_name == constant.IDE_ECLIPSE: 404 return source_locator.EclipseModuleData( 405 module_name, module_data, self.project_relative_path) 406 depth = project_config.ProjectConfig.get_instance().depth 407 return source_locator.ModuleData(module_name, module_data, depth) 408 409 def _append_jars_as_dependencies(self, module): 410 """Add given module's jar files into dependent_data as dependencies. 411 412 Args: 413 module: A ModuleData instance. 414 """ 415 if module.jar_files: 416 self.source_path['jar_path'].update(module.jar_files) 417 for jar in list(module.jar_files): 418 self.source_path['jar_module_path'].update({ 419 jar: 420 module.module_path 421 }) 422 # Collecting the jar files of default core modules as dependencies. 423 if constant.KEY_DEPENDENCIES in module.module_data: 424 self.source_path['jar_path'].update([ 425 x for x in module.module_data[constant.KEY_DEPENDENCIES] 426 if common_util.is_target(x, constant.TARGET_LIBS) 427 ]) 428 429 def _get_rel_project_out_soong_jar_path(self): 430 """Gets the projects' jar path in 'out/soong/.intermediates' folder. 431 432 Gets the relative project's jar path in the 'out/soong/.intermediates' 433 directory. For example, if the self.project_relative_path is 434 'frameworks/base', the returned value should be 435 'out/soong/.intermediates/frameworks/base/'. 436 437 Returns: 438 A string of relative project path in out/soong/.intermediates/ 439 directory, e.g. 'out/soong/.intermediates/frameworks/base/'. 440 """ 441 rdir = os.path.relpath(common_util.get_soong_out_path(), 442 common_util.get_android_root_dir()) 443 return os.sep.join( 444 [rdir, constant.INTERMEDIATES, self.project_relative_path]) + os.sep 445 446 @classmethod 447 def multi_projects_locate_source(cls, projects): 448 """Locate the paths of dependent source folders and jar files. 449 450 Args: 451 projects: A list of ProjectInfo instances. Information of a project 452 such as project relative path, project real path, project 453 dependencies. 454 """ 455 cls.projects = projects 456 for project in projects: 457 project.locate_source() 458 _update_iml_dep_modules(project) 459 460 461class MultiProjectsInfo(ProjectInfo): 462 """Multiple projects info. 463 464 Usage example: 465 if folder_base: 466 project = MultiProjectsInfo(['module_name']) 467 project.collect_all_dep_modules() 468 project.gen_folder_base_dependencies() 469 else: 470 ProjectInfo.generate_projects(['module_name']) 471 472 Attributes: 473 _targets: A list of module names or project paths. 474 path_to_sources: A dictionary of modules' sources, the module's path 475 as key and the sources as value. 476 e.g. 477 { 478 'frameworks/base': { 479 'src_dirs': [], 480 'test_dirs': [], 481 'r_java_paths': [], 482 'srcjar_paths': [], 483 'jar_files': [], 484 'dep_paths': [], 485 } 486 } 487 """ 488 489 def __init__(self, targets=None): 490 """MultiProjectsInfo initialize. 491 492 Args: 493 targets: A list of module names or project paths from user's input. 494 """ 495 super().__init__(targets[0], True) 496 self._targets = targets 497 self.path_to_sources = {} 498 499 def _clear_srcjar_paths(self, module): 500 """Clears the srcjar_paths. 501 502 Args: 503 module: A ModuleData instance. 504 """ 505 module.srcjar_paths = [] 506 507 def _collect_framework_srcjar_info(self, module): 508 """Clears the framework's srcjars. 509 510 Args: 511 module: A ModuleData instance. 512 """ 513 if module.module_path == constant.FRAMEWORK_PATH: 514 framework_srcjar_path = os.path.join(constant.FRAMEWORK_PATH, 515 constant.FRAMEWORK_SRCJARS) 516 if module.module_name == constant.FRAMEWORK_ALL: 517 self.path_to_sources[framework_srcjar_path] = { 518 'src_dirs': [], 519 'test_dirs': [], 520 'r_java_paths': [], 521 'srcjar_paths': module.srcjar_paths, 522 'jar_files': [], 523 'dep_paths': [constant.FRAMEWORK_PATH], 524 } 525 # In the folder base case, AIDEGen has to ignore all module's srcjar 526 # files under the frameworks/base except the framework-all. Because 527 # there are too many duplicate srcjars of modules under the 528 # frameworks/base. So that AIDEGen keeps the srcjar files only from 529 # the framework-all module. Other modeuls' srcjar files will be 530 # removed. However, when users choose the module base case, srcjar 531 # files will be collected by the ProjectInfo class, so that the 532 # removing srcjar_paths in this class does not impact the 533 # srcjar_paths collection of modules in the ProjectInfo class. 534 self._clear_srcjar_paths(module) 535 536 def collect_all_dep_modules(self): 537 """Collects all dependency modules for the projects.""" 538 self.project_module_names.clear() 539 module_names = set(_CORE_MODULES) 540 for target in self._targets: 541 relpath, _ = common_util.get_related_paths(self.modules_info, 542 target) 543 module_names.update(self._get_modules_under_project_path(relpath)) 544 module_names.update(self._get_robolectric_dep_module(module_names)) 545 self.dep_modules = self.get_dep_modules(module_names) 546 547 def gen_folder_base_dependencies(self, module): 548 """Generates the folder base dependencies dictionary. 549 550 Args: 551 module: A ModuleData instance. 552 """ 553 mod_path = module.module_path 554 if not mod_path: 555 logging.debug('The %s\'s path is empty.', module.module_name) 556 return 557 self._collect_framework_srcjar_info(module) 558 if mod_path not in self.path_to_sources: 559 self.path_to_sources[mod_path] = { 560 'src_dirs': module.src_dirs, 561 'test_dirs': module.test_dirs, 562 'r_java_paths': module.r_java_paths, 563 'srcjar_paths': module.srcjar_paths, 564 'jar_files': module.jar_files, 565 'dep_paths': module.dep_paths, 566 } 567 else: 568 for key, val in self.path_to_sources[mod_path].items(): 569 val.extend([v for v in getattr(module, key) if v not in val]) 570 571 572def batch_build_dependencies(rebuild_targets): 573 """Batch build the jar or srcjar files of the modules if they don't exist. 574 575 Command line has the max length limit, MAX_ARG_STRLEN, and 576 MAX_ARG_STRLEN = (PAGE_SIZE * 32). 577 If the build command is longer than MAX_ARG_STRLEN, this function will 578 separate the rebuild_targets into chunks with size less or equal to 579 MAX_ARG_STRLEN to make sure it can be built successfully. 580 581 Args: 582 rebuild_targets: A set of jar or srcjar files which do not exist. 583 """ 584 start_time = time.time() 585 logging.info('Ready to build the jar or srcjar files. Files count = %s', 586 str(len(rebuild_targets))) 587 arg_max = os.sysconf('SC_PAGE_SIZE') * 32 - _CMD_LENGTH_BUFFER 588 rebuild_targets = list(rebuild_targets) 589 for start, end in iter(_separate_build_targets(rebuild_targets, arg_max)): 590 _build_target(rebuild_targets[start:end]) 591 duration = time.time() - start_time 592 logging.debug('Build Time, duration = %s', str(duration)) 593 aidegen_metrics.performance_metrics(constant.TYPE_AIDEGEN_BUILD_TIME, 594 duration) 595 596 597def _build_target(targets): 598 """Build the jar or srcjar files. 599 600 Use -k to keep going when some targets can't be built or build failed. 601 Use -j to speed up building. 602 603 Args: 604 targets: A list of jar or srcjar files which need to build. 605 """ 606 build_cmd = ['-k', '-j'] 607 build_cmd.extend(list(targets)) 608 verbose = True 609 if not atest_utils.build(build_cmd, verbose, _BUILD_BP_JSON_ENV_ON): 610 message = ('Build failed!\n{}\nAIDEGen will proceed but dependency ' 611 'correctness is not guaranteed if not all targets being ' 612 'built successfully.'.format('\n'.join(targets))) 613 print('\n{} {}\n'.format(common_util.COLORED_INFO('Warning:'), message)) 614 615 616def _separate_build_targets(build_targets, max_length): 617 """Separate the build_targets by limit the command size to max command 618 length. 619 620 Args: 621 build_targets: A list to be separated. 622 max_length: The max number of each build command length. 623 624 Yields: 625 The start index and end index of build_targets. 626 """ 627 arg_len = 0 628 first_item_index = 0 629 for i, item in enumerate(build_targets): 630 arg_len = arg_len + len(item) + _BLANK_SIZE 631 if arg_len > max_length: 632 yield first_item_index, i 633 first_item_index = i 634 arg_len = len(item) + _BLANK_SIZE 635 if first_item_index < len(build_targets): 636 yield first_item_index, len(build_targets) 637 638 639def _update_iml_dep_modules(project): 640 """Gets the dependent modules in the project's iml file. 641 642 The jar files which have the same source codes as cls.projects' source files 643 should be removed from the dependencies.iml file's jar paths. The codes are 644 written in aidegen.project.project_splitter.py. 645 We should also add the jar project's unique iml name into self.dependencies 646 which later will be written into its own iml project file. If we don't 647 remove these files in dependencies.iml, it will cause the duplicated codes 648 in IDE and raise issues. For example, when users do 'refactor' and rename a 649 class in the IDE, it will search all sources and dependencies' jar paths and 650 lead to the error. 651 """ 652 keys = ('source_folder_path', 'test_folder_path', 'r_java_path', 653 'srcjar_path', 'jar_path') 654 for key in keys: 655 for jar in project.source_path[key]: 656 for prj in ProjectInfo.projects: 657 if prj is project: 658 continue 659 if (prj.rel_out_soong_jar_path in jar and 660 jar.endswith(constant.JAR_EXT)): 661 if prj.iml_name not in project.dependencies: 662 project.dependencies.append(prj.iml_name) 663 break 664