1#!/usr/bin/env python3 2# 3# Copyright 2018 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Project information.""" 18 19from __future__ import absolute_import 20 21import logging 22import os 23import time 24 25from aidegen import constant 26from aidegen.lib import aidegen_metrics 27from aidegen.lib import common_util 28from aidegen.lib import errors 29from aidegen.lib import module_info 30from aidegen.lib import project_config 31from aidegen.lib import source_locator 32from aidegen.idea import iml 33 34from atest import atest_utils 35 36_CONVERT_MK_URL = ('https://android.googlesource.com/platform/build/soong/' 37 '#convert-android_mk-files') 38_ROBOLECTRIC_MODULE = 'Robolectric_all' 39_NOT_TARGET = ('The module %s does not contain any Java or Kotlin file, ' 40 'therefore we skip this module in the project.') 41# The module fake-framework have the same package name with framework but empty 42# content. It will impact the dependency for framework when referencing the 43# package from fake-framework in IntelliJ. 44_EXCLUDE_MODULES = ['fake-framework'] 45# When we use atest_utils.build(), there is a command length limit on 46# soong_ui.bash. We reserve 5000 characters for rewriting the command line 47# in soong_ui.bash. 48_CMD_LENGTH_BUFFER = 5000 49# For each argument, it need a space to separate following argument. 50_BLANK_SIZE = 1 51_CORE_MODULES = [constant.FRAMEWORK_ALL, constant.CORE_ALL, 52 'org.apache.http.legacy.stubs.system'] 53 54 55class ProjectInfo: 56 """Project information. 57 58 Users should call config_project first before starting using ProjectInfo. 59 60 Class attributes: 61 modules_info: An AidegenModuleInfo instance whose name_to_module_info is 62 combining module-info.json with module_bp_java_deps.json. 63 projects: A list of instances of ProjectInfo that are generated in an 64 AIDEGen command. 65 66 Attributes: 67 project_absolute_path: The absolute path of the project. 68 project_relative_path: The relative path of the project to 69 common_util.get_android_root_dir(). 70 project_module_names: A set of module names under project_absolute_path 71 directory or it's subdirectories. 72 dep_modules: A dict has recursively dependent modules of 73 project_module_names. 74 iml_path: The project's iml file path. 75 source_path: A dictionary to keep following data: 76 source_folder_path: A set contains the source folder 77 relative paths. 78 test_folder_path: A set contains the test folder relative 79 paths. 80 jar_path: A set contains the jar file paths. 81 jar_module_path: A dictionary contains the jar file and 82 the module's path mapping, only used in 83 Eclipse. 84 r_java_path: A set contains the relative path to the 85 R.java files, only used in Eclipse. 86 srcjar_path: A source content descriptor only used in 87 IntelliJ. 88 e.g. out/.../aapt2.srcjar!/ 89 The "!/" is a content descriptor for 90 compressed files in IntelliJ. 91 is_main_project: A boolean to verify the project is main project. 92 dependencies: A list of dependency projects' iml file names, e.g. base, 93 framework-all. 94 iml_name: The iml project file name of this project. 95 rel_out_soong_jar_path: A string of relative project path in the 96 'out/soong/.intermediates' directory, e.g., if 97 self.project_relative_path = 'frameworks/base' 98 the rel_out_soong_jar_path should be 99 'out/soong/.intermediates/frameworks/base/'. 100 """ 101 102 modules_info = None 103 104 def __init__(self, target=None, is_main_project=False): 105 """ProjectInfo initialize. 106 107 Args: 108 target: Includes target module or project path from user input, when 109 locating the target, project with matching module name of 110 the given target has a higher priority than project path. 111 is_main_project: A boolean, default is False. True if the target is 112 the main project, otherwise False. 113 """ 114 rel_path, abs_path = common_util.get_related_paths( 115 self.modules_info, target) 116 self.module_name = self.get_target_name(target, abs_path) 117 self.is_main_project = is_main_project 118 self.project_module_names = set( 119 self.modules_info.get_module_names(rel_path)) 120 self.project_relative_path = rel_path 121 self.project_absolute_path = abs_path 122 self.iml_path = '' 123 self._set_default_modues() 124 self._init_source_path() 125 if target == constant.FRAMEWORK_ALL: 126 self.dep_modules = self.get_dep_modules([target]) 127 else: 128 self.dep_modules = self.get_dep_modules() 129 self._filter_out_modules() 130 self.dependencies = [] 131 self.iml_name = iml.IMLGenerator.get_unique_iml_name(abs_path) 132 self.rel_out_soong_jar_path = self._get_rel_project_out_soong_jar_path() 133 134 def _set_default_modues(self): 135 """Append default hard-code modules, source paths and jar files. 136 137 1. framework: Framework module is always needed for dependencies but it 138 might not always be located by module dependency. 139 2. org.apache.http.legacy.stubs.system: The module can't be located 140 through module dependency. Without it, a lot of java files will have 141 error of "cannot resolve symbol" in IntelliJ since they import 142 packages android.Manifest and com.android.internal.R. 143 """ 144 # Set the default modules framework-all and core-all as the core 145 # dependency modules. 146 self.project_module_names.update(_CORE_MODULES) 147 148 def _init_source_path(self): 149 """Initialize source_path dictionary.""" 150 self.source_path = { 151 'source_folder_path': set(), 152 'test_folder_path': set(), 153 'jar_path': set(), 154 'jar_module_path': dict(), 155 'r_java_path': set(), 156 'srcjar_path': set() 157 } 158 159 def _search_android_make_files(self): 160 """Search project and dependency modules contain Android.mk files. 161 162 If there is only Android.mk but no Android.bp, we'll show the warning 163 message, otherwise we won't. 164 165 Yields: 166 A string: the relative path of Android.mk. 167 """ 168 if (common_util.exist_android_mk(self.project_absolute_path) and 169 not common_util.exist_android_bp(self.project_absolute_path)): 170 yield '\t' + os.path.join(self.project_relative_path, 171 constant.ANDROID_MK) 172 for mod_name in self.dep_modules: 173 rel_path, abs_path = common_util.get_related_paths( 174 self.modules_info, mod_name) 175 if rel_path and abs_path: 176 if (common_util.exist_android_mk(abs_path) 177 and not common_util.exist_android_bp(abs_path)): 178 yield '\t' + os.path.join(rel_path, constant.ANDROID_MK) 179 180 def _get_modules_under_project_path(self, rel_path): 181 """Find qualified modules under the rel_path. 182 183 Find modules which contain any Java or Kotlin file as a target module. 184 If it's the whole source tree project, add all modules into it. 185 186 Args: 187 rel_path: A string, the project's relative path. 188 189 Returns: 190 A set of module names. 191 """ 192 logging.info('Find modules contain any Java or Kotlin file under %s.', 193 rel_path) 194 if rel_path == '': 195 return self.modules_info.name_to_module_info.keys() 196 modules = set() 197 root_dir = common_util.get_android_root_dir() 198 for name, data in self.modules_info.name_to_module_info.items(): 199 if module_info.AidegenModuleInfo.is_project_path_relative_module( 200 data, rel_path): 201 if common_util.check_java_or_kotlin_file_exists( 202 os.path.join(root_dir, data[constant.KEY_PATH][0])): 203 modules.add(name) 204 else: 205 logging.debug(_NOT_TARGET, name) 206 return modules 207 208 def _get_robolectric_dep_module(self, modules): 209 """Return the robolectric module set as dependency if any module is a 210 robolectric test. 211 212 Args: 213 modules: A set of modules. 214 215 Returns: 216 A set with a robolectric_all module name if one of the modules 217 needs the robolectric test module. Otherwise return empty list. 218 """ 219 for module in modules: 220 if self.modules_info.is_robolectric_test(module): 221 return {_ROBOLECTRIC_MODULE} 222 return set() 223 224 def _filter_out_modules(self): 225 """Filter out unnecessary modules.""" 226 for module in _EXCLUDE_MODULES: 227 self.dep_modules.pop(module, None) 228 229 def get_dep_modules(self, module_names=None, depth=0): 230 """Recursively find dependent modules of the project. 231 232 Find dependent modules by dependencies parameter of each module. 233 For example: 234 The module_names is ['m1']. 235 The modules_info is 236 { 237 'm1': {'dependencies': ['m2'], 'path': ['path_to_m1']}, 238 'm2': {'path': ['path_to_m4']}, 239 'm3': {'path': ['path_to_m1']} 240 'm4': {'path': []} 241 } 242 The result dependent modules are: 243 { 244 'm1': {'dependencies': ['m2'], 'path': ['path_to_m1'] 245 'depth': 0}, 246 'm2': {'path': ['path_to_m4'], 'depth': 1}, 247 'm3': {'path': ['path_to_m1'], 'depth': 0} 248 } 249 Note that: 250 1. m4 is not in the result as it's not among dependent modules. 251 2. m3 is in the result as it has the same path to m1. 252 253 Args: 254 module_names: A set of module names. 255 depth: An integer shows the depth of module dependency referenced by 256 source. Zero means the max module depth. 257 258 Returns: 259 deps: A dict contains all dependent modules data of given modules. 260 """ 261 dep = {} 262 children = set() 263 if not module_names: 264 module_names = self.project_module_names 265 module_names.update( 266 self._get_modules_under_project_path( 267 self.project_relative_path)) 268 module_names.update(self._get_robolectric_dep_module(module_names)) 269 self.project_module_names = set() 270 for name in module_names: 271 if (name in self.modules_info.name_to_module_info 272 and name not in self.project_module_names): 273 dep[name] = self.modules_info.name_to_module_info[name] 274 dep[name][constant.KEY_DEPTH] = depth 275 self.project_module_names.add(name) 276 if (constant.KEY_DEPENDENCIES in dep[name] 277 and dep[name][constant.KEY_DEPENDENCIES]): 278 children.update(dep[name][constant.KEY_DEPENDENCIES]) 279 if children: 280 dep.update(self.get_dep_modules(children, depth + 1)) 281 return dep 282 283 @staticmethod 284 def generate_projects(targets): 285 """Generate a list of projects in one time by a list of module names. 286 287 Args: 288 targets: A list of target modules or project paths from user input, 289 when locating the target, project with matched module name 290 of the target has a higher priority than project path. 291 292 Returns: 293 List: A list of ProjectInfo instances. 294 """ 295 return [ProjectInfo(target, i == 0) for i, target in enumerate(targets)] 296 297 @staticmethod 298 def get_target_name(target, abs_path): 299 """Get target name from target's absolute path. 300 301 If the project is for entire Android source tree, change the target to 302 source tree's root folder name. In this way, we give IDE project file 303 a more specific name. e.g, master.iml. 304 305 Args: 306 target: Includes target module or project path from user input, when 307 locating the target, project with matching module name of 308 the given target has a higher priority than project path. 309 abs_path: A string, target's absolute path. 310 311 Returns: 312 A string, the target name. 313 """ 314 if abs_path == common_util.get_android_root_dir(): 315 return os.path.basename(abs_path) 316 return target 317 318 def locate_source(self, build=True): 319 """Locate the paths of dependent source folders and jar files. 320 321 Try to reference source folder path as dependent module unless the 322 dependent module should be referenced to a jar file, such as modules 323 have jars and jarjar_rules parameter. 324 For example: 325 Module: asm-6.0 326 java_import { 327 name: 'asm-6.0', 328 host_supported: true, 329 jars: ['asm-6.0.jar'], 330 } 331 Module: bouncycastle 332 java_library { 333 name: 'bouncycastle', 334 ... 335 target: { 336 android: { 337 jarjar_rules: 'jarjar-rules.txt', 338 }, 339 }, 340 } 341 342 Args: 343 build: A boolean default to true. If false, skip building jar and 344 srcjar files, otherwise build them. 345 346 Example usage: 347 project.source_path = project.locate_source() 348 E.g. 349 project.source_path = { 350 'source_folder_path': ['path/to/source/folder1', 351 'path/to/source/folder2', ...], 352 'test_folder_path': ['path/to/test/folder', ...], 353 'jar_path': ['path/to/jar/file1', 'path/to/jar/file2', ...] 354 } 355 """ 356 if not hasattr(self, 'dep_modules') or not self.dep_modules: 357 raise errors.EmptyModuleDependencyError( 358 'Dependent modules dictionary is empty.') 359 rebuild_targets = set() 360 for module_name, module_data in self.dep_modules.items(): 361 module = self._generate_moduledata(module_name, module_data) 362 module.locate_sources_path() 363 self.source_path['source_folder_path'].update(set(module.src_dirs)) 364 self.source_path['test_folder_path'].update(set(module.test_dirs)) 365 self.source_path['r_java_path'].update(set(module.r_java_paths)) 366 self.source_path['srcjar_path'].update(set(module.srcjar_paths)) 367 self._append_jars_as_dependencies(module) 368 rebuild_targets.update(module.build_targets) 369 config = project_config.ProjectConfig.get_instance() 370 if config.is_skip_build: 371 return 372 if rebuild_targets: 373 if build: 374 logging.info('\nThe batch_build_dependencies function is ' 375 'called by ProjectInfo\'s locate_source method.') 376 batch_build_dependencies(rebuild_targets) 377 self.locate_source(build=False) 378 else: 379 logging.warning('Jar or srcjar files build skipped:\n\t%s.', 380 '\n\t'.join(rebuild_targets)) 381 382 def _generate_moduledata(self, module_name, module_data): 383 """Generate a module class to collect dependencies in IDE. 384 385 The rules of initialize a module data instance: if ide_object isn't None 386 and its ide_name is 'eclipse', we'll create an EclipseModuleData 387 instance otherwise create a ModuleData instance. 388 389 Args: 390 module_name: Name of the module. 391 module_data: A dictionary holding a module information. 392 393 Returns: 394 A ModuleData class. 395 """ 396 ide_name = project_config.ProjectConfig.get_instance().ide_name 397 if ide_name == constant.IDE_ECLIPSE: 398 return source_locator.EclipseModuleData( 399 module_name, module_data, self.project_relative_path) 400 depth = project_config.ProjectConfig.get_instance().depth 401 return source_locator.ModuleData(module_name, module_data, depth) 402 403 def _append_jars_as_dependencies(self, module): 404 """Add given module's jar files into dependent_data as dependencies. 405 406 Args: 407 module: A ModuleData instance. 408 """ 409 if module.jar_files: 410 self.source_path['jar_path'].update(module.jar_files) 411 for jar in list(module.jar_files): 412 self.source_path['jar_module_path'].update({ 413 jar: 414 module.module_path 415 }) 416 # Collecting the jar files of default core modules as dependencies. 417 if constant.KEY_DEPENDENCIES in module.module_data: 418 self.source_path['jar_path'].update([ 419 x for x in module.module_data[constant.KEY_DEPENDENCIES] 420 if common_util.is_target(x, constant.TARGET_LIBS) 421 ]) 422 423 def _get_rel_project_out_soong_jar_path(self): 424 """Gets the projects' jar path in 'out/soong/.intermediates' folder. 425 426 Gets the relative project's jar path in the 'out/soong/.intermediates' 427 directory. For example, if the self.project_relative_path is 428 'frameworks/base', the returned value should be 429 'out/soong/.intermediates/frameworks/base/'. 430 431 Returns: 432 A string of relative project path in out/soong/.intermediates/ 433 directory, e.g. 'out/soong/.intermediates/frameworks/base/'. 434 """ 435 rdir = os.path.relpath(common_util.get_soong_out_path(), 436 common_util.get_android_root_dir()) 437 return os.sep.join( 438 [rdir, constant.INTERMEDIATES, self.project_relative_path]) + os.sep 439 440 @classmethod 441 def multi_projects_locate_source(cls, projects): 442 """Locate the paths of dependent source folders and jar files. 443 444 Args: 445 projects: A list of ProjectInfo instances. Information of a project 446 such as project relative path, project real path, project 447 dependencies. 448 """ 449 cls.projects = projects 450 for project in projects: 451 project.locate_source() 452 _update_iml_dep_modules(project) 453 454 455class MultiProjectsInfo(ProjectInfo): 456 """Multiple projects info. 457 458 Usage example: 459 if folder_base: 460 project = MultiProjectsInfo(['module_name']) 461 project.collect_all_dep_modules() 462 project.gen_folder_base_dependencies() 463 else: 464 ProjectInfo.generate_projects(['module_name']) 465 466 Attributes: 467 _targets: A list of module names or project paths. 468 path_to_sources: A dictionary of modules' sources, the module's path 469 as key and the sources as value. 470 e.g. 471 { 472 'frameworks/base': { 473 'src_dirs': [], 474 'test_dirs': [], 475 'r_java_paths': [], 476 'srcjar_paths': [], 477 'jar_files': [], 478 'dep_paths': [], 479 } 480 } 481 """ 482 483 def __init__(self, targets=None): 484 """MultiProjectsInfo initialize. 485 486 Args: 487 targets: A list of module names or project paths from user's input. 488 """ 489 super().__init__(targets[0], True) 490 self._targets = targets 491 self.path_to_sources = {} 492 493 def _clear_srcjar_paths(self, module): 494 """Clears the srcjar_paths. 495 496 Args: 497 module: A ModuleData instance. 498 """ 499 module.srcjar_paths = [] 500 501 def _collect_framework_srcjar_info(self, module): 502 """Clears the framework's srcjars. 503 504 Args: 505 module: A ModuleData instance. 506 """ 507 if module.module_path == constant.FRAMEWORK_PATH: 508 framework_srcjar_path = os.path.join(constant.FRAMEWORK_PATH, 509 constant.FRAMEWORK_SRCJARS) 510 if module.module_name == constant.FRAMEWORK_ALL: 511 self.path_to_sources[framework_srcjar_path] = { 512 'src_dirs': [], 513 'test_dirs': [], 514 'r_java_paths': [], 515 'srcjar_paths': module.srcjar_paths, 516 'jar_files': [], 517 'dep_paths': [constant.FRAMEWORK_PATH], 518 } 519 # In the folder base case, AIDEGen has to ignore all module's srcjar 520 # files under the frameworks/base except the framework-all. Because 521 # there are too many duplicate srcjars of modules under the 522 # frameworks/base. So that AIDEGen keeps the srcjar files only from 523 # the framework-all module. Other modeuls' srcjar files will be 524 # removed. However, when users choose the module base case, srcjar 525 # files will be collected by the ProjectInfo class, so that the 526 # removing srcjar_paths in this class does not impact the 527 # srcjar_paths collection of modules in the ProjectInfo class. 528 self._clear_srcjar_paths(module) 529 530 def collect_all_dep_modules(self): 531 """Collects all dependency modules for the projects.""" 532 self.project_module_names.clear() 533 module_names = set(_CORE_MODULES) 534 for target in self._targets: 535 relpath, _ = common_util.get_related_paths(self.modules_info, 536 target) 537 module_names.update(self._get_modules_under_project_path(relpath)) 538 module_names.update(self._get_robolectric_dep_module(module_names)) 539 self.dep_modules = self.get_dep_modules(module_names) 540 541 def gen_folder_base_dependencies(self, module): 542 """Generates the folder base dependencies dictionary. 543 544 Args: 545 module: A ModuleData instance. 546 """ 547 mod_path = module.module_path 548 if not mod_path: 549 logging.debug('The %s\'s path is empty.', module.module_name) 550 return 551 self._collect_framework_srcjar_info(module) 552 if mod_path not in self.path_to_sources: 553 self.path_to_sources[mod_path] = { 554 'src_dirs': module.src_dirs, 555 'test_dirs': module.test_dirs, 556 'r_java_paths': module.r_java_paths, 557 'srcjar_paths': module.srcjar_paths, 558 'jar_files': module.jar_files, 559 'dep_paths': module.dep_paths, 560 } 561 else: 562 for key, val in self.path_to_sources[mod_path].items(): 563 val.extend([v for v in getattr(module, key) if v not in val]) 564 565 566def batch_build_dependencies(rebuild_targets): 567 """Batch build the jar or srcjar files of the modules if they don't exist. 568 569 Command line has the max length limit, MAX_ARG_STRLEN, and 570 MAX_ARG_STRLEN = (PAGE_SIZE * 32). 571 If the build command is longer than MAX_ARG_STRLEN, this function will 572 separate the rebuild_targets into chunks with size less or equal to 573 MAX_ARG_STRLEN to make sure it can be built successfully. 574 575 Args: 576 rebuild_targets: A set of jar or srcjar files which do not exist. 577 """ 578 start_time = time.time() 579 logging.info('Ready to build the jar or srcjar files. Files count = %s', 580 str(len(rebuild_targets))) 581 arg_max = os.sysconf('SC_PAGE_SIZE') * 32 - _CMD_LENGTH_BUFFER 582 rebuild_targets = list(rebuild_targets) 583 for start, end in iter(_separate_build_targets(rebuild_targets, arg_max)): 584 _build_target(rebuild_targets[start:end]) 585 duration = time.time() - start_time 586 logging.debug('Build Time, duration = %s', str(duration)) 587 aidegen_metrics.performance_metrics(constant.TYPE_AIDEGEN_BUILD_TIME, 588 duration) 589 590 591def _build_target(targets): 592 """Build the jar or srcjar files. 593 594 Use -k to keep going when some targets can't be built or build failed. 595 Use -j to speed up building. 596 597 Args: 598 targets: A list of jar or srcjar files which need to build. 599 """ 600 build_cmd = ['-k', '-j'] 601 build_cmd.extend(list(targets)) 602 verbose = True 603 if not atest_utils.build(build_cmd, verbose): 604 message = ('Build failed!\n{}\nAIDEGen will proceed but dependency ' 605 'correctness is not guaranteed if not all targets being ' 606 'built successfully.'.format('\n'.join(targets))) 607 print('\n{} {}\n'.format(common_util.COLORED_INFO('Warning:'), message)) 608 609 610def _separate_build_targets(build_targets, max_length): 611 """Separate the build_targets by limit the command size to max command 612 length. 613 614 Args: 615 build_targets: A list to be separated. 616 max_length: The max number of each build command length. 617 618 Yields: 619 The start index and end index of build_targets. 620 """ 621 arg_len = 0 622 first_item_index = 0 623 for i, item in enumerate(build_targets): 624 arg_len = arg_len + len(item) + _BLANK_SIZE 625 if arg_len > max_length: 626 yield first_item_index, i 627 first_item_index = i 628 arg_len = len(item) + _BLANK_SIZE 629 if first_item_index < len(build_targets): 630 yield first_item_index, len(build_targets) 631 632 633def _update_iml_dep_modules(project): 634 """Gets the dependent modules in the project's iml file. 635 636 The jar files which have the same source codes as cls.projects' source files 637 should be removed from the dependencies.iml file's jar paths. The codes are 638 written in aidegen.project.project_splitter.py. 639 We should also add the jar project's unique iml name into self.dependencies 640 which later will be written into its own iml project file. If we don't 641 remove these files in dependencies.iml, it will cause the duplicated codes 642 in IDE and raise issues. For example, when users do 'refactor' and rename a 643 class in the IDE, it will search all sources and dependencies' jar paths and 644 lead to the error. 645 """ 646 keys = ('source_folder_path', 'test_folder_path', 'r_java_path', 647 'srcjar_path', 'jar_path') 648 for key in keys: 649 for jar in project.source_path[key]: 650 for prj in ProjectInfo.projects: 651 if prj is project: 652 continue 653 if (prj.rel_out_soong_jar_path in jar and 654 jar.endswith(constant.JAR_EXT)): 655 if prj.iml_name not in project.dependencies: 656 project.dependencies.append(prj.iml_name) 657 break 658