1#!/usr/bin/env python3 2# 3# Copyright 2020 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Separate the sources from multiple projects.""" 18 19import logging 20import os 21import shutil 22 23from pathlib import Path 24 25from aidegen import constant 26from aidegen.idea import iml 27from aidegen.lib import common_util 28from aidegen.lib import project_config 29 30_KEY_SOURCE_PATH = 'source_folder_path' 31_KEY_TEST_PATH = 'test_folder_path' 32_SOURCE_FOLDERS = [_KEY_SOURCE_PATH, _KEY_TEST_PATH] 33_KEY_SRCJAR_PATH = 'srcjar_path' 34_KEY_R_PATH = 'r_java_path' 35_KEY_JAR_PATH = 'jar_path' 36_EXCLUDE_ITEM = '\n <excludeFolder url="file://%s" />' 37# Temporarily exclude test-dump and src_stub folders to prevent symbols from 38# resolving failure by incorrect reference. These two folders should be removed 39# after b/136982078 is resolved. 40_EXCLUDE_FOLDERS = ['.idea', '.repo', 'art', 'bionic', 'bootable', 'build', 41 'dalvik', 'developers', 'device', 'hardware', 'kernel', 42 'libnativehelper', 'pdk', 'prebuilts', 'sdk', 'system', 43 'toolchain', 'tools', 'vendor', 'out', 'external', 44 'art/tools/ahat/src/test-dump', 45 'cts/common/device-side/device-info/src_stub', 46 'external/gson/gson/src/main/java' 47 ] 48_PERMISSION_DEFINED_PATH = ('frameworks/base/core/res/framework-res/' 49 'android_common/gen/') 50_ANDROID = 'android' 51_R = 'R' 52 53 54class ProjectSplitter: 55 """Splits the sources from multiple projects. 56 57 It's a specific solution to deal with the source folders in multiple 58 project case. Since the IntelliJ does not allow duplicate source folders, 59 AIDEGen needs to separate the source folders for each project. The single 60 project case has no different with current structure. 61 62 Usage: 63 project_splitter = ProjectSplitter(projects) 64 65 # Find the dependencies between the projects. 66 project_splitter.get_dependencies() 67 68 # Clear the source folders for each project. 69 project_splitter.revise_source_folders() 70 71 Attributes: 72 _projects: A list of ProjectInfo. 73 _all_srcs: A dictionary contains all sources of multiple projects. 74 e.g. 75 { 76 'module_name': 'test', 77 'path': ['path/to/module'], 78 'srcs': ['src_folder1', 'src_folder2'], 79 'tests': ['test_folder1', 'test_folder2'] 80 'jars': ['jar1.jar'], 81 'srcjars': ['1.srcjar', '2.srcjar'], 82 'dependencies': ['framework_srcjars', 'base'], 83 'iml_name': '/abs/path/to/iml.iml' 84 } 85 _framework_exist: A boolean, True if framework is one of the projects. 86 _framework_iml: A string, the name of the framework's iml. 87 _full_repo: A boolean, True if loading with full Android sources. 88 _full_repo_iml: A string, the name of the Android folder's iml. 89 _permission_r_srcjar: A string, the absolute path of R.srcjar file where 90 the permission relative constants are defined. 91 _permission_aapt2: A string, the absolute path of aapt2/R directory 92 where the permission relative constants are defined. 93 """ 94 def __init__(self, projects): 95 """ProjectSplitter initialize. 96 97 Args: 98 projects: A list of ProjectInfo object. 99 """ 100 self._projects = projects 101 self._all_srcs = dict(projects[0].source_path) 102 self._framework_iml = None 103 self._framework_exist = any( 104 {p.project_relative_path == constant.FRAMEWORK_PATH 105 for p in self._projects}) 106 if self._framework_exist: 107 self._framework_iml = iml.IMLGenerator.get_unique_iml_name( 108 os.path.join(common_util.get_android_root_dir(), 109 constant.FRAMEWORK_PATH)) 110 self._full_repo = project_config.ProjectConfig.get_instance().full_repo 111 if self._full_repo: 112 self._full_repo_iml = os.path.basename( 113 common_util.get_android_root_dir()) 114 self._permission_r_srcjar = _get_permission_r_srcjar_rel_path() 115 self._permission_aapt2 = _get_permission_aapt2_rel_path() 116 117 def revise_source_folders(self): 118 """Resets the source folders of each project. 119 120 There should be no duplicate source root path in IntelliJ. The issue 121 doesn't happen in single project case. Once users choose multiple 122 projects, there could be several same source paths of different 123 projects. In order to prevent that, we should remove the source paths 124 in dependencies.iml which are duplicate with the paths in [module].iml 125 files. 126 127 Steps to prevent the duplicate source root path in IntelliJ: 128 1. Copy all sources from sub-projects to main project. 129 2. Delete the source and test folders which are not under the 130 sub-projects. 131 3. Delete the sub-projects' source and test paths from the main project. 132 """ 133 self._collect_all_srcs() 134 self._keep_local_sources() 135 self._remove_duplicate_sources() 136 137 def _collect_all_srcs(self): 138 """Copies all projects' sources to a dictionary.""" 139 for project in self._projects[1:]: 140 for key, value in project.source_path.items(): 141 self._all_srcs[key].update(value) 142 143 def _keep_local_sources(self): 144 """Removes source folders which are not under the project's path. 145 146 1. Remove the source folders which are not under the project. 147 2. Remove the duplicate project's source folders from the _all_srcs. 148 """ 149 for project in self._projects: 150 srcs = project.source_path 151 relpath = project.project_relative_path 152 is_root = not relpath 153 for key in _SOURCE_FOLDERS: 154 srcs[key] = {s for s in srcs[key] 155 if common_util.is_source_under_relative_path( 156 s, relpath) or is_root} 157 self._all_srcs[key] -= srcs[key] 158 159 def _remove_duplicate_sources(self): 160 """Removes the duplicate source folders from each sub-project. 161 162 Priority processing with the longest path length, e.g. 163 frameworks/base/packages/SettingsLib must have priority over 164 frameworks/base. 165 (b/160303006): Remove the parent project's source and test paths under 166 the child's project path. 167 """ 168 root = common_util.get_android_root_dir() 169 projects = sorted(self._projects, key=lambda k: len( 170 k.project_relative_path), reverse=True) 171 for child in projects: 172 for parent in self._projects: 173 is_root = not parent.project_relative_path 174 if parent is child: 175 continue 176 if (common_util.is_source_under_relative_path( 177 child.project_relative_path, 178 parent.project_relative_path) or is_root): 179 for key in _SOURCE_FOLDERS: 180 parent.source_path[key] -= child.source_path[key] 181 rm_paths = _remove_child_duplicate_sources_from_parent( 182 child, parent.source_path[key], root) 183 parent.source_path[key] -= rm_paths 184 185 def get_dependencies(self): 186 """Gets the dependencies between the projects. 187 188 Check if the current project's source folder exists in other projects. 189 If so, the current project is a dependency module to the other. 190 """ 191 projects = sorted(self._projects, key=lambda k: len( 192 k.project_relative_path)) 193 for project in projects: 194 proj_path = project.project_relative_path 195 project.dependencies = [constant.FRAMEWORK_SRCJARS] 196 if self._framework_exist and proj_path != constant.FRAMEWORK_PATH: 197 project.dependencies.append(self._framework_iml) 198 if self._full_repo and proj_path: 199 project.dependencies.append(self._full_repo_iml) 200 srcs = (project.source_path[_KEY_SOURCE_PATH] 201 | project.source_path[_KEY_TEST_PATH]) 202 dep_projects = sorted(self._projects, key=lambda k: len( 203 k.project_relative_path)) 204 for dep_proj in dep_projects: 205 dep_path = dep_proj.project_relative_path 206 is_root = not dep_path 207 is_child = common_util.is_source_under_relative_path(dep_path, 208 proj_path) 209 is_dep = any({s for s in srcs 210 if common_util.is_source_under_relative_path( 211 s, dep_path) or is_root}) 212 if dep_proj is project or is_child or not is_dep: 213 continue 214 dep = iml.IMLGenerator.get_unique_iml_name(os.path.join( 215 common_util.get_android_root_dir(), dep_path)) 216 if dep not in project.dependencies: 217 project.dependencies.append(dep) 218 project.dependencies.append(constant.KEY_DEPENDENCIES) 219 220 def gen_framework_srcjars_iml(self): 221 """Generates the framework_srcjars.iml. 222 223 Create the iml file with only the srcjars of module framework-all. These 224 srcjars will be separated from the modules under frameworks/base. 225 226 Returns: 227 A string of the framework_srcjars.iml's absolute path. 228 """ 229 self._remove_permission_definition_srcjar_path() 230 mod = dict(self._projects[0].dep_modules[constant.FRAMEWORK_ALL]) 231 mod[constant.KEY_DEPENDENCIES] = [] 232 mod[constant.KEY_IML_NAME] = constant.FRAMEWORK_SRCJARS 233 if self._framework_exist: 234 mod[constant.KEY_DEPENDENCIES].append(self._framework_iml) 235 if self._full_repo: 236 mod[constant.KEY_DEPENDENCIES].append(self._full_repo_iml) 237 mod[constant.KEY_DEPENDENCIES].append(constant.KEY_DEPENDENCIES) 238 permission_src = self._get_permission_defined_source_path() 239 if permission_src: 240 mod[constant.KEY_SRCS] = [permission_src] 241 srcjar_dict = {constant.KEY_DEP_SRCS: True, 242 constant.KEY_SRCJARS: True, 243 constant.KEY_DEPENDENCIES: True} 244 else: 245 logging.warning('The permission definition relative paths are ' 246 'missing.') 247 srcjar_dict = {constant.KEY_SRCJARS: True, 248 constant.KEY_DEPENDENCIES: True} 249 framework_srcjars_iml = iml.IMLGenerator(mod) 250 framework_srcjars_iml.create(srcjar_dict) 251 self._all_srcs[_KEY_SRCJAR_PATH] -= set(mod.get(constant.KEY_SRCJARS, 252 [])) 253 return framework_srcjars_iml.iml_path 254 255 def _get_permission_defined_source_path(self): 256 """Gets the source path where permission relative constants are defined. 257 258 For the definition permission constants, the priority is, 259 1) If framework-res/android_common/gen/aapt2/R directory exists, return 260 it. 261 2) If the framework-res/android_common/gen/android/R.srcjar file exists, 262 unzip it to 'aidegen_r.srcjar' folder and return the path. 263 264 Returns: 265 A string of the path of aapt2/R or android/aidegen_r.srcjar folder, 266 else None. 267 """ 268 if os.path.isdir(self._permission_aapt2): 269 return self._permission_aapt2 270 if os.path.isfile(self._permission_r_srcjar): 271 dest = os.path.join( 272 os.path.dirname(self._permission_r_srcjar), 273 ''.join([constant.UNZIP_SRCJAR_PATH_HEAD, 274 os.path.basename(self._permission_r_srcjar).lower()])) 275 if os.path.isdir(dest): 276 shutil.rmtree(dest) 277 common_util.unzip_file(self._permission_r_srcjar, dest) 278 return dest 279 return None 280 281 def _gen_dependencies_iml(self): 282 """Generates the dependencies.iml.""" 283 rel_project_soong_paths = self._get_rel_project_soong_paths() 284 self._unzip_all_scrjars() 285 mod = { 286 constant.KEY_SRCS: _get_real_dependencies_jars( 287 rel_project_soong_paths, self._all_srcs[_KEY_SOURCE_PATH]), 288 constant.KEY_TESTS: _get_real_dependencies_jars( 289 rel_project_soong_paths, self._all_srcs[_KEY_TEST_PATH]), 290 constant.KEY_JARS: _get_real_dependencies_jars( 291 rel_project_soong_paths, self._all_srcs[_KEY_JAR_PATH]), 292 constant.KEY_SRCJARS: _get_real_dependencies_jars( 293 rel_project_soong_paths, 294 self._all_srcs[_KEY_R_PATH] | self._all_srcs[_KEY_SRCJAR_PATH]), 295 constant.KEY_DEPENDENCIES: _get_real_dependencies_jars( 296 rel_project_soong_paths, [constant.FRAMEWORK_SRCJARS]), 297 constant.KEY_PATH: [self._projects[0].project_relative_path], 298 constant.KEY_MODULE_NAME: constant.KEY_DEPENDENCIES, 299 constant.KEY_IML_NAME: constant.KEY_DEPENDENCIES 300 } 301 if self._framework_exist: 302 mod[constant.KEY_DEPENDENCIES].append(self._framework_iml) 303 if self._full_repo: 304 mod[constant.KEY_DEPENDENCIES].append(self._full_repo_iml) 305 dep_iml = iml.IMLGenerator(mod) 306 dep_iml.create({constant.KEY_DEP_SRCS: True, 307 constant.KEY_SRCJARS: True, 308 constant.KEY_JARS: True, 309 constant.KEY_DEPENDENCIES: True}) 310 311 def _unzip_all_scrjars(self): 312 """Unzips all scrjar files to a specific folder 'aidegen_r.srcjar'. 313 314 For some versions of IntelliJ no more supports unzipping srcjar files 315 automatically, we have to unzip it to a 'aidegen_r.srcjar' directory. 316 The rules of the unzip process are, 317 1) If it's a aapt2/R type jar or other directory type sources, add them 318 into self._all_srcs[_KEY_SOURCE_PATH]. 319 2) If it's an R.srcjar file, check if the same path of aapt2/R directory 320 exists if so add aapt2/R path into the 321 self._all_srcs[_KEY_SOURCE_PATH], otherwise unzip R.srcjar into 322 the 'aidegen_r.srcjar' directory and add the unzipped path into 323 self._all_srcs[_KEY_SOURCE_PATH]. 324 """ 325 sjars = self._all_srcs[_KEY_R_PATH] | self._all_srcs[_KEY_SRCJAR_PATH] 326 self._all_srcs[_KEY_R_PATH] = set() 327 self._all_srcs[_KEY_SRCJAR_PATH] = set() 328 for sjar in sjars: 329 if not os.path.exists(sjar): 330 continue 331 if os.path.isdir(sjar): 332 self._all_srcs[_KEY_SOURCE_PATH].add(sjar) 333 continue 334 sjar_dir = os.path.dirname(sjar) 335 sjar_name = os.path.basename(sjar).lower() 336 aapt2 = os.path.join( 337 os.path.dirname(sjar_dir), constant.NAME_AAPT2, _R) 338 if os.path.isdir(aapt2): 339 self._all_srcs[_KEY_SOURCE_PATH].add(aapt2) 340 continue 341 dest = os.path.join( 342 sjar_dir, ''.join([constant.UNZIP_SRCJAR_PATH_HEAD, sjar_name])) 343 if os.path.isdir(dest): 344 shutil.rmtree(dest) 345 common_util.unzip_file(sjar, dest) 346 self._all_srcs[_KEY_SOURCE_PATH].add(dest) 347 348 def gen_projects_iml(self): 349 """Generates the projects' iml file.""" 350 root_path = common_util.get_android_root_dir() 351 excludes = project_config.ProjectConfig.get_instance().exclude_paths 352 for project in self._projects: 353 relpath = project.project_relative_path 354 exclude_folders = [] 355 if not relpath: 356 exclude_folders.extend(get_exclude_content(root_path)) 357 if excludes: 358 exclude_folders.extend(get_exclude_content(root_path, excludes)) 359 mod_info = { 360 constant.KEY_EXCLUDES: ''.join(exclude_folders), 361 constant.KEY_SRCS: project.source_path[_KEY_SOURCE_PATH], 362 constant.KEY_TESTS: project.source_path[_KEY_TEST_PATH], 363 constant.KEY_DEPENDENCIES: project.dependencies, 364 constant.KEY_PATH: [relpath], 365 constant.KEY_MODULE_NAME: project.module_name, 366 constant.KEY_IML_NAME: iml.IMLGenerator.get_unique_iml_name( 367 str(Path(root_path, relpath))) 368 } 369 dep_iml = iml.IMLGenerator(mod_info) 370 dep_iml.create({constant.KEY_SRCS: True, 371 constant.KEY_DEPENDENCIES: True}) 372 project.iml_path = dep_iml.iml_path 373 self._gen_dependencies_iml() 374 375 def _get_rel_project_soong_paths(self): 376 """Gets relative projects' paths in 'out/soong/.intermediates' folder. 377 378 Gets relative projects' paths in the 'out/soong/.intermediates' 379 directory. For example, if the self.projects = ['frameworks/base'] the 380 returned list should be ['out/soong/.intermediates/frameworks/base/']. 381 382 Returns: 383 A list of relative projects' paths in out/soong/.intermediates. 384 """ 385 out_soong_dir = os.path.relpath(common_util.get_soong_out_path(), 386 common_util.get_android_root_dir()) 387 rel_project_soong_paths = [] 388 for project in self._projects: 389 relpath = project.project_relative_path 390 rel_project_soong_paths.append(os.sep.join( 391 [out_soong_dir, constant.INTERMEDIATES, relpath]) + os.sep) 392 return rel_project_soong_paths 393 394 def _remove_permission_definition_srcjar_path(self): 395 """Removes android.Manifest.permission definition srcjar path. 396 397 If framework-res/android_common/gen/aapt2/R directory or 398 framework-res/android_common/gen/android/R.srcjar file exists in 399 self._all_srcs[_KEY_SRCJAR_PATH], remove them. 400 """ 401 if self._permission_aapt2 in self._all_srcs[_KEY_SRCJAR_PATH]: 402 self._all_srcs[_KEY_SRCJAR_PATH].remove(self._permission_aapt2) 403 if self._permission_r_srcjar in self._all_srcs[_KEY_SRCJAR_PATH]: 404 self._all_srcs[_KEY_SRCJAR_PATH].remove(self._permission_r_srcjar) 405 406 407def _get_real_dependencies_jars(list_to_check, list_to_be_checked): 408 """Gets real dependencies' jar from the input list. 409 410 There are jar files which have the same source codes as the 411 self.projects should be removed from dependencies. Otherwise these files 412 will cause the duplicated codes in IDE and lead to issues: b/158583214 is an 413 example. 414 415 Args: 416 list_to_check: A list of relative projects' paths in the folder 417 out/soong/.intermediates to be checked if are contained 418 in the list_to_be_checked list. 419 list_to_be_checked: A list of dependencies' paths to be checked. 420 421 Returns: 422 A list of dependency jar paths after duplicated ones removed. 423 """ 424 file_exts = [constant.JAR_EXT] 425 real_jars = list_to_be_checked.copy() 426 for jar in list_to_be_checked: 427 ext = os.path.splitext(jar)[-1] 428 for check_path in list_to_check: 429 if check_path in jar and ext in file_exts: 430 real_jars.remove(jar) 431 break 432 return real_jars 433 434 435def get_exclude_content(root_path, excludes=None): 436 """Get the exclude folder content list. 437 438 It returns the exclude folders content list. 439 e.g. 440 ['<excludeFolder url="file://a/.idea" />', 441 '<excludeFolder url="file://a/.repo" />'] 442 443 Args: 444 root_path: Android source file path. 445 excludes: A list of exclusive directories, the default value is None but 446 will be assigned to _EXCLUDE_FOLDERS. 447 448 Returns: 449 String: exclude folder content list. 450 """ 451 exclude_items = [] 452 if not excludes: 453 excludes = _EXCLUDE_FOLDERS 454 for folder in excludes: 455 folder_path = os.path.join(root_path, folder) 456 if os.path.isdir(folder_path): 457 exclude_items.append(_EXCLUDE_ITEM % folder_path) 458 return exclude_items 459 460 461def _remove_child_duplicate_sources_from_parent(child, parent_sources, root): 462 """Removes the child's duplicate source folders from the parent source list. 463 464 Remove all the child's subdirectories from the parent's source list if there 465 is any. 466 467 Args: 468 child: A child project of ProjectInfo instance. 469 parent_sources: The parent project sources of the ProjectInfo instance. 470 root: A string of the Android root. 471 472 Returns: 473 A set of the sources to be removed. 474 """ 475 rm_paths = set() 476 for path in parent_sources: 477 if (common_util.is_source_under_relative_path( 478 os.path.relpath(path, root), child.project_relative_path)): 479 rm_paths.add(path) 480 return rm_paths 481 482 483def _get_permission_aapt2_rel_path(): 484 """Gets android.Manifest.permission definition srcjar path.""" 485 out_soong_dir = os.path.relpath(common_util.get_soong_out_path(), 486 common_util.get_android_root_dir()) 487 return os.path.join(out_soong_dir, constant.INTERMEDIATES, 488 _PERMISSION_DEFINED_PATH, constant.NAME_AAPT2, _R) 489 490 491def _get_permission_r_srcjar_rel_path(): 492 """Gets android.Manifest.permission definition srcjar path.""" 493 out_soong_dir = os.path.relpath(common_util.get_soong_out_path(), 494 common_util.get_android_root_dir()) 495 return os.path.join(out_soong_dir, constant.INTERMEDIATES, 496 _PERMISSION_DEFINED_PATH, _ANDROID, 497 constant.TARGET_R_SRCJAR) 498