1#!/usr/bin/env python3 2# 3# Copyright 2020 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Separate the sources from multiple projects.""" 18 19import logging 20import os 21import shutil 22 23from aidegen import constant 24from aidegen.idea import iml 25from aidegen.lib import common_util 26from aidegen.lib import project_config 27 28_KEY_SOURCE_PATH = 'source_folder_path' 29_KEY_TEST_PATH = 'test_folder_path' 30_SOURCE_FOLDERS = [_KEY_SOURCE_PATH, _KEY_TEST_PATH] 31_KEY_SRCJAR_PATH = 'srcjar_path' 32_KEY_R_PATH = 'r_java_path' 33_KEY_JAR_PATH = 'jar_path' 34_EXCLUDE_ITEM = '\n <excludeFolder url="file://%s" />' 35# Temporarily exclude test-dump and src_stub folders to prevent symbols from 36# resolving failure by incorrect reference. These two folders should be removed 37# after b/136982078 is resolved. 38_EXCLUDE_FOLDERS = ['.idea', '.repo', 'art', 'bionic', 'bootable', 'build', 39 'dalvik', 'developers', 'device', 'hardware', 'kernel', 40 'libnativehelper', 'pdk', 'prebuilts', 'sdk', 'system', 41 'toolchain', 'tools', 'vendor', 'out', 'external', 42 'art/tools/ahat/src/test-dump', 43 'cts/common/device-side/device-info/src_stub'] 44_PERMISSION_DEFINED_PATH = ('frameworks/base/core/res/framework-res/' 45 'android_common/gen/') 46_ANDROID = 'android' 47_R = 'R' 48 49 50class ProjectSplitter: 51 """Splits the sources from multiple projects. 52 53 It's a specific solution to deal with the source folders in multiple 54 project case. Since the IntelliJ does not allow duplicate source folders, 55 AIDEGen needs to separate the source folders for each project. The single 56 project case has no different with current structure. 57 58 Usage: 59 project_splitter = ProjectSplitter(projects) 60 61 # Find the dependencies between the projects. 62 project_splitter.get_dependencies() 63 64 # Clear the source folders for each project. 65 project_splitter.revise_source_folders() 66 67 Attributes: 68 _projects: A list of ProjectInfo. 69 _all_srcs: A dictionary contains all sources of multiple projects. 70 e.g. 71 { 72 'module_name': 'test', 73 'path': ['path/to/module'], 74 'srcs': ['src_folder1', 'src_folder2'], 75 'tests': ['test_folder1', 'test_folder2'] 76 'jars': ['jar1.jar'], 77 'srcjars': ['1.srcjar', '2.srcjar'], 78 'dependencies': ['framework_srcjars', 'base'], 79 'iml_name': '/abs/path/to/iml.iml' 80 } 81 _framework_exist: A boolean, True if framework is one of the projects. 82 _framework_iml: A string, the name of the framework's iml. 83 _full_repo: A boolean, True if loading with full Android sources. 84 _full_repo_iml: A string, the name of the Android folder's iml. 85 _permission_r_srcjar: A string, the absolute path of R.srcjar file where 86 the permission relative constants are defined. 87 _permission_aapt2: A string, the absolute path of aapt2/R directory 88 where the permission relative constants are defined. 89 """ 90 def __init__(self, projects): 91 """ProjectSplitter initialize. 92 93 Args: 94 projects: A list of ProjectInfo object. 95 """ 96 self._projects = projects 97 self._all_srcs = dict(projects[0].source_path) 98 self._framework_iml = None 99 self._framework_exist = any( 100 {p.project_relative_path == constant.FRAMEWORK_PATH 101 for p in self._projects}) 102 if self._framework_exist: 103 self._framework_iml = iml.IMLGenerator.get_unique_iml_name( 104 os.path.join(common_util.get_android_root_dir(), 105 constant.FRAMEWORK_PATH)) 106 self._full_repo = project_config.ProjectConfig.get_instance().full_repo 107 if self._full_repo: 108 self._full_repo_iml = os.path.basename( 109 common_util.get_android_root_dir()) 110 self._permission_r_srcjar = _get_permission_r_srcjar_rel_path() 111 self._permission_aapt2 = _get_permission_aapt2_rel_path() 112 113 def revise_source_folders(self): 114 """Resets the source folders of each project. 115 116 There should be no duplicate source root path in IntelliJ. The issue 117 doesn't happen in single project case. Once users choose multiple 118 projects, there could be several same source paths of different 119 projects. In order to prevent that, we should remove the source paths 120 in dependencies.iml which are duplicate with the paths in [module].iml 121 files. 122 123 Steps to prevent the duplicate source root path in IntelliJ: 124 1. Copy all sources from sub-projects to main project. 125 2. Delete the source and test folders which are not under the 126 sub-projects. 127 3. Delete the sub-projects' source and test paths from the main project. 128 """ 129 self._collect_all_srcs() 130 self._keep_local_sources() 131 self._remove_duplicate_sources() 132 133 def _collect_all_srcs(self): 134 """Copies all projects' sources to a dictionary.""" 135 for project in self._projects[1:]: 136 for key, value in project.source_path.items(): 137 self._all_srcs[key].update(value) 138 139 def _keep_local_sources(self): 140 """Removes source folders which are not under the project's path. 141 142 1. Remove the source folders which are not under the project. 143 2. Remove the duplicate project's source folders from the _all_srcs. 144 """ 145 for project in self._projects: 146 srcs = project.source_path 147 relpath = project.project_relative_path 148 is_root = not relpath 149 for key in _SOURCE_FOLDERS: 150 srcs[key] = {s for s in srcs[key] 151 if common_util.is_source_under_relative_path( 152 s, relpath) or is_root} 153 self._all_srcs[key] -= srcs[key] 154 155 def _remove_duplicate_sources(self): 156 """Removes the duplicate source folders from each sub project. 157 158 Priority processing with the longest path length, e.g. 159 frameworks/base/packages/SettingsLib must have priority over 160 frameworks/base. 161 (b/160303006): Remove the parent project's source and test paths under 162 the child's project path. 163 """ 164 root = common_util.get_android_root_dir() 165 projects = sorted(self._projects, key=lambda k: len( 166 k.project_relative_path), reverse=True) 167 for child in projects: 168 for parent in self._projects: 169 is_root = not parent.project_relative_path 170 if parent is child: 171 continue 172 if (common_util.is_source_under_relative_path( 173 child.project_relative_path, 174 parent.project_relative_path) or is_root): 175 for key in _SOURCE_FOLDERS: 176 parent.source_path[key] -= child.source_path[key] 177 rm_paths = _remove_child_duplicate_sources_from_parent( 178 child, parent.source_path[key], root) 179 parent.source_path[key] -= rm_paths 180 181 def get_dependencies(self): 182 """Gets the dependencies between the projects. 183 184 Check if the current project's source folder exists in other projects. 185 If do, the current project is a dependency module to the other. 186 """ 187 projects = sorted(self._projects, key=lambda k: len( 188 k.project_relative_path)) 189 for project in projects: 190 proj_path = project.project_relative_path 191 project.dependencies = [constant.FRAMEWORK_SRCJARS] 192 if self._framework_exist and proj_path != constant.FRAMEWORK_PATH: 193 project.dependencies.append(self._framework_iml) 194 if self._full_repo and proj_path: 195 project.dependencies.append(self._full_repo_iml) 196 srcs = (project.source_path[_KEY_SOURCE_PATH] 197 | project.source_path[_KEY_TEST_PATH]) 198 dep_projects = sorted(self._projects, key=lambda k: len( 199 k.project_relative_path)) 200 for dep_proj in dep_projects: 201 dep_path = dep_proj.project_relative_path 202 is_root = not dep_path 203 is_child = common_util.is_source_under_relative_path(dep_path, 204 proj_path) 205 is_dep = any({s for s in srcs 206 if common_util.is_source_under_relative_path( 207 s, dep_path) or is_root}) 208 if dep_proj is project or is_child or not is_dep: 209 continue 210 dep = iml.IMLGenerator.get_unique_iml_name(os.path.join( 211 common_util.get_android_root_dir(), dep_path)) 212 if dep not in project.dependencies: 213 project.dependencies.append(dep) 214 project.dependencies.append(constant.KEY_DEPENDENCIES) 215 216 def gen_framework_srcjars_iml(self): 217 """Generates the framework_srcjars.iml. 218 219 Create the iml file with only the srcjars of module framework-all. These 220 srcjars will be separated from the modules under frameworks/base. 221 222 Returns: 223 A string of the framework_srcjars.iml's absolute path. 224 """ 225 self._remove_permission_definition_srcjar_path() 226 mod = dict(self._projects[0].dep_modules[constant.FRAMEWORK_ALL]) 227 mod[constant.KEY_DEPENDENCIES] = [] 228 mod[constant.KEY_IML_NAME] = constant.FRAMEWORK_SRCJARS 229 if self._framework_exist: 230 mod[constant.KEY_DEPENDENCIES].append(self._framework_iml) 231 if self._full_repo: 232 mod[constant.KEY_DEPENDENCIES].append(self._full_repo_iml) 233 mod[constant.KEY_DEPENDENCIES].append(constant.KEY_DEPENDENCIES) 234 srcjar_dict = dict() 235 permission_src = self._get_permission_defined_source_path() 236 if permission_src: 237 mod[constant.KEY_SRCS] = [permission_src] 238 srcjar_dict = {constant.KEY_DEP_SRCS: True, 239 constant.KEY_SRCJARS: True, 240 constant.KEY_DEPENDENCIES: True} 241 else: 242 logging.warning('The permission definition relative paths are ' 243 'missing.') 244 srcjar_dict = {constant.KEY_SRCJARS: True, 245 constant.KEY_DEPENDENCIES: True} 246 framework_srcjars_iml = iml.IMLGenerator(mod) 247 framework_srcjars_iml.create(srcjar_dict) 248 self._all_srcs[_KEY_SRCJAR_PATH] -= set(mod.get(constant.KEY_SRCJARS, 249 [])) 250 return framework_srcjars_iml.iml_path 251 252 def _get_permission_defined_source_path(self): 253 """Gets the source path where permission relative constants are defined. 254 255 For the definition permission constants, the priority is, 256 1) If framework-res/android_common/gen/aapt2/R directory exists, return 257 it. 258 2) If the framework-res/android_common/gen/android/R.srcjar file exists, 259 unzip it to 'aidegen_r.srcjar' folder and return the path. 260 261 Returns: 262 A string of the path of aapt2/R or android/aidegen_r.srcjar folder, 263 else None. 264 """ 265 if os.path.isdir(self._permission_aapt2): 266 return self._permission_aapt2 267 if os.path.isfile(self._permission_r_srcjar): 268 dest = os.path.join( 269 os.path.dirname(self._permission_r_srcjar), 270 ''.join([constant.UNZIP_SRCJAR_PATH_HEAD, 271 os.path.basename(self._permission_r_srcjar).lower()])) 272 if os.path.isdir(dest): 273 shutil.rmtree(dest) 274 common_util.unzip_file(self._permission_r_srcjar, dest) 275 return dest 276 return None 277 278 def _gen_dependencies_iml(self): 279 """Generates the dependencies.iml.""" 280 rel_project_soong_paths = self._get_rel_project_soong_paths() 281 self._unzip_all_scrjars() 282 mod = { 283 constant.KEY_SRCS: _get_real_dependencies_jars( 284 rel_project_soong_paths, self._all_srcs[_KEY_SOURCE_PATH]), 285 constant.KEY_TESTS: _get_real_dependencies_jars( 286 rel_project_soong_paths, self._all_srcs[_KEY_TEST_PATH]), 287 constant.KEY_JARS: _get_real_dependencies_jars( 288 rel_project_soong_paths, self._all_srcs[_KEY_JAR_PATH]), 289 constant.KEY_SRCJARS: _get_real_dependencies_jars( 290 rel_project_soong_paths, 291 self._all_srcs[_KEY_R_PATH] | self._all_srcs[_KEY_SRCJAR_PATH]), 292 constant.KEY_DEPENDENCIES: _get_real_dependencies_jars( 293 rel_project_soong_paths, [constant.FRAMEWORK_SRCJARS]), 294 constant.KEY_PATH: [self._projects[0].project_relative_path], 295 constant.KEY_MODULE_NAME: constant.KEY_DEPENDENCIES, 296 constant.KEY_IML_NAME: constant.KEY_DEPENDENCIES 297 } 298 if self._framework_exist: 299 mod[constant.KEY_DEPENDENCIES].append(self._framework_iml) 300 if self._full_repo: 301 mod[constant.KEY_DEPENDENCIES].append(self._full_repo_iml) 302 dep_iml = iml.IMLGenerator(mod) 303 dep_iml.create({constant.KEY_DEP_SRCS: True, 304 constant.KEY_SRCJARS: True, 305 constant.KEY_JARS: True, 306 constant.KEY_DEPENDENCIES: True}) 307 308 def _unzip_all_scrjars(self): 309 """Unzips all scrjar files to a specific folder 'aidegen_r.srcjar'. 310 311 For some versions of IntelliJ no more supports unzipping srcjar files 312 automatically, we have to unzip it to a 'aidegen_r.srcjar' directory. 313 The rules of the unzip process are, 314 1) If it's a aapt2/R type jar or other directory type sources, add them 315 into self._all_srcs[_KEY_SOURCE_PATH]. 316 2) If it's an R.srcjar file, check if the same path of aapt2/R directory 317 exists if so add aapt2/R path into into the 318 self._all_srcs[_KEY_SOURCE_PATH], otherwise unzip R.srcjar into 319 the 'aidegen_r.srcjar' directory and add the unzipped path into 320 self._all_srcs[_KEY_SOURCE_PATH]. 321 """ 322 sjars = self._all_srcs[_KEY_R_PATH] | self._all_srcs[_KEY_SRCJAR_PATH] 323 self._all_srcs[_KEY_R_PATH] = set() 324 self._all_srcs[_KEY_SRCJAR_PATH] = set() 325 for sjar in sjars: 326 if not os.path.exists(sjar): 327 continue 328 if os.path.isdir(sjar): 329 self._all_srcs[_KEY_SOURCE_PATH].add(sjar) 330 continue 331 sjar_dir = os.path.dirname(sjar) 332 sjar_name = os.path.basename(sjar).lower() 333 aapt2 = os.path.join( 334 os.path.dirname(sjar_dir), constant.NAME_AAPT2, _R) 335 if os.path.isdir(aapt2): 336 self._all_srcs[_KEY_SOURCE_PATH].add(aapt2) 337 continue 338 dest = os.path.join( 339 sjar_dir, ''.join([constant.UNZIP_SRCJAR_PATH_HEAD, sjar_name])) 340 if os.path.isdir(dest): 341 shutil.rmtree(dest) 342 common_util.unzip_file(sjar, dest) 343 self._all_srcs[_KEY_SOURCE_PATH].add(dest) 344 345 def gen_projects_iml(self): 346 """Generates the projects' iml file.""" 347 root_path = common_util.get_android_root_dir() 348 excludes = project_config.ProjectConfig.get_instance().exclude_paths 349 for project in self._projects: 350 relpath = project.project_relative_path 351 exclude_folders = [] 352 if not relpath: 353 exclude_folders.extend(get_exclude_content(root_path)) 354 if excludes: 355 exclude_folders.extend(get_exclude_content(root_path, excludes)) 356 mod_info = { 357 constant.KEY_EXCLUDES: ''.join(exclude_folders), 358 constant.KEY_SRCS: project.source_path[_KEY_SOURCE_PATH], 359 constant.KEY_TESTS: project.source_path[_KEY_TEST_PATH], 360 constant.KEY_DEPENDENCIES: project.dependencies, 361 constant.KEY_PATH: [relpath], 362 constant.KEY_MODULE_NAME: project.module_name, 363 constant.KEY_IML_NAME: iml.IMLGenerator.get_unique_iml_name( 364 os.path.join(root_path, relpath)) 365 } 366 dep_iml = iml.IMLGenerator(mod_info) 367 dep_iml.create({constant.KEY_SRCS: True, 368 constant.KEY_DEPENDENCIES: True}) 369 project.iml_path = dep_iml.iml_path 370 self._gen_dependencies_iml() 371 372 def _get_rel_project_soong_paths(self): 373 """Gets relative projects' paths in 'out/soong/.intermediates' folder. 374 375 Gets relative projects' paths in the 'out/soong/.intermediates' 376 directory. For example, if the self.projects = ['frameworks/base'] the 377 returned list should be ['out/soong/.intermediates/frameworks/base/']. 378 379 Returns: 380 A list of relative projects' paths in out/soong/.intermediates. 381 """ 382 out_soong_dir = os.path.relpath(common_util.get_soong_out_path(), 383 common_util.get_android_root_dir()) 384 rel_project_soong_paths = [] 385 for project in self._projects: 386 relpath = project.project_relative_path 387 rel_project_soong_paths.append(os.sep.join( 388 [out_soong_dir, constant.INTERMEDIATES, relpath]) + os.sep) 389 return rel_project_soong_paths 390 391 def _remove_permission_definition_srcjar_path(self): 392 """Removes android.Manifest.permission definition srcjar path. 393 394 If framework-res/android_common/gen/aapt2/R directory or 395 framework-res/android_common/gen/android/R.srcjar file exists in 396 self._all_srcs[_KEY_SRCJAR_PATH], remove them. 397 """ 398 if self._permission_aapt2 in self._all_srcs[_KEY_SRCJAR_PATH]: 399 self._all_srcs[_KEY_SRCJAR_PATH].remove(self._permission_aapt2) 400 if self._permission_r_srcjar in self._all_srcs[_KEY_SRCJAR_PATH]: 401 self._all_srcs[_KEY_SRCJAR_PATH].remove(self._permission_r_srcjar) 402 403 404def _get_real_dependencies_jars(list_to_check, list_to_be_checked): 405 """Gets real dependencies' jar from the input list. 406 407 There are jar files which have the same source codes as the 408 self.projects should be removed from dependencies. Otherwise these files 409 will cause the duplicated codes in IDE and lead to issues: b/158583214 is an 410 example. 411 412 Args: 413 list_to_check: A list of relative projects' paths in the folder 414 out/soong/.intermediates to be checked if are contained 415 in the list_to_be_checked list. 416 list_to_be_checked: A list of dependencies' paths to be checked. 417 418 Returns: 419 A list of dependency jar paths after duplicated ones removed. 420 """ 421 file_exts = [constant.JAR_EXT] 422 real_jars = list_to_be_checked.copy() 423 for jar in list_to_be_checked: 424 ext = os.path.splitext(jar)[-1] 425 for check_path in list_to_check: 426 if check_path in jar and ext in file_exts: 427 real_jars.remove(jar) 428 break 429 return real_jars 430 431 432def get_exclude_content(root_path, excludes=None): 433 """Get the exclude folder content list. 434 435 It returns the exclude folders content list. 436 e.g. 437 ['<excludeFolder url="file://a/.idea" />', 438 '<excludeFolder url="file://a/.repo" />'] 439 440 Args: 441 root_path: Android source file path. 442 excludes: A list of exclusive directories, the default value is None but 443 will be assigned to _EXCLUDE_FOLDERS. 444 445 Returns: 446 String: exclude folder content list. 447 """ 448 exclude_items = [] 449 if not excludes: 450 excludes = _EXCLUDE_FOLDERS 451 for folder in excludes: 452 folder_path = os.path.join(root_path, folder) 453 if os.path.isdir(folder_path): 454 exclude_items.append(_EXCLUDE_ITEM % folder_path) 455 return exclude_items 456 457 458def _remove_child_duplicate_sources_from_parent(child, parent_sources, root): 459 """Removes the child's duplicate source folders from the parent source list. 460 461 Remove all the child's subdirectories from the parent's source list if there 462 is any. 463 464 Args: 465 child: A child project of ProjectInfo instance. 466 parent_sources: The parent project sources of the ProjectInfo instance. 467 root: A string of the Android root. 468 469 Returns: 470 A set of the sources to be removed. 471 """ 472 rm_paths = set() 473 for path in parent_sources: 474 if (common_util.is_source_under_relative_path( 475 os.path.relpath(path, root), child.project_relative_path)): 476 rm_paths.add(path) 477 return rm_paths 478 479 480def _get_permission_aapt2_rel_path(): 481 """Gets android.Manifest.permission definition srcjar path.""" 482 out_soong_dir = os.path.relpath(common_util.get_soong_out_path(), 483 common_util.get_android_root_dir()) 484 return os.path.join(out_soong_dir, constant.INTERMEDIATES, 485 _PERMISSION_DEFINED_PATH, constant.NAME_AAPT2, _R) 486 487 488def _get_permission_r_srcjar_rel_path(): 489 """Gets android.Manifest.permission definition srcjar path.""" 490 out_soong_dir = os.path.relpath(common_util.get_soong_out_path(), 491 common_util.get_android_root_dir()) 492 return os.path.join(out_soong_dir, constant.INTERMEDIATES, 493 _PERMISSION_DEFINED_PATH, _ANDROID, 494 constant.TARGET_R_SRCJAR) 495