1#!/usr/bin/env python3 2# 3# Copyright 2020 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Separate the sources from multiple projects.""" 18 19import logging 20import os 21import shutil 22 23from pathlib import Path 24 25from aidegen import constant 26from aidegen.idea import iml 27from aidegen.lib import common_util 28from aidegen.lib import project_config 29 30_KEY_SOURCE_PATH = 'source_folder_path' 31_KEY_TEST_PATH = 'test_folder_path' 32_SOURCE_FOLDERS = [_KEY_SOURCE_PATH, _KEY_TEST_PATH] 33_KEY_SRCJAR_PATH = 'srcjar_path' 34_KEY_R_PATH = 'r_java_path' 35_KEY_JAR_PATH = 'jar_path' 36_EXCLUDE_ITEM = '\n <excludeFolder url="file://%s" />' 37# Temporarily exclude test-dump and src_stub folders to prevent symbols from 38# resolving failure by incorrect reference. These two folders should be removed 39# after b/136982078 is resolved. 40_EXCLUDE_FOLDERS = ['.idea', '.repo', 'art', 'bionic', 'bootable', 'build', 41 'dalvik', 'developers', 'device', 'hardware', 'kernel', 42 'libnativehelper', 'pdk', 'prebuilts', 'sdk', 'system', 43 'toolchain', 'tools', 'vendor', 'out', 'external', 44 'art/tools/ahat/src/test-dump', 45 'cts/common/device-side/device-info/src_stub'] 46_PERMISSION_DEFINED_PATH = ('frameworks/base/core/res/framework-res/' 47 'android_common/gen/') 48_ANDROID = 'android' 49_R = 'R' 50 51 52class ProjectSplitter: 53 """Splits the sources from multiple projects. 54 55 It's a specific solution to deal with the source folders in multiple 56 project case. Since the IntelliJ does not allow duplicate source folders, 57 AIDEGen needs to separate the source folders for each project. The single 58 project case has no different with current structure. 59 60 Usage: 61 project_splitter = ProjectSplitter(projects) 62 63 # Find the dependencies between the projects. 64 project_splitter.get_dependencies() 65 66 # Clear the source folders for each project. 67 project_splitter.revise_source_folders() 68 69 Attributes: 70 _projects: A list of ProjectInfo. 71 _all_srcs: A dictionary contains all sources of multiple projects. 72 e.g. 73 { 74 'module_name': 'test', 75 'path': ['path/to/module'], 76 'srcs': ['src_folder1', 'src_folder2'], 77 'tests': ['test_folder1', 'test_folder2'] 78 'jars': ['jar1.jar'], 79 'srcjars': ['1.srcjar', '2.srcjar'], 80 'dependencies': ['framework_srcjars', 'base'], 81 'iml_name': '/abs/path/to/iml.iml' 82 } 83 _framework_exist: A boolean, True if framework is one of the projects. 84 _framework_iml: A string, the name of the framework's iml. 85 _full_repo: A boolean, True if loading with full Android sources. 86 _full_repo_iml: A string, the name of the Android folder's iml. 87 _permission_r_srcjar: A string, the absolute path of R.srcjar file where 88 the permission relative constants are defined. 89 _permission_aapt2: A string, the absolute path of aapt2/R directory 90 where the permission relative constants are defined. 91 """ 92 def __init__(self, projects): 93 """ProjectSplitter initialize. 94 95 Args: 96 projects: A list of ProjectInfo object. 97 """ 98 self._projects = projects 99 self._all_srcs = dict(projects[0].source_path) 100 self._framework_iml = None 101 self._framework_exist = any( 102 {p.project_relative_path == constant.FRAMEWORK_PATH 103 for p in self._projects}) 104 if self._framework_exist: 105 self._framework_iml = iml.IMLGenerator.get_unique_iml_name( 106 os.path.join(common_util.get_android_root_dir(), 107 constant.FRAMEWORK_PATH)) 108 self._full_repo = project_config.ProjectConfig.get_instance().full_repo 109 if self._full_repo: 110 self._full_repo_iml = os.path.basename( 111 common_util.get_android_root_dir()) 112 self._permission_r_srcjar = _get_permission_r_srcjar_rel_path() 113 self._permission_aapt2 = _get_permission_aapt2_rel_path() 114 115 def revise_source_folders(self): 116 """Resets the source folders of each project. 117 118 There should be no duplicate source root path in IntelliJ. The issue 119 doesn't happen in single project case. Once users choose multiple 120 projects, there could be several same source paths of different 121 projects. In order to prevent that, we should remove the source paths 122 in dependencies.iml which are duplicate with the paths in [module].iml 123 files. 124 125 Steps to prevent the duplicate source root path in IntelliJ: 126 1. Copy all sources from sub-projects to main project. 127 2. Delete the source and test folders which are not under the 128 sub-projects. 129 3. Delete the sub-projects' source and test paths from the main project. 130 """ 131 self._collect_all_srcs() 132 self._keep_local_sources() 133 self._remove_duplicate_sources() 134 135 def _collect_all_srcs(self): 136 """Copies all projects' sources to a dictionary.""" 137 for project in self._projects[1:]: 138 for key, value in project.source_path.items(): 139 self._all_srcs[key].update(value) 140 141 def _keep_local_sources(self): 142 """Removes source folders which are not under the project's path. 143 144 1. Remove the source folders which are not under the project. 145 2. Remove the duplicate project's source folders from the _all_srcs. 146 """ 147 for project in self._projects: 148 srcs = project.source_path 149 relpath = project.project_relative_path 150 is_root = not relpath 151 for key in _SOURCE_FOLDERS: 152 srcs[key] = {s for s in srcs[key] 153 if common_util.is_source_under_relative_path( 154 s, relpath) or is_root} 155 self._all_srcs[key] -= srcs[key] 156 157 def _remove_duplicate_sources(self): 158 """Removes the duplicate source folders from each sub project. 159 160 Priority processing with the longest path length, e.g. 161 frameworks/base/packages/SettingsLib must have priority over 162 frameworks/base. 163 (b/160303006): Remove the parent project's source and test paths under 164 the child's project path. 165 """ 166 root = common_util.get_android_root_dir() 167 projects = sorted(self._projects, key=lambda k: len( 168 k.project_relative_path), reverse=True) 169 for child in projects: 170 for parent in self._projects: 171 is_root = not parent.project_relative_path 172 if parent is child: 173 continue 174 if (common_util.is_source_under_relative_path( 175 child.project_relative_path, 176 parent.project_relative_path) or is_root): 177 for key in _SOURCE_FOLDERS: 178 parent.source_path[key] -= child.source_path[key] 179 rm_paths = _remove_child_duplicate_sources_from_parent( 180 child, parent.source_path[key], root) 181 parent.source_path[key] -= rm_paths 182 183 def get_dependencies(self): 184 """Gets the dependencies between the projects. 185 186 Check if the current project's source folder exists in other projects. 187 If do, the current project is a dependency module to the other. 188 """ 189 projects = sorted(self._projects, key=lambda k: len( 190 k.project_relative_path)) 191 for project in projects: 192 proj_path = project.project_relative_path 193 project.dependencies = [constant.FRAMEWORK_SRCJARS] 194 if self._framework_exist and proj_path != constant.FRAMEWORK_PATH: 195 project.dependencies.append(self._framework_iml) 196 if self._full_repo and proj_path: 197 project.dependencies.append(self._full_repo_iml) 198 srcs = (project.source_path[_KEY_SOURCE_PATH] 199 | project.source_path[_KEY_TEST_PATH]) 200 dep_projects = sorted(self._projects, key=lambda k: len( 201 k.project_relative_path)) 202 for dep_proj in dep_projects: 203 dep_path = dep_proj.project_relative_path 204 is_root = not dep_path 205 is_child = common_util.is_source_under_relative_path(dep_path, 206 proj_path) 207 is_dep = any({s for s in srcs 208 if common_util.is_source_under_relative_path( 209 s, dep_path) or is_root}) 210 if dep_proj is project or is_child or not is_dep: 211 continue 212 dep = iml.IMLGenerator.get_unique_iml_name(os.path.join( 213 common_util.get_android_root_dir(), dep_path)) 214 if dep not in project.dependencies: 215 project.dependencies.append(dep) 216 project.dependencies.append(constant.KEY_DEPENDENCIES) 217 218 def gen_framework_srcjars_iml(self): 219 """Generates the framework_srcjars.iml. 220 221 Create the iml file with only the srcjars of module framework-all. These 222 srcjars will be separated from the modules under frameworks/base. 223 224 Returns: 225 A string of the framework_srcjars.iml's absolute path. 226 """ 227 self._remove_permission_definition_srcjar_path() 228 mod = dict(self._projects[0].dep_modules[constant.FRAMEWORK_ALL]) 229 mod[constant.KEY_DEPENDENCIES] = [] 230 mod[constant.KEY_IML_NAME] = constant.FRAMEWORK_SRCJARS 231 if self._framework_exist: 232 mod[constant.KEY_DEPENDENCIES].append(self._framework_iml) 233 if self._full_repo: 234 mod[constant.KEY_DEPENDENCIES].append(self._full_repo_iml) 235 mod[constant.KEY_DEPENDENCIES].append(constant.KEY_DEPENDENCIES) 236 srcjar_dict = dict() 237 permission_src = self._get_permission_defined_source_path() 238 if permission_src: 239 mod[constant.KEY_SRCS] = [permission_src] 240 srcjar_dict = {constant.KEY_DEP_SRCS: True, 241 constant.KEY_SRCJARS: True, 242 constant.KEY_DEPENDENCIES: True} 243 else: 244 logging.warning('The permission definition relative paths are ' 245 'missing.') 246 srcjar_dict = {constant.KEY_SRCJARS: True, 247 constant.KEY_DEPENDENCIES: True} 248 framework_srcjars_iml = iml.IMLGenerator(mod) 249 framework_srcjars_iml.create(srcjar_dict) 250 self._all_srcs[_KEY_SRCJAR_PATH] -= set(mod.get(constant.KEY_SRCJARS, 251 [])) 252 return framework_srcjars_iml.iml_path 253 254 def _get_permission_defined_source_path(self): 255 """Gets the source path where permission relative constants are defined. 256 257 For the definition permission constants, the priority is, 258 1) If framework-res/android_common/gen/aapt2/R directory exists, return 259 it. 260 2) If the framework-res/android_common/gen/android/R.srcjar file exists, 261 unzip it to 'aidegen_r.srcjar' folder and return the path. 262 263 Returns: 264 A string of the path of aapt2/R or android/aidegen_r.srcjar folder, 265 else None. 266 """ 267 if os.path.isdir(self._permission_aapt2): 268 return self._permission_aapt2 269 if os.path.isfile(self._permission_r_srcjar): 270 dest = os.path.join( 271 os.path.dirname(self._permission_r_srcjar), 272 ''.join([constant.UNZIP_SRCJAR_PATH_HEAD, 273 os.path.basename(self._permission_r_srcjar).lower()])) 274 if os.path.isdir(dest): 275 shutil.rmtree(dest) 276 common_util.unzip_file(self._permission_r_srcjar, dest) 277 return dest 278 return None 279 280 def _gen_dependencies_iml(self): 281 """Generates the dependencies.iml.""" 282 rel_project_soong_paths = self._get_rel_project_soong_paths() 283 self._unzip_all_scrjars() 284 mod = { 285 constant.KEY_SRCS: _get_real_dependencies_jars( 286 rel_project_soong_paths, self._all_srcs[_KEY_SOURCE_PATH]), 287 constant.KEY_TESTS: _get_real_dependencies_jars( 288 rel_project_soong_paths, self._all_srcs[_KEY_TEST_PATH]), 289 constant.KEY_JARS: _get_real_dependencies_jars( 290 rel_project_soong_paths, self._all_srcs[_KEY_JAR_PATH]), 291 constant.KEY_SRCJARS: _get_real_dependencies_jars( 292 rel_project_soong_paths, 293 self._all_srcs[_KEY_R_PATH] | self._all_srcs[_KEY_SRCJAR_PATH]), 294 constant.KEY_DEPENDENCIES: _get_real_dependencies_jars( 295 rel_project_soong_paths, [constant.FRAMEWORK_SRCJARS]), 296 constant.KEY_PATH: [self._projects[0].project_relative_path], 297 constant.KEY_MODULE_NAME: constant.KEY_DEPENDENCIES, 298 constant.KEY_IML_NAME: constant.KEY_DEPENDENCIES 299 } 300 if self._framework_exist: 301 mod[constant.KEY_DEPENDENCIES].append(self._framework_iml) 302 if self._full_repo: 303 mod[constant.KEY_DEPENDENCIES].append(self._full_repo_iml) 304 dep_iml = iml.IMLGenerator(mod) 305 dep_iml.create({constant.KEY_DEP_SRCS: True, 306 constant.KEY_SRCJARS: True, 307 constant.KEY_JARS: True, 308 constant.KEY_DEPENDENCIES: True}) 309 310 def _unzip_all_scrjars(self): 311 """Unzips all scrjar files to a specific folder 'aidegen_r.srcjar'. 312 313 For some versions of IntelliJ no more supports unzipping srcjar files 314 automatically, we have to unzip it to a 'aidegen_r.srcjar' directory. 315 The rules of the unzip process are, 316 1) If it's a aapt2/R type jar or other directory type sources, add them 317 into self._all_srcs[_KEY_SOURCE_PATH]. 318 2) If it's an R.srcjar file, check if the same path of aapt2/R directory 319 exists if so add aapt2/R path into into the 320 self._all_srcs[_KEY_SOURCE_PATH], otherwise unzip R.srcjar into 321 the 'aidegen_r.srcjar' directory and add the unzipped path into 322 self._all_srcs[_KEY_SOURCE_PATH]. 323 """ 324 sjars = self._all_srcs[_KEY_R_PATH] | self._all_srcs[_KEY_SRCJAR_PATH] 325 self._all_srcs[_KEY_R_PATH] = set() 326 self._all_srcs[_KEY_SRCJAR_PATH] = set() 327 for sjar in sjars: 328 if not os.path.exists(sjar): 329 continue 330 if os.path.isdir(sjar): 331 self._all_srcs[_KEY_SOURCE_PATH].add(sjar) 332 continue 333 sjar_dir = os.path.dirname(sjar) 334 sjar_name = os.path.basename(sjar).lower() 335 aapt2 = os.path.join( 336 os.path.dirname(sjar_dir), constant.NAME_AAPT2, _R) 337 if os.path.isdir(aapt2): 338 self._all_srcs[_KEY_SOURCE_PATH].add(aapt2) 339 continue 340 dest = os.path.join( 341 sjar_dir, ''.join([constant.UNZIP_SRCJAR_PATH_HEAD, sjar_name])) 342 if os.path.isdir(dest): 343 shutil.rmtree(dest) 344 common_util.unzip_file(sjar, dest) 345 self._all_srcs[_KEY_SOURCE_PATH].add(dest) 346 347 def gen_projects_iml(self): 348 """Generates the projects' iml file.""" 349 root_path = common_util.get_android_root_dir() 350 excludes = project_config.ProjectConfig.get_instance().exclude_paths 351 for project in self._projects: 352 relpath = project.project_relative_path 353 exclude_folders = [] 354 if not relpath: 355 exclude_folders.extend(get_exclude_content(root_path)) 356 if excludes: 357 exclude_folders.extend(get_exclude_content(root_path, excludes)) 358 mod_info = { 359 constant.KEY_EXCLUDES: ''.join(exclude_folders), 360 constant.KEY_SRCS: project.source_path[_KEY_SOURCE_PATH], 361 constant.KEY_TESTS: project.source_path[_KEY_TEST_PATH], 362 constant.KEY_DEPENDENCIES: project.dependencies, 363 constant.KEY_PATH: [relpath], 364 constant.KEY_MODULE_NAME: project.module_name, 365 constant.KEY_IML_NAME: iml.IMLGenerator.get_unique_iml_name( 366 str(Path(root_path, relpath))) 367 } 368 dep_iml = iml.IMLGenerator(mod_info) 369 dep_iml.create({constant.KEY_SRCS: True, 370 constant.KEY_DEPENDENCIES: True}) 371 project.iml_path = dep_iml.iml_path 372 self._gen_dependencies_iml() 373 374 def _get_rel_project_soong_paths(self): 375 """Gets relative projects' paths in 'out/soong/.intermediates' folder. 376 377 Gets relative projects' paths in the 'out/soong/.intermediates' 378 directory. For example, if the self.projects = ['frameworks/base'] the 379 returned list should be ['out/soong/.intermediates/frameworks/base/']. 380 381 Returns: 382 A list of relative projects' paths in out/soong/.intermediates. 383 """ 384 out_soong_dir = os.path.relpath(common_util.get_soong_out_path(), 385 common_util.get_android_root_dir()) 386 rel_project_soong_paths = [] 387 for project in self._projects: 388 relpath = project.project_relative_path 389 rel_project_soong_paths.append(os.sep.join( 390 [out_soong_dir, constant.INTERMEDIATES, relpath]) + os.sep) 391 return rel_project_soong_paths 392 393 def _remove_permission_definition_srcjar_path(self): 394 """Removes android.Manifest.permission definition srcjar path. 395 396 If framework-res/android_common/gen/aapt2/R directory or 397 framework-res/android_common/gen/android/R.srcjar file exists in 398 self._all_srcs[_KEY_SRCJAR_PATH], remove them. 399 """ 400 if self._permission_aapt2 in self._all_srcs[_KEY_SRCJAR_PATH]: 401 self._all_srcs[_KEY_SRCJAR_PATH].remove(self._permission_aapt2) 402 if self._permission_r_srcjar in self._all_srcs[_KEY_SRCJAR_PATH]: 403 self._all_srcs[_KEY_SRCJAR_PATH].remove(self._permission_r_srcjar) 404 405 406def _get_real_dependencies_jars(list_to_check, list_to_be_checked): 407 """Gets real dependencies' jar from the input list. 408 409 There are jar files which have the same source codes as the 410 self.projects should be removed from dependencies. Otherwise these files 411 will cause the duplicated codes in IDE and lead to issues: b/158583214 is an 412 example. 413 414 Args: 415 list_to_check: A list of relative projects' paths in the folder 416 out/soong/.intermediates to be checked if are contained 417 in the list_to_be_checked list. 418 list_to_be_checked: A list of dependencies' paths to be checked. 419 420 Returns: 421 A list of dependency jar paths after duplicated ones removed. 422 """ 423 file_exts = [constant.JAR_EXT] 424 real_jars = list_to_be_checked.copy() 425 for jar in list_to_be_checked: 426 ext = os.path.splitext(jar)[-1] 427 for check_path in list_to_check: 428 if check_path in jar and ext in file_exts: 429 real_jars.remove(jar) 430 break 431 return real_jars 432 433 434def get_exclude_content(root_path, excludes=None): 435 """Get the exclude folder content list. 436 437 It returns the exclude folders content list. 438 e.g. 439 ['<excludeFolder url="file://a/.idea" />', 440 '<excludeFolder url="file://a/.repo" />'] 441 442 Args: 443 root_path: Android source file path. 444 excludes: A list of exclusive directories, the default value is None but 445 will be assigned to _EXCLUDE_FOLDERS. 446 447 Returns: 448 String: exclude folder content list. 449 """ 450 exclude_items = [] 451 if not excludes: 452 excludes = _EXCLUDE_FOLDERS 453 for folder in excludes: 454 folder_path = os.path.join(root_path, folder) 455 if os.path.isdir(folder_path): 456 exclude_items.append(_EXCLUDE_ITEM % folder_path) 457 return exclude_items 458 459 460def _remove_child_duplicate_sources_from_parent(child, parent_sources, root): 461 """Removes the child's duplicate source folders from the parent source list. 462 463 Remove all the child's subdirectories from the parent's source list if there 464 is any. 465 466 Args: 467 child: A child project of ProjectInfo instance. 468 parent_sources: The parent project sources of the ProjectInfo instance. 469 root: A string of the Android root. 470 471 Returns: 472 A set of the sources to be removed. 473 """ 474 rm_paths = set() 475 for path in parent_sources: 476 if (common_util.is_source_under_relative_path( 477 os.path.relpath(path, root), child.project_relative_path)): 478 rm_paths.add(path) 479 return rm_paths 480 481 482def _get_permission_aapt2_rel_path(): 483 """Gets android.Manifest.permission definition srcjar path.""" 484 out_soong_dir = os.path.relpath(common_util.get_soong_out_path(), 485 common_util.get_android_root_dir()) 486 return os.path.join(out_soong_dir, constant.INTERMEDIATES, 487 _PERMISSION_DEFINED_PATH, constant.NAME_AAPT2, _R) 488 489 490def _get_permission_r_srcjar_rel_path(): 491 """Gets android.Manifest.permission definition srcjar path.""" 492 out_soong_dir = os.path.relpath(common_util.get_soong_out_path(), 493 common_util.get_android_root_dir()) 494 return os.path.join(out_soong_dir, constant.INTERMEDIATES, 495 _PERMISSION_DEFINED_PATH, _ANDROID, 496 constant.TARGET_R_SRCJAR) 497