1#!/usr/bin/env python3 2# 3# Copyright 2018 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""ModuleData information.""" 18 19from __future__ import absolute_import 20 21import glob 22import logging 23import os 24import re 25 26from aidegen import constant 27from aidegen.lib import common_util 28from aidegen.lib import module_info 29from aidegen.lib import project_config 30 31# Parse package name from the package declaration line of a java. 32# Group matches "foo.bar" of line "package foo.bar;" or "package foo.bar" 33_PACKAGE_RE = re.compile(r'\s*package\s+(?P<package>[^(;|\s)]+)\s*', re.I) 34_ANDROID_SUPPORT_PATH_KEYWORD = 'prebuilts/sdk/current/' 35 36# File extensions 37_JAVA_EXT = '.java' 38_KOTLIN_EXT = '.kt' 39_TARGET_FILES = [_JAVA_EXT, _KOTLIN_EXT] 40_JARJAR_RULES_FILE = 'jarjar-rules.txt' 41_KEY_JARJAR_RULES = 'jarjar_rules' 42_TARGET_AAPT2_SRCJAR = constant.NAME_AAPT2 + constant.SRCJAR_EXT 43_TARGET_BUILD_FILES = [_TARGET_AAPT2_SRCJAR, constant.TARGET_R_SRCJAR] 44_IGNORE_DIRS = [ 45 # The java files under this directory have to be ignored because it will 46 # cause duplicated classes by libcore/ojluni/src/main/java. 47 'libcore/ojluni/src/lambda/java' 48] 49_ANDROID = 'android' 50_REPACKAGES = 'repackaged' 51_FRAMEWORK_SRCJARS_PATH = os.path.join(constant.FRAMEWORK_PATH, 52 constant.FRAMEWORK_SRCJARS) 53 54 55class ModuleData: 56 """ModuleData class. 57 58 Attributes: 59 All following relative paths stand for the path relative to the android 60 repo root. 61 62 module_path: A string of the relative path to the module. 63 src_dirs: A list to keep the unique source folder relative paths. 64 test_dirs: A list to keep the unique test folder relative paths. 65 jar_files: A list to keep the unique jar file relative paths. 66 r_java_paths: A list to keep the R folder paths to use in Eclipse. 67 srcjar_paths: A list to keep the srcjar source root paths to use in 68 IntelliJ. Some modules' srcjar_paths will be removed when 69 run with the MultiProjectInfo. 70 dep_paths: A list to keep the dependency modules' path. 71 referenced_by_jar: A boolean to check if the module is referenced by a 72 jar file. 73 build_targets: A set to keep the unique build target jar or srcjar file 74 relative paths which are ready to be rebuld. 75 missing_jars: A set to keep the jar file relative paths if it doesn't 76 exist. 77 specific_soong_path: A string of the relative path to the module's 78 intermediates folder under out/. 79 """ 80 81 def __init__(self, module_name, module_data, depth): 82 """Initialize ModuleData. 83 84 Args: 85 module_name: Name of the module. 86 module_data: A dictionary holding a module information. 87 depth: An integer shows the depth of module dependency referenced by 88 source. Zero means the max module depth. 89 For example: 90 { 91 'class': ['APPS'], 92 'path': ['path/to/the/module'], 93 'depth': 0, 94 'dependencies': ['bouncycastle', 'ims-common'], 95 'srcs': [ 96 'path/to/the/module/src/com/android/test.java', 97 'path/to/the/module/src/com/google/test.java', 98 'out/soong/.intermediates/path/to/the/module/test/src/ 99 com/android/test.srcjar' 100 ], 101 'installed': ['out/target/product/generic_x86_64/ 102 system/framework/framework.jar'], 103 'jars': ['settings.jar'], 104 'jarjar_rules': ['jarjar-rules.txt'] 105 } 106 """ 107 assert module_name, 'Module name can\'t be null.' 108 assert module_data, 'Module data of %s can\'t be null.' % module_name 109 self.module_name = module_name 110 self.module_data = module_data 111 self._init_module_path() 112 self._init_module_depth(depth) 113 self.src_dirs = [] 114 self.test_dirs = [] 115 self.jar_files = [] 116 self.r_java_paths = [] 117 self.srcjar_paths = [] 118 self.dep_paths = [] 119 self.referenced_by_jar = False 120 self.build_targets = set() 121 self.missing_jars = set() 122 self.specific_soong_path = os.path.join( 123 'out/soong/.intermediates', self.module_path, self.module_name) 124 125 def _is_app_module(self): 126 """Check if the current module's class is APPS""" 127 return self._check_key('class') and 'APPS' in self.module_data['class'] 128 129 def _is_target_module(self): 130 """Check if the current module is a target module. 131 132 A target module is the target project or a module under the 133 target project and it's module depth is 0. 134 For example: aidegen Settings framework 135 The target projects are Settings and framework so they are also 136 target modules. And the dependent module SettingsUnitTests's path 137 is packages/apps/Settings/tests/unit so it also a target module. 138 """ 139 return self.module_depth == 0 140 141 def _collect_r_srcs_paths(self): 142 """Collect the source folder of R.java. 143 144 Check if the path of aapt2.srcjar or R.srcjar exists, these are both the 145 values of key "srcjars" in module_data. If neither of the cases exists, 146 build it onto an intermediates directory. 147 148 For IntelliJ, we can set the srcjar file as a source root for 149 dependency. For Eclipse, we still use the R folder as dependencies until 150 we figure out how to set srcjar file as dependency. 151 # TODO(b/135594800): Set aapt2.srcjar or R.srcjar as a dependency in 152 Eclipse. 153 """ 154 if (self._is_app_module() and self._is_target_module() 155 and self._check_key(constant.KEY_SRCJARS)): 156 for srcjar in self.module_data[constant.KEY_SRCJARS]: 157 if not os.path.exists(common_util.get_abs_path(srcjar)): 158 self.build_targets.add(srcjar) 159 self._collect_srcjar_path(srcjar) 160 r_dir = self._get_r_dir(srcjar) 161 if r_dir and r_dir not in self.r_java_paths: 162 self.r_java_paths.append(r_dir) 163 164 def _collect_srcjar_path(self, srcjar): 165 """Collect the source folders from a srcjar path. 166 167 Set the aapt2.srcjar or R.srcjar as source root: 168 Case aapt2.srcjar: 169 The source path string is 170 out/.../Bluetooth_intermediates/aapt2.srcjar. 171 Case R.srcjar: 172 The source path string is out/soong/.../gen/android/R.srcjar. 173 174 Args: 175 srcjar: A file path string relative to ANDROID_BUILD_TOP, the build 176 target of the module to generate R.java. 177 """ 178 if (os.path.basename(srcjar) in _TARGET_BUILD_FILES 179 and srcjar not in self.srcjar_paths): 180 self.srcjar_paths.append(srcjar) 181 182 def _collect_all_srcjar_paths(self): 183 """Collect all srcjar files of target module as source folders. 184 185 Since the aidl files are built to *.java and collected in the 186 aidl.srcjar file by the build system. AIDEGen needs to collect these 187 aidl.srcjar files as the source root folders in IntelliJ. Furthermore, 188 AIDEGen collects all *.srcjar files for other cases to fulfil the same 189 purpose. 190 """ 191 if self._is_target_module() and self._check_key(constant.KEY_SRCJARS): 192 for srcjar in self.module_data[constant.KEY_SRCJARS]: 193 if not os.path.exists(common_util.get_abs_path(srcjar)): 194 self.build_targets.add(srcjar) 195 if srcjar not in self.srcjar_paths: 196 self.srcjar_paths.append(srcjar) 197 198 @staticmethod 199 def _get_r_dir(srcjar): 200 """Get the source folder of R.java for Eclipse. 201 202 Get the folder contains the R.java of aapt2.srcjar or R.srcjar: 203 Case aapt2.srcjar: 204 If the relative path of the aapt2.srcjar is a/b/aapt2.srcjar, the 205 source root of the R.java is a/b/aapt2 206 Case R.srcjar: 207 If the relative path of the R.srcjar is a/b/android/R.srcjar, the 208 source root of the R.java is a/b/aapt2/R 209 210 Args: 211 srcjar: A file path string, the build target of the module to 212 generate R.java. 213 214 Returns: 215 A relative source folder path string, and return None if the target 216 file name is not aapt2.srcjar or R.srcjar. 217 """ 218 target_folder, target_file = os.path.split(srcjar) 219 base_dirname = os.path.basename(target_folder) 220 if target_file == _TARGET_AAPT2_SRCJAR: 221 return os.path.join(target_folder, constant.NAME_AAPT2) 222 if target_file == constant.TARGET_R_SRCJAR and base_dirname == _ANDROID: 223 return os.path.join(os.path.dirname(target_folder), 224 constant.NAME_AAPT2, 'R') 225 return None 226 227 def _init_module_path(self): 228 """Inintialize self.module_path.""" 229 self.module_path = (self.module_data[constant.KEY_PATH][0] 230 if self._check_key(constant.KEY_PATH) else '') 231 232 def _init_module_depth(self, depth): 233 """Initialize module depth's settings. 234 235 Set the module's depth from module info when user have -d parameter. 236 Set the -d value from user input, default to 0. 237 238 Args: 239 depth: the depth to be set. 240 """ 241 self.module_depth = (int(self.module_data[constant.KEY_DEPTH]) 242 if depth else 0) 243 self.depth_by_source = depth 244 245 def _is_android_supported_module(self): 246 """Determine if this is an Android supported module.""" 247 return common_util.is_source_under_relative_path( 248 self.module_path, _ANDROID_SUPPORT_PATH_KEYWORD) 249 250 def _check_jarjar_rules_exist(self): 251 """Check if jarjar rules exist.""" 252 return (_KEY_JARJAR_RULES in self.module_data and 253 self.module_data[_KEY_JARJAR_RULES][0] == _JARJAR_RULES_FILE) 254 255 def _check_jars_exist(self): 256 """Check if jars exist.""" 257 return self._check_key(constant.KEY_JARS) 258 259 def _check_classes_jar_exist(self): 260 """Check if classes_jar exist.""" 261 return self._check_key(constant.KEY_CLASSES_JAR) 262 263 def _collect_srcs_paths(self): 264 """Collect source folder paths in src_dirs from module_data['srcs'].""" 265 if self._check_key(constant.KEY_SRCS): 266 scanned_dirs = set() 267 for src_item in self.module_data[constant.KEY_SRCS]: 268 src_dir = None 269 src_item = os.path.relpath(src_item) 270 if common_util.is_target(src_item, _TARGET_FILES): 271 # Only scan one java file in each source directories. 272 src_item_dir = os.path.dirname(src_item) 273 if src_item_dir not in scanned_dirs: 274 scanned_dirs.add(src_item_dir) 275 src_dir = self._get_source_folder(src_item) 276 else: 277 # To record what files except java and kt in the srcs. 278 logging.debug('%s is not in parsing scope.', src_item) 279 if src_dir: 280 self._add_to_source_or_test_dirs( 281 self._switch_repackaged(src_dir)) 282 283 def _check_key(self, key): 284 """Check if key is in self.module_data and not empty. 285 286 Args: 287 key: the key to be checked. 288 """ 289 return key in self.module_data and self.module_data[key] 290 291 def _add_to_source_or_test_dirs(self, src_dir): 292 """Add folder to source or test directories. 293 294 Args: 295 src_dir: the directory to be added. 296 """ 297 if (src_dir not in _IGNORE_DIRS and src_dir not in self.src_dirs 298 and src_dir not in self.test_dirs): 299 if self._is_test_module(src_dir): 300 self.test_dirs.append(src_dir) 301 else: 302 self.src_dirs.append(src_dir) 303 304 @staticmethod 305 def _is_test_module(src_dir): 306 """Check if the module path is a test module path. 307 308 Args: 309 src_dir: the directory to be checked. 310 311 Returns: 312 True if module path is a test module path, otherwise False. 313 """ 314 return constant.KEY_TESTS in src_dir.split(os.sep) 315 316 def _get_source_folder(self, java_file): 317 """Parsing a java to get the package name to filter out source path. 318 319 Args: 320 java_file: A string, the java file with relative path. 321 e.g. path/to/the/java/file.java 322 323 Returns: 324 source_folder: A string of path to source folder(e.g. src/main/java) 325 or none when it failed to get package name. 326 """ 327 abs_java_path = common_util.get_abs_path(java_file) 328 if os.path.exists(abs_java_path): 329 package_name = self._get_package_name(abs_java_path) 330 if package_name: 331 return self._parse_source_path(java_file, package_name) 332 return None 333 334 @staticmethod 335 def _parse_source_path(java_file, package_name): 336 """Parse the source path by filter out the package name. 337 338 Case 1: 339 java file: a/b/c/d/e.java 340 package name: c.d 341 The source folder is a/b. 342 343 Case 2: 344 java file: a/b/c.d/e.java 345 package name: c.d 346 The source folder is a/b. 347 348 Case 3: 349 java file: a/b/c/d/e.java 350 package name: x.y 351 The source folder is a/b/c/d. 352 353 Case 4: 354 java file: a/b/c.d/e/c/d/f.java 355 package name: c.d 356 The source folder is a/b/c.d/e. 357 358 Case 5: 359 java file: a/b/c.d/e/c.d/e/f.java 360 package name: c.d.e 361 The source folder is a/b/c.d/e. 362 363 Args: 364 java_file: A string of the java file relative path. 365 package_name: A string of the java file's package name. 366 367 Returns: 368 A string, the source folder path. 369 """ 370 java_file_name = os.path.basename(java_file) 371 pattern = r'%s/%s$' % (package_name, java_file_name) 372 search_result = re.search(pattern, java_file) 373 if search_result: 374 return java_file[:search_result.start()].strip(os.sep) 375 return os.path.dirname(java_file) 376 377 @staticmethod 378 def _switch_repackaged(src_dir): 379 """Changes the directory to repackaged if it does exist. 380 381 Args: 382 src_dir: a string of relative path. 383 384 Returns: 385 The source folder under repackaged if it exists, otherwise the 386 original one. 387 """ 388 root_path = common_util.get_android_root_dir() 389 dir_list = src_dir.split(os.sep) 390 for i in range(1, len(dir_list)): 391 tmp_dir = dir_list.copy() 392 tmp_dir.insert(i, _REPACKAGES) 393 real_path = os.path.join(root_path, os.path.join(*tmp_dir)) 394 if os.path.exists(real_path): 395 return os.path.relpath(real_path, root_path) 396 return src_dir 397 398 @staticmethod 399 def _get_package_name(abs_java_path): 400 """Get the package name by parsing a java file. 401 402 Args: 403 abs_java_path: A string of the java file with absolute path. 404 e.g. /root/path/to/the/java/file.java 405 406 Returns: 407 package_name: A string of package name. 408 """ 409 package_name = None 410 with open(abs_java_path, encoding='utf8') as data: 411 for line in data.read().splitlines(): 412 match = _PACKAGE_RE.match(line) 413 if match: 414 package_name = match.group('package') 415 break 416 return package_name 417 418 def _append_jar_file(self, jar_path): 419 """Append a path to the jar file into self.jar_files if it's exists. 420 421 Args: 422 jar_path: A path supposed to be a jar file. 423 424 Returns: 425 Boolean: True if jar_path is an existing jar file. 426 """ 427 if common_util.is_target(jar_path, constant.TARGET_LIBS): 428 self.referenced_by_jar = True 429 if os.path.isfile(common_util.get_abs_path(jar_path)): 430 if jar_path not in self.jar_files: 431 self.jar_files.append(jar_path) 432 else: 433 self.missing_jars.add(jar_path) 434 return True 435 return False 436 437 def _append_classes_jar(self): 438 """Append the jar file as dependency for prebuilt modules.""" 439 for jar in self.module_data[constant.KEY_CLASSES_JAR]: 440 if self._append_jar_file(jar): 441 break 442 443 def _append_jar_from_installed(self, specific_dir=None): 444 """Append a jar file's path to the list of jar_files with matching 445 path_prefix. 446 447 There might be more than one jar in "installed" parameter and only the 448 first jar file is returned. If specific_dir is set, the jar file must be 449 under the specific directory or its sub-directory. 450 451 Args: 452 specific_dir: A string of path. 453 """ 454 if self._check_key(constant.KEY_INSTALLED): 455 for jar in self.module_data[constant.KEY_INSTALLED]: 456 if specific_dir and not jar.startswith(specific_dir): 457 continue 458 if self._append_jar_file(jar): 459 break 460 461 def _set_jars_jarfile(self): 462 """Append prebuilt jars of module into self.jar_files. 463 464 Some modules' sources are prebuilt jar files instead of source java 465 files. The jar files can be imported into IntelliJ as a dependency 466 directly. There is only jar file name in self.module_data['jars'], it 467 has to be combined with self.module_data['path'] to append into 468 self.jar_files. 469 Once the file doesn't exist, it's not assumed to be a prebuilt jar so 470 that we can ignore it. 471 # TODO(b/141959125): Collect the correct prebuilt jar files by jdeps.go. 472 473 For example: 474 'asm-6.0': { 475 'jars': [ 476 'asm-6.0.jar' 477 ], 478 'path': [ 479 'prebuilts/misc/common/asm' 480 ], 481 }, 482 Path to the jar file is prebuilts/misc/common/asm/asm-6.0.jar. 483 """ 484 if self._check_key(constant.KEY_JARS): 485 for jar_name in self.module_data[constant.KEY_JARS]: 486 if self._check_key(constant.KEY_INSTALLED): 487 self._append_jar_from_installed() 488 else: 489 jar_path = os.path.join(self.module_path, jar_name) 490 jar_abs = common_util.get_abs_path(jar_path) 491 if not os.path.isfile(jar_abs) and jar_name.endswith( 492 'prebuilt.jar'): 493 rel_path = self._get_jar_path_from_prebuilts(jar_name) 494 if rel_path: 495 jar_path = rel_path 496 if os.path.exists(common_util.get_abs_path(jar_path)): 497 self._append_jar_file(jar_path) 498 499 @staticmethod 500 def _get_jar_path_from_prebuilts(jar_name): 501 """Get prebuilt jar file from prebuilts folder. 502 503 If the prebuilt jar file we get from method _set_jars_jarfile() does not 504 exist, we should search the prebuilt jar file in prebuilts folder. 505 For example: 506 'platformprotos': { 507 'jars': [ 508 'platformprotos-prebuilt.jar' 509 ], 510 'path': [ 511 'frameworks/base' 512 ], 513 }, 514 We get an incorrect path: 'frameworks/base/platformprotos-prebuilt.jar' 515 If the file does not exist, we should search the file name from 516 prebuilts folder. If we can get the correct path from 'prebuilts', we 517 can replace it with the incorrect path. 518 519 Args: 520 jar_name: The prebuilt jar file name. 521 522 Return: 523 A relative prebuilt jar file path if found, otherwise None. 524 """ 525 rel_path = '' 526 search = os.sep.join( 527 [common_util.get_android_root_dir(), 'prebuilts/**', jar_name]) 528 results = glob.glob(search, recursive=True) 529 if results: 530 jar_abs = results[0] 531 rel_path = os.path.relpath( 532 jar_abs, common_util.get_android_root_dir()) 533 return rel_path 534 535 def _collect_specific_jars(self): 536 """Collect specific types of jar files.""" 537 if self._is_android_supported_module(): 538 self._append_jar_from_installed() 539 elif self._check_jarjar_rules_exist(): 540 self._append_jar_from_installed(self.specific_soong_path) 541 elif self._check_jars_exist(): 542 self._set_jars_jarfile() 543 544 def _collect_classes_jars(self): 545 """Collect classes jar files.""" 546 # If there is no source/tests folder of the module, reference the 547 # module by jar. 548 if not self.src_dirs and not self.test_dirs: 549 # Add the classes.jar from the classes_jar attribute as 550 # dependency if it exists. If the classes.jar doesn't exist, 551 # find the jar file from the installed attribute and add the jar 552 # as dependency. 553 if self._check_classes_jar_exist(): 554 self._append_classes_jar() 555 else: 556 self._append_jar_from_installed() 557 558 def _collect_srcs_and_r_srcs_paths(self): 559 """Collect source and R source folder paths for the module.""" 560 self._collect_specific_jars() 561 self._collect_srcs_paths() 562 self._collect_classes_jars() 563 self._collect_r_srcs_paths() 564 self._collect_all_srcjar_paths() 565 566 def _collect_missing_jars(self): 567 """Collect missing jar files to rebuild them.""" 568 if self.referenced_by_jar and self.missing_jars: 569 self.build_targets |= self.missing_jars 570 571 def _collect_dep_paths(self): 572 """Collects the path of dependency modules.""" 573 config = project_config.ProjectConfig.get_instance() 574 modules_info = config.atest_module_info 575 self.dep_paths = [] 576 if self.module_path != constant.FRAMEWORK_PATH: 577 self.dep_paths.append(constant.FRAMEWORK_PATH) 578 self.dep_paths.append(_FRAMEWORK_SRCJARS_PATH) 579 if self.module_path != constant.LIBCORE_PATH: 580 self.dep_paths.append(constant.LIBCORE_PATH) 581 for module in self.module_data.get(constant.KEY_DEPENDENCIES, []): 582 for path in modules_info.get_paths(module): 583 if path not in self.dep_paths and path != self.module_path: 584 self.dep_paths.append(path) 585 586 def locate_sources_path(self): 587 """Locate source folders' paths or jar files.""" 588 # Check if users need to reference source according to source depth. 589 if not self.module_depth <= self.depth_by_source: 590 self._append_jar_from_installed(self.specific_soong_path) 591 else: 592 self._collect_srcs_and_r_srcs_paths() 593 self._collect_missing_jars() 594 595 596class EclipseModuleData(ModuleData): 597 """Deal with modules data for Eclipse 598 599 Only project target modules use source folder type and the other ones use 600 jar as their source. We'll combine both to establish the whole project's 601 dependencies. If the source folder used to build dependency jar file exists 602 in Android, we should provide the jar file path as <linkedResource> item in 603 source data. 604 """ 605 606 def __init__(self, module_name, module_data, project_relpath): 607 """Initialize EclipseModuleData. 608 609 Only project target modules apply source folder type, so set the depth 610 of module referenced by source to 0. 611 612 Args: 613 module_name: String type, name of the module. 614 module_data: A dictionary contains a module information. 615 project_relpath: A string stands for the project's relative path. 616 """ 617 super().__init__(module_name, module_data, depth=0) 618 related = module_info.AidegenModuleInfo.is_project_path_relative_module( 619 module_data, project_relpath) 620 self.is_project = related 621 622 def locate_sources_path(self): 623 """Locate source folders' paths or jar files. 624 625 Only collect source folders for the project modules and collect jar 626 files for the other dependent modules. 627 """ 628 if self.is_project: 629 self._locate_project_source_path() 630 else: 631 self._locate_jar_path() 632 self._collect_classes_jars() 633 self._collect_missing_jars() 634 635 def _add_to_source_or_test_dirs(self, src_dir): 636 """Add a folder to source list if it is not in ignored directories. 637 638 Override the parent method since the tests folder has no difference 639 with source folder in Eclipse. 640 641 Args: 642 src_dir: a string of relative path to the Android root. 643 """ 644 if src_dir not in _IGNORE_DIRS and src_dir not in self.src_dirs: 645 self.src_dirs.append(src_dir) 646 647 def _locate_project_source_path(self): 648 """Locate the source folder paths of the project module. 649 650 A project module is the target modules or paths that users key in 651 aidegen command. Collecting the source folders is necessary for 652 developers to edit code. And also collect the central R folder for the 653 dependency of resources. 654 """ 655 self._collect_srcs_paths() 656 self._collect_r_srcs_paths() 657 658 def _locate_jar_path(self): 659 """Locate the jar path of the module. 660 661 Use jar files for dependency modules for Eclipse. Collect the jar file 662 path with different cases. 663 """ 664 if self._check_jarjar_rules_exist(): 665 self._append_jar_from_installed(self.specific_soong_path) 666 elif self._check_jars_exist(): 667 self._set_jars_jarfile() 668 elif self._check_classes_jar_exist(): 669 self._append_classes_jar() 670 else: 671 self._append_jar_from_installed() 672