1# Copyright 2018, The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15""" 16Module Info class used to hold cached module-info.json. 17""" 18 19# pylint: disable=line-too-long,too-many-lines 20 21import json 22import logging 23import os 24import pickle 25import re 26import shutil 27import tempfile 28import time 29 30from pathlib import Path 31from typing import Any, Dict, List, Set 32 33from atest import atest_utils 34from atest import constants 35 36from atest.atest_enum import DetectType 37from atest.metrics import metrics 38 39 40# JSON file generated by build system that lists all buildable targets. 41_MODULE_INFO = 'module-info.json' 42# JSON file generated by build system that lists dependencies for java. 43_JAVA_DEP_INFO = 'module_bp_java_deps.json' 44# JSON file generated by build system that lists dependencies for cc. 45_CC_DEP_INFO = 'module_bp_cc_deps.json' 46# JSON file generated by atest merged the content from module-info, 47# module_bp_java_deps.json, and module_bp_cc_deps. 48_MERGED_INFO = 'atest_merged_dep.json' 49 50 51Module = Dict[str, Any] 52 53 54class ModuleInfo: 55 """Class that offers fast/easy lookup for Module related details.""" 56 57 def __init__( 58 self, 59 force_build=False, 60 module_file=None, 61 index_dir=None, 62 no_generate=False): 63 """Initialize the ModuleInfo object. 64 65 Load up the module-info.json file and initialize the helper vars. 66 Note that module-info.json does not contain all module dependencies, 67 therefore, Atest needs to accumulate dependencies defined in bp files. 68 69 +----------------------+ +----------------------------+ 70 | $ANDROID_PRODUCT_OUT | |$ANDROID_BUILD_TOP/out/soong| 71 | /module-info.json | | /module_bp_java_deps.json | 72 +-----------+----------+ +-------------+--------------+ 73 | _merge_soong_info() | 74 +------------------------------+ 75 | 76 v 77 +----------------------------+ +----------------------------+ 78 |tempfile.NamedTemporaryFile | |$ANDROID_BUILD_TOP/out/soong| 79 +-------------+--------------+ | /module_bp_cc_deps.json | 80 | +-------------+--------------+ 81 | _merge_soong_info() | 82 +-------------------------------+ 83 | 84 +-------| 85 v 86 +============================+ 87 | $ANDROID_PRODUCT_OUT | 88 | /atest_merged_dep.json |--> load as module info. 89 +============================+ 90 91 Args: 92 force_build: Boolean to indicate if we should rebuild the 93 module_info file regardless if it's created or not. 94 module_file: String of path to file to load up. Used for testing. 95 index_dir: String of path to store testable module index and md5. 96 no_generate: Boolean to indicate if we should populate module info 97 from the soong artifacts; setting to true will 98 leave module info empty. 99 """ 100 # TODO(b/263199608): Refactor the ModuleInfo constructor. 101 # The module-info constructor does too much. We should never be doing 102 # real work in a constructor and should only use it to inject 103 # dependencies. 104 105 # force_build could be from "-m" or smart_build(build files change). 106 self.force_build = force_build 107 # update_merge_info flag will merge dep files only when any of them have 108 # changed even force_build == True. 109 self.update_merge_info = False 110 self.roboleaf_tests = {} 111 112 # Index and checksum files that will be used. 113 index_dir = ( 114 Path(index_dir) if index_dir else 115 Path(os.getenv(constants.ANDROID_HOST_OUT)).joinpath('indexes') 116 ) 117 if not index_dir.is_dir(): 118 index_dir.mkdir(parents=True) 119 self.module_index = index_dir.joinpath(constants.MODULE_INDEX) 120 self.module_info_checksum = index_dir.joinpath(constants.MODULE_INFO_MD5) 121 122 # Paths to java, cc and merged module info json files. 123 self.java_dep_path = Path( 124 atest_utils.get_build_out_dir()).joinpath('soong', _JAVA_DEP_INFO) 125 self.cc_dep_path = Path( 126 atest_utils.get_build_out_dir()).joinpath('soong', _CC_DEP_INFO) 127 self.merged_dep_path = Path( 128 os.getenv(constants.ANDROID_PRODUCT_OUT, '')).joinpath(_MERGED_INFO) 129 130 self.mod_info_file_path = Path(module_file) if module_file else None 131 132 if no_generate: 133 self.name_to_module_info = {} 134 return 135 136 module_info_target, name_to_module_info = self._load_module_info_file( 137 module_file) 138 self.name_to_module_info = name_to_module_info 139 self.module_info_target = module_info_target 140 self.path_to_module_info = self._get_path_to_module_info( 141 self.name_to_module_info) 142 self.root_dir = os.environ.get(constants.ANDROID_BUILD_TOP) 143 self.module_index_proc = None 144 if self.update_merge_info or not self.module_index.is_file(): 145 # Assumably null module_file reflects a common run, and index testable 146 # modules only when common runs. 147 if not module_file: 148 self.module_index_proc = atest_utils.run_multi_proc( 149 func=self._get_testable_modules, 150 kwargs={'index': True}) 151 152 @staticmethod 153 def _discover_mod_file_and_target(force_build): 154 """Find the module file. 155 156 Args: 157 force_build: Boolean to indicate if we should rebuild the 158 module_info file regardless of the existence of it. 159 160 Returns: 161 Tuple of module_info_target and path to module file. 162 """ 163 logging.debug('Probing and validating module info...') 164 module_info_target = None 165 root_dir = os.environ.get(constants.ANDROID_BUILD_TOP, '/') 166 out_dir = os.environ.get(constants.ANDROID_PRODUCT_OUT, root_dir) 167 module_file_path = os.path.join(out_dir, _MODULE_INFO) 168 169 # Check if the user set a custom out directory by comparing the out_dir 170 # to the root_dir. 171 if out_dir.find(root_dir) == 0: 172 # Make target is simply file path no-absolute to root 173 module_info_target = os.path.relpath(module_file_path, root_dir) 174 else: 175 # If the user has set a custom out directory, generate an absolute 176 # path for module info targets. 177 logging.debug('User customized out dir!') 178 module_file_path = os.path.join( 179 os.environ.get(constants.ANDROID_PRODUCT_OUT), _MODULE_INFO) 180 module_info_target = module_file_path 181 if force_build: 182 atest_utils.build_module_info_target(module_info_target) 183 return module_info_target, module_file_path 184 185 def _load_module_info_file(self, module_file): 186 """Load the module file. 187 188 No matter whether passing module_file or not, ModuleInfo will load 189 atest_merged_dep.json as module info eventually. 190 191 +--------------+ +----------------------------------+ 192 | ModuleInfo() | | ModuleInfo(module_file=foo.json) | 193 +-------+------+ +----------------+-----------------+ 194 | _discover_mod_file_and_target() | 195 | atest_utils.build() | load 196 v V 197 +--------------------------+ +--------------------------+ 198 | module-info.json | | foo.json | 199 | module_bp_cc_deps.json | | module_bp_cc_deps.json | 200 | module_bp_java_deps.json | | module_bp_java_deps.json | 201 +--------------------------+ +--------------------------+ 202 | | 203 | _merge_soong_info() <--------------------+ 204 v 205 +============================+ 206 | $ANDROID_PRODUCT_OUT | 207 | /atest_merged_dep.json |--> load as module info. 208 +============================+ 209 210 Args: 211 module_file: String of path to file to load up. Used for testing. 212 Note: if set, ModuleInfo will skip build process. 213 214 Returns: 215 Tuple of module_info_target and dict of json. 216 """ 217 # If module_file is specified, we're gonna test it so we don't care if 218 # module_info_target stays None. 219 module_info_target = None 220 file_path = module_file 221 previous_checksum = atest_utils.load_json_safely( 222 self.module_info_checksum) 223 if not file_path: 224 module_info_target, file_path = self._discover_mod_file_and_target( 225 self.force_build) 226 self.mod_info_file_path = Path(file_path) 227 # Even undergone a rebuild after _discover_mod_file_and_target(), merge 228 # atest_merged_dep.json only when module_deps_infos actually change so 229 # that Atest can decrease disk I/O and ensure data accuracy at all. 230 self.update_merge_info = self.need_update_merged_file(previous_checksum) 231 start = time.time() 232 if self.update_merge_info: 233 # Load the $ANDROID_PRODUCT_OUT/module-info.json for merging. 234 module_info_json = atest_utils.load_json_safely(file_path) 235 if Path(file_path).name == _MODULE_INFO and not module_info_json: 236 # Rebuild module-info.json when it has invalid format. However, 237 # if the file_path doesn't end with module-info.json, it could 238 # be from unit tests and won't trigger rebuild. 239 atest_utils.build_module_info_target(module_info_target) 240 start = time.time() 241 module_info_json = atest_utils.load_json_safely(file_path) 242 mod_info = self._merge_build_system_infos(module_info_json) 243 duration = time.time() - start 244 logging.debug('Merging module info took %ss', duration) 245 metrics.LocalDetectEvent( 246 detect_type=DetectType.MODULE_MERGE_MS, result=int(duration*1000)) 247 else: 248 # Load $ANDROID_PRODUCT_OUT/atest_merged_dep.json directly. 249 with open(self.merged_dep_path, encoding='utf-8') as merged_info_json: 250 mod_info = json.load(merged_info_json) 251 duration = time.time() - start 252 logging.debug('Loading module info took %ss', duration) 253 metrics.LocalDetectEvent( 254 detect_type=DetectType.MODULE_LOAD_MS, result=int(duration*1000)) 255 _add_missing_variant_modules(mod_info) 256 logging.debug('Loading %s as module-info.', self.merged_dep_path) 257 return module_info_target, mod_info 258 259 def _save_module_info_checksum(self): 260 """Dump the checksum of essential module info files. 261 * module-info.json 262 * module_bp_cc_deps.json 263 * module_bp_java_deps.json 264 """ 265 dirname = Path(self.module_info_checksum).parent 266 if not dirname.is_dir(): 267 dirname.mkdir(parents=True) 268 atest_utils.save_md5([ 269 self.mod_info_file_path, 270 self.java_dep_path, 271 self.cc_dep_path], self.module_info_checksum) 272 273 @staticmethod 274 def _get_path_to_module_info(name_to_module_info): 275 """Return the path_to_module_info dict. 276 277 Args: 278 name_to_module_info: Dict of module name to module info dict. 279 280 Returns: 281 Dict of module path to module info dict. 282 """ 283 path_to_module_info = {} 284 for mod_name, mod_info in name_to_module_info.items(): 285 # Cross-compiled and multi-arch modules actually all belong to 286 # a single target so filter out these extra modules. 287 if mod_name != mod_info.get(constants.MODULE_NAME, ''): 288 continue 289 for path in mod_info.get(constants.MODULE_PATH, []): 290 mod_info[constants.MODULE_NAME] = mod_name 291 # There could be multiple modules in a path. 292 if path in path_to_module_info: 293 path_to_module_info[path].append(mod_info) 294 else: 295 path_to_module_info[path] = [mod_info] 296 return path_to_module_info 297 298 def _index_testable_modules(self, content): 299 """Dump testable modules. 300 301 Args: 302 content: An object that will be written to the index file. 303 """ 304 logging.debug(r'Indexing testable modules... ' 305 r'(This is required whenever module-info.json ' 306 r'was rebuilt.)') 307 Path(self.module_index).parent.mkdir(parents=True, exist_ok=True) 308 with open(self.module_index, 'wb') as cache: 309 try: 310 pickle.dump(content, cache, protocol=2) 311 except IOError: 312 logging.error('Failed in dumping %s', cache) 313 os.remove(cache) 314 315 def _get_testable_modules(self, index=False, suite=None): 316 """Return all available testable modules and index them. 317 318 Args: 319 index: boolean that determines running _index_testable_modules(). 320 suite: string for the suite name. 321 322 Returns: 323 Set of all testable modules. 324 """ 325 modules = set() 326 begin = time.time() 327 for _, info in self.name_to_module_info.items(): 328 if self.is_testable_module(info): 329 modules.add(info.get(constants.MODULE_NAME)) 330 logging.debug('Probing all testable modules took %ss', 331 time.time() - begin) 332 if index: 333 self._index_testable_modules(modules) 334 if suite: 335 _modules = set() 336 for module_name in modules: 337 info = self.get_module_info(module_name) 338 if self.is_suite_in_compatibility_suites(suite, info): 339 _modules.add(info.get(constants.MODULE_NAME)) 340 return _modules 341 return modules 342 343 def is_module(self, name): 344 """Return True if name is a module, False otherwise.""" 345 info = self.get_module_info(name) 346 # From aosp/2293302 it started merging all modules' dependency in bp 347 # even the module is not be exposed to make, and those modules could not 348 # be treated as a build target using m. Only treat input name as module 349 # if it also has the module_name attribute which means it could be a 350 # build target for m. 351 if info and info.get(constants.MODULE_NAME): 352 return True 353 return False 354 355 def get_paths(self, name): 356 """Return paths of supplied module name, Empty list if non-existent.""" 357 info = self.get_module_info(name) 358 if info: 359 return info.get(constants.MODULE_PATH, []) 360 return [] 361 362 def get_module_names(self, rel_module_path): 363 """Get the modules that all have module_path. 364 365 Args: 366 rel_module_path: path of module in module-info.json 367 368 Returns: 369 List of module names. 370 """ 371 return [m.get(constants.MODULE_NAME) 372 for m in self.path_to_module_info.get(rel_module_path, [])] 373 374 def get_module_info(self, mod_name): 375 """Return dict of info for given module name, None if non-existence.""" 376 return self.name_to_module_info.get(mod_name) 377 378 def is_suite_in_compatibility_suites(self, suite, mod_info): 379 """Check if suite exists in the compatibility_suites of module-info. 380 381 Args: 382 suite: A string of suite name. 383 mod_info: Dict of module info to check. 384 385 Returns: 386 True if it exists in mod_info, False otherwise. 387 """ 388 if not isinstance(mod_info, dict): 389 return False 390 return suite in mod_info.get( 391 constants.MODULE_COMPATIBILITY_SUITES, []) 392 393 def get_testable_modules(self, suite=None): 394 """Return the testable modules of the given suite name. 395 396 Atest does not index testable modules against compatibility_suites. When 397 suite was given, or the index file was interrupted, always run 398 _get_testable_modules() and re-index. 399 400 Args: 401 suite: A string of suite name. 402 403 Returns: 404 If suite is not given, return all the testable modules in module 405 info, otherwise return only modules that belong to the suite. 406 """ 407 modules = set() 408 start = time.time() 409 if self.module_index_proc: 410 self.module_index_proc.join() 411 412 if self.module_index.is_file(): 413 if not suite: 414 with open(self.module_index, 'rb') as cache: 415 try: 416 modules = pickle.load(cache, encoding="utf-8") 417 except UnicodeDecodeError: 418 modules = pickle.load(cache) 419 # when module indexing was interrupted. 420 except EOFError: 421 pass 422 else: 423 modules = self._get_testable_modules(suite=suite) 424 # If the modules.idx does not exist or invalid for any reason, generate 425 # a new one arbitrarily. 426 if not modules: 427 if not suite: 428 modules = self._get_testable_modules(index=True) 429 else: 430 modules = self._get_testable_modules(index=True, suite=suite) 431 duration = time.time() - start 432 metrics.LocalDetectEvent( 433 detect_type=DetectType.TESTABLE_MODULES, 434 result=int(duration)) 435 return modules 436 437 def is_tradefed_testable_module(self, info: Dict[str, Any]) -> bool: 438 """Check whether the module is a Tradefed executable test.""" 439 if not info: 440 return False 441 if not info.get(constants.MODULE_INSTALLED, []): 442 return False 443 return self.has_test_config(info) 444 445 def is_testable_module(self, info: Dict[str, Any]) -> bool: 446 """Check if module is something we can test. 447 448 A module is testable if: 449 - it's a tradefed testable module, or 450 - it's a robolectric module (or shares path with one). 451 452 Args: 453 info: Dict of module info to check. 454 455 Returns: 456 True if we can test this module, False otherwise. 457 """ 458 if not info: 459 return False 460 if self.is_tradefed_testable_module(info): 461 return True 462 if self.is_legacy_robolectric_test(info.get(constants.MODULE_NAME)): 463 return True 464 return False 465 466 def has_test_config(self, info: Dict[str, Any]) -> bool: 467 """Validate if this module has a test config. 468 469 A module can have a test config in the following manner: 470 - test_config be set in module-info.json. 471 - Auto-generated config via the auto_test_config key 472 in module-info.json. 473 474 Args: 475 info: Dict of module info to check. 476 477 Returns: 478 True if this module has a test config, False otherwise. 479 """ 480 return bool(info.get(constants.MODULE_TEST_CONFIG, []) or 481 info.get('auto_test_config', [])) 482 483 def is_legacy_robolectric_test(self, module_name: str) -> bool: 484 """Return whether the module_name is a legacy Robolectric test""" 485 return bool(self.get_robolectric_test_name(module_name)) 486 487 def get_robolectric_test_name(self, module_name: str) -> str: 488 """Returns runnable robolectric module name. 489 490 This method is for legacy robolectric tests and returns one of associated 491 modules. The pattern is determined by the amount of shards: 492 493 10 shards: 494 FooTests -> RunFooTests0, RunFooTests1 ... RunFooTests9 495 No shard: 496 FooTests -> RunFooTests 497 498 Arg: 499 module_name: String of module. 500 501 Returns: 502 String of the first-matched associated module that belongs to the 503 actual robolectric module, None if nothing has been found. 504 """ 505 info = self.get_module_info(module_name) or {} 506 module_paths = info.get(constants.MODULE_PATH, []) 507 if not module_paths: 508 return '' 509 filtered_module_names = [ 510 name 511 for name in self.get_module_names(module_paths[0]) 512 if name.startswith("Run") 513 ] 514 return next( 515 ( 516 name 517 for name in filtered_module_names 518 if self.is_legacy_robolectric_class(self.get_module_info(name)) 519 ), 520 '', 521 ) 522 523 def is_robolectric_test(self, module_name): 524 """Check if the given module is a robolectric test. 525 526 Args: 527 module_name: String of module to check. 528 529 Returns: 530 Boolean whether it's a robotest or not. 531 """ 532 if self.get_robolectric_type(module_name): 533 return True 534 return False 535 536 def get_robolectric_type(self, module_name): 537 """Check if the given module is a robolectric test and return type of it. 538 539 Robolectric declaration is converting from Android.mk to Android.bp, and 540 in the interim Atest needs to support testing both types of tests. 541 542 The modern robolectric tests defined by 'android_robolectric_test' in an 543 Android.bp file can can be run in Tradefed Test Runner: 544 545 SettingsRoboTests -> Tradefed Test Runner 546 547 Legacy tests defined in an Android.mk can only run with the 'make' way. 548 549 SettingsRoboTests -> make RunSettingsRoboTests0 550 551 To determine whether the test is a modern/legacy robolectric test: 552 1. Traverse all modules share the module path. If one of the 553 modules has a ROBOLECTRIC class, it is a robolectric test. 554 2. If the 'robolectric-test` in the compatibility_suites, it's a 555 modern one, otherwise it's a legacy test. This is accurate since 556 aosp/2308586 already set the test suite of `robolectric-test` 557 for all `modern` Robolectric tests in Soong. 558 559 Args: 560 module_name: String of module to check. 561 562 Returns: 563 0: not a robolectric test. 564 1: a modern robolectric test(defined in Android.bp) 565 2: a legacy robolectric test(defined in Android.mk) 566 """ 567 info = self.get_module_info(module_name) 568 if not info: 569 return 0 570 # Some Modern mode Robolectric test has related module which compliant 571 # with the Legacy Robolectric test. In this case, the Modern mode 572 # Robolectric tests should prior to Legacy mode. 573 if self.is_modern_robolectric_test(info): 574 return constants.ROBOTYPE_MODERN 575 if self.is_legacy_robolectric_test(module_name): 576 return constants.ROBOTYPE_LEGACY 577 return 0 578 579 def get_instrumentation_target_apps(self, module_name: str) -> Dict: 580 """Return target APKs of an instrumentation test. 581 582 Returns: 583 A dict of target module and target APK(s). e.g. 584 {"FooService": {"/path/to/the/FooService.apk"}} 585 """ 586 # 1. Determine the actual manifest filename from an Android.bp(if any) 587 manifest = self.get_filepath_from_module(module_name, 588 'AndroidManifest.xml') 589 bpfile = self.get_filepath_from_module(module_name, 'Android.bp') 590 if bpfile.is_file(): 591 bp_info = atest_utils.get_bp_content(bpfile, 'android_test') 592 if not bp_info or not bp_info.get(module_name): 593 return {} 594 manifest = self.get_filepath_from_module( 595 module_name, 596 bp_info.get(module_name).get('manifest')) 597 xml_info = atest_utils.get_manifest_info(manifest) 598 # 2. Translate package name to a module name. 599 package = xml_info.get('package') 600 target_package = xml_info.get('target_package') 601 # Ensure it's an instrumentation test(excluding self-instrmented) 602 if target_package and package != target_package: 603 logging.debug('Found %s an instrumentation test.', module_name) 604 metrics.LocalDetectEvent( 605 detect_type=DetectType.FOUND_INSTRUMENTATION_TEST, result=1) 606 target_module = self.get_target_module_by_pkg( 607 package=target_package, 608 search_from=manifest.parent) 609 if target_module: 610 return self.get_artifact_map(target_module) 611 return {} 612 613 # pylint: disable=anomalous-backslash-in-string 614 def get_target_module_by_pkg(self, package: str, search_from: Path) -> str: 615 """Translate package name to the target module name. 616 617 This method is dedicated to determine the target module by translating 618 a package name. 619 620 Phase 1: Find out possible manifest files among parent directories. 621 Phase 2. Look for the defined package fits the given name, and ensure 622 it is not a persistent app. 623 Phase 3: Translate the manifest path to possible modules. A valid module 624 must fulfill: 625 1. The 'class' type must be ['APPS']. 626 2. It is not a Robolectric test. 627 628 Returns: 629 A string of module name. 630 """ 631 xmls = [] 632 for pth in search_from.parents: 633 if pth == Path(self.root_dir): 634 break 635 for name in os.listdir(pth): 636 if pth.joinpath(name).is_file(): 637 match = re.match('.*AndroidManifest.*\.xml$', name) 638 if match: 639 xmls.append(os.path.join(pth, name)) 640 possible_modules = [] 641 for xml in xmls: 642 rel_dir = str(Path(xml).relative_to(self.root_dir).parent) 643 logging.debug('Looking for package "%s" in %s...', package, xml) 644 xml_info = atest_utils.get_manifest_info(xml) 645 if xml_info.get('package') == package: 646 if xml_info.get('persistent'): 647 logging.debug('%s is a persistent app.', package) 648 continue 649 for _m in self.path_to_module_info.get(rel_dir): 650 possible_modules.append(_m) 651 if possible_modules: 652 for mod in possible_modules: 653 name = mod.get('module_name') 654 if (mod.get('class') == ['APPS'] and 655 not self.is_robolectric_test(name)): 656 return name 657 return '' 658 659 def get_artifact_map(self, module_name: str) -> Dict: 660 """Get the installed APK path of the given module.""" 661 target_mod_info = self.get_module_info(module_name) 662 artifact_map = {} 663 if target_mod_info: 664 apks = set() 665 artifacts = target_mod_info.get('installed') 666 for artifact in artifacts: 667 if Path(artifact).suffix == '.apk': 668 apks.add(os.path.join(self.root_dir, artifact)) 669 artifact_map.update({module_name: apks}) 670 return artifact_map 671 672 def is_auto_gen_test_config(self, module_name): 673 """Check if the test config file will be generated automatically. 674 675 Args: 676 module_name: A string of the module name. 677 678 Returns: 679 True if the test config file will be generated automatically. 680 """ 681 if self.is_module(module_name): 682 mod_info = self.get_module_info(module_name) 683 auto_test_config = mod_info.get('auto_test_config', []) 684 return auto_test_config and auto_test_config[0] 685 return False 686 687 def is_legacy_robolectric_class(self, info: Dict[str, Any]) -> bool: 688 """Check if the class is `ROBOLECTRIC` 689 690 This method is for legacy robolectric tests that the associated modules 691 contain: 692 'class': ['ROBOLECTRIC'] 693 694 Args: 695 info: ModuleInfo to check. 696 697 Returns: 698 True if the attribute class in mod_info is ROBOLECTRIC, False 699 otherwise. 700 """ 701 if info: 702 module_classes = info.get(constants.MODULE_CLASS, []) 703 return (module_classes and 704 module_classes[0] == constants.MODULE_CLASS_ROBOLECTRIC) 705 return False 706 707 def is_native_test(self, module_name): 708 """Check if the input module is a native test. 709 710 Args: 711 module_name: A string of the module name. 712 713 Returns: 714 True if the test is a native test, False otherwise. 715 """ 716 mod_info = self.get_module_info(module_name) 717 return constants.MODULE_CLASS_NATIVE_TESTS in mod_info.get( 718 constants.MODULE_CLASS, []) 719 720 def has_mainline_modules(self, 721 module_name: str, mainline_binaries: List[str]) -> bool: 722 """Check if the mainline modules are in module-info. 723 724 Args: 725 module_name: A string of the module name. 726 mainline_binaries: A list of mainline module binaries. 727 728 Returns: 729 True if mainline_binaries is in module-info, False otherwise. 730 """ 731 mod_info = self.get_module_info(module_name) 732 # Check 'test_mainline_modules' attribute of the module-info.json. 733 mm_in_mf = mod_info.get(constants.MODULE_MAINLINE_MODULES, []) 734 ml_modules_set = set(mainline_binaries) 735 if mm_in_mf: 736 return contains_same_mainline_modules( 737 ml_modules_set, set(mm_in_mf)) 738 for test_config in mod_info.get(constants.MODULE_TEST_CONFIG, []): 739 # Check the value of 'mainline-param' in the test config. 740 if not self.is_auto_gen_test_config(module_name): 741 return contains_same_mainline_modules( 742 ml_modules_set, 743 atest_utils.get_mainline_param( 744 os.path.join(self.root_dir, test_config))) 745 # Unable to verify mainline modules in an auto-gen test config. 746 logging.debug('%s is associated with an auto-generated test config.', 747 module_name) 748 return True 749 return False 750 751 def _merge_build_system_infos(self, name_to_module_info, 752 java_bp_info_path=None, cc_bp_info_path=None): 753 """Merge the content of module-info.json and CC/Java dependency files 754 to name_to_module_info. 755 756 Args: 757 name_to_module_info: Dict of module name to module info dict. 758 java_bp_info_path: String of path to java dep file to load up. 759 Used for testing. 760 cc_bp_info_path: String of path to cc dep file to load up. 761 Used for testing. 762 763 Returns: 764 Dict of updated name_to_module_info. 765 """ 766 # Merge _JAVA_DEP_INFO 767 if not java_bp_info_path: 768 java_bp_info_path = self.java_dep_path 769 java_bp_infos = atest_utils.load_json_safely(java_bp_info_path) 770 if java_bp_infos: 771 logging.debug('Merging Java build info: %s', java_bp_info_path) 772 name_to_module_info = self._merge_soong_info( 773 name_to_module_info, java_bp_infos) 774 # Merge _CC_DEP_INFO 775 if not cc_bp_info_path: 776 cc_bp_info_path = self.cc_dep_path 777 cc_bp_infos = atest_utils.load_json_safely(cc_bp_info_path) 778 if cc_bp_infos: 779 logging.debug('Merging CC build info: %s', cc_bp_info_path) 780 # CC's dep json format is different with java. 781 # Below is the example content: 782 # { 783 # "clang": "${ANDROID_ROOT}/bin/clang", 784 # "clang++": "${ANDROID_ROOT}/bin/clang++", 785 # "modules": { 786 # "ACameraNdkVendorTest": { 787 # "path": [ 788 # "frameworks/av/camera/ndk" 789 # ], 790 # "srcs": [ 791 # "frameworks/tests/AImageVendorTest.cpp", 792 # "frameworks/tests/ACameraManagerTest.cpp" 793 # ], 794 name_to_module_info = self._merge_soong_info( 795 name_to_module_info, cc_bp_infos.get('modules', {})) 796 # If $ANDROID_PRODUCT_OUT was not created in pyfakefs, simply return it 797 # without dumping atest_merged_dep.json in real. 798 799 # Adds the key into module info as a unique ID. 800 for key, info in name_to_module_info.items(): 801 info[constants.MODULE_INFO_ID] = key 802 803 if not self.merged_dep_path.parent.is_dir(): 804 return name_to_module_info 805 # b/178559543 saving merged module info in a temp file and copying it to 806 # atest_merged_dep.json can eliminate the possibility of accessing it 807 # concurrently and resulting in invalid JSON format. 808 with tempfile.NamedTemporaryFile() as temp_file: 809 with open(temp_file.name, 'w', encoding='utf-8') as _temp: 810 json.dump(name_to_module_info, _temp, indent=0) 811 shutil.copy(temp_file.name, self.merged_dep_path) 812 return name_to_module_info 813 814 def _merge_soong_info(self, name_to_module_info, mod_bp_infos): 815 """Merge the dependency and srcs in mod_bp_infos to name_to_module_info. 816 817 Args: 818 name_to_module_info: Dict of module name to module info dict. 819 mod_bp_infos: Dict of module name to bp's module info dict. 820 821 Returns: 822 Dict of updated name_to_module_info. 823 """ 824 merge_items = [constants.MODULE_DEPENDENCIES, constants.MODULE_SRCS, 825 constants.MODULE_LIBS, constants.MODULE_STATIC_LIBS, 826 constants.MODULE_STATIC_DEPS, constants.MODULE_PATH] 827 for module_name, dep_info in mod_bp_infos.items(): 828 mod_info = name_to_module_info.setdefault(module_name, {}) 829 for merge_item in merge_items: 830 dep_info_values = dep_info.get(merge_item, []) 831 mod_info_values = mod_info.get(merge_item, []) 832 mod_info_values.extend(dep_info_values) 833 mod_info_values.sort() 834 # deduplicate values just in case. 835 mod_info_values = list(dict.fromkeys(mod_info_values)) 836 name_to_module_info[ 837 module_name][merge_item] = mod_info_values 838 return name_to_module_info 839 840 def get_filepath_from_module(self, module_name: str, filename: str) -> Path: 841 """Return absolute path of the given module and filename.""" 842 mod_path = self.get_paths(module_name) 843 if mod_path: 844 return Path(self.root_dir).joinpath(mod_path[0], filename) 845 return Path() 846 847 def get_module_dependency(self, module_name, depend_on=None): 848 """Get the dependency sets for input module. 849 850 Recursively find all the dependencies of the input module. 851 852 Args: 853 module_name: String of module to check. 854 depend_on: The list of parent dependencies. 855 856 Returns: 857 Set of dependency modules. 858 """ 859 if not depend_on: 860 depend_on = set() 861 deps = set() 862 mod_info = self.get_module_info(module_name) 863 if not mod_info: 864 return deps 865 mod_deps = set(mod_info.get(constants.MODULE_DEPENDENCIES, [])) 866 # Remove item in deps if it already in depend_on: 867 mod_deps = mod_deps - depend_on 868 deps = deps.union(mod_deps) 869 for mod_dep in mod_deps: 870 deps = deps.union(set(self.get_module_dependency( 871 mod_dep, depend_on=depend_on.union(deps)))) 872 return deps 873 874 def get_install_module_dependency(self, module_name, depend_on=None): 875 """Get the dependency set for the given modules with installed path. 876 877 Args: 878 module_name: String of module to check. 879 depend_on: The list of parent dependencies. 880 881 Returns: 882 Set of dependency modules which has installed path. 883 """ 884 install_deps = set() 885 deps = self.get_module_dependency(module_name, depend_on) 886 logging.debug('%s depends on: %s', module_name, deps) 887 for module in deps: 888 mod_info = self.get_module_info(module) 889 if mod_info and mod_info.get(constants.MODULE_INSTALLED, []): 890 install_deps.add(module) 891 logging.debug('modules %s required by %s were not installed', 892 install_deps, module_name) 893 return install_deps 894 895 def need_update_merged_file(self, checksum): 896 """Check if need to update/generated atest_merged_dep. 897 898 There are 2 scienarios that atest_merged_dep.json will be updated. 899 1. One of the checksum of module-info.json, module_bp_java_deps.json and 900 module_cc_java_deps.json have changed. 901 2. atest_merged_deps.json does not exist. 902 903 If fits one of above scienarios, it is recognized to update. 904 905 Returns: 906 True if one of the scienarios reaches, False otherwise. 907 """ 908 current_checksum = {str(name): atest_utils.md5sum(name) for name in [ 909 self.mod_info_file_path, 910 self.java_dep_path, 911 self.cc_dep_path]} 912 return (checksum != current_checksum or 913 not Path(self.merged_dep_path).is_file()) 914 915 def is_unit_test(self, mod_info): 916 """Return True if input module is unit test, False otherwise. 917 918 Args: 919 mod_info: ModuleInfo to check. 920 921 Returns: 922 True if input module is unit test, False otherwise. 923 """ 924 return mod_info.get(constants.MODULE_IS_UNIT_TEST, '') == 'true' 925 926 def is_host_unit_test(self, info: Dict[str, Any]) -> bool: 927 """Return True if input module is host unit test, False otherwise. 928 929 Args: 930 info: ModuleInfo to check. 931 932 Returns: 933 True if input module is host unit test, False otherwise. 934 """ 935 return self.is_tradefed_testable_module(info) and \ 936 self.is_suite_in_compatibility_suites('host-unit-tests', info) 937 938 def is_modern_robolectric_test(self, info: Dict[str, Any]) -> bool: 939 """Return whether 'robolectric-tests' is in 'compatibility_suites'.""" 940 return self.is_tradefed_testable_module(info) and \ 941 self.is_robolectric_test_suite(info) 942 943 def is_robolectric_test_suite(self, mod_info) -> bool: 944 """Return True if 'robolectric-tests' in the compatibility_suites. 945 946 Args: 947 mod_info: ModuleInfo to check. 948 949 Returns: 950 True if the 'robolectric-tests' is in the compatibility_suites, 951 False otherwise. 952 """ 953 return self.is_suite_in_compatibility_suites('robolectric-tests', 954 mod_info) 955 956 def is_device_driven_test(self, mod_info): 957 """Return True if input module is device driven test, False otherwise. 958 959 Args: 960 mod_info: ModuleInfo to check. 961 962 Returns: 963 True if input module is device driven test, False otherwise. 964 """ 965 if self.is_robolectric_test_suite(mod_info): 966 return False 967 968 return self.is_tradefed_testable_module(mod_info) and \ 969 'DEVICE' in mod_info.get(constants.MODULE_SUPPORTED_VARIANTS, []) 970 971 def is_host_driven_test(self, mod_info): 972 """Return True if input module is host driven test, False otherwise. 973 974 Args: 975 mod_info: ModuleInfo to check. 976 977 Returns: 978 True if input module is host driven test, False otherwise. 979 """ 980 return self.is_tradefed_testable_module(mod_info) and \ 981 'HOST' in mod_info.get(constants.MODULE_SUPPORTED_VARIANTS, []) 982 983 def _any_module(self, _: Module) -> bool: 984 return True 985 986 def get_all_tests(self): 987 """Get a list of all the module names which are tests.""" 988 return self._get_all_modules(type_predicate=self.is_testable_module) 989 990 def get_all_unit_tests(self): 991 """Get a list of all the module names which are unit tests.""" 992 return self._get_all_modules(type_predicate=self.is_unit_test) 993 994 def get_all_host_unit_tests(self): 995 """Get a list of all the module names which are host unit tests.""" 996 return self._get_all_modules(type_predicate=self.is_host_unit_test) 997 998 def get_all_device_driven_tests(self): 999 """Get a list of all the module names which are device driven tests.""" 1000 return self._get_all_modules(type_predicate=self.is_device_driven_test) 1001 1002 def _get_all_modules(self, type_predicate=None): 1003 """Get a list of all the module names that passed the predicate.""" 1004 modules = [] 1005 type_predicate = type_predicate or self._any_module 1006 for mod_name, mod_info in self.name_to_module_info.items(): 1007 if mod_info.get(constants.MODULE_NAME, '') == mod_name: 1008 if type_predicate(mod_info): 1009 modules.append(mod_name) 1010 return modules 1011 1012 def get_modules_by_path_in_srcs(self, path: str) -> Set: 1013 """Get the module name that the given path belongs to.(in 'srcs') 1014 1015 Args: 1016 path: Relative path to ANDROID_BUILD_TOP of a file. 1017 1018 Returns: 1019 A set of string for matched module names, empty set if nothing find. 1020 """ 1021 modules = set() 1022 for _, mod_info in self.name_to_module_info.items(): 1023 if str(path) in mod_info.get(constants.MODULE_SRCS, []): 1024 modules.add(mod_info.get(constants.MODULE_NAME)) 1025 return modules 1026 1027 def get_modules_by_include_deps( 1028 self, deps: Set[str], 1029 testable_module_only: bool = False) -> Set[str]: 1030 """Get the matched module names for the input dependencies. 1031 1032 Args: 1033 deps: A set of string for dependencies. 1034 testable_module_only: Option if only want to get testable module. 1035 1036 Returns: 1037 A set of matched module names for the input dependencies. 1038 """ 1039 modules = set() 1040 1041 for mod_name in (self.get_testable_modules() if testable_module_only 1042 else self.name_to_module_info.keys()): 1043 mod_info = self.get_module_info(mod_name) 1044 if mod_info and deps.intersection( 1045 set(mod_info.get(constants.MODULE_DEPENDENCIES, []))): 1046 modules.add(mod_info.get(constants.MODULE_NAME)) 1047 return modules 1048 1049 1050def _add_missing_variant_modules(name_to_module_info: Dict[str, Module]): 1051 missing_modules = {} 1052 1053 # Android's build system automatically adds a suffix for some build module 1054 # variants. For example, a module-info entry for a module originally named 1055 # 'HelloWorldTest' might appear as 'HelloWorldTest_32' and which Atest would 1056 # not be able to find. We add such entries if not already present so they 1057 # can be looked up using their declared module name. 1058 for mod_name, mod_info in name_to_module_info.items(): 1059 declared_module_name = mod_info.get(constants.MODULE_NAME, mod_name) 1060 if declared_module_name in name_to_module_info: 1061 continue 1062 missing_modules.setdefault(declared_module_name, mod_info) 1063 1064 name_to_module_info.update(missing_modules) 1065 1066def contains_same_mainline_modules(mainline_modules: Set[str], module_lists: Set[str]): 1067 """Check if mainline modules listed on command line is 1068 the same set as config. 1069 1070 Args: 1071 mainline_modules: A list of mainline modules from triggered test. 1072 module_lists: A list of concatenate mainline module string from test configs. 1073 1074 Returns 1075 True if the set mainline modules from triggered test is in the test configs. 1076 """ 1077 for module_string in module_lists: 1078 if mainline_modules == set(module_string.split('+')): 1079 return True 1080 return False 1081