1# Copyright 2018, The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15""" 16Module Info class used to hold cached module-info.json. 17""" 18 19# pylint: disable=line-too-long 20 21import json 22import logging 23import os 24import pickle 25import shutil 26import sys 27import tempfile 28import time 29 30from pathlib import Path 31from typing import Any, Dict 32 33import atest_utils 34import constants 35 36from atest_enum import DetectType, ExitCode 37from metrics import metrics 38 39# JSON file generated by build system that lists all buildable targets. 40_MODULE_INFO = 'module-info.json' 41# JSON file generated by build system that lists dependencies for java. 42_JAVA_DEP_INFO = 'module_bp_java_deps.json' 43# JSON file generated by build system that lists dependencies for cc. 44_CC_DEP_INFO = 'module_bp_cc_deps.json' 45# JSON file generated by atest merged the content from module-info, 46# module_bp_java_deps.json, and module_bp_cc_deps. 47_MERGED_INFO = 'atest_merged_dep.json' 48 49 50Module = Dict[str, Any] 51 52 53class ModuleInfo: 54 """Class that offers fast/easy lookup for Module related details.""" 55 56 def __init__(self, force_build=False, module_file=None, index_dir=None): 57 """Initialize the ModuleInfo object. 58 59 Load up the module-info.json file and initialize the helper vars. 60 Note that module-info.json does not contain all module dependencies, 61 therefore, Atest needs to accumulate dependencies defined in bp files. 62 63 +----------------------+ +----------------------------+ 64 | $ANDROID_PRODUCT_OUT | |$ANDROID_BUILD_TOP/out/soong| 65 | /module-info.json | | /module_bp_java_deps.json | 66 +-----------+----------+ +-------------+--------------+ 67 | _merge_soong_info() | 68 +------------------------------+ 69 | 70 v 71 +----------------------------+ +----------------------------+ 72 |tempfile.NamedTemporaryFile | |$ANDROID_BUILD_TOP/out/soong| 73 +-------------+--------------+ | /module_bp_cc_deps.json | 74 | +-------------+--------------+ 75 | _merge_soong_info() | 76 +-------------------------------+ 77 | 78 +-------| 79 v 80 +============================+ 81 | $ANDROID_PRODUCT_OUT | 82 | /atest_merged_dep.json |--> load as module info. 83 +============================+ 84 85 Args: 86 force_build: Boolean to indicate if we should rebuild the 87 module_info file regardless if it's created or not. 88 module_file: String of path to file to load up. Used for testing. 89 index_dir: String of path to store testable module index and md5. 90 """ 91 # force_build could be from "-m" or smart_build(build files change). 92 self.force_build = force_build 93 # update_merge_info flag will merge dep files only when any of them have 94 # changed even force_build == True. 95 self.update_merge_info = False 96 # Index and checksum files that will be used. 97 if not index_dir: 98 index_dir = Path( 99 os.getenv(constants.ANDROID_HOST_OUT, 100 tempfile.TemporaryDirectory().name)).joinpath('indexes') 101 index_dir = Path(index_dir) 102 if not index_dir.is_dir(): 103 index_dir.mkdir(parents=True) 104 self.module_index = index_dir.joinpath(constants.MODULE_INDEX) 105 self.module_info_checksum = index_dir.joinpath(constants.MODULE_INFO_MD5) 106 107 # Paths to java, cc and merged module info json files. 108 self.java_dep_path = Path( 109 atest_utils.get_build_out_dir()).joinpath('soong', _JAVA_DEP_INFO) 110 self.cc_dep_path = Path( 111 atest_utils.get_build_out_dir()).joinpath('soong', _CC_DEP_INFO) 112 self.merged_dep_path = Path( 113 os.getenv(constants.ANDROID_PRODUCT_OUT, '')).joinpath(_MERGED_INFO) 114 115 self.mod_info_file_path = Path(module_file) if module_file else None 116 module_info_target, name_to_module_info = self._load_module_info_file( 117 module_file) 118 self.name_to_module_info = name_to_module_info 119 self.module_info_target = module_info_target 120 self.path_to_module_info = self._get_path_to_module_info( 121 self.name_to_module_info) 122 self.root_dir = os.environ.get(constants.ANDROID_BUILD_TOP) 123 self.module_index_proc = None 124 if self.update_merge_info or not self.module_index.is_file(): 125 # Assumably null module_file reflects a common run, and index testable 126 # modules only when common runs. 127 if not module_file: 128 self.module_index_proc = atest_utils.run_multi_proc( 129 func=self._get_testable_modules, 130 kwargs={'index': True}) 131 132 @staticmethod 133 def _discover_mod_file_and_target(force_build): 134 """Find the module file. 135 136 Args: 137 force_build: Boolean to indicate if we should rebuild the 138 module_info file regardless of the existence of it. 139 140 Returns: 141 Tuple of module_info_target and path to module file. 142 """ 143 logging.debug('Probing and validating module info...') 144 module_info_target = None 145 root_dir = os.environ.get(constants.ANDROID_BUILD_TOP, '/') 146 out_dir = os.environ.get(constants.ANDROID_PRODUCT_OUT, root_dir) 147 module_file_path = os.path.join(out_dir, _MODULE_INFO) 148 149 # Check if the user set a custom out directory by comparing the out_dir 150 # to the root_dir. 151 if out_dir.find(root_dir) == 0: 152 # Make target is simply file path no-absolute to root 153 module_info_target = os.path.relpath(module_file_path, root_dir) 154 else: 155 # If the user has set a custom out directory, generate an absolute 156 # path for module info targets. 157 logging.debug('User customized out dir!') 158 module_file_path = os.path.join( 159 os.environ.get(constants.ANDROID_PRODUCT_OUT), _MODULE_INFO) 160 module_info_target = module_file_path 161 # Make sure module-info exist and could be load properly. 162 if not atest_utils.is_valid_json_file(module_file_path) or force_build: 163 logging.debug('Generating %s - this is required for ' 164 'initial runs or forced rebuilds.', _MODULE_INFO) 165 build_start = time.time() 166 if not atest_utils.build([module_info_target], 167 verbose=logging.getLogger().isEnabledFor( 168 logging.DEBUG)): 169 sys.exit(ExitCode.BUILD_FAILURE) 170 build_duration = time.time() - build_start 171 metrics.LocalDetectEvent( 172 detect_type=DetectType.ONLY_BUILD_MODULE_INFO, 173 result=int(build_duration)) 174 return module_info_target, module_file_path 175 176 def _load_module_info_file(self, module_file): 177 """Load the module file. 178 179 No matter whether passing module_file or not, ModuleInfo will load 180 atest_merged_dep.json as module info eventually. 181 182 +--------------+ +----------------------------------+ 183 | ModuleInfo() | | ModuleInfo(module_file=foo.json) | 184 +-------+------+ +----------------+-----------------+ 185 | _discover_mod_file_and_target() | 186 | atest_utils.build() | load 187 v V 188 +--------------------------+ +--------------------------+ 189 | module-info.json | | foo.json | 190 | module_bp_cc_deps.json | | module_bp_cc_deps.json | 191 | module_bp_java_deps.json | | module_bp_java_deps.json | 192 +--------------------------+ +--------------------------+ 193 | | 194 | _merge_soong_info() <--------------------+ 195 v 196 +============================+ 197 | $ANDROID_PRODUCT_OUT | 198 | /atest_merged_dep.json |--> load as module info. 199 +============================+ 200 201 Args: 202 module_file: String of path to file to load up. Used for testing. 203 Note: if set, ModuleInfo will skip build process. 204 205 Returns: 206 Tuple of module_info_target and dict of json. 207 """ 208 # If module_file is specified, we're gonna test it so we don't care if 209 # module_info_target stays None. 210 module_info_target = None 211 file_path = module_file 212 previous_checksum = self._get_module_info_checksums() 213 if not file_path: 214 module_info_target, file_path = self._discover_mod_file_and_target( 215 self.force_build) 216 self.mod_info_file_path = Path(file_path) 217 # Even undergone a rebuild after _discover_mod_file_and_target(), merge 218 # atest_merged_dep.json only when module_deps_infos actually change so 219 # that Atest can decrease disk I/O and ensure data accuracy at all. 220 module_deps_infos = [file_path, self.java_dep_path, self.cc_dep_path] 221 self._save_module_info_checksum(module_deps_infos) 222 self.update_merge_info = self.need_update_merged_file(previous_checksum) 223 if self.update_merge_info: 224 # Load the $ANDROID_PRODUCT_OUT/module-info.json for merging. 225 with open(file_path) as module_info_json: 226 mod_info = self._merge_build_system_infos( 227 json.load(module_info_json)) 228 else: 229 # Load $ANDROID_PRODUCT_OUT/atest_merged_dep.json directly. 230 with open(self.merged_dep_path) as merged_info_json: 231 mod_info = json.load(merged_info_json) 232 _add_missing_variant_modules(mod_info) 233 logging.debug('Loading %s as module-info.', self.merged_dep_path) 234 return module_info_target, mod_info 235 236 def _get_module_info_checksums(self): 237 """Load the module-info.md5 and return the content. 238 239 Returns: 240 A dict of filename and checksum. 241 """ 242 if os.path.exists(self.module_info_checksum): 243 with open(self.module_info_checksum) as cache: 244 try: 245 content = json.load(cache) 246 return content 247 except json.JSONDecodeError: 248 pass 249 return {} 250 251 def _save_module_info_checksum(self, filenames): 252 """Dump the checksum of essential module info files. 253 * module-info.json 254 * module_bp_cc_deps.json 255 * module_bp_java_deps.json 256 """ 257 dirname = Path(self.module_info_checksum).parent 258 if not dirname.is_dir(): 259 dirname.mkdir(parents=True) 260 atest_utils.save_md5(filenames, self.module_info_checksum) 261 262 @staticmethod 263 def _get_path_to_module_info(name_to_module_info): 264 """Return the path_to_module_info dict. 265 266 Args: 267 name_to_module_info: Dict of module name to module info dict. 268 269 Returns: 270 Dict of module path to module info dict. 271 """ 272 path_to_module_info = {} 273 for mod_name, mod_info in name_to_module_info.items(): 274 # Cross-compiled and multi-arch modules actually all belong to 275 # a single target so filter out these extra modules. 276 if mod_name != mod_info.get(constants.MODULE_NAME, ''): 277 continue 278 for path in mod_info.get(constants.MODULE_PATH, []): 279 mod_info[constants.MODULE_NAME] = mod_name 280 # There could be multiple modules in a path. 281 if path in path_to_module_info: 282 path_to_module_info[path].append(mod_info) 283 else: 284 path_to_module_info[path] = [mod_info] 285 return path_to_module_info 286 287 def _index_testable_modules(self, content): 288 """Dump testable modules. 289 290 Args: 291 content: An object that will be written to the index file. 292 """ 293 logging.debug(r'Indexing testable modules... ' 294 r'(This is required whenever module-info.json ' 295 r'was rebuilt.)') 296 with open(self.module_index, 'wb') as cache: 297 try: 298 pickle.dump(content, cache, protocol=2) 299 except IOError: 300 logging.error('Failed in dumping %s', cache) 301 os.remove(cache) 302 303 def _get_testable_modules(self, index=False, suite=None): 304 """Return all available testable modules and index them. 305 306 Args: 307 index: boolean that determines running _index_testable_modules(). 308 suite: string for the suite name. 309 310 Returns: 311 Set of all testable modules. 312 """ 313 modules = set() 314 begin = time.time() 315 for _, info in self.name_to_module_info.items(): 316 if self.is_testable_module(info): 317 modules.add(info.get(constants.MODULE_NAME)) 318 logging.debug('Probing all testable modules took %ss', 319 time.time() - begin) 320 if index: 321 self._index_testable_modules(modules) 322 if suite: 323 _modules = set() 324 for module_name in modules: 325 info = self.get_module_info(module_name) 326 if self.is_suite_in_compatibility_suites(suite, info): 327 _modules.add(info.get(constants.MODULE_NAME)) 328 return _modules 329 return modules 330 331 def is_module(self, name): 332 """Return True if name is a module, False otherwise.""" 333 if self.get_module_info(name): 334 return True 335 return False 336 337 def get_paths(self, name): 338 """Return paths of supplied module name, Empty list if non-existent.""" 339 info = self.get_module_info(name) 340 if info: 341 return info.get(constants.MODULE_PATH, []) 342 return [] 343 344 def get_module_names(self, rel_module_path): 345 """Get the modules that all have module_path. 346 347 Args: 348 rel_module_path: path of module in module-info.json 349 350 Returns: 351 List of module names. 352 """ 353 return [m.get(constants.MODULE_NAME) 354 for m in self.path_to_module_info.get(rel_module_path, [])] 355 356 def get_module_info(self, mod_name): 357 """Return dict of info for given module name, None if non-existence.""" 358 return self.name_to_module_info.get(mod_name) 359 360 def is_suite_in_compatibility_suites(self, suite, mod_info): 361 """Check if suite exists in the compatibility_suites of module-info. 362 363 Args: 364 suite: A string of suite name. 365 mod_info: Dict of module info to check. 366 367 Returns: 368 True if it exists in mod_info, False otherwise. 369 """ 370 if mod_info: 371 return suite in mod_info.get( 372 constants.MODULE_COMPATIBILITY_SUITES, []) 373 return [] 374 375 def get_testable_modules(self, suite=None): 376 """Return the testable modules of the given suite name. 377 378 Atest does not index testable modules against compatibility_suites. When 379 suite was given, or the index file was interrupted, always run 380 _get_testable_modules() and re-index. 381 382 Args: 383 suite: A string of suite name. 384 385 Returns: 386 If suite is not given, return all the testable modules in module 387 info, otherwise return only modules that belong to the suite. 388 """ 389 modules = set() 390 start = time.time() 391 if self.module_index_proc: 392 self.module_index_proc.join() 393 394 if self.module_index.is_file(): 395 if not suite: 396 with open(self.module_index, 'rb') as cache: 397 try: 398 modules = pickle.load(cache, encoding="utf-8") 399 except UnicodeDecodeError: 400 modules = pickle.load(cache) 401 # when module indexing was interrupted. 402 except EOFError: 403 pass 404 else: 405 modules = self._get_testable_modules(suite=suite) 406 # If the modules.idx does not exist or invalid for any reason, generate 407 # a new one arbitrarily. 408 if not modules: 409 if not suite: 410 modules = self._get_testable_modules(index=True) 411 else: 412 modules = self._get_testable_modules(index=True, suite=suite) 413 duration = time.time() - start 414 metrics.LocalDetectEvent( 415 detect_type=DetectType.TESTABLE_MODULES, 416 result=int(duration)) 417 return modules 418 419 def is_testable_module(self, mod_info): 420 """Check if module is something we can test. 421 422 A module is testable if: 423 - it's installed, or 424 - it's a robolectric module (or shares path with one). 425 426 Args: 427 mod_info: Dict of module info to check. 428 429 Returns: 430 True if we can test this module, False otherwise. 431 """ 432 if not mod_info: 433 return False 434 if mod_info.get(constants.MODULE_INSTALLED) and self.has_test_config(mod_info): 435 return True 436 if self.is_robolectric_test(mod_info.get(constants.MODULE_NAME)): 437 return True 438 return False 439 440 def has_test_config(self, mod_info): 441 """Validate if this module has a test config. 442 443 A module can have a test config in the following manner: 444 - AndroidTest.xml at the module path. 445 - test_config be set in module-info.json. 446 - Auto-generated config via the auto_test_config key 447 in module-info.json. 448 449 Args: 450 mod_info: Dict of module info to check. 451 452 Returns: 453 True if this module has a test config, False otherwise. 454 """ 455 # Check if test_config in module-info is set. 456 for test_config in mod_info.get(constants.MODULE_TEST_CONFIG, []): 457 if os.path.isfile(os.path.join(self.root_dir, test_config)): 458 return True 459 # Check for AndroidTest.xml at the module path. 460 for path in mod_info.get(constants.MODULE_PATH, []): 461 if os.path.isfile(os.path.join(self.root_dir, path, 462 constants.MODULE_CONFIG)): 463 return True 464 # Check if the module has an auto-generated config. 465 return self.is_auto_gen_test_config(mod_info.get(constants.MODULE_NAME)) 466 467 def get_robolectric_test_name(self, module_name): 468 """Returns runnable robolectric module name. 469 470 This method is for legacy robolectric tests and returns one of associated 471 modules. The pattern is determined by the amount of shards: 472 473 10 shards: 474 FooTests -> RunFooTests0, RunFooTests1 ... RunFooTests9 475 No shard: 476 FooTests -> RunFooTests 477 478 Arg: 479 module_name: String of module. 480 481 Returns: 482 String of the first-matched associated module that belongs to the 483 actual robolectric module, None if nothing has been found. 484 """ 485 module_name_info = self.get_module_info(module_name) 486 if not module_name_info: 487 return None 488 module_paths = module_name_info.get(constants.MODULE_PATH, []) 489 if module_paths: 490 for mod in self.get_module_names(module_paths[0]): 491 mod_info = self.get_module_info(mod) 492 if self.is_robolectric_module(mod_info): 493 return mod 494 return None 495 496 def is_robolectric_test(self, module_name): 497 """Check if the given module is a robolectric test. 498 499 Args: 500 module_name: String of module to check. 501 502 Returns: 503 Boolean whether it's a robotest or not. 504 """ 505 if self.get_robolectric_type(module_name): 506 return True 507 return False 508 509 def get_robolectric_type(self, module_name): 510 """Check if the given module is a robolectric test and return type of it. 511 512 Robolectric declaration is converting from Android.mk to Android.bp, and 513 in the interim Atest needs to support testing both types of tests. 514 515 The modern robolectric tests defined by 'android_robolectric_test' in an 516 Android.bp file can can be run in Tradefed Test Runner: 517 518 SettingsRoboTests -> Tradefed Test Runner 519 520 Legacy tests defined in an Android.mk can only run with the 'make' way. 521 522 SettingsRoboTests -> make RunSettingsRoboTests0 523 524 To determine whether the test is a modern/legacy robolectric test: 525 1. Traverse all modules share the module path. If one of the 526 modules has a ROBOLECTRIC class, it is a robolectric test. 527 2. If found an Android.bp in that path, it's a modern one, otherwise 528 it's a legacy test and will go to the build route. 529 530 Args: 531 module_name: String of module to check. 532 533 Returns: 534 0: not a robolectric test. 535 1: a modern robolectric test(defined in Android.bp) 536 2: a legacy robolectric test(defined in Android.mk) 537 """ 538 not_a_robo_test = 0 539 module_name_info = self.get_module_info(module_name) 540 if not module_name_info: 541 return not_a_robo_test 542 mod_path = module_name_info.get(constants.MODULE_PATH, []) 543 if mod_path: 544 # Check1: If the associated modules are "ROBOLECTRIC". 545 is_a_robotest = False 546 modules_in_path = self.get_module_names(mod_path[0]) 547 for mod in modules_in_path: 548 mod_info = self.get_module_info(mod) 549 if self.is_robolectric_module(mod_info): 550 is_a_robotest = True 551 break 552 if not is_a_robotest: 553 return not_a_robo_test 554 # Check 2: If found Android.bp in path, call it a modern test. 555 bpfile = os.path.join(self.root_dir, mod_path[0], 'Android.bp') 556 if os.path.isfile(bpfile): 557 return constants.ROBOTYPE_MODERN 558 return constants.ROBOTYPE_LEGACY 559 return not_a_robo_test 560 561 def is_auto_gen_test_config(self, module_name): 562 """Check if the test config file will be generated automatically. 563 564 Args: 565 module_name: A string of the module name. 566 567 Returns: 568 True if the test config file will be generated automatically. 569 """ 570 if self.is_module(module_name): 571 mod_info = self.get_module_info(module_name) 572 auto_test_config = mod_info.get('auto_test_config', []) 573 return auto_test_config and auto_test_config[0] 574 return False 575 576 def is_robolectric_module(self, mod_info): 577 """Check if a module is a robolectric module. 578 579 This method is for legacy robolectric tests that the associated modules 580 contain: 581 'class': ['ROBOLECTRIC'] 582 583 Args: 584 mod_info: ModuleInfo to check. 585 586 Returns: 587 True if module is a robolectric module, False otherwise. 588 """ 589 if mod_info: 590 return (mod_info.get(constants.MODULE_CLASS, [None])[0] == 591 constants.MODULE_CLASS_ROBOLECTRIC) 592 return False 593 594 def is_native_test(self, module_name): 595 """Check if the input module is a native test. 596 597 Args: 598 module_name: A string of the module name. 599 600 Returns: 601 True if the test is a native test, False otherwise. 602 """ 603 mod_info = self.get_module_info(module_name) 604 return constants.MODULE_CLASS_NATIVE_TESTS in mod_info.get( 605 constants.MODULE_CLASS, []) 606 607 def has_mainline_modules(self, module_name, mainline_modules): 608 """Check if the mainline modules are in module-info. 609 610 Args: 611 module_name: A string of the module name. 612 mainline_modules: A list of mainline modules. 613 614 Returns: 615 True if mainline_modules is in module-info, False otherwise. 616 """ 617 mod_info = self.get_module_info(module_name) 618 # Check 'test_mainline_modules' attribute of the module-info.json. 619 if mainline_modules in mod_info.get(constants.MODULE_MAINLINE_MODULES, 620 []): 621 return True 622 for test_config in mod_info.get(constants.MODULE_TEST_CONFIG, []): 623 # Check the value of 'mainline-param' in the test config. 624 if not self.is_auto_gen_test_config(module_name): 625 return mainline_modules in atest_utils.get_mainline_param( 626 os.path.join(self.root_dir, test_config)) 627 # Unable to verify mainline modules in an auto-gen test config. 628 logging.debug('%s is associated with an auto-generated test config.', 629 module_name) 630 return True 631 632 def _merge_build_system_infos(self, name_to_module_info, 633 java_bp_info_path=None, cc_bp_info_path=None): 634 """Merge the content of module-info.json and CC/Java dependency files 635 to name_to_module_info. 636 637 Args: 638 name_to_module_info: Dict of module name to module info dict. 639 java_bp_info_path: String of path to java dep file to load up. 640 Used for testing. 641 cc_bp_info_path: String of path to cc dep file to load up. 642 Used for testing. 643 644 Returns: 645 Dict of updated name_to_module_info. 646 """ 647 start = time.time() 648 # Merge _JAVA_DEP_INFO 649 if not java_bp_info_path: 650 java_bp_info_path = self.java_dep_path 651 if atest_utils.is_valid_json_file(java_bp_info_path): 652 with open(java_bp_info_path) as json_file: 653 java_bp_infos = json.load(json_file) 654 logging.debug('Merging Java build info: %s', java_bp_info_path) 655 name_to_module_info = self._merge_soong_info( 656 name_to_module_info, java_bp_infos) 657 # Merge _CC_DEP_INFO 658 if not cc_bp_info_path: 659 cc_bp_info_path = self.cc_dep_path 660 if atest_utils.is_valid_json_file(cc_bp_info_path): 661 with open(cc_bp_info_path) as json_file: 662 cc_bp_infos = json.load(json_file) 663 logging.debug('Merging CC build info: %s', cc_bp_info_path) 664 # CC's dep json format is different with java. 665 # Below is the example content: 666 # { 667 # "clang": "${ANDROID_ROOT}/bin/clang", 668 # "clang++": "${ANDROID_ROOT}/bin/clang++", 669 # "modules": { 670 # "ACameraNdkVendorTest": { 671 # "path": [ 672 # "frameworks/av/camera/ndk" 673 # ], 674 # "srcs": [ 675 # "frameworks/tests/AImageVendorTest.cpp", 676 # "frameworks/tests/ACameraManagerTest.cpp" 677 # ], 678 name_to_module_info = self._merge_soong_info( 679 name_to_module_info, cc_bp_infos.get('modules', {})) 680 # If $ANDROID_PRODUCT_OUT was not created in pyfakefs, simply return it 681 # without dumping atest_merged_dep.json in real. 682 if not self.merged_dep_path.parent.is_dir(): 683 return name_to_module_info 684 # b/178559543 saving merged module info in a temp file and copying it to 685 # atest_merged_dep.json can eliminate the possibility of accessing it 686 # concurrently and resulting in invalid JSON format. 687 temp_file = tempfile.NamedTemporaryFile() 688 with open(temp_file.name, 'w') as _temp: 689 json.dump(name_to_module_info, _temp, indent=0) 690 shutil.copy(temp_file.name, self.merged_dep_path) 691 temp_file.close() 692 duration = time.time() - start 693 logging.debug('Merging module info took %ss', duration) 694 metrics.LocalDetectEvent( 695 detect_type=DetectType.MODULE_MERGE, result=int(duration)) 696 return name_to_module_info 697 698 def _merge_soong_info(self, name_to_module_info, mod_bp_infos): 699 """Merge the dependency and srcs in mod_bp_infos to name_to_module_info. 700 701 Args: 702 name_to_module_info: Dict of module name to module info dict. 703 mod_bp_infos: Dict of module name to bp's module info dict. 704 705 Returns: 706 Dict of updated name_to_module_info. 707 """ 708 merge_items = [constants.MODULE_DEPENDENCIES, constants.MODULE_SRCS] 709 for module_name, dep_info in mod_bp_infos.items(): 710 if name_to_module_info.get(module_name, None): 711 mod_info = name_to_module_info.get(module_name) 712 for merge_item in merge_items: 713 dep_info_values = dep_info.get(merge_item, []) 714 mod_info_values = mod_info.get(merge_item, []) 715 mod_info_values.extend(dep_info_values) 716 mod_info_values.sort() 717 # deduplicate values just in case. 718 mod_info_values = list(dict.fromkeys(mod_info_values)) 719 name_to_module_info[ 720 module_name][merge_item] = mod_info_values 721 return name_to_module_info 722 723 def get_module_dependency(self, module_name, depend_on=None): 724 """Get the dependency sets for input module. 725 726 Recursively find all the dependencies of the input module. 727 728 Args: 729 module_name: String of module to check. 730 depend_on: The list of parent dependencies. 731 732 Returns: 733 Set of dependency modules. 734 """ 735 if not depend_on: 736 depend_on = set() 737 deps = set() 738 mod_info = self.get_module_info(module_name) 739 if not mod_info: 740 return deps 741 mod_deps = set(mod_info.get(constants.MODULE_DEPENDENCIES, [])) 742 # Remove item in deps if it already in depend_on: 743 mod_deps = mod_deps - depend_on 744 deps = deps.union(mod_deps) 745 for mod_dep in mod_deps: 746 deps = deps.union(set(self.get_module_dependency( 747 mod_dep, depend_on=depend_on.union(deps)))) 748 return deps 749 750 def get_install_module_dependency(self, module_name, depend_on=None): 751 """Get the dependency set for the given modules with installed path. 752 753 Args: 754 module_name: String of module to check. 755 depend_on: The list of parent dependencies. 756 757 Returns: 758 Set of dependency modules which has installed path. 759 """ 760 install_deps = set() 761 deps = self.get_module_dependency(module_name, depend_on) 762 logging.debug('%s depends on: %s', module_name, deps) 763 for module in deps: 764 mod_info = self.get_module_info(module) 765 if mod_info and mod_info.get(constants.MODULE_INSTALLED, []): 766 install_deps.add(module) 767 logging.debug('modules %s required by %s were not installed', 768 install_deps, module_name) 769 return install_deps 770 771 def need_update_merged_file(self, checksum): 772 """Check if need to update/generated atest_merged_dep. 773 774 There are 2 scienarios that atest_merged_dep.json will be updated. 775 1. One of the checksum of module-info.json, module_bp_java_deps.json and 776 module_cc_java_deps.json have changed. 777 2. atest_merged_deps.json does not exist. 778 779 If fits one of above scienarios, it is recognized to update. 780 781 Returns: 782 True if one of the scienarios reaches, False otherwise. 783 """ 784 return (checksum != self._get_module_info_checksums() or 785 not Path(self.merged_dep_path).is_file()) 786 787 def is_unit_test(self, mod_info): 788 """Return True if input module is unit test, False otherwise. 789 790 Args: 791 mod_info: ModuleInfo to check. 792 793 Returns: 794 True if if input module is unit test, False otherwise. 795 """ 796 return mod_info.get(constants.MODULE_IS_UNIT_TEST, '') == 'true' 797 798 def is_host_unit_test(self, mod_info): 799 """Return True if input module is host unit test, False otherwise. 800 801 Args: 802 mod_info: ModuleInfo to check. 803 804 Returns: 805 True if if input module is host unit test, False otherwise. 806 """ 807 return self.is_suite_in_compatibility_suites( 808 'host-unit-tests', mod_info) 809 810 def is_device_driven_test(self, mod_info): 811 """Return True if input module is device driven test, False otherwise. 812 813 Args: 814 mod_info: ModuleInfo to check. 815 816 Returns: 817 True if if input module is device driven test, False otherwise. 818 """ 819 return self.is_testable_module(mod_info) and 'DEVICE' in mod_info.get( 820 constants.MODULE_SUPPORTED_VARIANTS, []) 821 822 def _any_module(self, _: Module) -> bool: 823 return True 824 825 def get_all_tests(self): 826 """Get a list of all the module names which are tests.""" 827 return self._get_all_modules(type_predicate=self.is_testable_module) 828 829 def get_all_unit_tests(self): 830 """Get a list of all the module names which are unit tests.""" 831 return self._get_all_modules(type_predicate=self.is_unit_test) 832 833 def get_all_host_unit_tests(self): 834 """Get a list of all the module names which are host unit tests.""" 835 return self._get_all_modules(type_predicate=self.is_host_unit_test) 836 837 def get_all_device_driven_tests(self): 838 """Get a list of all the module names which are device driven tests.""" 839 return self._get_all_modules(type_predicate=self.is_device_driven_test) 840 841 def _get_all_modules(self, type_predicate=None): 842 """Get a list of all the module names that passed the predicate.""" 843 modules = [] 844 type_predicate = type_predicate or self._any_module 845 for mod_name, mod_info in self.name_to_module_info.items(): 846 if mod_info.get(constants.MODULE_NAME, '') == mod_name: 847 if type_predicate(mod_info): 848 modules.append(mod_name) 849 return modules 850 851 852def _add_missing_variant_modules(name_to_module_info: Dict[str, Module]): 853 missing_modules = dict() 854 855 # Android's build system automatically adds a suffix for some build module 856 # variants. For example, a module-info entry for a module originally named 857 # 'HelloWorldTest' might appear as 'HelloWorldTest_32' and which Atest would 858 # not be able to find. We add such entries if not already present so they 859 # can be looked up using their declared module name. 860 for mod_name, mod_info in name_to_module_info.items(): 861 declared_module_name = mod_info.get(constants.MODULE_NAME) 862 if declared_module_name == mod_name: 863 continue 864 if declared_module_name in name_to_module_info: 865 continue 866 missing_modules.setdefault(declared_module_name, mod_info) 867 868 name_to_module_info.update(missing_modules) 869