1# Copyright 2018, The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15""" 16Module Finder class. 17""" 18 19# pylint: disable=line-too-long 20 21import logging 22import os 23import time 24 25from typing import List 26 27from atest import atest_configs 28from atest import atest_error 29from atest import atest_utils 30from atest import constants 31 32from atest.atest_enum import DetectType 33from atest.metrics import metrics 34from atest.test_finders import test_info 35from atest.test_finders import test_finder_base 36from atest.test_finders import test_finder_utils 37from atest.test_runners import atest_tf_test_runner 38from atest.test_runners import robolectric_test_runner 39from atest.test_runners import vts_tf_test_runner 40 41# These are suites in LOCAL_COMPATIBILITY_SUITE that aren't really suites so 42# we can ignore them. 43_SUITES_TO_IGNORE = frozenset({'general-tests', 'device-tests', 'tests'}) 44 45class ModuleFinder(test_finder_base.TestFinderBase): 46 """Module finder class.""" 47 NAME = 'MODULE' 48 _TEST_RUNNER = atest_tf_test_runner.AtestTradefedTestRunner.NAME 49 _ROBOLECTRIC_RUNNER = robolectric_test_runner.RobolectricTestRunner.NAME 50 _VTS_TEST_RUNNER = vts_tf_test_runner.VtsTradefedTestRunner.NAME 51 52 def __init__(self, module_info=None): 53 super().__init__() 54 self.root_dir = os.environ.get(constants.ANDROID_BUILD_TOP) 55 self.module_info = module_info 56 57 def _determine_testable_module(self, path: str, 58 file_path: str = None) -> List: 59 """Determine which module the user is trying to test. 60 61 Returns the modules to test. If there are multiple possibilities, will 62 ask the user. Otherwise will return the only module found. 63 64 Args: 65 path: String path of module to look for. 66 file_path: String path of input file. 67 68 Returns: 69 A list of the module names. 70 """ 71 testable_modules = [] 72 # A list to save those testable modules but srcs information is empty. 73 testable_modules_no_srcs = [] 74 for mod in self.module_info.get_module_names(path): 75 mod_info = self.module_info.get_module_info(mod) 76 if self.module_info.is_testable_module(mod_info): 77 # If test module defined srcs, input file_path should be defined 78 # in the src list of module. 79 module_srcs = mod_info.get(constants.MODULE_SRCS, []) 80 if file_path and os.path.relpath( 81 file_path, self.root_dir) not in module_srcs: 82 logging.debug('Skip module: %s for %s', mod, file_path) 83 # Collect those modules if they don't have srcs information 84 # in module-info, use this list if there's no other matched 85 # module with src information. 86 if not module_srcs: 87 testable_modules_no_srcs.append( 88 mod_info.get(constants.MODULE_NAME)) 89 continue 90 testable_modules.append(mod_info.get(constants.MODULE_NAME)) 91 if not testable_modules: 92 testable_modules.extend(testable_modules_no_srcs) 93 return test_finder_utils.extract_test_from_tests(testable_modules) 94 95 def _is_vts_module(self, module_name): 96 """Returns True if the module is a vts10 module, else False.""" 97 mod_info = self.module_info.get_module_info(module_name) 98 suites = [] 99 if mod_info: 100 suites = mod_info.get(constants.MODULE_COMPATIBILITY_SUITES, []) 101 # Pull out all *ts (cts, tvts, etc) suites. 102 suites = [suite for suite in suites if suite not in _SUITES_TO_IGNORE] 103 return len(suites) == 1 and 'vts10' in suites 104 105 def _update_to_vts_test_info(self, test): 106 """Fill in the fields with vts10 specific info. 107 108 We need to update the runner to use the vts10 runner and also find the 109 test specific dependencies. 110 111 Args: 112 test: TestInfo to update with vts10 specific details. 113 114 Return: 115 TestInfo that is ready for the vts10 test runner. 116 """ 117 test.test_runner = self._VTS_TEST_RUNNER 118 config_file = os.path.join(self.root_dir, 119 test.data[constants.TI_REL_CONFIG]) 120 # Need to get out dir (special logic is to account for custom out dirs). 121 # The out dir is used to construct the build targets for the test deps. 122 out_dir = os.environ.get(constants.ANDROID_HOST_OUT) 123 custom_out_dir = os.environ.get(constants.ANDROID_OUT_DIR) 124 # If we're not an absolute custom out dir, get no-absolute out dir path. 125 if custom_out_dir is None or not os.path.isabs(custom_out_dir): 126 out_dir = os.path.relpath(out_dir, self.root_dir) 127 vts_out_dir = os.path.join(out_dir, 'vts10', 'android-vts10', 'testcases') 128 # Parse dependency of default staging plans. 129 xml_paths = test_finder_utils.search_integration_dirs( 130 constants.VTS_STAGING_PLAN, 131 self.module_info.get_paths(constants.VTS_TF_MODULE)) 132 vts_xmls = set() 133 vts_xmls.add(config_file) 134 for xml_path in xml_paths: 135 vts_xmls |= test_finder_utils.get_plans_from_vts_xml(xml_path) 136 for config_file in vts_xmls: 137 # Add in vts10 test build targets. 138 for target in test_finder_utils.get_targets_from_vts_xml( 139 config_file, vts_out_dir, self.module_info): 140 test.add_build_target(target) 141 test.add_build_target('vts-test-core') 142 test.add_build_target(test.test_name) 143 return test 144 145 def _update_legacy_robolectric_test_info(self, test): 146 """Update the fields for a legacy robolectric test. 147 148 This method is updating test_name when the given is a legacy robolectric 149 test, and assigning Robolectric Runner for it. 150 151 e.g. WallPaperPicker2RoboTests is a legacy robotest, and the test_name 152 will become RunWallPaperPicker2RoboTests and run it with Robolectric 153 Runner. 154 155 Args: 156 test: TestInfo to be updated with robolectric fields. 157 158 Returns: 159 TestInfo with updated robolectric fields. 160 """ 161 test.test_runner = self._ROBOLECTRIC_RUNNER 162 test.test_name = self.module_info.get_robolectric_test_name(test.test_name) 163 return test 164 165 # pylint: disable=too-many-branches 166 def _process_test_info(self, test): 167 """Process the test info and return some fields updated/changed. 168 169 We need to check if the test found is a special module (like vts10) and 170 update the test_info fields (like test_runner) appropriately. 171 172 Args: 173 test: TestInfo that has been filled out by a find method. 174 175 Return: 176 TestInfo that has been modified as needed and return None if 177 this module can't be found in the module_info. 178 """ 179 module_name = test.test_name 180 mod_info = self.module_info.get_module_info(module_name) 181 if not mod_info: 182 return None 183 test.module_class = mod_info['class'] 184 test.install_locations = test_finder_utils.get_install_locations( 185 mod_info['installed']) 186 # Check if this is only a vts10 module. 187 if self._is_vts_module(test.test_name): 188 return self._update_to_vts_test_info(test) 189 test.robo_type = self.module_info.get_robolectric_type(test.test_name) 190 if test.robo_type: 191 test.install_locations = {constants.DEVICELESS_TEST} 192 if test.robo_type == constants.ROBOTYPE_MODERN: 193 test.add_build_target(test.test_name) 194 return test 195 if test.robo_type == constants.ROBOTYPE_LEGACY: 196 return self._update_legacy_robolectric_test_info(test) 197 rel_config = test.data[constants.TI_REL_CONFIG] 198 for target in self._get_build_targets(module_name, rel_config): 199 test.add_build_target(target) 200 # (b/177626045) Probe target APK for running instrumentation tests to 201 # prevent RUNNER ERROR by adding target application(module) to the 202 # build_targets, and install these target apks before testing. 203 artifact_map = self.module_info.get_instrumentation_target_apps( 204 module_name) 205 if artifact_map: 206 logging.debug('Found %s an instrumentation test.', module_name) 207 for art in artifact_map.keys(): 208 test.add_build_target(art) 209 logging.debug('Add %s to build targets...', 210 ', '.join(artifact_map.keys())) 211 test.artifacts = [apk for p in artifact_map.values() for apk in p] 212 logging.debug('Will install target APK: %s\n', test.artifacts) 213 metrics.LocalDetectEvent( 214 detect_type=DetectType.FOUND_TARGET_ARTIFACTS, 215 result=len(test.artifacts)) 216 # For device side java test, it will use 217 # com.android.compatibility.testtype.DalvikTest as test runner in 218 # cts-dalvik-device-test-runner.jar 219 if self.module_info.is_auto_gen_test_config(module_name): 220 if constants.MODULE_CLASS_JAVA_LIBRARIES in test.module_class: 221 for dalvik_dep in test_finder_utils.DALVIK_TEST_DEPS: 222 if self.module_info.is_module(dalvik_dep): 223 test.add_build_target(dalvik_dep) 224 # Update test name if the test belong to extra config which means it's 225 # test config name is not the same as module name. For extra config, it 226 # index will be greater or equal to 1. 227 try: 228 if (mod_info.get(constants.MODULE_TEST_CONFIG, []).index(rel_config) 229 > 0): 230 config_test_name = os.path.splitext(os.path.basename( 231 rel_config))[0] 232 logging.debug('Replace test_info.name(%s) to %s', 233 test.test_name, config_test_name) 234 test.test_name = config_test_name 235 except ValueError: 236 pass 237 return test 238 239 def _get_build_targets(self, module_name, rel_config): 240 """Get the test deps. 241 242 Args: 243 module_name: name of the test. 244 rel_config: XML for the given test. 245 246 Returns: 247 Set of build targets. 248 """ 249 targets = set() 250 if not self.module_info.is_auto_gen_test_config(module_name): 251 config_file = os.path.join(self.root_dir, rel_config) 252 targets = test_finder_utils.get_targets_from_xml(config_file, 253 self.module_info) 254 if constants.VTS_CORE_SUITE in self.module_info.get_module_info( 255 module_name).get(constants.MODULE_COMPATIBILITY_SUITES, []): 256 targets.add(constants.VTS_CORE_TF_MODULE) 257 for suite in self.module_info.get_module_info( 258 module_name).get(constants.MODULE_COMPATIBILITY_SUITES, []): 259 targets.update(constants.SUITE_DEPS.get(suite, [])) 260 for module_path in self.module_info.get_paths(module_name): 261 mod_dir = module_path.replace('/', '-') 262 targets.add(constants.MODULES_IN + mod_dir) 263 # (b/156457698) Force add vts_kernel_ltp_tests as build target if our 264 # test belongs to REQUIRED_LTP_TEST_MODULES due to required_module 265 # option not working for sh_test in soong. Ditto for kselftest. 266 if module_name in constants.REQUIRED_LTP_TEST_MODULES: 267 targets.add('vts_kernel_ltp_tests') 268 if module_name in constants.REQUIRED_KSELFTEST_TEST_MODULES: 269 targets.add('vts_kernel_kselftest_tests') 270 # (b/184567849) Force adding module_name as a build_target. This will 271 # allow excluding MODULES-IN-* and prevent from missing build targets. 272 if module_name and self.module_info.is_module(module_name): 273 targets.add(module_name) 274 # If it's a MTS test, add cts-tradefed as test dependency. 275 if constants.MTS_SUITE in self.module_info.get_module_info( 276 module_name).get(constants.MODULE_COMPATIBILITY_SUITES, []): 277 if self.module_info.is_module(constants.CTS_JAR): 278 targets.add(constants.CTS_JAR) 279 return targets 280 281 def _get_module_test_config(self, module_name, rel_config=None): 282 """Get the value of test_config in module_info. 283 284 Get the value of 'test_config' in module_info if its 285 auto_test_config is not true. 286 In this case, the test_config is specified by user. 287 If not, return rel_config. 288 289 Args: 290 module_name: A string of the test's module name. 291 rel_config: XML for the given test. 292 293 Returns: 294 A list of string of test_config path if found, else return rel_config. 295 """ 296 default_all_config = not (atest_configs.GLOBAL_ARGS and 297 atest_configs.GLOBAL_ARGS.test_config_select) 298 mod_info = self.module_info.get_module_info(module_name) 299 if mod_info: 300 test_configs = [] 301 test_config_list = mod_info.get(constants.MODULE_TEST_CONFIG, []) 302 if test_config_list: 303 # multiple test configs 304 if len(test_config_list) > 1: 305 test_configs = test_finder_utils.extract_test_from_tests( 306 test_config_list, default_all=default_all_config) 307 else: 308 test_configs = test_config_list 309 if test_configs: 310 return test_configs 311 # Double check if below section is needed. 312 if (not self.module_info.is_auto_gen_test_config(module_name) 313 and len(test_configs) > 0): 314 return test_configs 315 return [rel_config] if rel_config else [] 316 317 # pylint: disable=too-many-branches 318 # pylint: disable=too-many-locals 319 def _get_test_info_filter(self, path, methods, **kwargs): 320 """Get test info filter. 321 322 Args: 323 path: A string of the test's path. 324 methods: A set of method name strings. 325 rel_module_dir: Optional. A string of the module dir no-absolute to 326 root. 327 class_name: Optional. A string of the class name. 328 is_native_test: Optional. A boolean variable of whether to search 329 for a native test or not. 330 331 Returns: 332 A set of test info filter. 333 """ 334 _, file_name = test_finder_utils.get_dir_path_and_filename(path) 335 ti_filter = frozenset() 336 if os.path.isfile(path) and kwargs.get('is_native_test', None): 337 class_info = test_finder_utils.get_cc_class_info(path) 338 ti_filter = frozenset([test_info.TestFilter( 339 test_finder_utils.get_cc_filter( 340 class_info, kwargs.get('class_name', '*'), methods), 341 frozenset())]) 342 # Path to java file. 343 elif file_name and constants.JAVA_EXT_RE.match(file_name): 344 full_class_name = test_finder_utils.get_fully_qualified_class_name( 345 path) 346 # If input class is parameterized java class, adding * to the end of 347 # method filter string to make sure the generated method name could 348 # be run. 349 if test_finder_utils.is_parameterized_java_class(path): 350 update_methods = [] 351 for method in methods: 352 update_methods.append(method + '*') 353 methods = frozenset(update_methods) 354 ti_filter = frozenset( 355 [test_info.TestFilter(full_class_name, methods)]) 356 # Path to cc file. 357 elif file_name and constants.CC_EXT_RE.match(file_name): 358 # TODO (b/173019813) Should setup correct filter for an input file. 359 if not test_finder_utils.has_cc_class(path): 360 raise atest_error.MissingCCTestCaseError( 361 "Can't find CC class in %s" % path) 362 class_info = test_finder_utils.get_cc_class_info(path) 363 cc_filters = [] 364 for classname, _ in class_info.items(): 365 cc_filters.append( 366 test_info.TestFilter( 367 test_finder_utils.get_cc_filter(class_info, classname, methods), 368 frozenset())) 369 ti_filter = frozenset(cc_filters) 370 # If input path is a folder and have class_name information. 371 elif (not file_name and kwargs.get('class_name', None)): 372 ti_filter = frozenset( 373 [test_info.TestFilter(kwargs.get('class_name', None), methods)]) 374 # Path to non-module dir, treat as package. 375 elif (not file_name 376 and kwargs.get('rel_module_dir', None) != 377 os.path.relpath(path, self.root_dir)): 378 dir_items = [os.path.join(path, f) for f in os.listdir(path)] 379 for dir_item in dir_items: 380 if constants.JAVA_EXT_RE.match(dir_item): 381 package_name = test_finder_utils.get_package_name(dir_item) 382 if package_name: 383 # methods should be empty frozenset for package. 384 if methods: 385 raise atest_error.MethodWithoutClassError( 386 '%s: Method filtering requires class' 387 % str(methods)) 388 ti_filter = frozenset( 389 [test_info.TestFilter(package_name, methods)]) 390 break 391 logging.debug('_get_test_info_filter() ti_filter: %s', ti_filter) 392 return ti_filter 393 394 def _get_rel_config(self, test_path): 395 """Get config file's no-absolute path. 396 397 Args: 398 test_path: A string of the test absolute path. 399 400 Returns: 401 A string of config's no-absolute path, else None. 402 """ 403 test_dir = os.path.dirname(test_path) 404 rel_module_dir = test_finder_utils.find_parent_module_dir( 405 self.root_dir, test_dir, self.module_info) 406 if rel_module_dir: 407 return os.path.join(rel_module_dir, constants.MODULE_CONFIG) 408 return None 409 410 def _get_test_infos(self, test_path, rel_config, module_name, test_filter): 411 """Get test_info for test_path. 412 413 Args: 414 test_path: A string of the test path. 415 rel_config: A string of rel path of config. 416 module_name: A string of the module name to use. 417 test_filter: A test info filter. 418 419 Returns: 420 A list of TestInfo namedtuple if found, else None. 421 """ 422 if not rel_config: 423 rel_config = self._get_rel_config(test_path) 424 if not rel_config: 425 return None 426 if module_name: 427 module_names = [module_name] 428 else: 429 module_names = self._determine_testable_module( 430 os.path.dirname(rel_config), 431 test_path if self._is_comparted_src(test_path) else None) 432 test_infos = [] 433 if module_names: 434 for mname in module_names: 435 # The real test config might be record in module-info. 436 rel_configs = self._get_module_test_config( 437 mname, rel_config=rel_config) 438 for rel_cfg in rel_configs: 439 mod_info = self.module_info.get_module_info(mname) 440 tinfo = self._process_test_info(test_info.TestInfo( 441 test_name=mname, 442 test_runner=self._TEST_RUNNER, 443 build_targets=set(), 444 data={constants.TI_FILTER: test_filter, 445 constants.TI_REL_CONFIG: rel_cfg}, 446 compatibility_suites=mod_info.get( 447 constants.MODULE_COMPATIBILITY_SUITES, []))) 448 if tinfo: 449 test_infos.append(tinfo) 450 return test_infos 451 452 def find_test_by_module_name(self, module_name): 453 """Find test for the given module name. 454 455 Args: 456 module_name: A string of the test's module name. 457 458 Returns: 459 A list that includes only 1 populated TestInfo namedtuple 460 if found, otherwise None. 461 """ 462 tinfos = [] 463 mod_info = self.module_info.get_module_info(module_name) 464 if self.module_info.is_testable_module(mod_info): 465 # path is a list with only 1 element. 466 rel_config = os.path.join(mod_info['path'][0], 467 constants.MODULE_CONFIG) 468 rel_configs = self._get_module_test_config(module_name, 469 rel_config=rel_config) 470 for rel_config in rel_configs: 471 tinfo = self._process_test_info(test_info.TestInfo( 472 test_name=module_name, 473 test_runner=self._TEST_RUNNER, 474 build_targets=set(), 475 data={constants.TI_REL_CONFIG: rel_config, 476 constants.TI_FILTER: frozenset()}, 477 compatibility_suites=mod_info.get( 478 constants.MODULE_COMPATIBILITY_SUITES, []))) 479 if tinfo: 480 tinfos.append(tinfo) 481 if tinfos: 482 return tinfos 483 return None 484 485 def find_test_by_kernel_class_name(self, module_name, class_name): 486 """Find kernel test for the given class name. 487 488 Args: 489 module_name: A string of the module name to use. 490 class_name: A string of the test's class name. 491 492 Returns: 493 A list of populated TestInfo namedtuple if test found, else None. 494 """ 495 496 class_name, methods = test_finder_utils.split_methods(class_name) 497 test_configs = self._get_module_test_config(module_name) 498 if not test_configs: 499 return None 500 tinfos = [] 501 for test_config in test_configs: 502 test_config_path = os.path.join(self.root_dir, test_config) 503 mod_info = self.module_info.get_module_info(module_name) 504 ti_filter = frozenset( 505 [test_info.TestFilter(class_name, methods)]) 506 if test_finder_utils.is_test_from_kernel_xml(test_config_path, class_name): 507 tinfo = self._process_test_info(test_info.TestInfo( 508 test_name=module_name, 509 test_runner=self._TEST_RUNNER, 510 build_targets=set(), 511 data={constants.TI_REL_CONFIG: test_config, 512 constants.TI_FILTER: ti_filter}, 513 compatibility_suites=mod_info.get( 514 constants.MODULE_COMPATIBILITY_SUITES, []))) 515 if tinfo: 516 tinfos.append(tinfo) 517 if tinfos: 518 return tinfos 519 return None 520 521 def find_test_by_class_name(self, class_name, module_name=None, 522 rel_config=None, is_native_test=False): 523 """Find test files given a class name. 524 525 If module_name and rel_config not given it will calculate it determine 526 it by looking up the tree from the class file. 527 528 Args: 529 class_name: A string of the test's class name. 530 module_name: Optional. A string of the module name to use. 531 rel_config: Optional. A string of module dir no-absolute to repo root. 532 is_native_test: A boolean variable of whether to search for a 533 native test or not. 534 535 Returns: 536 A list of populated TestInfo namedtuple if test found, else None. 537 """ 538 class_name, methods = test_finder_utils.split_methods(class_name) 539 search_class_name = class_name 540 # For parameterized gtest, test class will be automerged to 541 # $(class_prefix)/$(base_class) name. Using $(base_class) for searching 542 # matched TEST_P to make sure test class is matched. 543 if '/' in search_class_name: 544 search_class_name = str(search_class_name).split('/')[-1] 545 if rel_config: 546 search_dir = os.path.join(self.root_dir, 547 os.path.dirname(rel_config)) 548 else: 549 search_dir = self.root_dir 550 test_paths = test_finder_utils.find_class_file(search_dir, search_class_name, 551 is_native_test, methods) 552 if not test_paths and rel_config: 553 logging.info('Did not find class (%s) under module path (%s), ' 554 'researching from repo root.', class_name, rel_config) 555 test_paths = test_finder_utils.find_class_file(self.root_dir, 556 search_class_name, 557 is_native_test, 558 methods) 559 test_paths = test_paths if test_paths is not None else [] 560 # If we already have module name, use path in module-info as test_path. 561 if not test_paths: 562 if not module_name: 563 return None 564 # Use the module path as test_path. 565 module_paths = self.module_info.get_paths(module_name) 566 test_paths = [] 567 for rel_module_path in module_paths: 568 test_paths.append(os.path.join(self.root_dir, rel_module_path)) 569 tinfos = [] 570 for test_path in test_paths: 571 test_filter = self._get_test_info_filter( 572 test_path, methods, class_name=class_name, 573 is_native_test=is_native_test) 574 test_infos = self._get_test_infos( 575 test_path, rel_config, module_name, test_filter) 576 # If input include methods, check if tinfo match. 577 if test_infos and len(test_infos) > 1 and methods: 578 test_infos = self._get_matched_test_infos(test_infos, methods) 579 if test_infos: 580 tinfos.extend(test_infos) 581 return tinfos if tinfos else None 582 583 def _get_matched_test_infos(self, test_infos, methods): 584 """Get the test_infos matched the given methods. 585 586 Args: 587 test_infos: A list of TestInfo obj. 588 methods: A set of method name strings. 589 590 Returns: 591 A list of matched TestInfo namedtuple, else None. 592 """ 593 matched_test_infos = set() 594 for tinfo in test_infos: 595 test_config, test_srcs = test_finder_utils.get_test_config_and_srcs( 596 tinfo, self.module_info) 597 if test_config: 598 filter_dict = atest_utils.get_android_junit_config_filters( 599 test_config) 600 # Always treat the test_info is matched if no filters found. 601 if not filter_dict.keys(): 602 matched_test_infos.add(tinfo) 603 continue 604 for method in methods: 605 if self._is_srcs_match_method_annotation(method, test_srcs, 606 filter_dict): 607 logging.debug('For method:%s Test:%s matched ' 608 'filter_dict: %s', method, 609 tinfo.test_name, filter_dict) 610 matched_test_infos.add(tinfo) 611 return list(matched_test_infos) 612 613 def _is_srcs_match_method_annotation(self, method, srcs, annotation_dict): 614 """Check if input srcs matched annotation. 615 616 Args: 617 method: A string of test method name. 618 srcs: A list of source file of test. 619 annotation_dict: A dictionary record the include and exclude 620 annotations. 621 622 Returns: 623 True if input method matched the annotation of input srcs, else 624 None. 625 """ 626 include_annotations = annotation_dict.get( 627 constants.INCLUDE_ANNOTATION, []) 628 exclude_annotations = annotation_dict.get( 629 constants.EXCLUDE_ANNOTATION, []) 630 for src in srcs: 631 include_methods = set() 632 src_path = os.path.join(self.root_dir, src) 633 # Add methods matched include_annotations. 634 for annotation in include_annotations: 635 include_methods.update( 636 test_finder_utils.get_annotated_methods( 637 annotation, src_path)) 638 if exclude_annotations: 639 # For exclude annotation, get all the method in the input srcs, 640 # and filter out the matched annotation. 641 exclude_methods = set() 642 all_methods = test_finder_utils.get_java_methods(src_path) 643 for annotation in exclude_annotations: 644 exclude_methods.update( 645 test_finder_utils.get_annotated_methods( 646 annotation, src_path)) 647 include_methods = all_methods - exclude_methods 648 if method in include_methods: 649 return True 650 return False 651 652 def find_test_by_module_and_class(self, module_class): 653 """Find the test info given a MODULE:CLASS string. 654 655 Args: 656 module_class: A string of form MODULE:CLASS or MODULE:CLASS#METHOD. 657 658 Returns: 659 A list of populated TestInfo namedtuple if found, else None. 660 """ 661 parse_result = test_finder_utils.parse_test_reference(module_class) 662 if not parse_result: 663 return None 664 module_name = parse_result['module_name'] 665 class_name = parse_result['pkg_class_name'] 666 method_name = parse_result.get('method_name', '') 667 if method_name: 668 class_name = class_name + '#' + method_name 669 670 # module_infos is a list with at most 1 element. 671 module_infos = self.find_test_by_module_name(module_name) 672 module_info = module_infos[0] if module_infos else None 673 if not module_info: 674 return None 675 find_result = None 676 # If the target module is JAVA or Python test, search class name. 677 find_result = self.find_test_by_class_name( 678 class_name, module_info.test_name, 679 module_info.data.get(constants.TI_REL_CONFIG), 680 self.module_info.is_native_test(module_name)) 681 # kernel target test is also define as NATIVE_TEST in build system. 682 # TODO (b/157210083) Update find_test_by_kernel_class_name method to 683 # support gen_rule use case. 684 if not find_result: 685 find_result = self.find_test_by_kernel_class_name( 686 module_name, class_name) 687 # Find by cc class. 688 if not find_result: 689 find_result = self.find_test_by_cc_class_name( 690 class_name, module_info.test_name, 691 module_info.data.get(constants.TI_REL_CONFIG)) 692 return find_result 693 694 def find_test_by_package_name(self, package, module_name=None, 695 rel_config=None): 696 """Find the test info given a PACKAGE string. 697 698 Args: 699 package: A string of the package name. 700 module_name: Optional. A string of the module name. 701 ref_config: Optional. A string of rel path of config. 702 703 Returns: 704 A list of populated TestInfo namedtuple if found, else None. 705 """ 706 _, methods = test_finder_utils.split_methods(package) 707 if methods: 708 raise atest_error.MethodWithoutClassError('%s: Method filtering ' 709 'requires class' % ( 710 methods)) 711 # Confirm that packages exists and get user input for multiples. 712 if rel_config: 713 search_dir = os.path.join(self.root_dir, 714 os.path.dirname(rel_config)) 715 else: 716 search_dir = self.root_dir 717 package_paths = test_finder_utils.run_find_cmd( 718 test_finder_utils.TestReferenceType.PACKAGE, search_dir, package) 719 package_paths = package_paths if package_paths is not None else [] 720 # Package path will be the full path to the dir represented by package. 721 if not package_paths: 722 if not module_name: 723 return None 724 module_paths = self.module_info.get_paths(module_name) 725 for rel_module_path in module_paths: 726 package_paths.append(os.path.join(self.root_dir, rel_module_path)) 727 test_filter = frozenset([test_info.TestFilter(package, frozenset())]) 728 test_infos = [] 729 for package_path in package_paths: 730 tinfo = self._get_test_infos(package_path, rel_config, 731 module_name, test_filter) 732 if tinfo: 733 test_infos.extend(tinfo) 734 return test_infos if test_infos else None 735 736 def find_test_by_module_and_package(self, module_package): 737 """Find the test info given a MODULE:PACKAGE string. 738 739 Args: 740 module_package: A string of form MODULE:PACKAGE 741 742 Returns: 743 A list of populated TestInfo namedtuple if found, else None. 744 """ 745 parse_result = test_finder_utils.parse_test_reference(module_package) 746 if not parse_result: 747 return None 748 module_name = parse_result['module_name'] 749 package = parse_result['pkg_class_name'] 750 method = parse_result.get('method_name', '') 751 if method: 752 package = package + '#' + method 753 754 # module_infos is a list with at most 1 element. 755 module_infos = self.find_test_by_module_name(module_name) 756 module_info = module_infos[0] if module_infos else None 757 if not module_info: 758 return None 759 return self.find_test_by_package_name( 760 package, module_info.test_name, 761 module_info.data.get(constants.TI_REL_CONFIG)) 762 763 def find_test_by_path(self, rel_path: str) -> List[test_info.TestInfo]: 764 """Find the first test info matching the given path. 765 766 Strategy: 767 path_to_java_file --> Resolve to CLASS 768 path_to_cc_file --> Resolve to CC CLASS 769 path_to_module_file -> Resolve to MODULE 770 path_to_module_dir -> Resolve to MODULE 771 path_to_dir_with_class_files--> Resolve to PACKAGE 772 path_to_any_other_dir --> Resolve as MODULE 773 774 Args: 775 rel_path: A string of the relative path to $BUILD_TOP. 776 777 Returns: 778 A list of populated TestInfo namedtuple if test found, else None 779 """ 780 logging.debug('Finding test by path: %s', rel_path) 781 path, methods = test_finder_utils.split_methods(rel_path) 782 # TODO: See if this can be generalized and shared with methods above 783 # create absolute path from cwd and remove symbolic links 784 path = os.path.realpath(path) 785 if not os.path.exists(path): 786 return None 787 if (methods and 788 not test_finder_utils.has_method_in_file(path, methods)): 789 return None 790 dir_path, _ = test_finder_utils.get_dir_path_and_filename(path) 791 # Module/Class 792 rel_module_dir = test_finder_utils.find_parent_module_dir( 793 self.root_dir, dir_path, self.module_info) 794 795 # If the input file path does not belong to a module(by searching 796 # upwards to the build_top), check whether it belongs to the dependency 797 # of modules. 798 if not rel_module_dir: 799 testable_modules = self.module_info.get_modules_by_include_deps( 800 self.module_info.get_modules_by_path_in_srcs(rel_path), 801 testable_module_only=True) 802 if testable_modules: 803 test_filter = self._get_test_info_filter( 804 path, methods, rel_module_dir=rel_module_dir) 805 tinfos = [] 806 for testable_module in testable_modules: 807 rel_config = os.path.join( 808 self.module_info.get_paths( 809 testable_module)[0], constants.MODULE_CONFIG) 810 tinfos.extend( 811 self._get_test_infos( 812 path, rel_config, testable_module, test_filter)) 813 metrics.LocalDetectEvent( 814 detect_type=DetectType.FIND_TEST_IN_DEPS, 815 result=1) 816 return tinfos 817 818 if not rel_module_dir: 819 # Try to find unit-test for input path. 820 path = os.path.relpath( 821 os.path.realpath(rel_path), 822 os.environ.get(constants.ANDROID_BUILD_TOP, '')) 823 unit_tests = test_finder_utils.find_host_unit_tests( 824 self.module_info, path) 825 if unit_tests: 826 tinfos = [] 827 for unit_test in unit_tests: 828 tinfo = self._get_test_infos(path, constants.MODULE_CONFIG, 829 unit_test, frozenset()) 830 if tinfo: 831 tinfos.extend(tinfo) 832 return tinfos 833 return None 834 rel_config = os.path.join(rel_module_dir, constants.MODULE_CONFIG) 835 test_filter = self._get_test_info_filter(path, methods, 836 rel_module_dir=rel_module_dir) 837 return self._get_test_infos(path, rel_config, None, test_filter) 838 839 def find_test_by_cc_class_name(self, class_name, module_name=None, 840 rel_config=None): 841 """Find test files given a cc class name. 842 843 If module_name and rel_config not given, test will be determined 844 by looking up the tree for files which has input class. 845 846 Args: 847 class_name: A string of the test's class name. 848 module_name: Optional. A string of the module name to use. 849 rel_config: Optional. A string of module dir no-absolute to repo root. 850 851 Returns: 852 A list of populated TestInfo namedtuple if test found, else None. 853 """ 854 # Check if class_name is prepended with file name. If so, trim the 855 # prefix and keep only the class_name. 856 if '.' in class_name: 857 # (b/202764540) Strip prefixes of a cc class. 858 # Assume the class name has a format of file_name.class_name 859 class_name = class_name[class_name.rindex('.')+1:] 860 logging.info('Search with updated class name: %s', class_name) 861 return self.find_test_by_class_name( 862 class_name, module_name, rel_config, is_native_test=True) 863 864 def get_testable_modules_with_ld(self, user_input, ld_range=0): 865 """Calculate the edit distances of the input and testable modules. 866 867 The user input will be calculated across all testable modules and 868 results in integers generated by Levenshtein Distance algorithm. 869 To increase the speed of the calculation, a bound can be applied to 870 this method to prevent from calculating every testable modules. 871 872 Guessing from typos, e.g. atest atest_unitests, implies a tangible range 873 of length that Atest only needs to search within it, and the default of 874 the bound is 2. 875 876 Guessing from keywords however, e.g. atest --search Camera, means that 877 the uncertainty of the module name is way higher, and Atest should walk 878 through all testable modules and return the highest possibilities. 879 880 Args: 881 user_input: A string of the user input. 882 ld_range: An integer that range the searching scope. If the length 883 of user_input is 10, then Atest will calculate modules of 884 which length is between 8 and 12. 0 is equivalent to 885 unlimited. 886 887 Returns: 888 A List of LDs and possible module names. If the user_input is "fax", 889 the output will be like: 890 [[2, "fog"], [2, "Fix"], [4, "duck"], [7, "Duckies"]] 891 892 Which means the most lilely names of "fax" are fog and Fix(LD=2), 893 while Dickies is the most unlikely one(LD=7). 894 """ 895 atest_utils.colorful_print('\nSearching for similar module names using ' 896 'fuzzy search...', constants.CYAN) 897 search_start = time.time() 898 testable_modules = sorted(self.module_info.get_testable_modules(), 899 key=len) 900 lower_bound = len(user_input) - ld_range 901 upper_bound = len(user_input) + ld_range 902 testable_modules_with_ld = [] 903 for module_name in testable_modules: 904 # Dispose those too short or too lengthy. 905 if ld_range != 0: 906 if len(module_name) < lower_bound: 907 continue 908 if len(module_name) > upper_bound: 909 break 910 testable_modules_with_ld.append( 911 [test_finder_utils.get_levenshtein_distance( 912 user_input, module_name), module_name]) 913 search_duration = time.time() - search_start 914 logging.debug('Fuzzy search took %ss', search_duration) 915 metrics.LocalDetectEvent( 916 detect_type=DetectType.FUZZY_SEARCH_TIME, 917 result=round(search_duration)) 918 return testable_modules_with_ld 919 920 def get_fuzzy_searching_results(self, user_input): 921 """Give results which have no more than allowance of edit distances. 922 923 Args: 924 user_input: the target module name for fuzzy searching. 925 926 Return: 927 A list of guessed modules. 928 """ 929 modules_with_ld = self.get_testable_modules_with_ld( 930 user_input, ld_range=constants.LD_RANGE) 931 guessed_modules = [] 932 for _distance, _module in modules_with_ld: 933 if _distance <= abs(constants.LD_RANGE): 934 guessed_modules.append(_module) 935 return guessed_modules 936 937 def find_test_by_config_name(self, config_name): 938 """Find test for the given config name. 939 940 Args: 941 config_name: A string of the test's config name. 942 943 Returns: 944 A list that includes only 1 populated TestInfo namedtuple 945 if found, otherwise None. 946 """ 947 for module_name, mod_info in self.module_info.name_to_module_info.items(): 948 test_configs = mod_info.get(constants.MODULE_TEST_CONFIG, []) 949 for test_config in test_configs: 950 test_config_name = os.path.splitext( 951 os.path.basename(test_config))[0] 952 if test_config_name == config_name: 953 tinfo = test_info.TestInfo( 954 test_name=test_config_name, 955 test_runner=self._TEST_RUNNER, 956 build_targets=self._get_build_targets(module_name, 957 test_config), 958 data={constants.TI_REL_CONFIG: test_config, 959 constants.TI_FILTER: frozenset()}, 960 compatibility_suites=mod_info.get( 961 constants.MODULE_COMPATIBILITY_SUITES, [])) 962 test_config_path = os.path.join(self.root_dir, test_config) 963 if test_finder_utils.need_aggregate_metrics_result(test_config_path): 964 tinfo.aggregate_metrics_result = True 965 if tinfo: 966 # There should have only one test_config with the same 967 # name in source tree. 968 return [tinfo] 969 return None 970 971 @staticmethod 972 def _is_comparted_src(path): 973 """Check if the input path need to match srcs information in module. 974 975 If path is a folder or android build file, we don't need to compart 976 with module's srcs. 977 978 Args: 979 path: A string of the test's path. 980 981 Returns: 982 True if input path need to match with module's src info, else False. 983 """ 984 if os.path.isdir(path): 985 return False 986 if atest_utils.is_build_file(path): 987 return False 988 return True 989 990class MainlineModuleFinder(ModuleFinder): 991 """Mainline Module finder class.""" 992 NAME = 'MAINLINE_MODULE' 993 994 def __init__(self, module_info=None): 995 super().__init__() 996