1# Copyright 2018, The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15""" 16Module Finder class. 17""" 18 19import logging 20import os 21import re 22 23# pylint: disable=import-error 24import atest_error 25import atest_utils 26import constants 27from test_finders import test_info 28from test_finders import test_finder_base 29from test_finders import test_finder_utils 30from test_runners import atest_tf_test_runner 31from test_runners import robolectric_test_runner 32from test_runners import vts_tf_test_runner 33 34_CC_EXT_RE = re.compile(r'.*(\.cc|\.cpp)$', re.I) 35_JAVA_EXT_RE = re.compile(r'.*(\.java|\.kt)$', re.I) 36 37_MODULES_IN = 'MODULES-IN-%s' 38_ANDROID_MK = 'Android.mk' 39 40# These are suites in LOCAL_COMPATIBILITY_SUITE that aren't really suites so 41# we can ignore them. 42_SUITES_TO_IGNORE = frozenset({'general-tests', 'device-tests', 'tests'}) 43 44class ModuleFinder(test_finder_base.TestFinderBase): 45 """Module finder class.""" 46 NAME = 'MODULE' 47 _TEST_RUNNER = atest_tf_test_runner.AtestTradefedTestRunner.NAME 48 _ROBOLECTRIC_RUNNER = robolectric_test_runner.RobolectricTestRunner.NAME 49 _VTS_TEST_RUNNER = vts_tf_test_runner.VtsTradefedTestRunner.NAME 50 51 def __init__(self, module_info=None): 52 super(ModuleFinder, self).__init__() 53 self.root_dir = os.environ.get(constants.ANDROID_BUILD_TOP) 54 self.module_info = module_info 55 56 def _determine_testable_module(self, path): 57 """Determine which module the user is trying to test. 58 59 Returns the module to test. If there are multiple possibilities, will 60 ask the user. Otherwise will return the only module found. 61 62 Args: 63 path: String path of module to look for. 64 65 Returns: 66 String of the module name. 67 """ 68 testable_modules = [] 69 for mod in self.module_info.get_module_names(path): 70 mod_info = self.module_info.get_module_info(mod) 71 # Robolectric tests always exist in pairs of 2, one module to build 72 # the test and another to run it. For now, we are assuming they are 73 # isolated in their own folders and will return if we find one. 74 if self.module_info.is_robolectric_test(mod): 75 return mod 76 if self.module_info.is_testable_module(mod_info): 77 testable_modules.append(mod_info.get(constants.MODULE_NAME)) 78 return test_finder_utils.extract_test_from_tests(testable_modules) 79 80 def _is_vts_module(self, module_name): 81 """Returns True if the module is a vts module, else False.""" 82 mod_info = self.module_info.get_module_info(module_name) 83 suites = [] 84 if mod_info: 85 suites = mod_info.get('compatibility_suites', []) 86 # Pull out all *ts (cts, tvts, etc) suites. 87 suites = [suite for suite in suites if suite not in _SUITES_TO_IGNORE] 88 return len(suites) == 1 and 'vts' in suites 89 90 def _update_to_vts_test_info(self, test): 91 """Fill in the fields with vts specific info. 92 93 We need to update the runner to use the vts runner and also find the 94 test specific depedencies 95 96 Args: 97 test: TestInfo to update with vts specific details. 98 99 Return: 100 TestInfo that is ready for the vts test runner. 101 """ 102 test.test_runner = self._VTS_TEST_RUNNER 103 config_file = os.path.join(self.root_dir, 104 test.data[constants.TI_REL_CONFIG]) 105 # Need to get out dir (special logic is to account for custom out dirs). 106 # The out dir is used to construct the build targets for the test deps. 107 out_dir = os.environ.get(constants.ANDROID_HOST_OUT) 108 custom_out_dir = os.environ.get(constants.ANDROID_OUT_DIR) 109 # If we're not an absolute custom out dir, get relative out dir path. 110 if custom_out_dir is None or not os.path.isabs(custom_out_dir): 111 out_dir = os.path.relpath(out_dir, self.root_dir) 112 vts_out_dir = os.path.join(out_dir, 'vts', 'android-vts', 'testcases') 113 # Parse dependency of default staging plans. 114 115 xml_path = test_finder_utils.search_integration_dirs( 116 constants.VTS_STAGING_PLAN, 117 self.module_info.get_paths(constants.VTS_TF_MODULE)) 118 vts_xmls = test_finder_utils.get_plans_from_vts_xml(xml_path) 119 vts_xmls.add(config_file) 120 for config_file in vts_xmls: 121 # Add in vts test build targets. 122 test.build_targets |= test_finder_utils.get_targets_from_vts_xml( 123 config_file, vts_out_dir, self.module_info) 124 test.build_targets.add('vts-test-core') 125 test.build_targets.add(test.test_name) 126 return test 127 128 def _update_to_robolectric_test_info(self, test): 129 """Update the fields for a robolectric test. 130 131 Args: 132 test: TestInfo to be updated with robolectric fields. 133 134 Returns: 135 TestInfo with robolectric fields. 136 """ 137 test.test_runner = self._ROBOLECTRIC_RUNNER 138 test.test_name = self.module_info.get_robolectric_test_name(test.test_name) 139 return test 140 141 def _process_test_info(self, test): 142 """Process the test info and return some fields updated/changed. 143 144 We need to check if the test found is a special module (like vts) and 145 update the test_info fields (like test_runner) appropriately. 146 147 Args: 148 test: TestInfo that has been filled out by a find method. 149 150 Return: 151 TestInfo that has been modified as needed and return None if 152 this module can't be found in the module_info. 153 """ 154 module_name = test.test_name 155 mod_info = self.module_info.get_module_info(module_name) 156 if not mod_info: 157 return None 158 test.module_class = mod_info['class'] 159 test.install_locations = test_finder_utils.get_install_locations( 160 mod_info['installed']) 161 # Check if this is only a vts module. 162 if self._is_vts_module(test.test_name): 163 return self._update_to_vts_test_info(test) 164 elif self.module_info.is_robolectric_test(test.test_name): 165 return self._update_to_robolectric_test_info(test) 166 rel_config = test.data[constants.TI_REL_CONFIG] 167 test.build_targets = self._get_build_targets(module_name, rel_config) 168 return test 169 170 def _get_build_targets(self, module_name, rel_config): 171 """Get the test deps. 172 173 Args: 174 module_name: name of the test. 175 rel_config: XML for the given test. 176 177 Returns: 178 Set of build targets. 179 """ 180 targets = set() 181 if not self.module_info.is_auto_gen_test_config(module_name): 182 config_file = os.path.join(self.root_dir, rel_config) 183 targets = test_finder_utils.get_targets_from_xml(config_file, 184 self.module_info) 185 for module_path in self.module_info.get_paths(module_name): 186 mod_dir = module_path.replace('/', '-') 187 targets.add(_MODULES_IN % mod_dir) 188 return targets 189 190 def _get_module_test_config(self, module_name, rel_config=None): 191 """Get the value of test_config in module_info. 192 193 Get the value of 'test_config' in module_info if its 194 auto_test_config is not true. 195 In this case, the test_config is specified by user. 196 If not, return rel_config. 197 198 Args: 199 module_name: A string of the test's module name. 200 rel_config: XML for the given test. 201 202 Returns: 203 A string of test_config path if found, else return rel_config. 204 """ 205 mod_info = self.module_info.get_module_info(module_name) 206 if mod_info: 207 test_config = '' 208 test_config_list = mod_info.get(constants.MODULE_TEST_CONFIG, []) 209 if test_config_list: 210 test_config = test_config_list[0] 211 if not self.module_info.is_auto_gen_test_config(module_name) and test_config != '': 212 return test_config 213 return rel_config 214 215 def _get_test_info_filter(self, path, methods, **kwargs): 216 """Get test info filter. 217 218 Args: 219 path: A string of the test's path. 220 methods: A set of method name strings. 221 rel_module_dir: Optional. A string of the module dir relative to 222 root. 223 class_name: Optional. A string of the class name. 224 is_native_test: Optional. A boolean variable of whether to search 225 for a native test or not. 226 227 Returns: 228 A set of test info filter. 229 """ 230 _, file_name = test_finder_utils.get_dir_path_and_filename(path) 231 ti_filter = frozenset() 232 if kwargs.get('is_native_test', None): 233 ti_filter = frozenset([test_info.TestFilter( 234 test_finder_utils.get_cc_filter( 235 kwargs.get('class_name', '*'), methods), frozenset())]) 236 # Path to java file. 237 elif file_name and _JAVA_EXT_RE.match(file_name): 238 full_class_name = test_finder_utils.get_fully_qualified_class_name( 239 path) 240 ti_filter = frozenset( 241 [test_info.TestFilter(full_class_name, methods)]) 242 # Path to cc file. 243 elif file_name and _CC_EXT_RE.match(file_name): 244 if not test_finder_utils.has_cc_class(path): 245 raise atest_error.MissingCCTestCaseError( 246 "Can't find CC class in %s" % path) 247 if methods: 248 ti_filter = frozenset( 249 [test_info.TestFilter(test_finder_utils.get_cc_filter( 250 kwargs.get('class_name', '*'), methods), frozenset())]) 251 # Path to non-module dir, treat as package. 252 elif (not file_name 253 and kwargs.get('rel_module_dir', None) != 254 os.path.relpath(path, self.root_dir)): 255 dir_items = [os.path.join(path, f) for f in os.listdir(path)] 256 for dir_item in dir_items: 257 if _JAVA_EXT_RE.match(dir_item): 258 package_name = test_finder_utils.get_package_name(dir_item) 259 if package_name: 260 # methods should be empty frozenset for package. 261 if methods: 262 raise atest_error.MethodWithoutClassError( 263 '%s: Method filtering requires class' 264 % str(methods)) 265 ti_filter = frozenset( 266 [test_info.TestFilter(package_name, methods)]) 267 break 268 return ti_filter 269 270 def _get_rel_config(self, test_path): 271 """Get config file's relative path. 272 273 Args: 274 test_path: A string of the test absolute path. 275 276 Returns: 277 A string of config's relative path, else None. 278 """ 279 test_dir = os.path.dirname(test_path) 280 rel_module_dir = test_finder_utils.find_parent_module_dir( 281 self.root_dir, test_dir, self.module_info) 282 if rel_module_dir: 283 return os.path.join(rel_module_dir, constants.MODULE_CONFIG) 284 return None 285 286 def _get_test_info(self, test_path, rel_config, module_name, test_filter): 287 """Get test_info for test_path. 288 289 Args: 290 test_path: A string of the test path. 291 rel_config: A string of rel path of config. 292 module_name: A string of the module name to use. 293 test_filter: A test info filter. 294 295 Returns: 296 TestInfo namedtuple if found, else None. 297 """ 298 if not rel_config: 299 rel_config = self._get_rel_config(test_path) 300 if not rel_config: 301 return None 302 if not module_name: 303 module_name = self._determine_testable_module( 304 os.path.dirname(rel_config)) 305 # The real test config might be recorded in module-info. 306 rel_config = self._get_module_test_config(module_name, 307 rel_config=rel_config) 308 return self._process_test_info(test_info.TestInfo( 309 test_name=module_name, 310 test_runner=self._TEST_RUNNER, 311 build_targets=set(), 312 data={constants.TI_FILTER: test_filter, 313 constants.TI_REL_CONFIG: rel_config})) 314 315 def find_test_by_module_name(self, module_name): 316 """Find test for the given module name. 317 318 Args: 319 module_name: A string of the test's module name. 320 321 Returns: 322 A populated TestInfo namedtuple if found, else None. 323 """ 324 mod_info = self.module_info.get_module_info(module_name) 325 if self.module_info.is_testable_module(mod_info): 326 # path is a list with only 1 element. 327 rel_config = os.path.join(mod_info['path'][0], 328 constants.MODULE_CONFIG) 329 rel_config = self._get_module_test_config(module_name, rel_config=rel_config) 330 return self._process_test_info(test_info.TestInfo( 331 test_name=module_name, 332 test_runner=self._TEST_RUNNER, 333 build_targets=set(), 334 data={constants.TI_REL_CONFIG: rel_config, 335 constants.TI_FILTER: frozenset()})) 336 return None 337 338 def find_test_by_class_name(self, class_name, module_name=None, 339 rel_config=None, is_native_test=False): 340 """Find test files given a class name. 341 342 If module_name and rel_config not given it will calculate it determine 343 it by looking up the tree from the class file. 344 345 Args: 346 class_name: A string of the test's class name. 347 module_name: Optional. A string of the module name to use. 348 rel_config: Optional. A string of module dir relative to repo root. 349 is_native_test: A boolean variable of whether to search for a 350 native test or not. 351 352 Returns: 353 A populated TestInfo namedtuple if test found, else None. 354 """ 355 class_name, methods = test_finder_utils.split_methods(class_name) 356 if rel_config: 357 search_dir = os.path.join(self.root_dir, 358 os.path.dirname(rel_config)) 359 else: 360 search_dir = self.root_dir 361 test_path = test_finder_utils.find_class_file(search_dir, class_name, 362 is_native_test) 363 if not test_path and rel_config: 364 logging.info('Did not find class (%s) under module path (%s), ' 365 'researching from repo root.', class_name, rel_config) 366 test_path = test_finder_utils.find_class_file(self.root_dir, 367 class_name, 368 is_native_test) 369 if not test_path: 370 return None 371 test_filter = self._get_test_info_filter( 372 test_path, methods, class_name=class_name, 373 is_native_test=is_native_test) 374 tinfo = self._get_test_info(test_path, rel_config, module_name, 375 test_filter) 376 return tinfo 377 378 def find_test_by_module_and_class(self, module_class): 379 """Find the test info given a MODULE:CLASS string. 380 381 Args: 382 module_class: A string of form MODULE:CLASS or MODULE:CLASS#METHOD. 383 384 Returns: 385 A populated TestInfo namedtuple if found, else None. 386 """ 387 if ':' not in module_class: 388 return None 389 module_name, class_name = module_class.split(':') 390 module_info = self.find_test_by_module_name(module_name) 391 if not module_info: 392 return None 393 # If the target module is NATIVE_TEST, search CC classes only. 394 find_result = None 395 if not self.module_info.is_native_test(module_name): 396 # Find by java class. 397 find_result = self.find_test_by_class_name( 398 class_name, module_info.test_name, 399 module_info.data.get(constants.TI_REL_CONFIG)) 400 # Find by cc class. 401 if not find_result: 402 find_result = self.find_test_by_cc_class_name( 403 class_name, module_info.test_name, 404 module_info.data.get(constants.TI_REL_CONFIG)) 405 return find_result 406 407 def find_test_by_package_name(self, package, module_name=None, 408 rel_config=None): 409 """Find the test info given a PACKAGE string. 410 411 Args: 412 package: A string of the package name. 413 module_name: Optional. A string of the module name. 414 ref_config: Optional. A string of rel path of config. 415 416 Returns: 417 A populated TestInfo namedtuple if found, else None. 418 """ 419 _, methods = test_finder_utils.split_methods(package) 420 if methods: 421 raise atest_error.MethodWithoutClassError('%s: Method filtering ' 422 'requires class' % ( 423 methods)) 424 # Confirm that packages exists and get user input for multiples. 425 if rel_config: 426 search_dir = os.path.join(self.root_dir, 427 os.path.dirname(rel_config)) 428 else: 429 search_dir = self.root_dir 430 package_path = test_finder_utils.run_find_cmd( 431 test_finder_utils.FIND_REFERENCE_TYPE.PACKAGE, search_dir, 432 package.replace('.', '/')) 433 # Package path will be the full path to the dir represented by package. 434 if not package_path: 435 return None 436 test_filter = frozenset([test_info.TestFilter(package, frozenset())]) 437 tinfo = self._get_test_info(package_path, rel_config, module_name, 438 test_filter) 439 return tinfo 440 441 def find_test_by_module_and_package(self, module_package): 442 """Find the test info given a MODULE:PACKAGE string. 443 444 Args: 445 module_package: A string of form MODULE:PACKAGE 446 447 Returns: 448 A populated TestInfo namedtuple if found, else None. 449 """ 450 module_name, package = module_package.split(':') 451 module_info = self.find_test_by_module_name(module_name) 452 if not module_info: 453 return None 454 return self.find_test_by_package_name( 455 package, module_info.test_name, 456 module_info.data.get(constants.TI_REL_CONFIG)) 457 458 def find_test_by_path(self, path): 459 """Find the first test info matching the given path. 460 461 Strategy: 462 path_to_java_file --> Resolve to CLASS 463 path_to_cc_file --> Resolve to CC CLASS 464 path_to_module_file -> Resolve to MODULE 465 path_to_module_dir -> Resolve to MODULE 466 path_to_dir_with_class_files--> Resolve to PACKAGE 467 path_to_any_other_dir --> Resolve as MODULE 468 469 Args: 470 path: A string of the test's path. 471 472 Returns: 473 A populated TestInfo namedtuple if test found, else None 474 """ 475 logging.debug('Finding test by path: %s', path) 476 path, methods = test_finder_utils.split_methods(path) 477 # TODO: See if this can be generalized and shared with methods above 478 # create absolute path from cwd and remove symbolic links 479 path = os.path.realpath(path) 480 if not os.path.exists(path): 481 return None 482 dir_path, _ = test_finder_utils.get_dir_path_and_filename(path) 483 # Module/Class 484 rel_module_dir = test_finder_utils.find_parent_module_dir( 485 self.root_dir, dir_path, self.module_info) 486 if not rel_module_dir: 487 return None 488 rel_config = os.path.join(rel_module_dir, constants.MODULE_CONFIG) 489 test_filter = self._get_test_info_filter(path, methods, 490 rel_module_dir=rel_module_dir) 491 return self._get_test_info(path, rel_config, None, test_filter) 492 493 def find_test_by_cc_class_name(self, class_name, module_name=None, 494 rel_config=None): 495 """Find test files given a cc class name. 496 497 If module_name and rel_config not given, test will be determined 498 by looking up the tree for files which has input class. 499 500 Args: 501 class_name: A string of the test's class name. 502 module_name: Optional. A string of the module name to use. 503 rel_config: Optional. A string of module dir relative to repo root. 504 505 Returns: 506 A populated TestInfo namedtuple if test found, else None. 507 """ 508 # Check if class_name is prepended with file name. If so, trim the 509 # prefix and keep only the class_name. 510 if '.' in class_name: 511 # Assume the class name has a format of file_name.class_name 512 class_name = class_name[class_name.rindex('.')+1:] 513 logging.info('Search with updated class name: %s', class_name) 514 return self.find_test_by_class_name( 515 class_name, module_name, rel_config, is_native_test=True) 516 517 def get_testable_modules_with_ld(self, user_input, ld_range=0): 518 """Calculate the edit distances of the input and testable modules. 519 520 The user input will be calculated across all testable modules and 521 results in integers generated by Levenshtein Distance algorithm. 522 To increase the speed of the calculation, a bound can be applied to 523 this method to prevent from calculating every testable modules. 524 525 Guessing from typos, e.g. atest atest_unitests, implies a tangible range 526 of length that Atest only needs to search within it, and the default of 527 the bound is 2. 528 529 Guessing from keywords however, e.g. atest --search Camera, means that 530 the uncertainty of the module name is way higher, and Atest should walk 531 through all testable modules and return the highest possibilities. 532 533 Args: 534 user_input: A string of the user input. 535 ld_range: An integer that range the searching scope. If the length of 536 user_input is 10, then Atest will calculate modules of which 537 length is between 8 and 12. 0 is equivalent to unlimited. 538 539 Returns: 540 A List of LDs and possible module names. If the user_input is "fax", 541 the output will be like: 542 [[2, "fog"], [2, "Fix"], [4, "duck"], [7, "Duckies"]] 543 544 Which means the most lilely names of "fax" are fog and Fix(LD=2), 545 while Dickies is the most unlikely one(LD=7). 546 """ 547 atest_utils.colorful_print('\nSearching for similar module names using ' 548 'fuzzy search...', constants.CYAN) 549 testable_modules = sorted(self.module_info.get_testable_modules(), key=len) 550 lower_bound = len(user_input) - ld_range 551 upper_bound = len(user_input) + ld_range 552 testable_modules_with_ld = [] 553 for module_name in testable_modules: 554 # Dispose those too short or too lengthy. 555 if ld_range != 0: 556 if len(module_name) < lower_bound: 557 continue 558 elif len(module_name) > upper_bound: 559 break 560 testable_modules_with_ld.append( 561 [test_finder_utils.get_levenshtein_distance( 562 user_input, module_name), module_name]) 563 return testable_modules_with_ld 564 565 def get_fuzzy_searching_results(self, user_input): 566 """Give results which have no more than allowance of edit distances. 567 568 Args: 569 user_input: the target module name for fuzzy searching. 570 571 Return: 572 A list of guessed modules. 573 """ 574 modules_with_ld = self.get_testable_modules_with_ld(user_input, 575 ld_range=constants.LD_RANGE) 576 guessed_modules = [] 577 for _distance, _module in modules_with_ld: 578 if _distance <= abs(constants.LD_RANGE): 579 guessed_modules.append(_module) 580 return guessed_modules 581