1# Copyright 2017, The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15"""Command Line Translator for atest.""" 16 17# pylint: disable=line-too-long 18# pylint: disable=too-many-lines 19 20from __future__ import print_function 21 22import fnmatch 23import json 24import logging 25import os 26import re 27import sys 28import time 29 30import atest_error 31import atest_utils 32import constants 33import test_finder_handler 34import test_mapping 35 36from metrics import metrics 37from metrics import metrics_utils 38from test_finders import module_finder 39from test_finders import test_finder_utils 40 41FUZZY_FINDER = 'FUZZY' 42CACHE_FINDER = 'CACHE' 43TESTNAME_CHARS = {'#', ':', '/'} 44 45# Pattern used to identify comments start with '//' or '#' in TEST_MAPPING. 46_COMMENTS_RE = re.compile(r'(?m)[\s\t]*(#|//).*|(\".*?\")') 47_COMMENTS = frozenset(['//', '#']) 48 49_MAINLINE_MODULES_EXT_RE = re.compile(r'(.apex|.apks|.apk)$') 50 51#pylint: disable=no-self-use 52class CLITranslator: 53 """ 54 CLITranslator class contains public method translate() and some private 55 helper methods. The atest tool can call the translate() method with a list 56 of strings, each string referencing a test to run. Translate() will 57 "translate" this list of test strings into a list of build targets and a 58 list of TradeFederation run commands. 59 60 Translation steps for a test string reference: 61 1. Narrow down the type of reference the test string could be, i.e. 62 whether it could be referencing a Module, Class, Package, etc. 63 2. Try to find the test files assuming the test string is one of these 64 types of reference. 65 3. If test files found, generate Build Targets and the Run Command. 66 """ 67 68 def __init__(self, module_info=None, print_cache_msg=True): 69 """CLITranslator constructor 70 71 Args: 72 module_info: ModuleInfo class that has cached module-info.json. 73 print_cache_msg: Boolean whether printing clear cache message or not. 74 True will print message while False won't print. 75 """ 76 self.mod_info = module_info 77 self.enable_file_patterns = False 78 self.msg = '' 79 if print_cache_msg: 80 self.msg = ('(Test info has been cached for speeding up the next ' 81 'run, if test info need to be updated, please add -c ' 82 'to clean the old cache.)') 83 84 # pylint: disable=too-many-locals 85 # pylint: disable=too-many-branches 86 # pylint: disable=too-many-statements 87 def _find_test_infos(self, test, tm_test_detail, 88 is_rebuild_module_info=False): 89 """Return set of TestInfos based on a given test. 90 91 Args: 92 test: A string representing test references. 93 tm_test_detail: The TestDetail of test configured in TEST_MAPPING 94 files. 95 is_rebuild_module_info: Boolean of args.is_rebuild_module_info 96 97 Returns: 98 Set of TestInfos based on the given test. 99 """ 100 test_infos = set() 101 test_find_starts = time.time() 102 test_found = False 103 test_finders = [] 104 test_info_str = '' 105 find_test_err_msg = None 106 mm_build_targets = [] 107 test, mainline_modules = atest_utils.parse_mainline_modules(test) 108 if not self._verified_mainline_modules(test, mainline_modules): 109 return test_infos 110 test_modules_to_build = [] 111 test_mainline_modules = [] 112 if self.mod_info and self.mod_info.get_module_info(test): 113 test_mainline_modules = self.mod_info.get_module_info(test).get( 114 constants.MODULE_MAINLINE_MODULES, []) 115 for modules in test_mainline_modules: 116 for module in modules.split('+'): 117 test_modules_to_build.append(re.sub( 118 _MAINLINE_MODULES_EXT_RE, '', module)) 119 if mainline_modules: 120 mm_build_targets = [re.sub(_MAINLINE_MODULES_EXT_RE, '', x) 121 for x in mainline_modules.split('+')] 122 for finder in test_finder_handler.get_find_methods_for_test( 123 self.mod_info, test): 124 # For tests in TEST_MAPPING, find method is only related to 125 # test name, so the details can be set after test_info object 126 # is created. 127 try: 128 found_test_infos = finder.find_method( 129 finder.test_finder_instance, test) 130 except atest_error.TestDiscoveryException as e: 131 find_test_err_msg = e 132 if found_test_infos: 133 finder_info = finder.finder_info 134 for test_info in found_test_infos: 135 test_deps = set() 136 if self.mod_info: 137 test_deps = self.mod_info.get_install_module_dependency( 138 test_info.test_name) 139 logging.debug('(%s) Test dependencies: %s', 140 test_info.test_name, test_deps) 141 if tm_test_detail: 142 test_info.data[constants.TI_MODULE_ARG] = ( 143 tm_test_detail.options) 144 test_info.from_test_mapping = True 145 test_info.host = tm_test_detail.host 146 if finder_info != CACHE_FINDER: 147 test_info.test_finder = finder_info 148 test_info.mainline_modules = mainline_modules 149 test_info.build_targets = { 150 x for x in test_info.build_targets 151 if x not in test_modules_to_build} 152 test_info.build_targets.update(mm_build_targets) 153 # Only add dependencies to build_targets when they are in 154 # module info 155 test_deps_in_mod_info = [ 156 test_dep for test_dep in test_deps 157 if self.mod_info.is_module(test_dep)] 158 test_info.build_targets.update(test_deps_in_mod_info) 159 test_infos.add(test_info) 160 test_found = True 161 print("Found '%s' as %s" % ( 162 atest_utils.colorize(test, constants.GREEN), 163 finder_info)) 164 if finder_info == CACHE_FINDER and test_infos: 165 test_finders.append(list(test_infos)[0].test_finder) 166 test_finders.append(finder_info) 167 test_info_str = ','.join([str(x) for x in found_test_infos]) 168 break 169 if not test_found: 170 f_results = self._fuzzy_search_and_msg(test, find_test_err_msg, 171 is_rebuild_module_info) 172 if f_results: 173 test_infos.update(f_results) 174 test_found = True 175 test_finders.append(FUZZY_FINDER) 176 metrics.FindTestFinishEvent( 177 duration=metrics_utils.convert_duration( 178 time.time() - test_find_starts), 179 success=test_found, 180 test_reference=test, 181 test_finders=test_finders, 182 test_info=test_info_str) 183 # Cache test_infos by default except running with TEST_MAPPING which may 184 # include customized flags and they are likely to mess up other 185 # non-test_mapping tests. 186 if test_infos and not tm_test_detail: 187 atest_utils.update_test_info_cache(test, test_infos) 188 if self.msg: 189 print(self.msg) 190 return test_infos 191 192 def _verified_mainline_modules(self, test, mainline_modules): 193 """ Verify the test with mainline modules is acceptable. 194 195 The test must be a module and mainline modules are in module-info. 196 The syntax rule of mainline modules will check in build process. 197 The rule includes mainline modules are sorted alphabetically, no space, 198 and no duplication. 199 200 Args: 201 test: A string representing test references 202 mainline_modules: A string of mainline_modules. 203 204 Returns: 205 True if this test is acceptable. Otherwise, print the reason and 206 return False. 207 """ 208 if not mainline_modules: 209 return True 210 if not self.mod_info.is_module(test): 211 print('Test mainline modules(%s) for: %s failed. Only support ' 212 'module tests.' 213 % (atest_utils.colorize(mainline_modules, constants.RED), 214 atest_utils.colorize(test, constants.RED))) 215 return False 216 if not self.mod_info.has_mainline_modules(test, mainline_modules): 217 print('Error: Test mainline modules(%s) not for %s.' 218 % (atest_utils.colorize(mainline_modules, constants.RED), 219 atest_utils.colorize(test, constants.RED))) 220 return False 221 return True 222 223 def _fuzzy_search_and_msg(self, test, find_test_err_msg, 224 is_rebuild_module_info=False): 225 """ Fuzzy search and print message. 226 227 Args: 228 test: A string representing test references 229 find_test_err_msg: A string of find test error message. 230 is_rebuild_module_info: Boolean of args.is_rebuild_module_info 231 232 Returns: 233 A list of TestInfos if found, otherwise None. 234 """ 235 print('No test found for: %s' % 236 atest_utils.colorize(test, constants.RED)) 237 # Currently we focus on guessing module names. Append names on 238 # results if more finders support fuzzy searching. 239 if atest_utils.has_chars(test, TESTNAME_CHARS): 240 return None 241 mod_finder = module_finder.ModuleFinder(self.mod_info) 242 results = mod_finder.get_fuzzy_searching_results(test) 243 if len(results) == 1 and self._confirm_running(results): 244 found_test_infos = mod_finder.find_test_by_module_name(results[0]) 245 # found_test_infos is a list with at most 1 element. 246 if found_test_infos: 247 return found_test_infos 248 elif len(results) > 1: 249 self._print_fuzzy_searching_results(results) 250 else: 251 print('No matching result for {0}.'.format(test)) 252 if find_test_err_msg: 253 print('%s\n' % (atest_utils.colorize( 254 find_test_err_msg, constants.MAGENTA))) 255 else: 256 if not is_rebuild_module_info: 257 print(constants.REBUILD_MODULE_INFO_MSG.format( 258 atest_utils.colorize(constants.REBUILD_MODULE_INFO_FLAG, 259 constants.RED))) 260 print('') 261 return None 262 263 def _get_test_infos(self, tests, test_mapping_test_details=None, 264 is_rebuild_module_info=False): 265 """Return set of TestInfos based on passed in tests. 266 267 Args: 268 tests: List of strings representing test references. 269 test_mapping_test_details: List of TestDetail for tests configured 270 in TEST_MAPPING files. 271 is_rebuild_module_info: Boolean of args.is_rebuild_module_info 272 273 Returns: 274 Set of TestInfos based on the passed in tests. 275 """ 276 test_infos = set() 277 if not test_mapping_test_details: 278 test_mapping_test_details = [None] * len(tests) 279 for test, tm_test_detail in zip(tests, test_mapping_test_details): 280 found_test_infos = self._find_test_infos(test, tm_test_detail, 281 is_rebuild_module_info) 282 test_infos.update(found_test_infos) 283 return test_infos 284 285 def _confirm_running(self, results): 286 """Listen to an answer from raw input. 287 288 Args: 289 results: A list of results. 290 291 Returns: 292 True is the answer is affirmative. 293 """ 294 return atest_utils.prompt_with_yn_result( 295 'Did you mean {0}?'.format( 296 atest_utils.colorize(results[0], constants.GREEN)), True) 297 298 def _print_fuzzy_searching_results(self, results): 299 """Print modules when fuzzy searching gives multiple results. 300 301 If the result is lengthy, just print the first 10 items only since we 302 have already given enough-accurate result. 303 304 Args: 305 results: A list of guessed testable module names. 306 307 """ 308 atest_utils.colorful_print('Did you mean the following modules?', 309 constants.WHITE) 310 for mod in results[:10]: 311 atest_utils.colorful_print(mod, constants.GREEN) 312 313 def filter_comments(self, test_mapping_file): 314 """Remove comments in TEST_MAPPING file to valid format. Only '//' and 315 '#' are regarded as comments. 316 317 Args: 318 test_mapping_file: Path to a TEST_MAPPING file. 319 320 Returns: 321 Valid json string without comments. 322 """ 323 def _replace(match): 324 """Replace comments if found matching the defined regular 325 expression. 326 327 Args: 328 match: The matched regex pattern 329 330 Returns: 331 "" if it matches _COMMENTS, otherwise original string. 332 """ 333 line = match.group(0).strip() 334 return "" if any(map(line.startswith, _COMMENTS)) else line 335 with open(test_mapping_file) as json_file: 336 return re.sub(_COMMENTS_RE, _replace, json_file.read()) 337 338 def _read_tests_in_test_mapping(self, test_mapping_file): 339 """Read tests from a TEST_MAPPING file. 340 341 Args: 342 test_mapping_file: Path to a TEST_MAPPING file. 343 344 Returns: 345 A tuple of (all_tests, imports), where 346 all_tests is a dictionary of all tests in the TEST_MAPPING file, 347 grouped by test group. 348 imports is a list of test_mapping.Import to include other test 349 mapping files. 350 """ 351 all_tests = {} 352 imports = [] 353 test_mapping_dict = json.loads(self.filter_comments(test_mapping_file)) 354 for test_group_name, test_list in test_mapping_dict.items(): 355 if test_group_name == constants.TEST_MAPPING_IMPORTS: 356 for import_detail in test_list: 357 imports.append( 358 test_mapping.Import(test_mapping_file, import_detail)) 359 else: 360 grouped_tests = all_tests.setdefault(test_group_name, set()) 361 tests = [] 362 for test in test_list: 363 # TODO: uncomment below when atest support testing mainline 364 # module in TEST_MAPPING files. 365 if constants.TEST_WITH_MAINLINE_MODULES_RE.match(test['name']): 366 logging.debug('Skipping mainline module: %s', 367 atest_utils.colorize(test['name'], 368 constants.RED)) 369 continue 370 if (self.enable_file_patterns and 371 not test_mapping.is_match_file_patterns( 372 test_mapping_file, test)): 373 continue 374 test_mod_info = self.mod_info.name_to_module_info.get( 375 test['name']) 376 if not test_mod_info: 377 print('WARNING: %s is not a valid build target and ' 378 'may not be discoverable by TreeHugger. If you ' 379 'want to specify a class or test-package, ' 380 'please set \'name\' to the test module and use ' 381 '\'options\' to specify the right tests via ' 382 '\'include-filter\'.\nNote: this can also occur ' 383 'if the test module is not built for your ' 384 'current lunch target.\n' % 385 atest_utils.colorize(test['name'], constants.RED)) 386 elif not any(x in test_mod_info['compatibility_suites'] for 387 x in constants.TEST_MAPPING_SUITES): 388 print('WARNING: Please add %s to either suite: %s for ' 389 'this TEST_MAPPING file to work with TreeHugger.' % 390 (atest_utils.colorize(test['name'], 391 constants.RED), 392 atest_utils.colorize(constants.TEST_MAPPING_SUITES, 393 constants.GREEN))) 394 tests.append(test_mapping.TestDetail(test)) 395 grouped_tests.update(tests) 396 return all_tests, imports 397 398 def _get_tests_from_test_mapping_files( 399 self, test_groups, test_mapping_files): 400 """Get tests in the given test mapping files with the match group. 401 402 Args: 403 test_groups: Groups of tests to run. Default is set to `presubmit` 404 and `presubmit-large`. 405 test_mapping_files: A list of path of TEST_MAPPING files. 406 407 Returns: 408 A tuple of (tests, all_tests, imports), where, 409 tests is a set of tests (test_mapping.TestDetail) defined in 410 TEST_MAPPING file of the given path, and its parent directories, 411 with matching test_group. 412 all_tests is a dictionary of all tests in TEST_MAPPING files, 413 grouped by test group. 414 imports is a list of test_mapping.Import objects that contains the 415 details of where to import a TEST_MAPPING file. 416 """ 417 all_imports = [] 418 # Read and merge the tests in all TEST_MAPPING files. 419 merged_all_tests = {} 420 for test_mapping_file in test_mapping_files: 421 all_tests, imports = self._read_tests_in_test_mapping( 422 test_mapping_file) 423 all_imports.extend(imports) 424 for test_group_name, test_list in all_tests.items(): 425 grouped_tests = merged_all_tests.setdefault( 426 test_group_name, set()) 427 grouped_tests.update(test_list) 428 tests = set() 429 for test_group in test_groups: 430 temp_tests = set(merged_all_tests.get(test_group, [])) 431 tests.update(temp_tests) 432 if test_group == constants.TEST_GROUP_ALL: 433 for grouped_tests in merged_all_tests.values(): 434 tests.update(grouped_tests) 435 return tests, merged_all_tests, all_imports 436 437 # pylint: disable=too-many-arguments 438 # pylint: disable=too-many-locals 439 def _find_tests_by_test_mapping( 440 self, path='', test_groups=None, 441 file_name=constants.TEST_MAPPING, include_subdirs=False, 442 checked_files=None): 443 """Find tests defined in TEST_MAPPING in the given path. 444 445 Args: 446 path: A string of path in source. Default is set to '', i.e., CWD. 447 test_groups: A List of test groups to run. 448 file_name: Name of TEST_MAPPING file. Default is set to 449 `TEST_MAPPING`. The argument is added for testing purpose. 450 include_subdirs: True to include tests in TEST_MAPPING files in sub 451 directories. 452 checked_files: Paths of TEST_MAPPING files that have been checked. 453 454 Returns: 455 A tuple of (tests, all_tests), where, 456 tests is a set of tests (test_mapping.TestDetail) defined in 457 TEST_MAPPING file of the given path, and its parent directories, 458 with matching test_group. 459 all_tests is a dictionary of all tests in TEST_MAPPING files, 460 grouped by test group. 461 """ 462 path = os.path.realpath(path) 463 # Default test_groups is set to [`presubmit`, `presubmit-large`]. 464 if not test_groups: 465 test_groups = constants.DEFAULT_TEST_GROUPS 466 test_mapping_files = set() 467 all_tests = {} 468 test_mapping_file = os.path.join(path, file_name) 469 if os.path.exists(test_mapping_file): 470 test_mapping_files.add(test_mapping_file) 471 # Include all TEST_MAPPING files in sub-directories if `include_subdirs` 472 # is set to True. 473 if include_subdirs: 474 test_mapping_files.update(atest_utils.find_files(path, file_name)) 475 # Include all possible TEST_MAPPING files in parent directories. 476 root_dir = os.environ.get(constants.ANDROID_BUILD_TOP, os.sep) 477 while path not in (root_dir, os.sep): 478 path = os.path.dirname(path) 479 test_mapping_file = os.path.join(path, file_name) 480 if os.path.exists(test_mapping_file): 481 test_mapping_files.add(test_mapping_file) 482 483 if checked_files is None: 484 checked_files = set() 485 test_mapping_files.difference_update(checked_files) 486 checked_files.update(test_mapping_files) 487 if not test_mapping_files: 488 return test_mapping_files, all_tests 489 490 tests, all_tests, imports = self._get_tests_from_test_mapping_files( 491 test_groups, test_mapping_files) 492 493 # Load TEST_MAPPING files from imports recursively. 494 if imports: 495 for import_detail in imports: 496 path = import_detail.get_path() 497 # (b/110166535 #19) Import path might not exist if a project is 498 # located in different directory in different branches. 499 if path is None: 500 logging.warning( 501 'Failed to import TEST_MAPPING at %s', import_detail) 502 continue 503 # Search for tests based on the imported search path. 504 import_tests, import_all_tests = ( 505 self._find_tests_by_test_mapping( 506 path, test_groups, file_name, include_subdirs, 507 checked_files)) 508 # Merge the collections 509 tests.update(import_tests) 510 for group, grouped_tests in import_all_tests.items(): 511 all_tests.setdefault(group, set()).update(grouped_tests) 512 513 return tests, all_tests 514 515 def _gather_build_targets(self, test_infos): 516 targets = set() 517 for test_info in test_infos: 518 targets |= test_info.build_targets 519 return targets 520 521 def _get_test_mapping_tests(self, args, exit_if_no_test_found=True): 522 """Find the tests in TEST_MAPPING files. 523 524 Args: 525 args: arg parsed object. 526 exit_if_no_test(s)_found: A flag to exit atest if no test mapping 527 tests found. 528 529 Returns: 530 A tuple of (test_names, test_details_list), where 531 test_names: a list of test name 532 test_details_list: a list of test_mapping.TestDetail objects for 533 the tests in TEST_MAPPING files with matching test group. 534 """ 535 # Pull out tests from test mapping 536 src_path = '' 537 test_groups = constants.DEFAULT_TEST_GROUPS 538 if args.tests: 539 if ':' in args.tests[0]: 540 src_path, test_group = args.tests[0].split(':') 541 test_groups = [test_group] 542 else: 543 src_path = args.tests[0] 544 545 test_details, all_test_details = self._find_tests_by_test_mapping( 546 path=src_path, test_groups=test_groups, 547 include_subdirs=args.include_subdirs, checked_files=set()) 548 test_details_list = list(test_details) 549 if not test_details_list and exit_if_no_test_found: 550 logging.warning( 551 'No tests of group `%s` found in TEST_MAPPING at %s or its ' 552 'parent directories.\nYou might be missing atest arguments,' 553 ' try `atest --help` for more information', 554 test_groups, os.path.realpath('')) 555 if all_test_details: 556 tests = '' 557 for test_group, test_list in all_test_details.items(): 558 tests += '%s:\n' % test_group 559 for test_detail in sorted(test_list, key=str): 560 tests += '\t%s\n' % test_detail 561 logging.warning( 562 'All available tests in TEST_MAPPING files are:\n%s', 563 tests) 564 metrics_utils.send_exit_event(constants.EXIT_CODE_TEST_NOT_FOUND) 565 sys.exit(constants.EXIT_CODE_TEST_NOT_FOUND) 566 567 logging.debug( 568 'Test details:\n%s', 569 '\n'.join([str(detail) for detail in test_details_list])) 570 test_names = [detail.name for detail in test_details_list] 571 return test_names, test_details_list 572 573 def _extract_testable_modules_by_wildcard(self, user_input): 574 """Extract the given string with wildcard symbols to testable 575 module names. 576 577 Assume the available testable modules is: 578 ['Google', 'google', 'G00gle', 'g00gle'] 579 and the user_input is: 580 ['*oo*', 'g00gle'] 581 This method will return: 582 ['Google', 'google', 'g00gle'] 583 584 Args: 585 user_input: A list of input. 586 587 Returns: 588 A list of testable modules. 589 """ 590 testable_mods = self.mod_info.get_testable_modules() 591 extracted_tests = [] 592 for test in user_input: 593 if atest_utils.has_wildcard(test): 594 extracted_tests.extend(fnmatch.filter(testable_mods, test)) 595 else: 596 extracted_tests.append(test) 597 return extracted_tests 598 599 def translate(self, args): 600 """Translate atest command line into build targets and run commands. 601 602 Args: 603 args: arg parsed object. 604 605 Returns: 606 A tuple with set of build_target strings and list of TestInfos. 607 """ 608 tests = args.tests 609 # Test details from TEST_MAPPING files 610 test_details_list = None 611 # Loading Host Unit Tests. 612 host_unit_tests = [] 613 if not args.tests: 614 logging.debug('Finding Host Unit Tests...') 615 path = os.path.relpath( 616 os.path.realpath(''), 617 os.environ.get(constants.ANDROID_BUILD_TOP, '')) 618 host_unit_tests = test_finder_utils.find_host_unit_tests( 619 self.mod_info, path) 620 logging.debug('Found host_unit_tests: %s', host_unit_tests) 621 if atest_utils.is_test_mapping(args): 622 if args.enable_file_patterns: 623 self.enable_file_patterns = True 624 tests, test_details_list = self._get_test_mapping_tests( 625 args, not bool(host_unit_tests)) 626 atest_utils.colorful_print("\nFinding Tests...", constants.CYAN) 627 logging.debug('Finding Tests: %s', tests) 628 start = time.time() 629 # Clear cache if user pass -c option 630 if args.clear_cache: 631 atest_utils.clean_test_info_caches(tests + host_unit_tests) 632 # Process tests which might contain wildcard symbols in advance. 633 if atest_utils.has_wildcard(tests): 634 tests = self._extract_testable_modules_by_wildcard(tests) 635 test_infos = self._get_test_infos(tests, test_details_list, 636 args.rebuild_module_info) 637 if host_unit_tests: 638 host_unit_test_details = [test_mapping.TestDetail( 639 {'name':test, 'host':True}) for test in host_unit_tests] 640 host_unit_test_infos = self._get_test_infos(host_unit_tests, 641 host_unit_test_details) 642 test_infos.update(host_unit_test_infos) 643 logging.debug('Found tests in %ss', time.time() - start) 644 for test_info in test_infos: 645 logging.debug('%s\n', test_info) 646 build_targets = self._gather_build_targets(test_infos) 647 return build_targets, test_infos 648