1# Copyright 2017, The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15"""Command Line Translator for atest.""" 16 17# pylint: disable=line-too-long 18# pylint: disable=too-many-lines 19 20from __future__ import print_function 21 22import fnmatch 23import json 24import logging 25import os 26import re 27import sys 28import time 29 30from typing import List 31 32import atest_error 33import atest_utils 34import bazel_mode 35import constants 36import test_finder_handler 37import test_mapping 38 39from atest_enum import DetectType, ExitCode 40from metrics import metrics 41from metrics import metrics_utils 42from test_finders import module_finder 43from test_finders import test_finder_utils 44 45FUZZY_FINDER = 'FUZZY' 46CACHE_FINDER = 'CACHE' 47TESTNAME_CHARS = {'#', ':', '/'} 48 49# Pattern used to identify comments start with '//' or '#' in TEST_MAPPING. 50_COMMENTS_RE = re.compile(r'(?m)[\s\t]*(#|//).*|(\".*?\")') 51_COMMENTS = frozenset(['//', '#']) 52 53 54#pylint: disable=no-self-use 55class CLITranslator: 56 """ 57 CLITranslator class contains public method translate() and some private 58 helper methods. The atest tool can call the translate() method with a list 59 of strings, each string referencing a test to run. Translate() will 60 "translate" this list of test strings into a list of build targets and a 61 list of TradeFederation run commands. 62 63 Translation steps for a test string reference: 64 1. Narrow down the type of reference the test string could be, i.e. 65 whether it could be referencing a Module, Class, Package, etc. 66 2. Try to find the test files assuming the test string is one of these 67 types of reference. 68 3. If test files found, generate Build Targets and the Run Command. 69 """ 70 71 def __init__(self, mod_info=None, print_cache_msg=True, 72 bazel_mode_enabled=False, host=False, 73 bazel_mode_features: List[bazel_mode.Features]=None): 74 """CLITranslator constructor 75 76 Args: 77 mod_info: ModuleInfo class that has cached module-info.json. 78 print_cache_msg: Boolean whether printing clear cache message or not. 79 True will print message while False won't print. 80 bazel_mode_enabled: Boolean of args.bazel_mode. 81 host: Boolean of args.host. 82 bazel_mode_features: List of args.bazel_mode_features. 83 """ 84 self.mod_info = mod_info 85 self._bazel_mode = bazel_mode_enabled 86 self._bazel_mode_features = bazel_mode_features or [] 87 self._host = host 88 self.enable_file_patterns = False 89 self.msg = '' 90 if print_cache_msg: 91 self.msg = ('(Test info has been cached for speeding up the next ' 92 'run, if test info need to be updated, please add -c ' 93 'to clean the old cache.)') 94 95 # pylint: disable=too-many-locals 96 # pylint: disable=too-many-branches 97 # pylint: disable=too-many-statements 98 def _find_test_infos(self, test, tm_test_detail): 99 """Return set of TestInfos based on a given test. 100 101 Args: 102 test: A string representing test references. 103 tm_test_detail: The TestDetail of test configured in TEST_MAPPING 104 files. 105 106 Returns: 107 Set of TestInfos based on the given test. 108 """ 109 test_infos = set() 110 test_find_starts = time.time() 111 test_found = False 112 test_finders = [] 113 test_info_str = '' 114 find_test_err_msg = None 115 test_name, mainline_modules = atest_utils.parse_mainline_modules(test) 116 if not self._verified_mainline_modules(test_name, mainline_modules): 117 return test_infos 118 find_methods = test_finder_handler.get_find_methods_for_test( 119 self.mod_info, test) 120 if self._bazel_mode: 121 find_methods = [bazel_mode.create_new_finder( 122 self.mod_info, 123 f, 124 host=self._host, 125 enabled_features=self._bazel_mode_features 126 ) for f in find_methods] 127 for finder in find_methods: 128 # For tests in TEST_MAPPING, find method is only related to 129 # test name, so the details can be set after test_info object 130 # is created. 131 try: 132 found_test_infos = finder.find_method( 133 finder.test_finder_instance, test_name) 134 except atest_error.TestDiscoveryException as e: 135 find_test_err_msg = e 136 if found_test_infos: 137 finder_info = finder.finder_info 138 for test_info in found_test_infos: 139 test_deps = set() 140 if self.mod_info: 141 test_deps = self.mod_info.get_install_module_dependency( 142 test_info.test_name) 143 logging.debug('(%s) Test dependencies: %s', 144 test_info.test_name, test_deps) 145 if tm_test_detail: 146 test_info.data[constants.TI_MODULE_ARG] = ( 147 tm_test_detail.options) 148 test_info.from_test_mapping = True 149 test_info.host = tm_test_detail.host 150 if finder_info != CACHE_FINDER: 151 test_info.test_finder = finder_info 152 if mainline_modules: 153 test_info.test_name = test 154 # TODO: remove below statement when soong can also 155 # parse TestConfig and inject mainline modules information 156 # to module-info. 157 test_info.mainline_modules = mainline_modules 158 # Only add dependencies to build_targets when they are in 159 # module info 160 test_deps_in_mod_info = [ 161 test_dep for test_dep in test_deps 162 if self.mod_info.is_module(test_dep)] 163 test_info.build_targets = set(test_info.build_targets) 164 test_info.build_targets.update(test_deps_in_mod_info) 165 test_infos.add(test_info) 166 test_found = True 167 print("Found '%s' as %s" % ( 168 atest_utils.colorize(test, constants.GREEN), 169 finder_info)) 170 if finder_info == CACHE_FINDER and test_infos: 171 test_finders.append(list(test_infos)[0].test_finder) 172 test_finders.append(finder_info) 173 test_info_str = ','.join([str(x) for x in found_test_infos]) 174 break 175 if not test_found: 176 f_results = self._fuzzy_search_and_msg(test, find_test_err_msg) 177 if f_results: 178 test_infos.update(f_results) 179 test_found = True 180 test_finders.append(FUZZY_FINDER) 181 metrics.FindTestFinishEvent( 182 duration=metrics_utils.convert_duration( 183 time.time() - test_find_starts), 184 success=test_found, 185 test_reference=test, 186 test_finders=test_finders, 187 test_info=test_info_str) 188 # Cache test_infos by default except running with TEST_MAPPING which may 189 # include customized flags and they are likely to mess up other 190 # non-test_mapping tests. 191 if test_infos and not tm_test_detail: 192 atest_utils.update_test_info_cache(test, test_infos) 193 if self.msg: 194 print(self.msg) 195 return test_infos 196 197 def _verified_mainline_modules(self, test, mainline_modules): 198 """ Verify the test with mainline modules is acceptable. 199 200 The test must be a module and mainline modules are in module-info. 201 The syntax rule of mainline modules will check in build process. 202 The rule includes mainline modules are sorted alphabetically, no space, 203 and no duplication. 204 205 Args: 206 test: A string representing test references 207 mainline_modules: A string of mainline_modules. 208 209 Returns: 210 True if this test is acceptable. Otherwise, print the reason and 211 return False. 212 """ 213 if not mainline_modules: 214 return True 215 if not self.mod_info.is_module(test): 216 print('Error: "%s" is not a testable module.' 217 % atest_utils.colorize(test, constants.RED)) 218 return False 219 if not self.mod_info.has_mainline_modules(test, mainline_modules): 220 print('Error: Mainline modules "%s" were not defined for %s in ' 221 'neither build file nor test config.' 222 % (atest_utils.colorize(mainline_modules, constants.RED), 223 atest_utils.colorize(test, constants.RED))) 224 return False 225 return True 226 227 def _fuzzy_search_and_msg(self, test, find_test_err_msg): 228 """ Fuzzy search and print message. 229 230 Args: 231 test: A string representing test references 232 find_test_err_msg: A string of find test error message. 233 234 Returns: 235 A list of TestInfos if found, otherwise None. 236 """ 237 print('No test found for: %s' % 238 atest_utils.colorize(test, constants.RED)) 239 # Currently we focus on guessing module names. Append names on 240 # results if more finders support fuzzy searching. 241 if atest_utils.has_chars(test, TESTNAME_CHARS): 242 return None 243 mod_finder = module_finder.ModuleFinder(self.mod_info) 244 results = mod_finder.get_fuzzy_searching_results(test) 245 if len(results) == 1 and self._confirm_running(results): 246 found_test_infos = mod_finder.find_test_by_module_name(results[0]) 247 # found_test_infos is a list with at most 1 element. 248 if found_test_infos: 249 return found_test_infos 250 elif len(results) > 1: 251 self._print_fuzzy_searching_results(results) 252 else: 253 print('No matching result for {0}.'.format(test)) 254 if find_test_err_msg: 255 print('%s\n' % (atest_utils.colorize( 256 find_test_err_msg, constants.MAGENTA))) 257 return None 258 259 def _get_test_infos(self, tests, test_mapping_test_details=None): 260 """Return set of TestInfos based on passed in tests. 261 262 Args: 263 tests: List of strings representing test references. 264 test_mapping_test_details: List of TestDetail for tests configured 265 in TEST_MAPPING files. 266 267 Returns: 268 Set of TestInfos based on the passed in tests. 269 """ 270 test_infos = set() 271 if not test_mapping_test_details: 272 test_mapping_test_details = [None] * len(tests) 273 for test, tm_test_detail in zip(tests, test_mapping_test_details): 274 found_test_infos = self._find_test_infos(test, tm_test_detail) 275 test_infos.update(found_test_infos) 276 return test_infos 277 278 def _confirm_running(self, results): 279 """Listen to an answer from raw input. 280 281 Args: 282 results: A list of results. 283 284 Returns: 285 True is the answer is affirmative. 286 """ 287 return atest_utils.prompt_with_yn_result( 288 'Did you mean {0}?'.format( 289 atest_utils.colorize(results[0], constants.GREEN)), True) 290 291 def _print_fuzzy_searching_results(self, results): 292 """Print modules when fuzzy searching gives multiple results. 293 294 If the result is lengthy, just print the first 10 items only since we 295 have already given enough-accurate result. 296 297 Args: 298 results: A list of guessed testable module names. 299 300 """ 301 atest_utils.colorful_print('Did you mean the following modules?', 302 constants.WHITE) 303 for mod in results[:10]: 304 atest_utils.colorful_print(mod, constants.GREEN) 305 306 def filter_comments(self, test_mapping_file): 307 """Remove comments in TEST_MAPPING file to valid format. Only '//' and 308 '#' are regarded as comments. 309 310 Args: 311 test_mapping_file: Path to a TEST_MAPPING file. 312 313 Returns: 314 Valid json string without comments. 315 """ 316 def _replace(match): 317 """Replace comments if found matching the defined regular 318 expression. 319 320 Args: 321 match: The matched regex pattern 322 323 Returns: 324 "" if it matches _COMMENTS, otherwise original string. 325 """ 326 line = match.group(0).strip() 327 return "" if any(map(line.startswith, _COMMENTS)) else line 328 with open(test_mapping_file) as json_file: 329 return re.sub(_COMMENTS_RE, _replace, json_file.read()) 330 331 def _read_tests_in_test_mapping(self, test_mapping_file): 332 """Read tests from a TEST_MAPPING file. 333 334 Args: 335 test_mapping_file: Path to a TEST_MAPPING file. 336 337 Returns: 338 A tuple of (all_tests, imports), where 339 all_tests is a dictionary of all tests in the TEST_MAPPING file, 340 grouped by test group. 341 imports is a list of test_mapping.Import to include other test 342 mapping files. 343 """ 344 all_tests = {} 345 imports = [] 346 test_mapping_dict = json.loads(self.filter_comments(test_mapping_file)) 347 for test_group_name, test_list in test_mapping_dict.items(): 348 if test_group_name == constants.TEST_MAPPING_IMPORTS: 349 for import_detail in test_list: 350 imports.append( 351 test_mapping.Import(test_mapping_file, import_detail)) 352 else: 353 grouped_tests = all_tests.setdefault(test_group_name, set()) 354 tests = [] 355 for test in test_list: 356 if (self.enable_file_patterns and 357 not test_mapping.is_match_file_patterns( 358 test_mapping_file, test)): 359 continue 360 test_mod_info = self.mod_info.name_to_module_info.get( 361 atest_utils.parse_mainline_modules(test['name'])[0]) 362 if not test_mod_info : 363 print('WARNING: %s is not a valid build target and ' 364 'may not be discoverable by TreeHugger. If you ' 365 'want to specify a class or test-package, ' 366 'please set \'name\' to the test module and use ' 367 '\'options\' to specify the right tests via ' 368 '\'include-filter\'.\nNote: this can also occur ' 369 'if the test module is not built for your ' 370 'current lunch target.\n' % 371 atest_utils.colorize(test['name'], constants.RED)) 372 elif not any(x in test_mod_info['compatibility_suites'] for 373 x in constants.TEST_MAPPING_SUITES): 374 print('WARNING: Please add %s to either suite: %s for ' 375 'this TEST_MAPPING file to work with TreeHugger.' % 376 (atest_utils.colorize(test['name'], 377 constants.RED), 378 atest_utils.colorize(constants.TEST_MAPPING_SUITES, 379 constants.GREEN))) 380 tests.append(test_mapping.TestDetail(test)) 381 grouped_tests.update(tests) 382 return all_tests, imports 383 384 def _get_tests_from_test_mapping_files( 385 self, test_groups, test_mapping_files): 386 """Get tests in the given test mapping files with the match group. 387 388 Args: 389 test_groups: Groups of tests to run. Default is set to `presubmit` 390 and `presubmit-large`. 391 test_mapping_files: A list of path of TEST_MAPPING files. 392 393 Returns: 394 A tuple of (tests, all_tests, imports), where, 395 tests is a set of tests (test_mapping.TestDetail) defined in 396 TEST_MAPPING file of the given path, and its parent directories, 397 with matching test_group. 398 all_tests is a dictionary of all tests in TEST_MAPPING files, 399 grouped by test group. 400 imports is a list of test_mapping.Import objects that contains the 401 details of where to import a TEST_MAPPING file. 402 """ 403 all_imports = [] 404 # Read and merge the tests in all TEST_MAPPING files. 405 merged_all_tests = {} 406 for test_mapping_file in test_mapping_files: 407 all_tests, imports = self._read_tests_in_test_mapping( 408 test_mapping_file) 409 all_imports.extend(imports) 410 for test_group_name, test_list in all_tests.items(): 411 grouped_tests = merged_all_tests.setdefault( 412 test_group_name, set()) 413 grouped_tests.update(test_list) 414 tests = set() 415 for test_group in test_groups: 416 temp_tests = set(merged_all_tests.get(test_group, [])) 417 tests.update(temp_tests) 418 if test_group == constants.TEST_GROUP_ALL: 419 for grouped_tests in merged_all_tests.values(): 420 tests.update(grouped_tests) 421 return tests, merged_all_tests, all_imports 422 423 # pylint: disable=too-many-arguments 424 # pylint: disable=too-many-locals 425 def _find_tests_by_test_mapping( 426 self, path='', test_groups=None, 427 file_name=constants.TEST_MAPPING, include_subdirs=False, 428 checked_files=None): 429 """Find tests defined in TEST_MAPPING in the given path. 430 431 Args: 432 path: A string of path in source. Default is set to '', i.e., CWD. 433 test_groups: A List of test groups to run. 434 file_name: Name of TEST_MAPPING file. Default is set to 435 `TEST_MAPPING`. The argument is added for testing purpose. 436 include_subdirs: True to include tests in TEST_MAPPING files in sub 437 directories. 438 checked_files: Paths of TEST_MAPPING files that have been checked. 439 440 Returns: 441 A tuple of (tests, all_tests), where, 442 tests is a set of tests (test_mapping.TestDetail) defined in 443 TEST_MAPPING file of the given path, and its parent directories, 444 with matching test_group. 445 all_tests is a dictionary of all tests in TEST_MAPPING files, 446 grouped by test group. 447 """ 448 path = os.path.realpath(path) 449 # Default test_groups is set to [`presubmit`, `presubmit-large`]. 450 if not test_groups: 451 test_groups = constants.DEFAULT_TEST_GROUPS 452 test_mapping_files = set() 453 all_tests = {} 454 test_mapping_file = os.path.join(path, file_name) 455 if os.path.exists(test_mapping_file): 456 test_mapping_files.add(test_mapping_file) 457 # Include all TEST_MAPPING files in sub-directories if `include_subdirs` 458 # is set to True. 459 if include_subdirs: 460 test_mapping_files.update(atest_utils.find_files(path, file_name)) 461 # Include all possible TEST_MAPPING files in parent directories. 462 root_dir = os.environ.get(constants.ANDROID_BUILD_TOP, os.sep) 463 while path not in (root_dir, os.sep): 464 path = os.path.dirname(path) 465 test_mapping_file = os.path.join(path, file_name) 466 if os.path.exists(test_mapping_file): 467 test_mapping_files.add(test_mapping_file) 468 469 if checked_files is None: 470 checked_files = set() 471 test_mapping_files.difference_update(checked_files) 472 checked_files.update(test_mapping_files) 473 if not test_mapping_files: 474 return test_mapping_files, all_tests 475 476 tests, all_tests, imports = self._get_tests_from_test_mapping_files( 477 test_groups, test_mapping_files) 478 479 # Load TEST_MAPPING files from imports recursively. 480 if imports: 481 for import_detail in imports: 482 path = import_detail.get_path() 483 # (b/110166535 #19) Import path might not exist if a project is 484 # located in different directory in different branches. 485 if path is None: 486 logging.warning( 487 'Failed to import TEST_MAPPING at %s', import_detail) 488 continue 489 # Search for tests based on the imported search path. 490 import_tests, import_all_tests = ( 491 self._find_tests_by_test_mapping( 492 path, test_groups, file_name, include_subdirs, 493 checked_files)) 494 # Merge the collections 495 tests.update(import_tests) 496 for group, grouped_tests in import_all_tests.items(): 497 all_tests.setdefault(group, set()).update(grouped_tests) 498 499 return tests, all_tests 500 501 def _gather_build_targets(self, test_infos): 502 targets = set() 503 for test_info in test_infos: 504 targets |= test_info.build_targets 505 return targets 506 507 def _get_test_mapping_tests(self, args, exit_if_no_test_found=True): 508 """Find the tests in TEST_MAPPING files. 509 510 Args: 511 args: arg parsed object. 512 exit_if_no_test(s)_found: A flag to exit atest if no test mapping 513 tests found. 514 515 Returns: 516 A tuple of (test_names, test_details_list), where 517 test_names: a list of test name 518 test_details_list: a list of test_mapping.TestDetail objects for 519 the tests in TEST_MAPPING files with matching test group. 520 """ 521 # Pull out tests from test mapping 522 src_path = '' 523 test_groups = constants.DEFAULT_TEST_GROUPS 524 if args.tests: 525 if ':' in args.tests[0]: 526 src_path, test_group = args.tests[0].split(':') 527 test_groups = [test_group] 528 else: 529 src_path = args.tests[0] 530 531 test_details, all_test_details = self._find_tests_by_test_mapping( 532 path=src_path, test_groups=test_groups, 533 include_subdirs=args.include_subdirs, checked_files=set()) 534 test_details_list = list(test_details) 535 if not test_details_list and exit_if_no_test_found: 536 logging.warning( 537 'No tests of group `%s` found in %s or its ' 538 'parent directories. (Available groups: %s)\n' 539 'You might be missing atest arguments,' 540 ' try `atest --help` for more information.', 541 test_groups, 542 os.path.join(src_path, constants.TEST_MAPPING), 543 ', '.join(all_test_details.keys())) 544 if all_test_details: 545 tests = '' 546 for test_group, test_list in all_test_details.items(): 547 tests += '%s:\n' % test_group 548 for test_detail in sorted(test_list, key=str): 549 tests += '\t%s\n' % test_detail 550 logging.warning( 551 'All available tests in TEST_MAPPING files are:\n%s', 552 tests) 553 metrics_utils.send_exit_event(ExitCode.TEST_NOT_FOUND) 554 sys.exit(ExitCode.TEST_NOT_FOUND) 555 556 logging.debug( 557 'Test details:\n%s', 558 '\n'.join([str(detail) for detail in test_details_list])) 559 test_names = [detail.name for detail in test_details_list] 560 return test_names, test_details_list 561 562 def _extract_testable_modules_by_wildcard(self, user_input): 563 """Extract the given string with wildcard symbols to testable 564 module names. 565 566 Assume the available testable modules is: 567 ['Google', 'google', 'G00gle', 'g00gle'] 568 and the user_input is: 569 ['*oo*', 'g00gle'] 570 This method will return: 571 ['Google', 'google', 'g00gle'] 572 573 Args: 574 user_input: A list of input. 575 576 Returns: 577 A list of testable modules. 578 """ 579 testable_mods = self.mod_info.get_testable_modules() 580 extracted_tests = [] 581 for test in user_input: 582 if atest_utils.has_wildcard(test): 583 extracted_tests.extend(fnmatch.filter(testable_mods, test)) 584 else: 585 extracted_tests.append(test) 586 return extracted_tests 587 588 def _has_host_unit_test(self, tests): 589 """Tell whether one of the given testis a host unit test. 590 591 Args: 592 tests: A list of test names. 593 594 Returns: 595 True when one of the given testis a host unit test. 596 """ 597 all_host_unit_tests = self.mod_info.get_all_host_unit_tests() 598 for test in tests: 599 if test in all_host_unit_tests: 600 return True 601 return False 602 603 def translate(self, args): 604 """Translate atest command line into build targets and run commands. 605 606 Args: 607 args: arg parsed object. 608 609 Returns: 610 A tuple with set of build_target strings and list of TestInfos. 611 """ 612 tests = args.tests 613 # Test details from TEST_MAPPING files 614 test_details_list = None 615 # Loading Host Unit Tests. 616 host_unit_tests = [] 617 detect_type = DetectType.TEST_WITH_ARGS 618 if not args.tests or atest_utils.is_test_mapping(args): 619 detect_type = DetectType.TEST_NULL_ARGS 620 start = time.time() 621 if not args.tests: 622 logging.debug('Finding Host Unit Tests...') 623 path = os.path.relpath( 624 os.path.realpath(''), 625 os.environ.get(constants.ANDROID_BUILD_TOP, '')) 626 host_unit_tests = test_finder_utils.find_host_unit_tests( 627 self.mod_info, path) 628 logging.debug('Found host_unit_tests: %s', host_unit_tests) 629 if atest_utils.is_test_mapping(args): 630 if args.enable_file_patterns: 631 self.enable_file_patterns = True 632 tests, test_details_list = self._get_test_mapping_tests( 633 args, not bool(host_unit_tests)) 634 atest_utils.colorful_print("\nFinding Tests...", constants.CYAN) 635 logging.debug('Finding Tests: %s', tests) 636 # Clear cache if user pass -c option 637 if args.clear_cache: 638 atest_utils.clean_test_info_caches(tests + host_unit_tests) 639 # Process tests which might contain wildcard symbols in advance. 640 if atest_utils.has_wildcard(tests): 641 tests = self._extract_testable_modules_by_wildcard(tests) 642 test_infos = self._get_test_infos(tests, test_details_list) 643 if host_unit_tests: 644 host_unit_test_details = [test_mapping.TestDetail( 645 {'name':test, 'host':True}) for test in host_unit_tests] 646 host_unit_test_infos = self._get_test_infos(host_unit_tests, 647 host_unit_test_details) 648 test_infos.update(host_unit_test_infos) 649 if atest_utils.has_mixed_type_filters(test_infos): 650 atest_utils.colorful_print( 651 'Mixed type filters found. ' 652 'Please separate tests into different runs.', 653 constants.YELLOW) 654 sys.exit(ExitCode.MIXED_TYPE_FILTER) 655 finished_time = time.time() - start 656 logging.debug('Finding tests finished in %ss', finished_time) 657 metrics.LocalDetectEvent( 658 detect_type=detect_type, 659 result=int(finished_time)) 660 for test_info in test_infos: 661 logging.debug('%s\n', test_info) 662 build_targets = self._gather_build_targets(test_infos) 663 if not self._bazel_mode: 664 if host_unit_tests or self._has_host_unit_test(tests): 665 msg = (r"It is recommended to run host unit tests with " 666 r"--bazel-mode.") 667 atest_utils.colorful_print(msg, constants.YELLOW) 668 return build_targets, test_infos 669