1# Copyright 2017, The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15#pylint: disable=too-many-lines 16""" 17Command Line Translator for atest. 18""" 19 20from __future__ import print_function 21 22import fnmatch 23import json 24import logging 25import os 26import sys 27import time 28 29import atest_error 30import atest_utils 31import constants 32import test_finder_handler 33import test_mapping 34 35from metrics import metrics 36from metrics import metrics_utils 37from test_finders import module_finder 38 39TEST_MAPPING = 'TEST_MAPPING' 40FUZZY_FINDER = 'FUZZY' 41 42 43#pylint: disable=no-self-use 44class CLITranslator(object): 45 """ 46 CLITranslator class contains public method translate() and some private 47 helper methods. The atest tool can call the translate() method with a list 48 of strings, each string referencing a test to run. Translate() will 49 "translate" this list of test strings into a list of build targets and a 50 list of TradeFederation run commands. 51 52 Translation steps for a test string reference: 53 1. Narrow down the type of reference the test string could be, i.e. 54 whether it could be referencing a Module, Class, Package, etc. 55 2. Try to find the test files assuming the test string is one of these 56 types of reference. 57 3. If test files found, generate Build Targets and the Run Command. 58 """ 59 60 def __init__(self, module_info=None): 61 """CLITranslator constructor 62 63 Args: 64 module_info: ModuleInfo class that has cached module-info.json. 65 """ 66 self.mod_info = module_info 67 68 def _find_test_infos(self, test, tm_test_detail): 69 """Return set of TestInfos based on a given test. 70 71 Args: 72 test: A string representing test references. 73 tm_test_detail: The TestDetail of test configured in TEST_MAPPING 74 files. 75 76 Returns: 77 Set of TestInfos based on the given test. 78 """ 79 test_infos = set() 80 test_find_starts = time.time() 81 test_found = False 82 test_finders = [] 83 test_info_str = '' 84 find_test_err_msg = None 85 for finder in test_finder_handler.get_find_methods_for_test( 86 self.mod_info, test): 87 # For tests in TEST_MAPPING, find method is only related to 88 # test name, so the details can be set after test_info object 89 # is created. 90 try: 91 test_info = finder.find_method(finder.test_finder_instance, 92 test) 93 except atest_error.TestDiscoveryException as e: 94 find_test_err_msg = e 95 if test_info: 96 if tm_test_detail: 97 test_info.data[constants.TI_MODULE_ARG] = ( 98 tm_test_detail.options) 99 test_info.from_test_mapping = True 100 test_info.host = tm_test_detail.host 101 test_infos.add(test_info) 102 test_found = True 103 finder_info = finder.finder_info 104 print("Found '%s' as %s" % ( 105 atest_utils.colorize(test, constants.GREEN), 106 finder_info)) 107 test_finders.append(finder_info) 108 test_info_str = str(test_info) 109 break 110 if not test_found: 111 f_results = self._fuzzy_search_and_msg(test, find_test_err_msg) 112 if f_results: 113 test_infos.add(f_results) 114 test_found = True 115 test_finders.append(FUZZY_FINDER) 116 metrics.FindTestFinishEvent( 117 duration=metrics_utils.convert_duration( 118 time.time() - test_find_starts), 119 success=test_found, 120 test_reference=test, 121 test_finders=test_finders, 122 test_info=test_info_str) 123 return test_infos 124 125 def _fuzzy_search_and_msg(self, test, find_test_err_msg): 126 """ Fuzzy search and print message. 127 128 Args: 129 test: A string representing test references 130 find_test_err_msg: A string of find test error message. 131 132 Returns: 133 A TestInfos if found, otherwise None. 134 """ 135 print('No test found for: %s' % 136 atest_utils.colorize(test, constants.RED)) 137 # Currently we focus on guessing module names. Append names on 138 # results if more finders support fuzzy searching. 139 mod_finder = module_finder.ModuleFinder(self.mod_info) 140 results = mod_finder.get_fuzzy_searching_results(test) 141 if len(results) == 1 and self._confirm_running(results): 142 test_info = mod_finder.find_test_by_module_name(results[0]) 143 if test_info: 144 return test_info 145 elif len(results) > 1: 146 self._print_fuzzy_searching_results(results) 147 else: 148 print('No matching result for {0}.'.format(test)) 149 if find_test_err_msg: 150 print('%s\n' % (atest_utils.colorize( 151 find_test_err_msg, constants.MAGENTA))) 152 else: 153 print('(This can happen after a repo sync or if the test' 154 ' is new. Running: with "%s" may resolve the issue.)' 155 '\n' % (atest_utils.colorize( 156 constants.REBUILD_MODULE_INFO_FLAG, 157 constants.RED))) 158 return None 159 160 def _get_test_infos(self, tests, test_mapping_test_details=None): 161 """Return set of TestInfos based on passed in tests. 162 163 Args: 164 tests: List of strings representing test references. 165 test_mapping_test_details: List of TestDetail for tests configured 166 in TEST_MAPPING files. 167 168 Returns: 169 Set of TestInfos based on the passed in tests. 170 """ 171 test_infos = set() 172 if not test_mapping_test_details: 173 test_mapping_test_details = [None] * len(tests) 174 for test, tm_test_detail in zip(tests, test_mapping_test_details): 175 found_test_infos = self._find_test_infos(test, tm_test_detail) 176 test_infos.update(found_test_infos) 177 return test_infos 178 179 def _confirm_running(self, results): 180 """Listen to an answer from raw input. 181 182 Args: 183 results: A list of results. 184 185 Returns: 186 True is the answer is affirmative. 187 """ 188 decision = raw_input('Did you mean {0}? [Y/n] '.format( 189 atest_utils.colorize(results[0], constants.GREEN))) 190 return decision in constants.AFFIRMATIVES 191 192 def _print_fuzzy_searching_results(self, results): 193 """Print modules when fuzzy searching gives multiple results. 194 195 If the result is lengthy, just print the first 10 items only since we 196 have already given enough-accurate result. 197 198 Args: 199 results: A list of guessed testable module names. 200 201 """ 202 atest_utils.colorful_print('Did you mean the following modules?', 203 constants.WHITE) 204 for mod in results[:10]: 205 atest_utils.colorful_print(mod, constants.GREEN) 206 207 def _read_tests_in_test_mapping(self, test_mapping_file): 208 """Read tests from a TEST_MAPPING file. 209 210 Args: 211 test_mapping_file: Path to a TEST_MAPPING file. 212 213 Returns: 214 A tuple of (all_tests, imports), where 215 all_tests is a dictionary of all tests in the TEST_MAPPING file, 216 grouped by test group. 217 imports is a list of test_mapping.Import to include other test 218 mapping files. 219 """ 220 all_tests = {} 221 imports = [] 222 test_mapping_dict = None 223 with open(test_mapping_file) as json_file: 224 test_mapping_dict = json.load(json_file) 225 for test_group_name, test_list in test_mapping_dict.items(): 226 if test_group_name == constants.TEST_MAPPING_IMPORTS: 227 for import_detail in test_list: 228 imports.append( 229 test_mapping.Import(test_mapping_file, import_detail)) 230 else: 231 grouped_tests = all_tests.setdefault(test_group_name, set()) 232 tests = [] 233 for test in test_list: 234 test_mod_info = self.mod_info.name_to_module_info.get( 235 test['name']) 236 if not test_mod_info: 237 print('WARNING: %s is not a valid build target and ' 238 'may not be discoverable by TreeHugger. If you ' 239 'want to specify a class or test-package, ' 240 'please set \'name\' to the test module and use ' 241 '\'options\' to specify the right tests via ' 242 '\'include-filter\'.\nNote: this can also occur ' 243 'if the test module is not built for your ' 244 'current lunch target.\n' % 245 atest_utils.colorize(test['name'], constants.RED)) 246 elif not any(x in test_mod_info['compatibility_suites'] for 247 x in constants.TEST_MAPPING_SUITES): 248 print('WARNING: Please add %s to either suite: %s for ' 249 'this TEST_MAPPING file to work with TreeHugger.' % 250 (atest_utils.colorize(test['name'], 251 constants.RED), 252 atest_utils.colorize(constants.TEST_MAPPING_SUITES, 253 constants.GREEN))) 254 tests.append(test_mapping.TestDetail(test)) 255 grouped_tests.update(tests) 256 return all_tests, imports 257 258 def _find_files(self, path, file_name=TEST_MAPPING): 259 """Find all files with given name under the given path. 260 261 Args: 262 path: A string of path in source. 263 264 Returns: 265 A list of paths of the files with the matching name under the given 266 path. 267 """ 268 test_mapping_files = [] 269 for root, _, filenames in os.walk(path): 270 for filename in fnmatch.filter(filenames, file_name): 271 test_mapping_files.append(os.path.join(root, filename)) 272 return test_mapping_files 273 274 def _get_tests_from_test_mapping_files( 275 self, test_group, test_mapping_files): 276 """Get tests in the given test mapping files with the match group. 277 278 Args: 279 test_group: Group of tests to run. Default is set to `presubmit`. 280 test_mapping_files: A list of path of TEST_MAPPING files. 281 282 Returns: 283 A tuple of (tests, all_tests, imports), where, 284 tests is a set of tests (test_mapping.TestDetail) defined in 285 TEST_MAPPING file of the given path, and its parent directories, 286 with matching test_group. 287 all_tests is a dictionary of all tests in TEST_MAPPING files, 288 grouped by test group. 289 imports is a list of test_mapping.Import objects that contains the 290 details of where to import a TEST_MAPPING file. 291 """ 292 all_imports = [] 293 # Read and merge the tests in all TEST_MAPPING files. 294 merged_all_tests = {} 295 for test_mapping_file in test_mapping_files: 296 all_tests, imports = self._read_tests_in_test_mapping( 297 test_mapping_file) 298 all_imports.extend(imports) 299 for test_group_name, test_list in all_tests.items(): 300 grouped_tests = merged_all_tests.setdefault( 301 test_group_name, set()) 302 grouped_tests.update(test_list) 303 304 tests = set(merged_all_tests.get(test_group, [])) 305 # Postsubmit tests shall include all presubmit tests as well. 306 if test_group == constants.TEST_GROUP_POSTSUBMIT: 307 tests.update(merged_all_tests.get( 308 constants.TEST_GROUP_PRESUBMIT, set())) 309 elif test_group == constants.TEST_GROUP_ALL: 310 for grouped_tests in merged_all_tests.values(): 311 tests.update(grouped_tests) 312 return tests, merged_all_tests, all_imports 313 314 # pylint: disable=too-many-arguments 315 # pylint: disable=too-many-locals 316 def _find_tests_by_test_mapping( 317 self, path='', test_group=constants.TEST_GROUP_PRESUBMIT, 318 file_name=TEST_MAPPING, include_subdirs=False, checked_files=None): 319 """Find tests defined in TEST_MAPPING in the given path. 320 321 Args: 322 path: A string of path in source. Default is set to '', i.e., CWD. 323 test_group: Group of tests to run. Default is set to `presubmit`. 324 file_name: Name of TEST_MAPPING file. Default is set to 325 `TEST_MAPPING`. The argument is added for testing purpose. 326 include_subdirs: True to include tests in TEST_MAPPING files in sub 327 directories. 328 checked_files: Paths of TEST_MAPPING files that have been checked. 329 330 Returns: 331 A tuple of (tests, all_tests), where, 332 tests is a set of tests (test_mapping.TestDetail) defined in 333 TEST_MAPPING file of the given path, and its parent directories, 334 with matching test_group. 335 all_tests is a dictionary of all tests in TEST_MAPPING files, 336 grouped by test group. 337 """ 338 path = os.path.realpath(path) 339 test_mapping_files = set() 340 all_tests = {} 341 test_mapping_file = os.path.join(path, file_name) 342 if os.path.exists(test_mapping_file): 343 test_mapping_files.add(test_mapping_file) 344 # Include all TEST_MAPPING files in sub-directories if `include_subdirs` 345 # is set to True. 346 if include_subdirs: 347 test_mapping_files.update(self._find_files(path, file_name)) 348 # Include all possible TEST_MAPPING files in parent directories. 349 root_dir = os.environ.get(constants.ANDROID_BUILD_TOP, os.sep) 350 while path != root_dir and path != os.sep: 351 path = os.path.dirname(path) 352 test_mapping_file = os.path.join(path, file_name) 353 if os.path.exists(test_mapping_file): 354 test_mapping_files.add(test_mapping_file) 355 356 if checked_files is None: 357 checked_files = set() 358 test_mapping_files.difference_update(checked_files) 359 checked_files.update(test_mapping_files) 360 if not test_mapping_files: 361 return test_mapping_files, all_tests 362 363 tests, all_tests, imports = self._get_tests_from_test_mapping_files( 364 test_group, test_mapping_files) 365 366 # Load TEST_MAPPING files from imports recursively. 367 if imports: 368 for import_detail in imports: 369 path = import_detail.get_path() 370 # (b/110166535 #19) Import path might not exist if a project is 371 # located in different directory in different branches. 372 if path is None: 373 logging.warn( 374 'Failed to import TEST_MAPPING at %s', import_detail) 375 continue 376 # Search for tests based on the imported search path. 377 import_tests, import_all_tests = ( 378 self._find_tests_by_test_mapping( 379 path, test_group, file_name, include_subdirs, 380 checked_files)) 381 # Merge the collections 382 tests.update(import_tests) 383 for group, grouped_tests in import_all_tests.items(): 384 all_tests.setdefault(group, set()).update(grouped_tests) 385 386 return tests, all_tests 387 388 def _gather_build_targets(self, test_infos): 389 targets = set() 390 for test_info in test_infos: 391 targets |= test_info.build_targets 392 return targets 393 394 def _get_test_mapping_tests(self, args): 395 """Find the tests in TEST_MAPPING files. 396 397 Args: 398 args: arg parsed object. 399 400 Returns: 401 A tuple of (test_names, test_details_list), where 402 test_names: a list of test name 403 test_details_list: a list of test_mapping.TestDetail objects for 404 the tests in TEST_MAPPING files with matching test group. 405 """ 406 # Pull out tests from test mapping 407 src_path = '' 408 test_group = constants.TEST_GROUP_PRESUBMIT 409 if args.tests: 410 if ':' in args.tests[0]: 411 src_path, test_group = args.tests[0].split(':') 412 else: 413 src_path = args.tests[0] 414 415 test_details, all_test_details = self._find_tests_by_test_mapping( 416 path=src_path, test_group=test_group, 417 include_subdirs=args.include_subdirs, checked_files=set()) 418 test_details_list = list(test_details) 419 if not test_details_list: 420 logging.warn( 421 'No tests of group `%s` found in TEST_MAPPING at %s or its ' 422 'parent directories.\nYou might be missing atest arguments,' 423 ' try `atest --help` for more information', 424 test_group, os.path.realpath('')) 425 if all_test_details: 426 tests = '' 427 for test_group, test_list in all_test_details.items(): 428 tests += '%s:\n' % test_group 429 for test_detail in sorted(test_list): 430 tests += '\t%s\n' % test_detail 431 logging.warn( 432 'All available tests in TEST_MAPPING files are:\n%s', 433 tests) 434 metrics_utils.send_exit_event(constants.EXIT_CODE_TEST_NOT_FOUND) 435 sys.exit(constants.EXIT_CODE_TEST_NOT_FOUND) 436 437 logging.debug( 438 'Test details:\n%s', 439 '\n'.join([str(detail) for detail in test_details_list])) 440 test_names = [detail.name for detail in test_details_list] 441 return test_names, test_details_list 442 443 444 def translate(self, args): 445 """Translate atest command line into build targets and run commands. 446 447 Args: 448 args: arg parsed object. 449 450 Returns: 451 A tuple with set of build_target strings and list of TestInfos. 452 """ 453 tests = args.tests 454 # Test details from TEST_MAPPING files 455 test_details_list = None 456 if atest_utils.is_test_mapping(args): 457 tests, test_details_list = self._get_test_mapping_tests(args) 458 atest_utils.colorful_print("\nFinding Tests...", constants.CYAN) 459 logging.debug('Finding Tests: %s', tests) 460 start = time.time() 461 test_infos = self._get_test_infos(tests, test_details_list) 462 logging.debug('Found tests in %ss', time.time() - start) 463 for test_info in test_infos: 464 logging.debug('%s\n', test_info) 465 build_targets = self._gather_build_targets(test_infos) 466 return build_targets, test_infos 467