1# Copyright 2018, The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15""" 16Utils for finder classes. 17""" 18 19from __future__ import print_function 20import logging 21import multiprocessing 22import os 23import pickle 24import re 25import subprocess 26import time 27import xml.etree.ElementTree as ET 28 29# pylint: disable=import-error 30import atest_decorator 31import atest_error 32import atest_enum 33import constants 34 35from metrics import metrics_utils 36 37# Helps find apk files listed in a test config (AndroidTest.xml) file. 38# Matches "filename.apk" in <option name="foo", value="filename.apk" /> 39# We want to make sure we don't grab apks with paths in their name since we 40# assume the apk name is the build target. 41_APK_RE = re.compile(r'^[^/]+\.apk$', re.I) 42# RE for checking if TEST or TEST_F is in a cc file or not. 43_CC_CLASS_RE = re.compile(r'^[ ]*TEST(_F|_P)?[ ]*\(', re.I) 44# RE for checking if there exists one of the methods in java file. 45_JAVA_METHODS_PATTERN = r'.*[ ]+({0})\(.*' 46# RE for checking if there exists one of the methods in cc file. 47_CC_METHODS_PATTERN = r'^[ ]*TEST(_F|_P)?[ ]*\(.*,[ ]*({0})\).*' 48# Parse package name from the package declaration line of a java or a kotlin file. 49# Group matches "foo.bar" of line "package foo.bar;" or "package foo.bar" 50_PACKAGE_RE = re.compile(r'\s*package\s+(?P<package>[^(;|\s)]+)\s*', re.I) 51# Matches install paths in module_info to install location(host or device). 52_HOST_PATH_RE = re.compile(r'.*\/host\/.*', re.I) 53_DEVICE_PATH_RE = re.compile(r'.*\/target\/.*', re.I) 54 55# Explanation of FIND_REFERENCE_TYPEs: 56# ---------------------------------- 57# 0. CLASS: Name of a java/kotlin class, usually file is named the same 58# (HostTest lives in HostTest.java or HostTest.kt) 59# 1. QUALIFIED_CLASS: Like CLASS but also contains the package in front like 60# com.android.tradefed.testtype.HostTest. 61# 2. PACKAGE: Name of a java package. 62# 3. INTEGRATION: XML file name in one of the 4 integration config directories. 63# 4. CC_CLASS: Name of a cc class. 64 65FIND_REFERENCE_TYPE = atest_enum.AtestEnum(['CLASS', 'QUALIFIED_CLASS', 66 'PACKAGE', 'INTEGRATION', 'CC_CLASS']) 67# Get cpu count. 68_CPU_COUNT = 0 if os.uname()[0] == 'Linux' else multiprocessing.cpu_count() 69 70# Unix find commands for searching for test files based on test type input. 71# Note: Find (unlike grep) exits with status 0 if nothing found. 72FIND_CMDS = { 73 FIND_REFERENCE_TYPE.CLASS: r"find {0} {1} -type f" 74 r"| egrep '.*/{2}\.(kt|java)$' || true", 75 FIND_REFERENCE_TYPE.QUALIFIED_CLASS: r"find {0} {1} -type f" 76 r"| egrep '.*{2}\.(kt|java)$' || true", 77 FIND_REFERENCE_TYPE.PACKAGE: r"find {0} {1} -wholename " 78 r"'*{2}' -type d -print", 79 FIND_REFERENCE_TYPE.INTEGRATION: r"find {0} {1} -wholename " 80 r"'*{2}.xml' -print", 81 # Searching a test among files where the absolute paths contain *test*. 82 # If users complain atest couldn't find a CC_CLASS, ask them to follow the 83 # convention that the filename or dirname must contain *test*, where *test* 84 # is case-insensitive. 85 FIND_REFERENCE_TYPE.CC_CLASS: r"find {0} {1} -type f -print" 86 r"| egrep -i '/*test.*\.(cc|cpp)$'" 87 r"| xargs -P" + str(_CPU_COUNT) + 88 r" egrep -sH '^[ ]*TEST(_F|_P)?[ ]*\({2}' || true" 89} 90 91# Map ref_type with its index file. 92FIND_INDEXES = { 93 FIND_REFERENCE_TYPE.CLASS: constants.CLASS_INDEX, 94 FIND_REFERENCE_TYPE.QUALIFIED_CLASS: constants.QCLASS_INDEX, 95 FIND_REFERENCE_TYPE.PACKAGE: constants.PACKAGE_INDEX, 96 FIND_REFERENCE_TYPE.INTEGRATION: constants.INT_INDEX, 97 FIND_REFERENCE_TYPE.CC_CLASS: constants.CC_CLASS_INDEX 98} 99 100# XML parsing related constants. 101_COMPATIBILITY_PACKAGE_PREFIX = "com.android.compatibility" 102_CTS_JAR = "cts-tradefed" 103_XML_PUSH_DELIM = '->' 104_APK_SUFFIX = '.apk' 105# Setup script for device perf tests. 106_PERF_SETUP_LABEL = 'perf-setup.sh' 107 108# XML tags. 109_XML_NAME = 'name' 110_XML_VALUE = 'value' 111 112# VTS xml parsing constants. 113_VTS_TEST_MODULE = 'test-module-name' 114_VTS_MODULE = 'module-name' 115_VTS_BINARY_SRC = 'binary-test-source' 116_VTS_PUSH_GROUP = 'push-group' 117_VTS_PUSH = 'push' 118_VTS_BINARY_SRC_DELIM = '::' 119_VTS_PUSH_DIR = os.path.join(os.environ.get(constants.ANDROID_BUILD_TOP, ''), 120 'test', 'vts', 'tools', 'vts-tradefed', 'res', 121 'push_groups') 122_VTS_PUSH_SUFFIX = '.push' 123_VTS_BITNESS = 'append-bitness' 124_VTS_BITNESS_TRUE = 'true' 125_VTS_BITNESS_32 = '32' 126_VTS_BITNESS_64 = '64' 127_VTS_TEST_FILE = 'test-file-name' 128_VTS_APK = 'apk' 129# Matches 'DATA/target' in '_32bit::DATA/target' 130_VTS_BINARY_SRC_DELIM_RE = re.compile(r'.*::(?P<target>.*)$') 131_VTS_OUT_DATA_APP_PATH = 'DATA/app' 132 133# pylint: disable=inconsistent-return-statements 134def split_methods(user_input): 135 """Split user input string into test reference and list of methods. 136 137 Args: 138 user_input: A string of the user's input. 139 Examples: 140 class_name 141 class_name#method1,method2 142 path 143 path#method1,method2 144 Returns: 145 A tuple. First element is String of test ref and second element is 146 a set of method name strings or empty list if no methods included. 147 Exception: 148 atest_error.TooManyMethodsError raised when input string is trying to 149 specify too many methods in a single positional argument. 150 151 Examples of unsupported input strings: 152 module:class#method,class#method 153 class1#method,class2#method 154 path1#method,path2#method 155 """ 156 parts = user_input.split('#') 157 if len(parts) == 1: 158 return parts[0], frozenset() 159 elif len(parts) == 2: 160 return parts[0], frozenset(parts[1].split(',')) 161 raise atest_error.TooManyMethodsError( 162 'Too many methods specified with # character in user input: %s.' 163 '\n\nOnly one class#method combination supported per positional' 164 ' argument. Multiple classes should be separated by spaces: ' 165 'class#method class#method') 166 167 168# pylint: disable=inconsistent-return-statements 169def get_fully_qualified_class_name(test_path): 170 """Parse the fully qualified name from the class java file. 171 172 Args: 173 test_path: A string of absolute path to the java class file. 174 175 Returns: 176 A string of the fully qualified class name. 177 178 Raises: 179 atest_error.MissingPackageName if no class name can be found. 180 """ 181 with open(test_path) as class_file: 182 for line in class_file: 183 match = _PACKAGE_RE.match(line) 184 if match: 185 package = match.group('package') 186 cls = os.path.splitext(os.path.split(test_path)[1])[0] 187 return '%s.%s' % (package, cls) 188 raise atest_error.MissingPackageNameError('%s: Test class java file' 189 'does not contain a package' 190 'name.'% test_path) 191 192 193def has_cc_class(test_path): 194 """Find out if there is any test case in the cc file. 195 196 Args: 197 test_path: A string of absolute path to the cc file. 198 199 Returns: 200 Boolean: has cc class in test_path or not. 201 """ 202 with open(test_path) as class_file: 203 for line in class_file: 204 match = _CC_CLASS_RE.match(line) 205 if match: 206 return True 207 return False 208 209 210def get_package_name(file_name): 211 """Parse the package name from a java file. 212 213 Args: 214 file_name: A string of the absolute path to the java file. 215 216 Returns: 217 A string of the package name or None 218 """ 219 with open(file_name) as data: 220 for line in data: 221 match = _PACKAGE_RE.match(line) 222 if match: 223 return match.group('package') 224 225 226def has_method_in_file(test_path, methods): 227 """Find out if there is at least one method in the file. 228 229 Note: This method doesn't handle if method is in comment sections or not. 230 If the file has any method(even in comment sections), it will return True. 231 232 Args: 233 test_path: A string of absolute path to the test file. 234 methods: A set of method names. 235 236 Returns: 237 Boolean: there is at least one method in test_path. 238 """ 239 if not os.path.isfile(test_path): 240 return False 241 methods_re = None 242 if constants.JAVA_EXT_RE.match(test_path): 243 methods_re = re.compile(_JAVA_METHODS_PATTERN.format( 244 '|'.join([r'%s' % x for x in methods]))) 245 elif constants.CC_EXT_RE.match(test_path): 246 methods_re = re.compile(_CC_METHODS_PATTERN.format( 247 '|'.join([r'%s' % x for x in methods]))) 248 if methods_re: 249 with open(test_path) as test_file: 250 for line in test_file: 251 match = re.match(methods_re, line) 252 if match: 253 return True 254 return False 255 256 257def extract_test_path(output, methods=None): 258 """Extract the test path from the output of a unix 'find' command. 259 260 Example of find output for CLASS find cmd: 261 /<some_root>/cts/tests/jank/src/android/jank/cts/ui/CtsDeviceJankUi.java 262 263 Args: 264 output: A string or list output of a unix 'find' command. 265 methods: A set of method names. 266 267 Returns: 268 A list of the test paths or None if output is '' or None. 269 """ 270 if not output: 271 return None 272 verified_tests = set() 273 if isinstance(output, str): 274 output = output.splitlines() 275 for test in output: 276 # compare CC_OUTPUT_RE with output 277 match_obj = constants.CC_OUTPUT_RE.match(test) 278 if match_obj: 279 # cc/cpp 280 fpath = match_obj.group('file_path') 281 if not methods or match_obj.group('method_name') in methods: 282 verified_tests.add(fpath) 283 else: 284 # TODO (b/138997521) - Atest checks has_method_in_file of a class 285 # without traversing its parent classes. A workaround for this is 286 # do not check has_method_in_file. Uncomment below when a solution 287 # to it is applied. 288 # java/kt 289 #if not methods or has_method_in_file(test, methods): 290 verified_tests.add(test) 291 return extract_test_from_tests(list(verified_tests)) 292 293 294def extract_test_from_tests(tests): 295 """Extract the test path from the tests. 296 297 Return the test to run from tests. If more than one option, prompt the user 298 to select multiple ones. Supporting formats: 299 - An integer. E.g. 0 300 - Comma-separated integers. E.g. 1,3,5 301 - A range of integers denoted by the starting integer separated from 302 the end integer by a dash, '-'. E.g. 1-3 303 304 Args: 305 tests: A string list which contains multiple test paths. 306 307 Returns: 308 A string list of paths. 309 """ 310 count = len(tests) 311 if count <= 1: 312 return tests if count else None 313 mtests = set() 314 try: 315 numbered_list = ['%s: %s' % (i, t) for i, t in enumerate(tests)] 316 numbered_list.append('%s: All' % count) 317 print('Multiple tests found:\n{0}'.format('\n'.join(numbered_list))) 318 test_indices = raw_input("Please enter numbers of test to use. " 319 "If none of above option matched, keep " 320 "searching for other possible tests." 321 "\n(multiple selection is supported," 322 " e.g. '1' or '0,1' or '0-2'): ") 323 for idx in re.sub(r'(\s)', '', test_indices).split(','): 324 indices = idx.split('-') 325 len_indices = len(indices) 326 if len_indices > 0: 327 start_index = min(int(indices[0]), int(indices[len_indices-1])) 328 end_index = max(int(indices[0]), int(indices[len_indices-1])) 329 # One of input is 'All', return all options. 330 if start_index == count or end_index == count: 331 return tests 332 mtests.update(tests[start_index:(end_index+1)]) 333 except (ValueError, IndexError, AttributeError, TypeError) as err: 334 logging.debug('%s', err) 335 print('None of above option matched, keep searching for other' 336 ' possible tests...') 337 return list(mtests) 338 339 340@atest_decorator.static_var("cached_ignore_dirs", []) 341def _get_ignored_dirs(): 342 """Get ignore dirs in find command. 343 344 Since we can't construct a single find cmd to find the target and 345 filter-out the dir with .out-dir, .find-ignore and $OUT-DIR. We have 346 to run the 1st find cmd to find these dirs. Then, we can use these 347 results to generate the real find cmd. 348 349 Return: 350 A list of the ignore dirs. 351 """ 352 out_dirs = _get_ignored_dirs.cached_ignore_dirs 353 if not out_dirs: 354 build_top = os.environ.get(constants.ANDROID_BUILD_TOP) 355 find_out_dir_cmd = (r'find %s -maxdepth 2 ' 356 r'-type f \( -name ".out-dir" -o -name ' 357 r'".find-ignore" \)') % build_top 358 out_files = subprocess.check_output(find_out_dir_cmd, shell=True) 359 # Get all dirs with .out-dir or .find-ignore 360 if out_files: 361 out_files = out_files.splitlines() 362 for out_file in out_files: 363 if out_file: 364 out_dirs.append(os.path.dirname(out_file.strip())) 365 # Get the out folder if user specified $OUT_DIR 366 custom_out_dir = os.environ.get(constants.ANDROID_OUT_DIR) 367 if custom_out_dir: 368 user_out_dir = None 369 if os.path.isabs(custom_out_dir): 370 user_out_dir = custom_out_dir 371 else: 372 user_out_dir = os.path.join(build_top, custom_out_dir) 373 # only ignore the out_dir when it under $ANDROID_BUILD_TOP 374 if build_top in user_out_dir: 375 if user_out_dir not in out_dirs: 376 out_dirs.append(user_out_dir) 377 _get_ignored_dirs.cached_ignore_dirs = out_dirs 378 return out_dirs 379 380 381def _get_prune_cond_of_ignored_dirs(): 382 """Get the prune condition of ignore dirs. 383 384 Generation a string of the prune condition in the find command. 385 It will filter-out the dir with .out-dir, .find-ignore and $OUT-DIR. 386 Because they are the out dirs, we don't have to find them. 387 388 Return: 389 A string of the prune condition of the ignore dirs. 390 """ 391 out_dirs = _get_ignored_dirs() 392 prune_cond = r'-type d \( -name ".*"' 393 for out_dir in out_dirs: 394 prune_cond += r' -o -path %s' % out_dir 395 prune_cond += r' \) -prune -o' 396 return prune_cond 397 398 399def run_find_cmd(ref_type, search_dir, target, methods=None): 400 """Find a path to a target given a search dir and a target name. 401 402 Args: 403 ref_type: An AtestEnum of the reference type. 404 search_dir: A string of the dirpath to search in. 405 target: A string of what you're trying to find. 406 methods: A set of method names. 407 408 Return: 409 A list of the path to the target. 410 If the search_dir is inexistent, None will be returned. 411 """ 412 # If module_info.json is outdated, finding in the search_dir can result in 413 # raising exception. Return null immediately can guild users to run 414 # --rebuild-module-info to resolve the problem. 415 if not os.path.isdir(search_dir): 416 logging.debug('\'%s\' does not exist!', search_dir) 417 return None 418 ref_name = FIND_REFERENCE_TYPE[ref_type] 419 start = time.time() 420 if os.path.isfile(FIND_INDEXES[ref_type]): 421 _dict, out = {}, None 422 with open(FIND_INDEXES[ref_type], 'rb') as index: 423 try: 424 _dict = pickle.load(index) 425 except (IOError, EOFError, pickle.UnpicklingError) as err: 426 logging.debug('Exception raised: %s', err) 427 metrics_utils.handle_exc_and_send_exit_event( 428 constants.ACCESS_CACHE_FAILURE) 429 os.remove(FIND_INDEXES[ref_type]) 430 if _dict.get(target): 431 logging.debug('Found %s in %s', target, FIND_INDEXES[ref_type]) 432 out = [path for path in _dict.get(target) if search_dir in path] 433 else: 434 prune_cond = _get_prune_cond_of_ignored_dirs() 435 if '.' in target: 436 target = target.replace('.', '/') 437 find_cmd = FIND_CMDS[ref_type].format(search_dir, prune_cond, target) 438 logging.debug('Executing %s find cmd: %s', ref_name, find_cmd) 439 out = subprocess.check_output(find_cmd, shell=True) 440 logging.debug('%s find cmd out: %s', ref_name, out) 441 logging.debug('%s find completed in %ss', ref_name, time.time() - start) 442 return extract_test_path(out, methods) 443 444 445def find_class_file(search_dir, class_name, is_native_test=False, methods=None): 446 """Find a path to a class file given a search dir and a class name. 447 448 Args: 449 search_dir: A string of the dirpath to search in. 450 class_name: A string of the class to search for. 451 is_native_test: A boolean variable of whether to search for a native 452 test or not. 453 methods: A set of method names. 454 455 Return: 456 A list of the path to the java/cc file. 457 """ 458 if is_native_test: 459 ref_type = FIND_REFERENCE_TYPE.CC_CLASS 460 elif '.' in class_name: 461 ref_type = FIND_REFERENCE_TYPE.QUALIFIED_CLASS 462 else: 463 ref_type = FIND_REFERENCE_TYPE.CLASS 464 return run_find_cmd(ref_type, search_dir, class_name, methods) 465 466 467def is_equal_or_sub_dir(sub_dir, parent_dir): 468 """Return True sub_dir is sub dir or equal to parent_dir. 469 470 Args: 471 sub_dir: A string of the sub directory path. 472 parent_dir: A string of the parent directory path. 473 474 Returns: 475 A boolean of whether both are dirs and sub_dir is sub of parent_dir 476 or is equal to parent_dir. 477 """ 478 # avoid symlink issues with real path 479 parent_dir = os.path.realpath(parent_dir) 480 sub_dir = os.path.realpath(sub_dir) 481 if not os.path.isdir(sub_dir) or not os.path.isdir(parent_dir): 482 return False 483 return os.path.commonprefix([sub_dir, parent_dir]) == parent_dir 484 485 486def find_parent_module_dir(root_dir, start_dir, module_info): 487 """From current dir search up file tree until root dir for module dir. 488 489 Args: 490 root_dir: A string of the dir that is the parent of the start dir. 491 start_dir: A string of the dir to start searching up from. 492 module_info: ModuleInfo object containing module information from the 493 build system. 494 495 Returns: 496 A string of the module dir relative to root, None if no Module Dir 497 found. There may be multiple testable modules at this level. 498 499 Exceptions: 500 ValueError: Raised if cur_dir not dir or not subdir of root dir. 501 """ 502 if not is_equal_or_sub_dir(start_dir, root_dir): 503 raise ValueError('%s not in repo %s' % (start_dir, root_dir)) 504 auto_gen_dir = None 505 current_dir = start_dir 506 while current_dir != root_dir: 507 # TODO (b/112904944) - migrate module_finder functions to here and 508 # reuse them. 509 rel_dir = os.path.relpath(current_dir, root_dir) 510 # Check if actual config file here 511 if os.path.isfile(os.path.join(current_dir, constants.MODULE_CONFIG)): 512 return rel_dir 513 # Check module_info if auto_gen config or robo (non-config) here 514 for mod in module_info.path_to_module_info.get(rel_dir, []): 515 if module_info.is_robolectric_module(mod): 516 return rel_dir 517 for test_config in mod.get(constants.MODULE_TEST_CONFIG, []): 518 if os.path.isfile(os.path.join(root_dir, test_config)): 519 return rel_dir 520 if mod.get('auto_test_config'): 521 auto_gen_dir = rel_dir 522 # Don't return for auto_gen, keep checking for real config, because 523 # common in cts for class in apk that's in hostside test setup. 524 current_dir = os.path.dirname(current_dir) 525 return auto_gen_dir 526 527 528def get_targets_from_xml(xml_file, module_info): 529 """Retrieve build targets from the given xml. 530 531 Just a helper func on top of get_targets_from_xml_root. 532 533 Args: 534 xml_file: abs path to xml file. 535 module_info: ModuleInfo class used to verify targets are valid modules. 536 537 Returns: 538 A set of build targets based on the signals found in the xml file. 539 """ 540 xml_root = ET.parse(xml_file).getroot() 541 return get_targets_from_xml_root(xml_root, module_info) 542 543 544def _get_apk_target(apk_target): 545 """Return the sanitized apk_target string from the xml. 546 547 The apk_target string can be of 2 forms: 548 - apk_target.apk 549 - apk_target.apk->/path/to/install/apk_target.apk 550 551 We want to return apk_target in both cases. 552 553 Args: 554 apk_target: String of target name to clean. 555 556 Returns: 557 String of apk_target to build. 558 """ 559 apk = apk_target.split(_XML_PUSH_DELIM, 1)[0].strip() 560 return apk[:-len(_APK_SUFFIX)] 561 562 563def _is_apk_target(name, value): 564 """Return True if XML option is an apk target. 565 566 We have some scenarios where an XML option can be an apk target: 567 - value is an apk file. 568 - name is a 'push' option where value holds the apk_file + other stuff. 569 570 Args: 571 name: String name of XML option. 572 value: String value of the XML option. 573 574 Returns: 575 True if it's an apk target we should build, False otherwise. 576 """ 577 if _APK_RE.match(value): 578 return True 579 if name == 'push' and value.endswith(_APK_SUFFIX): 580 return True 581 return False 582 583 584def get_targets_from_xml_root(xml_root, module_info): 585 """Retrieve build targets from the given xml root. 586 587 We're going to pull the following bits of info: 588 - Parse any .apk files listed in the config file. 589 - Parse option value for "test-module-name" (for vts10 tests). 590 - Look for the perf script. 591 592 Args: 593 module_info: ModuleInfo class used to verify targets are valid modules. 594 xml_root: ElementTree xml_root for us to look through. 595 596 Returns: 597 A set of build targets based on the signals found in the xml file. 598 """ 599 targets = set() 600 option_tags = xml_root.findall('.//option') 601 for tag in option_tags: 602 target_to_add = None 603 name = tag.attrib[_XML_NAME].strip() 604 value = tag.attrib[_XML_VALUE].strip() 605 if _is_apk_target(name, value): 606 target_to_add = _get_apk_target(value) 607 elif _PERF_SETUP_LABEL in value: 608 targets.add(_PERF_SETUP_LABEL) 609 continue 610 611 # Let's make sure we can actually build the target. 612 if target_to_add and module_info.is_module(target_to_add): 613 targets.add(target_to_add) 614 elif target_to_add: 615 logging.warning('Build target (%s) not present in module info, ' 616 'skipping build', target_to_add) 617 618 # TODO (b/70813166): Remove this lookup once all runtime dependencies 619 # can be listed as a build dependencies or are in the base test harness. 620 nodes_with_class = xml_root.findall(".//*[@class]") 621 for class_attr in nodes_with_class: 622 fqcn = class_attr.attrib['class'].strip() 623 if fqcn.startswith(_COMPATIBILITY_PACKAGE_PREFIX): 624 targets.add(_CTS_JAR) 625 logging.debug('Targets found in config file: %s', targets) 626 return targets 627 628 629def _get_vts_push_group_targets(push_file, rel_out_dir): 630 """Retrieve vts10 push group build targets. 631 632 A push group file is a file that list out test dependencies and other push 633 group files. Go through the push file and gather all the test deps we need. 634 635 Args: 636 push_file: Name of the push file in the VTS 637 rel_out_dir: Abs path to the out dir to help create vts10 build targets. 638 639 Returns: 640 Set of string which represent build targets. 641 """ 642 targets = set() 643 full_push_file_path = os.path.join(_VTS_PUSH_DIR, push_file) 644 # pylint: disable=invalid-name 645 with open(full_push_file_path) as f: 646 for line in f: 647 target = line.strip() 648 # Skip empty lines. 649 if not target: 650 continue 651 652 # This is a push file, get the targets from it. 653 if target.endswith(_VTS_PUSH_SUFFIX): 654 targets |= _get_vts_push_group_targets(line.strip(), 655 rel_out_dir) 656 continue 657 sanitized_target = target.split(_XML_PUSH_DELIM, 1)[0].strip() 658 targets.add(os.path.join(rel_out_dir, sanitized_target)) 659 return targets 660 661 662def _specified_bitness(xml_root): 663 """Check if the xml file contains the option append-bitness. 664 665 Args: 666 xml_root: abs path to xml file. 667 668 Returns: 669 True if xml specifies to append-bitness, False otherwise. 670 """ 671 option_tags = xml_root.findall('.//option') 672 for tag in option_tags: 673 value = tag.attrib[_XML_VALUE].strip() 674 name = tag.attrib[_XML_NAME].strip() 675 if name == _VTS_BITNESS and value == _VTS_BITNESS_TRUE: 676 return True 677 return False 678 679 680def _get_vts_binary_src_target(value, rel_out_dir): 681 """Parse out the vts10 binary src target. 682 683 The value can be in the following pattern: 684 - {_32bit,_64bit,_IPC32_32bit}::DATA/target (DATA/target) 685 - DATA/target->/data/target (DATA/target) 686 - out/host/linx-x86/bin/VtsSecuritySelinuxPolicyHostTest (the string as 687 is) 688 689 Args: 690 value: String of the XML option value to parse. 691 rel_out_dir: String path of out dir to prepend to target when required. 692 693 Returns: 694 String of the target to build. 695 """ 696 # We'll assume right off the bat we can use the value as is and modify it if 697 # necessary, e.g. out/host/linux-x86/bin... 698 target = value 699 # _32bit::DATA/target 700 match = _VTS_BINARY_SRC_DELIM_RE.match(value) 701 if match: 702 target = os.path.join(rel_out_dir, match.group('target')) 703 # DATA/target->/data/target 704 elif _XML_PUSH_DELIM in value: 705 target = value.split(_XML_PUSH_DELIM, 1)[0].strip() 706 target = os.path.join(rel_out_dir, target) 707 return target 708 709 710def get_plans_from_vts_xml(xml_file): 711 """Get configs which are included by xml_file. 712 713 We're looking for option(include) to get all dependency plan configs. 714 715 Args: 716 xml_file: Absolute path to xml file. 717 718 Returns: 719 A set of plan config paths which are depended by xml_file. 720 """ 721 if not os.path.exists(xml_file): 722 raise atest_error.XmlNotExistError('%s: The xml file does' 723 'not exist' % xml_file) 724 plans = set() 725 xml_root = ET.parse(xml_file).getroot() 726 plans.add(xml_file) 727 option_tags = xml_root.findall('.//include') 728 if not option_tags: 729 return plans 730 # Currently, all vts10 xmls live in the same dir : 731 # https://android.googlesource.com/platform/test/vts/+/master/tools/vts-tradefed/res/config/ 732 # If the vts10 plans start using folders to organize the plans, the logic here 733 # should be changed. 734 xml_dir = os.path.dirname(xml_file) 735 for tag in option_tags: 736 name = tag.attrib[_XML_NAME].strip() 737 plans |= get_plans_from_vts_xml(os.path.join(xml_dir, name + ".xml")) 738 return plans 739 740 741def get_targets_from_vts_xml(xml_file, rel_out_dir, module_info): 742 """Parse a vts10 xml for test dependencies we need to build. 743 744 We have a separate vts10 parsing function because we make a big assumption 745 on the targets (the way they're formatted and what they represent) and we 746 also create these build targets in a very special manner as well. 747 The 6 options we're looking for are: 748 - binary-test-source 749 - push-group 750 - push 751 - test-module-name 752 - test-file-name 753 - apk 754 755 Args: 756 module_info: ModuleInfo class used to verify targets are valid modules. 757 rel_out_dir: Abs path to the out dir to help create vts10 build targets. 758 xml_file: abs path to xml file. 759 760 Returns: 761 A set of build targets based on the signals found in the xml file. 762 """ 763 xml_root = ET.parse(xml_file).getroot() 764 targets = set() 765 option_tags = xml_root.findall('.//option') 766 for tag in option_tags: 767 value = tag.attrib[_XML_VALUE].strip() 768 name = tag.attrib[_XML_NAME].strip() 769 if name in [_VTS_TEST_MODULE, _VTS_MODULE]: 770 if module_info.is_module(value): 771 targets.add(value) 772 else: 773 logging.warning('vts10 test module (%s) not present in module ' 774 'info, skipping build', value) 775 elif name == _VTS_BINARY_SRC: 776 targets.add(_get_vts_binary_src_target(value, rel_out_dir)) 777 elif name == _VTS_PUSH_GROUP: 778 # Look up the push file and parse out build artifacts (as well as 779 # other push group files to parse). 780 targets |= _get_vts_push_group_targets(value, rel_out_dir) 781 elif name == _VTS_PUSH: 782 # Parse out the build artifact directly. 783 push_target = value.split(_XML_PUSH_DELIM, 1)[0].strip() 784 # If the config specified append-bitness, append the bits suffixes 785 # to the target. 786 if _specified_bitness(xml_root): 787 targets.add(os.path.join(rel_out_dir, push_target + _VTS_BITNESS_32)) 788 targets.add(os.path.join(rel_out_dir, push_target + _VTS_BITNESS_64)) 789 else: 790 targets.add(os.path.join(rel_out_dir, push_target)) 791 elif name == _VTS_TEST_FILE: 792 # The _VTS_TEST_FILE values can be set in 2 possible ways: 793 # 1. test_file.apk 794 # 2. DATA/app/test_file/test_file.apk 795 # We'll assume that test_file.apk (#1) is in an expected path (but 796 # that is not true, see b/76158619) and create the full path for it 797 # and then append the _VTS_TEST_FILE value to targets to build. 798 target = os.path.join(rel_out_dir, value) 799 # If value is just an APK, specify the path that we expect it to be in 800 # e.g. out/host/linux-x86/vts10/android-vts10/testcases/DATA/app/test_file/test_file.apk 801 head, _ = os.path.split(value) 802 if not head: 803 target = os.path.join(rel_out_dir, _VTS_OUT_DATA_APP_PATH, 804 _get_apk_target(value), value) 805 targets.add(target) 806 elif name == _VTS_APK: 807 targets.add(os.path.join(rel_out_dir, value)) 808 logging.debug('Targets found in config file: %s', targets) 809 return targets 810 811 812def get_dir_path_and_filename(path): 813 """Return tuple of dir and file name from given path. 814 815 Args: 816 path: String of path to break up. 817 818 Returns: 819 Tuple of (dir, file) paths. 820 """ 821 if os.path.isfile(path): 822 dir_path, file_path = os.path.split(path) 823 else: 824 dir_path, file_path = path, None 825 return dir_path, file_path 826 827 828def get_cc_filter(class_name, methods): 829 """Get the cc filter. 830 831 Args: 832 class_name: class name of the cc test. 833 methods: a list of method names. 834 835 Returns: 836 A formatted string for cc filter. 837 Ex: "class1.method1:class1.method2" or "class1.*" 838 """ 839 if methods: 840 return ":".join(["%s.%s" % (class_name, x) for x in methods]) 841 return "%s.*" % class_name 842 843 844def search_integration_dirs(name, int_dirs): 845 """Search integration dirs for name and return full path. 846 847 Args: 848 name: A string of plan name needed to be found. 849 int_dirs: A list of path needed to be searched. 850 851 Returns: 852 A list of the test path. 853 Ask user to select if multiple tests are found. 854 None if no matched test found. 855 """ 856 root_dir = os.environ.get(constants.ANDROID_BUILD_TOP) 857 test_files = [] 858 for integration_dir in int_dirs: 859 abs_path = os.path.join(root_dir, integration_dir) 860 test_paths = run_find_cmd(FIND_REFERENCE_TYPE.INTEGRATION, abs_path, 861 name) 862 if test_paths: 863 test_files.extend(test_paths) 864 return extract_test_from_tests(test_files) 865 866 867def get_int_dir_from_path(path, int_dirs): 868 """Search integration dirs for the given path and return path of dir. 869 870 Args: 871 path: A string of path needed to be found. 872 int_dirs: A list of path needed to be searched. 873 874 Returns: 875 A string of the test dir. None if no matched path found. 876 """ 877 root_dir = os.environ.get(constants.ANDROID_BUILD_TOP) 878 if not os.path.exists(path): 879 return None 880 dir_path, file_name = get_dir_path_and_filename(path) 881 int_dir = None 882 for possible_dir in int_dirs: 883 abs_int_dir = os.path.join(root_dir, possible_dir) 884 if is_equal_or_sub_dir(dir_path, abs_int_dir): 885 int_dir = abs_int_dir 886 break 887 if not file_name: 888 logging.warn('Found dir (%s) matching input (%s).' 889 ' Referencing an entire Integration/Suite dir' 890 ' is not supported. If you are trying to reference' 891 ' a test by its path, please input the path to' 892 ' the integration/suite config file itself.', 893 int_dir, path) 894 return None 895 return int_dir 896 897 898def get_install_locations(installed_paths): 899 """Get install locations from installed paths. 900 901 Args: 902 installed_paths: List of installed_paths from module_info. 903 904 Returns: 905 Set of install locations from module_info installed_paths. e.g. 906 set(['host', 'device']) 907 """ 908 install_locations = set() 909 for path in installed_paths: 910 if _HOST_PATH_RE.match(path): 911 install_locations.add(constants.DEVICELESS_TEST) 912 elif _DEVICE_PATH_RE.match(path): 913 install_locations.add(constants.DEVICE_TEST) 914 return install_locations 915 916 917def get_levenshtein_distance(test_name, module_name, dir_costs=constants.COST_TYPO): 918 """Return an edit distance between test_name and module_name. 919 920 Levenshtein Distance has 3 actions: delete, insert and replace. 921 dis_costs makes each action weigh differently. 922 923 Args: 924 test_name: A keyword from the users. 925 module_name: A testable module name. 926 dir_costs: A tuple which contains 3 integer, where dir represents 927 Deletion, Insertion and Replacement respectively. 928 For guessing typos: (1, 1, 1) gives the best result. 929 For searching keywords, (8, 1, 5) gives the best result. 930 931 Returns: 932 An edit distance integer between test_name and module_name. 933 """ 934 rows = len(test_name) + 1 935 cols = len(module_name) + 1 936 deletion, insertion, replacement = dir_costs 937 938 # Creating a Dynamic Programming Matrix and weighting accordingly. 939 dp_matrix = [[0 for _ in range(cols)] for _ in range(rows)] 940 # Weigh rows/deletion 941 for row in range(1, rows): 942 dp_matrix[row][0] = row * deletion 943 # Weigh cols/insertion 944 for col in range(1, cols): 945 dp_matrix[0][col] = col * insertion 946 # The core logic of LD 947 for col in range(1, cols): 948 for row in range(1, rows): 949 if test_name[row-1] == module_name[col-1]: 950 cost = 0 951 else: 952 cost = replacement 953 dp_matrix[row][col] = min(dp_matrix[row-1][col] + deletion, 954 dp_matrix[row][col-1] + insertion, 955 dp_matrix[row-1][col-1] + cost) 956 957 return dp_matrix[row][col] 958 959 960def is_test_from_kernel_xml(xml_file, test_name): 961 """Check if test defined in xml_file. 962 963 A kernel test can be defined like: 964 <option name="test-command-line" key="test_class_1" value="command 1" /> 965 where key is the name of test class and method of the runner. This method 966 returns True if the test_name was defined in the given xml_file. 967 968 Args: 969 xml_file: Absolute path to xml file. 970 test_name: test_name want to find. 971 972 Returns: 973 True if test_name in xml_file, False otherwise. 974 """ 975 if not os.path.exists(xml_file): 976 raise atest_error.XmlNotExistError('%s: The xml file does' 977 'not exist' % xml_file) 978 xml_root = ET.parse(xml_file).getroot() 979 option_tags = xml_root.findall('.//option') 980 for option_tag in option_tags: 981 if option_tag.attrib['name'] == 'test-command-line': 982 if option_tag.attrib['key'] == test_name: 983 return True 984 return False 985