1# Copyright 2018, The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15""" 16Utils for finder classes. 17""" 18 19# pylint: disable=line-too-long 20# pylint: disable=too-many-lines 21 22from __future__ import print_function 23 24import logging 25import multiprocessing 26import os 27import pickle 28import re 29import subprocess 30import time 31import xml.etree.ElementTree as ET 32 33import atest_decorator 34import atest_error 35import atest_enum 36import atest_utils 37import constants 38 39from metrics import metrics_utils 40 41# Helps find apk files listed in a test config (AndroidTest.xml) file. 42# Matches "filename.apk" in <option name="foo", value="filename.apk" /> 43# We want to make sure we don't grab apks with paths in their name since we 44# assume the apk name is the build target. 45_APK_RE = re.compile(r'^[^/]+\.apk$', re.I) 46# Group matches "class" of line "TEST_F(class, " 47_CC_CLASS_METHOD_RE = re.compile( 48 r'^\s*TEST(_F|_P)?\s*\(\s*(?P<class>\w+)\s*,\s*(?P<method>\w+)\s*\)', re.M) 49# Group matches parameterized "class" of line "INSTANTIATE_TEST_CASE_P( ,class " 50_PARA_CC_CLASS_RE = re.compile( 51 r'^\s*INSTANTIATE[_TYPED]*_TEST_(SUITE|CASE)_P\s*\(\s*(?P<instantiate>\w+)\s*,' 52 r'\s*(?P<class>\w+)\s*\,', re.M) 53# Group that matches java/kt method. 54_JAVA_METHODS_RE = r'.*\s+(fun|void)\s+(?P<methods>\w+)\(\)' 55# Parse package name from the package declaration line of a java or 56# a kotlin file. 57# Group matches "foo.bar" of line "package foo.bar;" or "package foo.bar" 58_PACKAGE_RE = re.compile(r'\s*package\s+(?P<package>[^(;|\s)]+)\s*', re.I) 59# Matches install paths in module_info to install location(host or device). 60_HOST_PATH_RE = re.compile(r'.*\/host\/.*', re.I) 61_DEVICE_PATH_RE = re.compile(r'.*\/target\/.*', re.I) 62# RE for checking if parameterized java class. 63_PARAMET_JAVA_CLASS_RE = re.compile( 64 r'^\s*@RunWith\s*\(\s*(Parameterized|TestParameterInjector|' 65 r'JUnitParamsRunner|DataProviderRunner|JukitoRunner|Theories|BedsteadJUnit4' 66 r').class\s*\)', re.I) 67_PARENT_CLS_RE = re.compile(r'.*class\s+\w+\s+extends\s+(?P<parent>[\w\.]+.*)\s\{') 68 69# Explanation of FIND_REFERENCE_TYPEs: 70# ---------------------------------- 71# 0. CLASS: Name of a java/kotlin class, usually file is named the same 72# (HostTest lives in HostTest.java or HostTest.kt) 73# 1. QUALIFIED_CLASS: Like CLASS but also contains the package in front like 74# com.android.tradefed.testtype.HostTest. 75# 2. PACKAGE: Name of a java package. 76# 3. INTEGRATION: XML file name in one of the 4 integration config directories. 77# 4. CC_CLASS: Name of a cc class. 78 79FIND_REFERENCE_TYPE = atest_enum.AtestEnum(['CLASS', 80 'QUALIFIED_CLASS', 81 'PACKAGE', 82 'INTEGRATION', 83 'CC_CLASS']) 84# Get cpu count. 85_CPU_COUNT = 0 if os.uname()[0] == 'Linux' else multiprocessing.cpu_count() 86 87# Unix find commands for searching for test files based on test type input. 88# Note: Find (unlike grep) exits with status 0 if nothing found. 89FIND_CMDS = { 90 FIND_REFERENCE_TYPE.CLASS: r"find {0} {1} -type f" 91 r"| egrep '.*/{2}\.(kt|java)$' || true", 92 FIND_REFERENCE_TYPE.QUALIFIED_CLASS: r"find {0} {1} -type f" 93 r"| egrep '.*{2}\.(kt|java)$' || true", 94 FIND_REFERENCE_TYPE.PACKAGE: r"find {0} {1} -wholename " 95 r"'*{2}' -type d -print", 96 FIND_REFERENCE_TYPE.INTEGRATION: r"find {0} {1} -wholename " 97 r"'*{2}.xml' -print", 98 # Searching a test among files where the absolute paths contain *test*. 99 # If users complain atest couldn't find a CC_CLASS, ask them to follow the 100 # convention that the filename or dirname must contain *test*, where *test* 101 # is case-insensitive. 102 FIND_REFERENCE_TYPE.CC_CLASS: r"find {0} {1} -type f -print" 103 r"| egrep -i '/*test.*\.(cc|cpp)$'" 104 r"| xargs -P" + str(_CPU_COUNT) + 105 r" egrep -sH '^[ ]*TEST(_F|_P)?[ ]*\({2}' " 106 " || true" 107} 108 109# Map ref_type with its index file. 110FIND_INDEXES = { 111 FIND_REFERENCE_TYPE.CLASS: constants.CLASS_INDEX, 112 FIND_REFERENCE_TYPE.QUALIFIED_CLASS: constants.QCLASS_INDEX, 113 FIND_REFERENCE_TYPE.PACKAGE: constants.PACKAGE_INDEX, 114 FIND_REFERENCE_TYPE.INTEGRATION: constants.INT_INDEX, 115 FIND_REFERENCE_TYPE.CC_CLASS: constants.CC_CLASS_INDEX 116} 117 118# XML parsing related constants. 119_COMPATIBILITY_PACKAGE_PREFIX = "com.android.compatibility" 120_CTS_JAR = "cts-tradefed" 121_XML_PUSH_DELIM = '->' 122_APK_SUFFIX = '.apk' 123DALVIK_TEST_RUNNER_CLASS = 'com.android.compatibility.testtype.DalvikTest' 124LIBCORE_TEST_RUNNER_CLASS = 'com.android.compatibility.testtype.LibcoreTest' 125DALVIK_TESTRUNNER_JAR_CLASSES = [DALVIK_TEST_RUNNER_CLASS, 126 LIBCORE_TEST_RUNNER_CLASS] 127DALVIK_DEVICE_RUNNER_JAR = 'cts-dalvik-device-test-runner' 128DALVIK_HOST_RUNNER_JAR = 'cts-dalvik-host-test-runner' 129DALVIK_TEST_DEPS = {DALVIK_DEVICE_RUNNER_JAR, 130 DALVIK_HOST_RUNNER_JAR, 131 _CTS_JAR} 132# Setup script for device perf tests. 133_PERF_SETUP_LABEL = 'perf-setup.sh' 134_PERF_SETUP_TARGET = 'perf-setup' 135 136# XML tags. 137_XML_NAME = 'name' 138_XML_VALUE = 'value' 139 140# VTS xml parsing constants. 141_VTS_TEST_MODULE = 'test-module-name' 142_VTS_MODULE = 'module-name' 143_VTS_BINARY_SRC = 'binary-test-source' 144_VTS_PUSH_GROUP = 'push-group' 145_VTS_PUSH = 'push' 146_VTS_BINARY_SRC_DELIM = '::' 147_VTS_PUSH_DIR = os.path.join(os.environ.get(constants.ANDROID_BUILD_TOP, ''), 148 'test', 'vts', 'tools', 'vts-tradefed', 'res', 149 'push_groups') 150_VTS_PUSH_SUFFIX = '.push' 151_VTS_BITNESS = 'append-bitness' 152_VTS_BITNESS_TRUE = 'true' 153_VTS_BITNESS_32 = '32' 154_VTS_BITNESS_64 = '64' 155_VTS_TEST_FILE = 'test-file-name' 156_VTS_APK = 'apk' 157# Matches 'DATA/target' in '_32bit::DATA/target' 158_VTS_BINARY_SRC_DELIM_RE = re.compile(r'.*::(?P<target>.*)$') 159_VTS_OUT_DATA_APP_PATH = 'DATA/app' 160 161# pylint: disable=inconsistent-return-statements 162def split_methods(user_input): 163 """Split user input string into test reference and list of methods. 164 165 Args: 166 user_input: A string of the user's input. 167 Examples: 168 class_name 169 class_name#method1,method2 170 path 171 path#method1,method2 172 Returns: 173 A tuple. First element is String of test ref and second element is 174 a set of method name strings or empty list if no methods included. 175 Exception: 176 atest_error.TooManyMethodsError raised when input string is trying to 177 specify too many methods in a single positional argument. 178 179 Examples of unsupported input strings: 180 module:class#method,class#method 181 class1#method,class2#method 182 path1#method,path2#method 183 """ 184 parts = user_input.split('#') 185 if len(parts) == 1: 186 return parts[0], frozenset() 187 if len(parts) == 2: 188 return parts[0], frozenset(parts[1].split(',')) 189 raise atest_error.TooManyMethodsError( 190 'Too many methods specified with # character in user input: %s.' 191 '\n\nOnly one class#method combination supported per positional' 192 ' argument. Multiple classes should be separated by spaces: ' 193 'class#method class#method') 194 195 196# pylint: disable=inconsistent-return-statements 197def get_fully_qualified_class_name(test_path): 198 """Parse the fully qualified name from the class java file. 199 200 Args: 201 test_path: A string of absolute path to the java class file. 202 203 Returns: 204 A string of the fully qualified class name. 205 206 Raises: 207 atest_error.MissingPackageName if no class name can be found. 208 """ 209 with open(test_path) as class_file: 210 for line in class_file: 211 match = _PACKAGE_RE.match(line) 212 if match: 213 package = match.group('package') 214 cls = os.path.splitext(os.path.split(test_path)[1])[0] 215 return '%s.%s' % (package, cls) 216 raise atest_error.MissingPackageNameError('%s: Test class java file' 217 'does not contain a package' 218 'name.'% test_path) 219 220 221def has_cc_class(test_path): 222 """Find out if there is any test case in the cc file. 223 224 Args: 225 test_path: A string of absolute path to the cc file. 226 227 Returns: 228 Boolean: has cc class in test_path or not. 229 """ 230 with open(test_path) as class_file: 231 for line in class_file: 232 match = _CC_CLASS_METHOD_RE.match(line) 233 if match: 234 return True 235 return False 236 237 238def get_package_name(file_name): 239 """Parse the package name from a java file. 240 241 Args: 242 file_name: A string of the absolute path to the java file. 243 244 Returns: 245 A string of the package name or None 246 """ 247 with open(file_name) as data: 248 for line in data: 249 match = _PACKAGE_RE.match(line) 250 if match: 251 return match.group('package') 252 253 254def get_parent_cls_name(file_name): 255 """Parse the parent class name from a java file. 256 257 Args: 258 file_name: A string of the absolute path to the java file. 259 260 Returns: 261 A string of the parent class name or None 262 """ 263 with open(file_name) as data: 264 for line in data: 265 match = _PARENT_CLS_RE.match(line) 266 if match: 267 return match.group('parent') 268 269# pylint: disable=too-many-branches 270def has_method_in_file(test_path, methods): 271 """Find out if every method can be found in the file. 272 273 Note: This method doesn't handle if method is in comment sections. 274 275 Args: 276 test_path: A string of absolute path to the test file. 277 methods: A set of method names. 278 279 Returns: 280 Boolean: there is at least one method in test_path. 281 """ 282 if not os.path.isfile(test_path): 283 return False 284 if constants.JAVA_EXT_RE.match(test_path): 285 # omit parameterized pattern: method[0] 286 _methods = set(re.sub(r'\[\S+\]', '', x) for x in methods) 287 if _methods.issubset(get_java_methods(test_path)): 288 return True 289 parent = get_parent_cls_name(test_path) 290 package = get_package_name(test_path) 291 if parent and package: 292 # Remove <Generics> when needed. 293 parent_cls = re.sub(r'\<\w+\>', '', parent) 294 # Use Full Qualified Class Name for searching precisely. 295 # package org.gnome; 296 # public class Foo extends com.android.Boo -> com.android.Boo 297 # public class Foo extends Boo -> org.gnome.Boo 298 if '.' in parent_cls: 299 parent_fqcn = parent_cls 300 else: 301 parent_fqcn = package + '.' + parent_cls 302 try: 303 logging.debug('Searching methods in %s', parent_fqcn) 304 return has_method_in_file( 305 run_find_cmd(FIND_REFERENCE_TYPE.QUALIFIED_CLASS, 306 os.environ.get(constants.ANDROID_BUILD_TOP), 307 parent_fqcn, 308 methods)[0], methods) 309 except TypeError: 310 logging.debug('Out of searching range: no test found.') 311 return False 312 if constants.CC_EXT_RE.match(test_path): 313 # omit parameterized pattern: method/argument 314 _methods = set(re.sub(r'\/.*', '', x) for x in methods) 315 _, cc_methods, _ = get_cc_test_classes_methods(test_path) 316 if _methods.issubset(cc_methods): 317 return True 318 return False 319 320 321def extract_test_path(output, methods=None): 322 """Extract the test path from the output of a unix 'find' command. 323 324 Example of find output for CLASS find cmd: 325 /<some_root>/cts/tests/jank/src/android/jank/cts/ui/CtsDeviceJankUi.java 326 327 Args: 328 output: A string or list output of a unix 'find' command. 329 methods: A set of method names. 330 331 Returns: 332 A list of the test paths or None if output is '' or None. 333 """ 334 if not output: 335 return None 336 verified_tests = set() 337 if isinstance(output, str): 338 output = output.splitlines() 339 for test in output: 340 match_obj = constants.CC_OUTPUT_RE.match(test) 341 # Legacy "find" cc output (with TEST_P() syntax): 342 if match_obj: 343 fpath = match_obj.group('file_path') 344 if not methods or match_obj.group('method_name') in methods: 345 verified_tests.add(fpath) 346 # "locate" output path for both java/cc. 347 elif not methods or has_method_in_file(test, methods): 348 verified_tests.add(test) 349 return extract_test_from_tests(sorted(list(verified_tests))) 350 351 352def extract_test_from_tests(tests, default_all=False): 353 """Extract the test path from the tests. 354 355 Return the test to run from tests. If more than one option, prompt the user 356 to select multiple ones. Supporting formats: 357 - An integer. E.g. 0 358 - Comma-separated integers. E.g. 1,3,5 359 - A range of integers denoted by the starting integer separated from 360 the end integer by a dash, '-'. E.g. 1-3 361 362 Args: 363 tests: A string list which contains multiple test paths. 364 365 Returns: 366 A string list of paths. 367 """ 368 count = len(tests) 369 if default_all or count <= 1: 370 return tests if count else None 371 mtests = set() 372 try: 373 numbered_list = ['%s: %s' % (i, t) for i, t in enumerate(tests)] 374 numbered_list.append('%s: All' % count) 375 print('Multiple tests found:\n{0}'.format('\n'.join(numbered_list))) 376 test_indices = input("Please enter numbers of test to use. If none of " 377 "above option matched, keep searching for other " 378 "possible tests.\n(multiple selection is supported, " 379 "e.g. '1' or '0,1' or '0-2'): ") 380 for idx in re.sub(r'(\s)', '', test_indices).split(','): 381 indices = idx.split('-') 382 len_indices = len(indices) 383 if len_indices > 0: 384 start_index = min(int(indices[0]), int(indices[len_indices-1])) 385 end_index = max(int(indices[0]), int(indices[len_indices-1])) 386 # One of input is 'All', return all options. 387 if count in (start_index, end_index): 388 return tests 389 mtests.update(tests[start_index:(end_index+1)]) 390 except (ValueError, IndexError, AttributeError, TypeError) as err: 391 logging.debug('%s', err) 392 print('None of above option matched, keep searching for other' 393 ' possible tests...') 394 return list(mtests) 395 396 397@atest_decorator.static_var("cached_ignore_dirs", []) 398def _get_ignored_dirs(): 399 """Get ignore dirs in find command. 400 401 Since we can't construct a single find cmd to find the target and 402 filter-out the dir with .out-dir, .find-ignore and $OUT-DIR. We have 403 to run the 1st find cmd to find these dirs. Then, we can use these 404 results to generate the real find cmd. 405 406 Return: 407 A list of the ignore dirs. 408 """ 409 out_dirs = _get_ignored_dirs.cached_ignore_dirs 410 if not out_dirs: 411 build_top = os.environ.get(constants.ANDROID_BUILD_TOP) 412 find_out_dir_cmd = (r'find %s -maxdepth 2 ' 413 r'-type f \( -name ".out-dir" -o -name ' 414 r'".find-ignore" \)') % build_top 415 out_files = subprocess.check_output(find_out_dir_cmd, shell=True) 416 if isinstance(out_files, bytes): 417 out_files = out_files.decode() 418 # Get all dirs with .out-dir or .find-ignore 419 if out_files: 420 out_files = out_files.splitlines() 421 for out_file in out_files: 422 if out_file: 423 out_dirs.append(os.path.dirname(out_file.strip())) 424 # Get the out folder if user specified $OUT_DIR 425 custom_out_dir = os.environ.get(constants.ANDROID_OUT_DIR) 426 if custom_out_dir: 427 user_out_dir = None 428 if os.path.isabs(custom_out_dir): 429 user_out_dir = custom_out_dir 430 else: 431 user_out_dir = os.path.join(build_top, custom_out_dir) 432 # only ignore the out_dir when it under $ANDROID_BUILD_TOP 433 if build_top in user_out_dir: 434 if user_out_dir not in out_dirs: 435 out_dirs.append(user_out_dir) 436 _get_ignored_dirs.cached_ignore_dirs = out_dirs 437 return out_dirs 438 439 440def _get_prune_cond_of_ignored_dirs(): 441 """Get the prune condition of ignore dirs. 442 443 Generation a string of the prune condition in the find command. 444 It will filter-out the dir with .out-dir, .find-ignore and $OUT-DIR. 445 Because they are the out dirs, we don't have to find them. 446 447 Return: 448 A string of the prune condition of the ignore dirs. 449 """ 450 out_dirs = _get_ignored_dirs() 451 prune_cond = r'-type d \( -name ".*"' 452 for out_dir in out_dirs: 453 prune_cond += r' -o -path %s' % out_dir 454 prune_cond += r' \) -prune -o' 455 return prune_cond 456 457 458def run_find_cmd(ref_type, search_dir, target, methods=None): 459 """Find a path to a target given a search dir and a target name. 460 461 Args: 462 ref_type: An AtestEnum of the reference type. 463 search_dir: A string of the dirpath to search in. 464 target: A string of what you're trying to find. 465 methods: A set of method names. 466 467 Return: 468 A list of the path to the target. 469 If the search_dir is inexistent, None will be returned. 470 """ 471 # If module_info.json is outdated, finding in the search_dir can result in 472 # raising exception. Return null immediately can guild users to run 473 # --rebuild-module-info to resolve the problem. 474 if not os.path.isdir(search_dir): 475 logging.debug('\'%s\' does not exist!', search_dir) 476 return None 477 ref_name = FIND_REFERENCE_TYPE[ref_type] 478 start = time.time() 479 # Validate mlocate.db before using 'locate' or 'find'. 480 # TODO: b/187146540 record abnormal mlocate.db in Metrics. 481 is_valid_mlocate = atest_utils.check_md5(constants.LOCATE_CACHE_MD5) 482 if os.path.isfile(FIND_INDEXES[ref_type]) and is_valid_mlocate: 483 _dict, out = {}, None 484 with open(FIND_INDEXES[ref_type], 'rb') as index: 485 try: 486 _dict = pickle.load(index, encoding='utf-8') 487 except (TypeError, IOError, EOFError, pickle.UnpicklingError) as err: 488 logging.debug('Exception raised: %s', err) 489 metrics_utils.handle_exc_and_send_exit_event( 490 constants.ACCESS_CACHE_FAILURE) 491 os.remove(FIND_INDEXES[ref_type]) 492 if _dict.get(target): 493 out = [path for path in _dict.get(target) if search_dir in path] 494 logging.debug('Found %s in %s', target, out) 495 else: 496 prune_cond = _get_prune_cond_of_ignored_dirs() 497 if '.' in target: 498 target = target.replace('.', '/') 499 find_cmd = FIND_CMDS[ref_type].format(search_dir, prune_cond, target) 500 logging.debug('Executing %s find cmd: %s', ref_name, find_cmd) 501 out = subprocess.check_output(find_cmd, shell=True) 502 if isinstance(out, bytes): 503 out = out.decode() 504 logging.debug('%s find cmd out: %s', ref_name, out) 505 logging.debug('%s find completed in %ss', ref_name, time.time() - start) 506 return extract_test_path(out, methods) 507 508 509def find_class_file(search_dir, class_name, is_native_test=False, methods=None): 510 """Find a path to a class file given a search dir and a class name. 511 512 Args: 513 search_dir: A string of the dirpath to search in. 514 class_name: A string of the class to search for. 515 is_native_test: A boolean variable of whether to search for a native 516 test or not. 517 methods: A set of method names. 518 519 Return: 520 A list of the path to the java/cc file. 521 """ 522 if is_native_test: 523 ref_type = FIND_REFERENCE_TYPE.CC_CLASS 524 elif '.' in class_name: 525 ref_type = FIND_REFERENCE_TYPE.QUALIFIED_CLASS 526 else: 527 ref_type = FIND_REFERENCE_TYPE.CLASS 528 return run_find_cmd(ref_type, search_dir, class_name, methods) 529 530 531def is_equal_or_sub_dir(sub_dir, parent_dir): 532 """Return True sub_dir is sub dir or equal to parent_dir. 533 534 Args: 535 sub_dir: A string of the sub directory path. 536 parent_dir: A string of the parent directory path. 537 538 Returns: 539 A boolean of whether both are dirs and sub_dir is sub of parent_dir 540 or is equal to parent_dir. 541 """ 542 # avoid symlink issues with real path 543 parent_dir = os.path.realpath(parent_dir) 544 sub_dir = os.path.realpath(sub_dir) 545 if not os.path.isdir(sub_dir) or not os.path.isdir(parent_dir): 546 return False 547 return os.path.commonprefix([sub_dir, parent_dir]) == parent_dir 548 549 550def find_parent_module_dir(root_dir, start_dir, module_info): 551 """From current dir search up file tree until root dir for module dir. 552 553 Args: 554 root_dir: A string of the dir that is the parent of the start dir. 555 start_dir: A string of the dir to start searching up from. 556 module_info: ModuleInfo object containing module information from the 557 build system. 558 559 Returns: 560 A string of the module dir relative to root, None if no Module Dir 561 found. There may be multiple testable modules at this level. 562 563 Exceptions: 564 ValueError: Raised if cur_dir not dir or not subdir of root dir. 565 """ 566 if not is_equal_or_sub_dir(start_dir, root_dir): 567 raise ValueError('%s not in repo %s' % (start_dir, root_dir)) 568 auto_gen_dir = None 569 current_dir = start_dir 570 while current_dir != root_dir: 571 # TODO (b/112904944) - migrate module_finder functions to here and 572 # reuse them. 573 rel_dir = os.path.relpath(current_dir, root_dir) 574 # Check if actual config file here but need to make sure that there 575 # exist module in module-info with the parent dir. 576 if (os.path.isfile(os.path.join(current_dir, constants.MODULE_CONFIG)) 577 and module_info.get_module_names(current_dir)): 578 return rel_dir 579 # Check module_info if auto_gen config or robo (non-config) here 580 for mod in module_info.path_to_module_info.get(rel_dir, []): 581 if module_info.is_robolectric_module(mod): 582 return rel_dir 583 for test_config in mod.get(constants.MODULE_TEST_CONFIG, []): 584 # If the test config doesn's exist until it was auto-generated 585 # in the build time(under <android_root>/out), atest still 586 # recognizes it testable. 587 if test_config: 588 return rel_dir 589 if mod.get('auto_test_config'): 590 auto_gen_dir = rel_dir 591 # Don't return for auto_gen, keep checking for real config, 592 # because common in cts for class in apk that's in hostside 593 # test setup. 594 current_dir = os.path.dirname(current_dir) 595 return auto_gen_dir 596 597 598def get_targets_from_xml(xml_file, module_info): 599 """Retrieve build targets from the given xml. 600 601 Just a helper func on top of get_targets_from_xml_root. 602 603 Args: 604 xml_file: abs path to xml file. 605 module_info: ModuleInfo class used to verify targets are valid modules. 606 607 Returns: 608 A set of build targets based on the signals found in the xml file. 609 """ 610 xml_root = ET.parse(xml_file).getroot() 611 return get_targets_from_xml_root(xml_root, module_info) 612 613 614def _get_apk_target(apk_target): 615 """Return the sanitized apk_target string from the xml. 616 617 The apk_target string can be of 2 forms: 618 - apk_target.apk 619 - apk_target.apk->/path/to/install/apk_target.apk 620 621 We want to return apk_target in both cases. 622 623 Args: 624 apk_target: String of target name to clean. 625 626 Returns: 627 String of apk_target to build. 628 """ 629 apk = apk_target.split(_XML_PUSH_DELIM, 1)[0].strip() 630 return apk[:-len(_APK_SUFFIX)] 631 632 633def _is_apk_target(name, value): 634 """Return True if XML option is an apk target. 635 636 We have some scenarios where an XML option can be an apk target: 637 - value is an apk file. 638 - name is a 'push' option where value holds the apk_file + other stuff. 639 640 Args: 641 name: String name of XML option. 642 value: String value of the XML option. 643 644 Returns: 645 True if it's an apk target we should build, False otherwise. 646 """ 647 if _APK_RE.match(value): 648 return True 649 if name == 'push' and value.endswith(_APK_SUFFIX): 650 return True 651 return False 652 653 654def get_targets_from_xml_root(xml_root, module_info): 655 """Retrieve build targets from the given xml root. 656 657 We're going to pull the following bits of info: 658 - Parse any .apk files listed in the config file. 659 - Parse option value for "test-module-name" (for vts10 tests). 660 - Look for the perf script. 661 662 Args: 663 module_info: ModuleInfo class used to verify targets are valid modules. 664 xml_root: ElementTree xml_root for us to look through. 665 666 Returns: 667 A set of build targets based on the signals found in the xml file. 668 """ 669 targets = set() 670 option_tags = xml_root.findall('.//option') 671 for tag in option_tags: 672 target_to_add = None 673 name = tag.attrib[_XML_NAME].strip() 674 value = tag.attrib[_XML_VALUE].strip() 675 if _is_apk_target(name, value): 676 target_to_add = _get_apk_target(value) 677 elif _PERF_SETUP_LABEL in value: 678 target_to_add = _PERF_SETUP_TARGET 679 680 # Let's make sure we can actually build the target. 681 if target_to_add and module_info.is_module(target_to_add): 682 targets.add(target_to_add) 683 elif target_to_add: 684 logging.warning('Build target (%s) not present in module info, ' 685 'skipping build', target_to_add) 686 687 # TODO (b/70813166): Remove this lookup once all runtime dependencies 688 # can be listed as a build dependencies or are in the base test harness. 689 nodes_with_class = xml_root.findall(".//*[@class]") 690 for class_attr in nodes_with_class: 691 fqcn = class_attr.attrib['class'].strip() 692 if fqcn.startswith(_COMPATIBILITY_PACKAGE_PREFIX): 693 targets.add(_CTS_JAR) 694 if fqcn in DALVIK_TESTRUNNER_JAR_CLASSES: 695 targets.update(DALVIK_TEST_DEPS) 696 logging.debug('Targets found in config file: %s', targets) 697 return targets 698 699 700def _get_vts_push_group_targets(push_file, rel_out_dir): 701 """Retrieve vts10 push group build targets. 702 703 A push group file is a file that list out test dependencies and other push 704 group files. Go through the push file and gather all the test deps we need. 705 706 Args: 707 push_file: Name of the push file in the VTS 708 rel_out_dir: Abs path to the out dir to help create vts10 build targets. 709 710 Returns: 711 Set of string which represent build targets. 712 """ 713 targets = set() 714 full_push_file_path = os.path.join(_VTS_PUSH_DIR, push_file) 715 # pylint: disable=invalid-name 716 with open(full_push_file_path) as f: 717 for line in f: 718 target = line.strip() 719 # Skip empty lines. 720 if not target: 721 continue 722 723 # This is a push file, get the targets from it. 724 if target.endswith(_VTS_PUSH_SUFFIX): 725 targets |= _get_vts_push_group_targets(line.strip(), 726 rel_out_dir) 727 continue 728 sanitized_target = target.split(_XML_PUSH_DELIM, 1)[0].strip() 729 targets.add(os.path.join(rel_out_dir, sanitized_target)) 730 return targets 731 732 733def _specified_bitness(xml_root): 734 """Check if the xml file contains the option append-bitness. 735 736 Args: 737 xml_root: abs path to xml file. 738 739 Returns: 740 True if xml specifies to append-bitness, False otherwise. 741 """ 742 option_tags = xml_root.findall('.//option') 743 for tag in option_tags: 744 value = tag.attrib[_XML_VALUE].strip() 745 name = tag.attrib[_XML_NAME].strip() 746 if name == _VTS_BITNESS and value == _VTS_BITNESS_TRUE: 747 return True 748 return False 749 750 751def _get_vts_binary_src_target(value, rel_out_dir): 752 """Parse out the vts10 binary src target. 753 754 The value can be in the following pattern: 755 - {_32bit,_64bit,_IPC32_32bit}::DATA/target (DATA/target) 756 - DATA/target->/data/target (DATA/target) 757 - out/host/linx-x86/bin/VtsSecuritySelinuxPolicyHostTest (the string as 758 is) 759 760 Args: 761 value: String of the XML option value to parse. 762 rel_out_dir: String path of out dir to prepend to target when required. 763 764 Returns: 765 String of the target to build. 766 """ 767 # We'll assume right off the bat we can use the value as is and modify it if 768 # necessary, e.g. out/host/linux-x86/bin... 769 target = value 770 # _32bit::DATA/target 771 match = _VTS_BINARY_SRC_DELIM_RE.match(value) 772 if match: 773 target = os.path.join(rel_out_dir, match.group('target')) 774 # DATA/target->/data/target 775 elif _XML_PUSH_DELIM in value: 776 target = value.split(_XML_PUSH_DELIM, 1)[0].strip() 777 target = os.path.join(rel_out_dir, target) 778 return target 779 780 781def get_plans_from_vts_xml(xml_file): 782 """Get configs which are included by xml_file. 783 784 We're looking for option(include) to get all dependency plan configs. 785 786 Args: 787 xml_file: Absolute path to xml file. 788 789 Returns: 790 A set of plan config paths which are depended by xml_file. 791 """ 792 if not os.path.exists(xml_file): 793 raise atest_error.XmlNotExistError('%s: The xml file does' 794 'not exist' % xml_file) 795 plans = set() 796 xml_root = ET.parse(xml_file).getroot() 797 plans.add(xml_file) 798 option_tags = xml_root.findall('.//include') 799 if not option_tags: 800 return plans 801 # Currently, all vts10 xmls live in the same dir : 802 # https://android.googlesource.com/platform/test/vts/+/master/tools/vts-tradefed/res/config/ 803 # If the vts10 plans start using folders to organize the plans, the logic here 804 # should be changed. 805 xml_dir = os.path.dirname(xml_file) 806 for tag in option_tags: 807 name = tag.attrib[_XML_NAME].strip() 808 plans |= get_plans_from_vts_xml(os.path.join(xml_dir, name + ".xml")) 809 return plans 810 811 812def get_targets_from_vts_xml(xml_file, rel_out_dir, module_info): 813 """Parse a vts10 xml for test dependencies we need to build. 814 815 We have a separate vts10 parsing function because we make a big assumption 816 on the targets (the way they're formatted and what they represent) and we 817 also create these build targets in a very special manner as well. 818 The 6 options we're looking for are: 819 - binary-test-source 820 - push-group 821 - push 822 - test-module-name 823 - test-file-name 824 - apk 825 826 Args: 827 module_info: ModuleInfo class used to verify targets are valid modules. 828 rel_out_dir: Abs path to the out dir to help create vts10 build targets. 829 xml_file: abs path to xml file. 830 831 Returns: 832 A set of build targets based on the signals found in the xml file. 833 """ 834 xml_root = ET.parse(xml_file).getroot() 835 targets = set() 836 option_tags = xml_root.findall('.//option') 837 for tag in option_tags: 838 value = tag.attrib[_XML_VALUE].strip() 839 name = tag.attrib[_XML_NAME].strip() 840 if name in [_VTS_TEST_MODULE, _VTS_MODULE]: 841 if module_info.is_module(value): 842 targets.add(value) 843 else: 844 logging.warning('vts10 test module (%s) not present in module ' 845 'info, skipping build', value) 846 elif name == _VTS_BINARY_SRC: 847 targets.add(_get_vts_binary_src_target(value, rel_out_dir)) 848 elif name == _VTS_PUSH_GROUP: 849 # Look up the push file and parse out build artifacts (as well as 850 # other push group files to parse). 851 targets |= _get_vts_push_group_targets(value, rel_out_dir) 852 elif name == _VTS_PUSH: 853 # Parse out the build artifact directly. 854 push_target = value.split(_XML_PUSH_DELIM, 1)[0].strip() 855 # If the config specified append-bitness, append the bits suffixes 856 # to the target. 857 if _specified_bitness(xml_root): 858 targets.add(os.path.join( 859 rel_out_dir, push_target + _VTS_BITNESS_32)) 860 targets.add(os.path.join( 861 rel_out_dir, push_target + _VTS_BITNESS_64)) 862 else: 863 targets.add(os.path.join(rel_out_dir, push_target)) 864 elif name == _VTS_TEST_FILE: 865 # The _VTS_TEST_FILE values can be set in 2 possible ways: 866 # 1. test_file.apk 867 # 2. DATA/app/test_file/test_file.apk 868 # We'll assume that test_file.apk (#1) is in an expected path (but 869 # that is not true, see b/76158619) and create the full path for it 870 # and then append the _VTS_TEST_FILE value to targets to build. 871 target = os.path.join(rel_out_dir, value) 872 # If value is just an APK, specify the path that we expect it to be in 873 # e.g. out/host/linux-x86/vts10/android-vts10/testcases/DATA/app/test_file/test_file.apk 874 head, _ = os.path.split(value) 875 if not head: 876 target = os.path.join(rel_out_dir, _VTS_OUT_DATA_APP_PATH, 877 _get_apk_target(value), value) 878 targets.add(target) 879 elif name == _VTS_APK: 880 targets.add(os.path.join(rel_out_dir, value)) 881 logging.debug('Targets found in config file: %s', targets) 882 return targets 883 884 885def get_dir_path_and_filename(path): 886 """Return tuple of dir and file name from given path. 887 888 Args: 889 path: String of path to break up. 890 891 Returns: 892 Tuple of (dir, file) paths. 893 """ 894 if os.path.isfile(path): 895 dir_path, file_path = os.path.split(path) 896 else: 897 dir_path, file_path = path, None 898 return dir_path, file_path 899 900 901def get_cc_filter(class_name, methods): 902 """Get the cc filter. 903 904 Args: 905 class_name: class name of the cc test. 906 methods: a list of method names. 907 908 Returns: 909 A formatted string for cc filter. 910 Ex: "class1.method1:class1.method2" or "class1.*" 911 """ 912 if methods: 913 sorted_methods = sorted(list(methods)) 914 return ":".join(["%s.%s" % (class_name, x) for x in sorted_methods]) 915 return "%s.*" % class_name 916 917 918def search_integration_dirs(name, int_dirs): 919 """Search integration dirs for name and return full path. 920 921 Args: 922 name: A string of plan name needed to be found. 923 int_dirs: A list of path needed to be searched. 924 925 Returns: 926 A list of the test path. 927 Ask user to select if multiple tests are found. 928 None if no matched test found. 929 """ 930 root_dir = os.environ.get(constants.ANDROID_BUILD_TOP) 931 test_files = [] 932 for integration_dir in int_dirs: 933 abs_path = os.path.join(root_dir, integration_dir) 934 test_paths = run_find_cmd(FIND_REFERENCE_TYPE.INTEGRATION, abs_path, 935 name) 936 if test_paths: 937 test_files.extend(test_paths) 938 return extract_test_from_tests(test_files) 939 940 941def get_int_dir_from_path(path, int_dirs): 942 """Search integration dirs for the given path and return path of dir. 943 944 Args: 945 path: A string of path needed to be found. 946 int_dirs: A list of path needed to be searched. 947 948 Returns: 949 A string of the test dir. None if no matched path found. 950 """ 951 root_dir = os.environ.get(constants.ANDROID_BUILD_TOP) 952 if not os.path.exists(path): 953 return None 954 dir_path, file_name = get_dir_path_and_filename(path) 955 int_dir = None 956 for possible_dir in int_dirs: 957 abs_int_dir = os.path.join(root_dir, possible_dir) 958 if is_equal_or_sub_dir(dir_path, abs_int_dir): 959 int_dir = abs_int_dir 960 break 961 if not file_name: 962 logging.warning('Found dir (%s) matching input (%s).' 963 ' Referencing an entire Integration/Suite dir' 964 ' is not supported. If you are trying to reference' 965 ' a test by its path, please input the path to' 966 ' the integration/suite config file itself.', 967 int_dir, path) 968 return None 969 return int_dir 970 971 972def get_install_locations(installed_paths): 973 """Get install locations from installed paths. 974 975 Args: 976 installed_paths: List of installed_paths from module_info. 977 978 Returns: 979 Set of install locations from module_info installed_paths. e.g. 980 set(['host', 'device']) 981 """ 982 install_locations = set() 983 for path in installed_paths: 984 if _HOST_PATH_RE.match(path): 985 install_locations.add(constants.DEVICELESS_TEST) 986 elif _DEVICE_PATH_RE.match(path): 987 install_locations.add(constants.DEVICE_TEST) 988 return install_locations 989 990 991def get_levenshtein_distance(test_name, module_name, 992 dir_costs=constants.COST_TYPO): 993 """Return an edit distance between test_name and module_name. 994 995 Levenshtein Distance has 3 actions: delete, insert and replace. 996 dis_costs makes each action weigh differently. 997 998 Args: 999 test_name: A keyword from the users. 1000 module_name: A testable module name. 1001 dir_costs: A tuple which contains 3 integer, where dir represents 1002 Deletion, Insertion and Replacement respectively. 1003 For guessing typos: (1, 1, 1) gives the best result. 1004 For searching keywords, (8, 1, 5) gives the best result. 1005 1006 Returns: 1007 An edit distance integer between test_name and module_name. 1008 """ 1009 rows = len(test_name) + 1 1010 cols = len(module_name) + 1 1011 deletion, insertion, replacement = dir_costs 1012 1013 # Creating a Dynamic Programming Matrix and weighting accordingly. 1014 dp_matrix = [[0 for _ in range(cols)] for _ in range(rows)] 1015 # Weigh rows/deletion 1016 for row in range(1, rows): 1017 dp_matrix[row][0] = row * deletion 1018 # Weigh cols/insertion 1019 for col in range(1, cols): 1020 dp_matrix[0][col] = col * insertion 1021 # The core logic of LD 1022 for col in range(1, cols): 1023 for row in range(1, rows): 1024 if test_name[row-1] == module_name[col-1]: 1025 cost = 0 1026 else: 1027 cost = replacement 1028 dp_matrix[row][col] = min(dp_matrix[row-1][col] + deletion, 1029 dp_matrix[row][col-1] + insertion, 1030 dp_matrix[row-1][col-1] + cost) 1031 1032 return dp_matrix[row][col] 1033 1034 1035def is_test_from_kernel_xml(xml_file, test_name): 1036 """Check if test defined in xml_file. 1037 1038 A kernel test can be defined like: 1039 <option name="test-command-line" key="test_class_1" value="command 1" /> 1040 where key is the name of test class and method of the runner. This method 1041 returns True if the test_name was defined in the given xml_file. 1042 1043 Args: 1044 xml_file: Absolute path to xml file. 1045 test_name: test_name want to find. 1046 1047 Returns: 1048 True if test_name in xml_file, False otherwise. 1049 """ 1050 if not os.path.exists(xml_file): 1051 raise atest_error.XmlNotExistError('%s: The xml file does' 1052 'not exist' % xml_file) 1053 xml_root = ET.parse(xml_file).getroot() 1054 option_tags = xml_root.findall('.//option') 1055 for option_tag in option_tags: 1056 if option_tag.attrib['name'] == 'test-command-line': 1057 if option_tag.attrib['key'] == test_name: 1058 return True 1059 return False 1060 1061 1062def is_parameterized_java_class(test_path): 1063 """Find out if input test path is a parameterized java class. 1064 1065 Args: 1066 test_path: A string of absolute path to the java file. 1067 1068 Returns: 1069 Boolean: Is parameterized class or not. 1070 """ 1071 with open(test_path) as class_file: 1072 for line in class_file: 1073 match = _PARAMET_JAVA_CLASS_RE.match(line) 1074 if match: 1075 return True 1076 return False 1077 1078 1079def get_java_methods(test_path): 1080 """Find out the java test class of input test_path. 1081 1082 Args: 1083 test_path: A string of absolute path to the java file. 1084 1085 Returns: 1086 A set of methods. 1087 """ 1088 with open(test_path) as class_file: 1089 content = class_file.read() 1090 matches = re.findall(_JAVA_METHODS_RE, content) 1091 if matches: 1092 methods = {match[1] for match in matches} 1093 logging.debug('Available methods: %s', methods) 1094 return methods 1095 return set() 1096 1097 1098def get_cc_test_classes_methods(test_path): 1099 """Find out the cc test class of input test_path. 1100 1101 Args: 1102 test_path: A string of absolute path to the cc file. 1103 1104 Returns: 1105 A tuple of sets: classes, methods and para_classes. 1106 """ 1107 classes = set() 1108 methods = set() 1109 para_classes = set() 1110 with open(test_path) as class_file: 1111 content = class_file.read() 1112 # Search matched CC CLASS/METHOD 1113 matches = re.findall(_CC_CLASS_METHOD_RE, content) 1114 logging.debug('Found cc classes: %s', matches) 1115 for match in matches: 1116 # The elements of `matches` will be "Group 1"(_F), 1117 # "Group class"(MyClass1) and "Group method"(MyMethod1) 1118 classes.update([match[1]]) 1119 methods.update([match[2]]) 1120 # Search matched parameterized CC CLASS. 1121 matches = re.findall(_PARA_CC_CLASS_RE, content) 1122 logging.debug('Found parameterized classes: %s', matches) 1123 for match in matches: 1124 # The elements of `matches` will be "Group 1"(_F), 1125 # "Group instantiate class"(MyInstantClass1) 1126 # and "Group class"(MyClass1) 1127 para_classes.update([match[2]]) 1128 return classes, methods, para_classes 1129 1130def find_host_unit_tests(module_info, path): 1131 """Find host unit tests for the input path. 1132 1133 Args: 1134 module_info: ModuleInfo obj. 1135 path: A string of the relative path from $BUILD_TOP we want to search. 1136 1137 Returns: 1138 A list that includes the module name of unit tests, otherwise an empty 1139 list. 1140 """ 1141 logging.debug('finding unit tests under %s', path) 1142 found_unit_tests = [] 1143 unit_test_names = module_info.get_all_unit_tests() 1144 logging.debug('All the unit tests: %s', unit_test_names) 1145 for unit_test_name in unit_test_names: 1146 for test_path in module_info.get_paths(unit_test_name): 1147 if test_path.find(path) == 0: 1148 found_unit_tests.append(unit_test_name) 1149 return found_unit_tests 1150 1151def get_annotated_methods(annotation, file_path): 1152 """Find all the methods annotated by the input annotation in the file_path. 1153 1154 Args: 1155 annotation: A string of the annotation class. 1156 file_path: A string of the file path. 1157 1158 Returns: 1159 A set of all the methods annotated. 1160 """ 1161 methods = set() 1162 annotation_name = '@' + str(annotation).split('.')[-1] 1163 with open(file_path) as class_file: 1164 enter_annotation_block = False 1165 for line in class_file: 1166 if str(line).strip().startswith(annotation_name): 1167 enter_annotation_block = True 1168 continue 1169 if enter_annotation_block: 1170 matches = re.findall(_JAVA_METHODS_RE, line) 1171 if matches: 1172 methods.update({match[1] for match in matches}) 1173 enter_annotation_block = False 1174 continue 1175 return methods 1176 1177def get_test_config_and_srcs(test_info, module_info): 1178 """Get the test config path for the input test_info. 1179 1180 The search rule will be: 1181 Check if test name in test_info could be found in module_info 1182 1. AndroidTest.xml under module path if no test config be set. 1183 2. The first test config defined in Android.bp if test config be set. 1184 If test name could not found matched module in module_info, search all the 1185 test config name if match. 1186 1187 Args: 1188 test_info: TestInfo obj. 1189 module_info: ModuleInfo obj. 1190 1191 Returns: 1192 A string of the config path and list of srcs, None if test config not 1193 exist. 1194 """ 1195 android_root_dir = os.environ.get(constants.ANDROID_BUILD_TOP) 1196 test_name = test_info.test_name 1197 mod_info = module_info.get_module_info(test_name) 1198 if mod_info: 1199 test_configs = mod_info.get(constants.MODULE_TEST_CONFIG, []) 1200 if len(test_configs) == 0: 1201 # Check for AndroidTest.xml at the module path. 1202 for path in mod_info.get(constants.MODULE_PATH, []): 1203 config_path = os.path.join( 1204 android_root_dir, path, constants.MODULE_CONFIG) 1205 if os.path.isfile(config_path): 1206 return config_path, mod_info.get(constants.MODULE_SRCS, []) 1207 if len(test_configs) >= 1: 1208 test_config = test_configs[0] 1209 config_path = os.path.join(android_root_dir, test_config) 1210 if os.path.isfile(config_path): 1211 return config_path, mod_info.get(constants.MODULE_SRCS, []) 1212 else: 1213 for _, info in module_info.name_to_module_info.items(): 1214 test_configs = info.get(constants.MODULE_TEST_CONFIG, []) 1215 for test_config in test_configs: 1216 config_path = os.path.join(android_root_dir, test_config) 1217 config_name = os.path.splitext(os.path.basename(config_path))[0] 1218 if config_name == test_name and os.path.isfile(config_path): 1219 return config_path, info.get(constants.MODULE_SRCS, []) 1220 return None, None 1221