1# Copyright 2015 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import collections 6import copy 7import logging 8import os 9import pickle 10import re 11 12from devil.android import apk_helper 13from devil.android import md5sum 14from pylib import constants 15from pylib.base import base_test_result 16from pylib.base import test_instance 17from pylib.constants import host_paths 18from pylib.instrumentation import test_result 19from pylib.instrumentation import instrumentation_parser 20from pylib.utils import isolator 21from pylib.utils import proguard 22 23with host_paths.SysPath(host_paths.BUILD_COMMON_PATH): 24 import unittest_util # pylint: disable=import-error 25 26# Ref: http://developer.android.com/reference/android/app/Activity.html 27_ACTIVITY_RESULT_CANCELED = 0 28_ACTIVITY_RESULT_OK = -1 29 30_COMMAND_LINE_PARAMETER = 'cmdlinearg-parameter' 31_DEFAULT_ANNOTATIONS = [ 32 'Smoke', 'SmallTest', 'MediumTest', 'LargeTest', 33 'EnormousTest', 'IntegrationTest'] 34_EXCLUDE_UNLESS_REQUESTED_ANNOTATIONS = [ 35 'DisabledTest', 'FlakyTest'] 36_VALID_ANNOTATIONS = set(['Manual', 'PerfTest'] + _DEFAULT_ANNOTATIONS + 37 _EXCLUDE_UNLESS_REQUESTED_ANNOTATIONS) 38_EXTRA_DRIVER_TEST_LIST = ( 39 'org.chromium.test.driver.OnDeviceInstrumentationDriver.TestList') 40_EXTRA_DRIVER_TEST_LIST_FILE = ( 41 'org.chromium.test.driver.OnDeviceInstrumentationDriver.TestListFile') 42_EXTRA_DRIVER_TARGET_PACKAGE = ( 43 'org.chromium.test.driver.OnDeviceInstrumentationDriver.TargetPackage') 44_EXTRA_DRIVER_TARGET_CLASS = ( 45 'org.chromium.test.driver.OnDeviceInstrumentationDriver.TargetClass') 46_EXTRA_TIMEOUT_SCALE = ( 47 'org.chromium.test.driver.OnDeviceInstrumentationDriver.TimeoutScale') 48 49_PARAMETERIZED_TEST_ANNOTATION = 'ParameterizedTest' 50_PARAMETERIZED_TEST_SET_ANNOTATION = 'ParameterizedTest$Set' 51_NATIVE_CRASH_RE = re.compile('native crash', re.IGNORECASE) 52_PICKLE_FORMAT_VERSION = 10 53 54 55class MissingSizeAnnotationError(Exception): 56 def __init__(self, class_name): 57 super(MissingSizeAnnotationError, self).__init__(class_name + 58 ': Test method is missing required size annotation. Add one of: ' + 59 ', '.join('@' + a for a in _VALID_ANNOTATIONS)) 60 61 62# TODO(jbudorick): Make these private class methods of 63# InstrumentationTestInstance once the instrumentation test_runner is 64# deprecated. 65def ParseAmInstrumentRawOutput(raw_output): 66 """Parses the output of an |am instrument -r| call. 67 68 Args: 69 raw_output: the output of an |am instrument -r| call as a list of lines 70 Returns: 71 A 3-tuple containing: 72 - the instrumentation code as an integer 73 - the instrumentation result as a list of lines 74 - the instrumentation statuses received as a list of 2-tuples 75 containing: 76 - the status code as an integer 77 - the bundle dump as a dict mapping string keys to a list of 78 strings, one for each line. 79 """ 80 parser = instrumentation_parser.InstrumentationParser(raw_output) 81 statuses = list(parser.IterStatus()) 82 code, bundle = parser.GetResult() 83 return (code, bundle, statuses) 84 85 86def GenerateTestResults( 87 result_code, result_bundle, statuses, start_ms, duration_ms): 88 """Generate test results from |statuses|. 89 90 Args: 91 result_code: The overall status code as an integer. 92 result_bundle: The summary bundle dump as a dict. 93 statuses: A list of 2-tuples containing: 94 - the status code as an integer 95 - the bundle dump as a dict mapping string keys to string values 96 Note that this is the same as the third item in the 3-tuple returned by 97 |_ParseAmInstrumentRawOutput|. 98 start_ms: The start time of the test in milliseconds. 99 duration_ms: The duration of the test in milliseconds. 100 101 Returns: 102 A list containing an instance of InstrumentationTestResult for each test 103 parsed. 104 """ 105 106 results = [] 107 108 current_result = None 109 110 for status_code, bundle in statuses: 111 test_class = bundle.get('class', '') 112 test_method = bundle.get('test', '') 113 if test_class and test_method: 114 test_name = '%s#%s' % (test_class, test_method) 115 else: 116 continue 117 118 if status_code == instrumentation_parser.STATUS_CODE_START: 119 if current_result: 120 results.append(current_result) 121 current_result = test_result.InstrumentationTestResult( 122 test_name, base_test_result.ResultType.UNKNOWN, start_ms, duration_ms) 123 else: 124 if status_code == instrumentation_parser.STATUS_CODE_OK: 125 if bundle.get('test_skipped', '').lower() in ('true', '1', 'yes'): 126 current_result.SetType(base_test_result.ResultType.SKIP) 127 elif current_result.GetType() == base_test_result.ResultType.UNKNOWN: 128 current_result.SetType(base_test_result.ResultType.PASS) 129 else: 130 if status_code not in (instrumentation_parser.STATUS_CODE_ERROR, 131 instrumentation_parser.STATUS_CODE_FAILURE): 132 logging.error('Unrecognized status code %d. Handling as an error.', 133 status_code) 134 current_result.SetType(base_test_result.ResultType.FAIL) 135 if 'stack' in bundle: 136 current_result.SetLog(bundle['stack']) 137 138 if current_result: 139 if current_result.GetType() == base_test_result.ResultType.UNKNOWN: 140 crashed = (result_code == _ACTIVITY_RESULT_CANCELED 141 and any(_NATIVE_CRASH_RE.search(l) 142 for l in result_bundle.itervalues())) 143 if crashed: 144 current_result.SetType(base_test_result.ResultType.CRASH) 145 146 results.append(current_result) 147 148 return results 149 150 151def ParseCommandLineFlagParameters(annotations): 152 """Determines whether the test is parameterized to be run with different 153 command-line flags. 154 155 Args: 156 annotations: The annotations of the test. 157 158 Returns: 159 If the test is parameterized, returns a list of named tuples 160 with lists of flags, e.g.: 161 162 [(add=['--flag-to-add']), (remove=['--flag-to-remove']), ()] 163 164 That means, the test must be run three times, the first time with 165 "--flag-to-add" added to command-line, the second time with 166 "--flag-to-remove" to be removed from command-line, and the third time 167 with default command-line args. If the same flag is listed both for adding 168 and for removing, it is left unchanged. 169 170 If the test is not parametrized, returns None. 171 172 """ 173 ParamsTuple = collections.namedtuple('ParamsTuple', ['add', 'remove']) 174 parameterized_tests = [] 175 if _PARAMETERIZED_TEST_SET_ANNOTATION in annotations: 176 if annotations[_PARAMETERIZED_TEST_SET_ANNOTATION]: 177 parameterized_tests = annotations[ 178 _PARAMETERIZED_TEST_SET_ANNOTATION].get('tests', []) 179 elif _PARAMETERIZED_TEST_ANNOTATION in annotations: 180 parameterized_tests = [annotations[_PARAMETERIZED_TEST_ANNOTATION]] 181 else: 182 return None 183 184 result = [] 185 for pt in parameterized_tests: 186 if not pt: 187 continue 188 for p in pt['parameters']: 189 if p['tag'] == _COMMAND_LINE_PARAMETER: 190 to_add = [] 191 to_remove = [] 192 for a in p.get('arguments', []): 193 if a['name'] == 'add': 194 to_add = ['--%s' % f for f in a['stringArray']] 195 elif a['name'] == 'remove': 196 to_remove = ['--%s' % f for f in a['stringArray']] 197 result.append(ParamsTuple(to_add, to_remove)) 198 return result if result else None 199 200 201class InstrumentationTestInstance(test_instance.TestInstance): 202 203 def __init__(self, args, isolate_delegate, error_func): 204 super(InstrumentationTestInstance, self).__init__() 205 206 self._additional_apks = [] 207 self._apk_under_test = None 208 self._apk_under_test_incremental_install_script = None 209 self._package_info = None 210 self._suite = None 211 self._test_apk = None 212 self._test_apk_incremental_install_script = None 213 self._test_jar = None 214 self._test_package = None 215 self._test_runner = None 216 self._test_support_apk = None 217 self._initializeApkAttributes(args, error_func) 218 219 self._data_deps = None 220 self._isolate_abs_path = None 221 self._isolate_delegate = None 222 self._isolated_abs_path = None 223 self._test_data = None 224 self._initializeDataDependencyAttributes(args, isolate_delegate) 225 226 self._annotations = None 227 self._excluded_annotations = None 228 self._test_filter = None 229 self._initializeTestFilterAttributes(args) 230 231 self._flags = None 232 self._initializeFlagAttributes(args) 233 234 self._driver_apk = None 235 self._driver_package = None 236 self._driver_name = None 237 self._initializeDriverAttributes() 238 239 self._timeout_scale = None 240 self._initializeTestControlAttributes(args) 241 242 def _initializeApkAttributes(self, args, error_func): 243 if args.apk_under_test: 244 apk_under_test_path = args.apk_under_test 245 if not args.apk_under_test.endswith('.apk'): 246 apk_under_test_path = os.path.join( 247 constants.GetOutDirectory(), constants.SDK_BUILD_APKS_DIR, 248 '%s.apk' % args.apk_under_test) 249 250 if not os.path.exists(apk_under_test_path): 251 error_func('Unable to find APK under test: %s' % apk_under_test_path) 252 253 self._apk_under_test = apk_helper.ToHelper(apk_under_test_path) 254 255 if args.test_apk.endswith('.apk'): 256 self._suite = os.path.splitext(os.path.basename(args.test_apk))[0] 257 self._test_apk = apk_helper.ToHelper(args.test_apk) 258 else: 259 self._suite = args.test_apk 260 self._test_apk = apk_helper.ToHelper(os.path.join( 261 constants.GetOutDirectory(), constants.SDK_BUILD_APKS_DIR, 262 '%s.apk' % args.test_apk)) 263 264 self._apk_under_test_incremental_install_script = ( 265 args.apk_under_test_incremental_install_script) 266 self._test_apk_incremental_install_script = ( 267 args.test_apk_incremental_install_script) 268 269 if self._test_apk_incremental_install_script: 270 assert self._suite.endswith('_incremental') 271 self._suite = self._suite[:-len('_incremental')] 272 273 self._test_jar = os.path.join( 274 constants.GetOutDirectory(), constants.SDK_BUILD_TEST_JAVALIB_DIR, 275 '%s.jar' % self._suite) 276 self._test_support_apk = apk_helper.ToHelper(os.path.join( 277 constants.GetOutDirectory(), constants.SDK_BUILD_TEST_JAVALIB_DIR, 278 '%sSupport.apk' % self._suite)) 279 280 if not os.path.exists(self._test_apk.path): 281 error_func('Unable to find test APK: %s' % self._test_apk.path) 282 if not os.path.exists(self._test_jar): 283 error_func('Unable to find test JAR: %s' % self._test_jar) 284 285 self._test_package = self._test_apk.GetPackageName() 286 self._test_runner = self._test_apk.GetInstrumentationName() 287 288 self._package_info = None 289 if self._apk_under_test: 290 package_under_test = self._apk_under_test.GetPackageName() 291 for package_info in constants.PACKAGE_INFO.itervalues(): 292 if package_under_test == package_info.package: 293 self._package_info = package_info 294 if not self._package_info: 295 logging.warning('Unable to find package info for %s', self._test_package) 296 297 for apk in args.additional_apks: 298 if not os.path.exists(apk): 299 error_func('Unable to find additional APK: %s' % apk) 300 self._additional_apks = ( 301 [apk_helper.ToHelper(x) for x in args.additional_apks]) 302 303 def _initializeDataDependencyAttributes(self, args, isolate_delegate): 304 self._data_deps = [] 305 if (args.isolate_file_path and 306 not isolator.IsIsolateEmpty(args.isolate_file_path)): 307 if os.path.isabs(args.isolate_file_path): 308 self._isolate_abs_path = args.isolate_file_path 309 else: 310 self._isolate_abs_path = os.path.join( 311 constants.DIR_SOURCE_ROOT, args.isolate_file_path) 312 self._isolate_delegate = isolate_delegate 313 self._isolated_abs_path = os.path.join( 314 constants.GetOutDirectory(), '%s.isolated' % self._test_package) 315 else: 316 self._isolate_delegate = None 317 318 # TODO(jbudorick): Deprecate and remove --test-data once data dependencies 319 # are fully converted to isolate. 320 if args.test_data: 321 logging.info('Data dependencies specified via --test-data') 322 self._test_data = args.test_data 323 else: 324 self._test_data = None 325 326 if not self._isolate_delegate and not self._test_data: 327 logging.warning('No data dependencies will be pushed.') 328 329 def _initializeTestFilterAttributes(self, args): 330 if args.test_filter: 331 self._test_filter = args.test_filter.replace('#', '.') 332 333 def annotation_dict_element(a): 334 a = a.split('=') 335 return (a[0], a[1] if len(a) == 2 else None) 336 337 if args.annotation_str: 338 self._annotations = dict( 339 annotation_dict_element(a) 340 for a in args.annotation_str.split(',')) 341 elif not self._test_filter: 342 self._annotations = dict( 343 annotation_dict_element(a) 344 for a in _DEFAULT_ANNOTATIONS) 345 else: 346 self._annotations = {} 347 348 if args.exclude_annotation_str: 349 self._excluded_annotations = dict( 350 annotation_dict_element(a) 351 for a in args.exclude_annotation_str.split(',')) 352 else: 353 self._excluded_annotations = {} 354 355 self._excluded_annotations.update( 356 { 357 a: None for a in _EXCLUDE_UNLESS_REQUESTED_ANNOTATIONS 358 if a not in self._annotations 359 }) 360 361 def _initializeFlagAttributes(self, args): 362 self._flags = ['--enable-test-intents'] 363 # TODO(jbudorick): Transition "--device-flags" to "--device-flags-file" 364 if hasattr(args, 'device_flags') and args.device_flags: 365 with open(args.device_flags) as device_flags_file: 366 stripped_lines = (l.strip() for l in device_flags_file) 367 self._flags.extend([flag for flag in stripped_lines if flag]) 368 if hasattr(args, 'device_flags_file') and args.device_flags_file: 369 with open(args.device_flags_file) as device_flags_file: 370 stripped_lines = (l.strip() for l in device_flags_file) 371 self._flags.extend([flag for flag in stripped_lines if flag]) 372 if (hasattr(args, 'strict_mode') and 373 args.strict_mode and 374 args.strict_mode != 'off'): 375 self._flags.append('--strict-mode=' + args.strict_mode) 376 377 def _initializeDriverAttributes(self): 378 self._driver_apk = os.path.join( 379 constants.GetOutDirectory(), constants.SDK_BUILD_APKS_DIR, 380 'OnDeviceInstrumentationDriver.apk') 381 if os.path.exists(self._driver_apk): 382 driver_apk = apk_helper.ApkHelper(self._driver_apk) 383 self._driver_package = driver_apk.GetPackageName() 384 self._driver_name = driver_apk.GetInstrumentationName() 385 else: 386 self._driver_apk = None 387 388 def _initializeTestControlAttributes(self, args): 389 self._timeout_scale = args.timeout_scale or 1 390 391 @property 392 def additional_apks(self): 393 return self._additional_apks 394 395 @property 396 def apk_under_test(self): 397 return self._apk_under_test 398 399 @property 400 def apk_under_test_incremental_install_script(self): 401 return self._apk_under_test_incremental_install_script 402 403 @property 404 def flags(self): 405 return self._flags 406 407 @property 408 def driver_apk(self): 409 return self._driver_apk 410 411 @property 412 def driver_package(self): 413 return self._driver_package 414 415 @property 416 def driver_name(self): 417 return self._driver_name 418 419 @property 420 def package_info(self): 421 return self._package_info 422 423 @property 424 def suite(self): 425 return self._suite 426 427 @property 428 def test_apk(self): 429 return self._test_apk 430 431 @property 432 def test_apk_incremental_install_script(self): 433 return self._test_apk_incremental_install_script 434 435 @property 436 def test_jar(self): 437 return self._test_jar 438 439 @property 440 def test_support_apk(self): 441 return self._test_support_apk 442 443 @property 444 def test_package(self): 445 return self._test_package 446 447 @property 448 def test_runner(self): 449 return self._test_runner 450 451 @property 452 def timeout_scale(self): 453 return self._timeout_scale 454 455 #override 456 def TestType(self): 457 return 'instrumentation' 458 459 #override 460 def SetUp(self): 461 if self._isolate_delegate: 462 self._isolate_delegate.Remap( 463 self._isolate_abs_path, self._isolated_abs_path) 464 self._isolate_delegate.MoveOutputDeps() 465 self._data_deps.extend([(self._isolate_delegate.isolate_deps_dir, None)]) 466 467 # TODO(jbudorick): Convert existing tests that depend on the --test-data 468 # mechanism to isolate, then remove this. 469 if self._test_data: 470 for t in self._test_data: 471 device_rel_path, host_rel_path = t.split(':') 472 host_abs_path = os.path.join(host_paths.DIR_SOURCE_ROOT, host_rel_path) 473 self._data_deps.extend( 474 [(host_abs_path, 475 [None, 'chrome', 'test', 'data', device_rel_path])]) 476 477 def GetDataDependencies(self): 478 return self._data_deps 479 480 def GetTests(self): 481 pickle_path = '%s-proguard.pickle' % self.test_jar 482 try: 483 tests = self._GetTestsFromPickle(pickle_path, self.test_jar) 484 except self.ProguardPickleException as e: 485 logging.info('Getting tests from JAR via proguard. (%s)', str(e)) 486 tests = self._GetTestsFromProguard(self.test_jar) 487 self._SaveTestsToPickle(pickle_path, self.test_jar, tests) 488 return self._ParametrizeTestsWithFlags( 489 self._InflateTests(self._FilterTests(tests))) 490 491 class ProguardPickleException(Exception): 492 pass 493 494 def _GetTestsFromPickle(self, pickle_path, jar_path): 495 if not os.path.exists(pickle_path): 496 raise self.ProguardPickleException('%s does not exist.' % pickle_path) 497 if os.path.getmtime(pickle_path) <= os.path.getmtime(jar_path): 498 raise self.ProguardPickleException( 499 '%s newer than %s.' % (jar_path, pickle_path)) 500 501 with open(pickle_path, 'r') as pickle_file: 502 pickle_data = pickle.loads(pickle_file.read()) 503 jar_md5 = md5sum.CalculateHostMd5Sums(jar_path)[jar_path] 504 505 try: 506 if pickle_data['VERSION'] != _PICKLE_FORMAT_VERSION: 507 raise self.ProguardPickleException('PICKLE_FORMAT_VERSION has changed.') 508 if pickle_data['JAR_MD5SUM'] != jar_md5: 509 raise self.ProguardPickleException('JAR file MD5 sum differs.') 510 return pickle_data['TEST_METHODS'] 511 except TypeError as e: 512 logging.error(pickle_data) 513 raise self.ProguardPickleException(str(e)) 514 515 # pylint: disable=no-self-use 516 def _GetTestsFromProguard(self, jar_path): 517 p = proguard.Dump(jar_path) 518 519 def is_test_class(c): 520 return c['class'].endswith('Test') 521 522 def is_test_method(m): 523 return m['method'].startswith('test') 524 525 class_lookup = dict((c['class'], c) for c in p['classes']) 526 def recursive_get_class_annotations(c): 527 s = c['superclass'] 528 if s in class_lookup: 529 a = recursive_get_class_annotations(class_lookup[s]) 530 else: 531 a = {} 532 a.update(c['annotations']) 533 return a 534 535 def stripped_test_class(c): 536 return { 537 'class': c['class'], 538 'annotations': recursive_get_class_annotations(c), 539 'methods': [m for m in c['methods'] if is_test_method(m)], 540 } 541 542 return [stripped_test_class(c) for c in p['classes'] 543 if is_test_class(c)] 544 545 def _SaveTestsToPickle(self, pickle_path, jar_path, tests): 546 jar_md5 = md5sum.CalculateHostMd5Sums(jar_path)[jar_path] 547 pickle_data = { 548 'VERSION': _PICKLE_FORMAT_VERSION, 549 'JAR_MD5SUM': jar_md5, 550 'TEST_METHODS': tests, 551 } 552 with open(pickle_path, 'w') as pickle_file: 553 pickle.dump(pickle_data, pickle_file) 554 555 def _FilterTests(self, tests): 556 557 def gtest_filter(c, m): 558 if not self._test_filter: 559 return True 560 # Allow fully-qualified name as well as an omitted package. 561 names = ['%s.%s' % (c['class'], m['method']), 562 '%s.%s' % (c['class'].split('.')[-1], m['method'])] 563 return unittest_util.FilterTestNames(names, self._test_filter) 564 565 def annotation_filter(all_annotations): 566 if not self._annotations: 567 return True 568 return any_annotation_matches(self._annotations, all_annotations) 569 570 def excluded_annotation_filter(all_annotations): 571 if not self._excluded_annotations: 572 return True 573 return not any_annotation_matches(self._excluded_annotations, 574 all_annotations) 575 576 def any_annotation_matches(annotations, all_annotations): 577 return any( 578 ak in all_annotations and (av is None or av == all_annotations[ak]) 579 for ak, av in annotations.iteritems()) 580 581 filtered_classes = [] 582 for c in tests: 583 filtered_methods = [] 584 for m in c['methods']: 585 # Gtest filtering 586 if not gtest_filter(c, m): 587 continue 588 589 all_annotations = dict(c['annotations']) 590 all_annotations.update(m['annotations']) 591 592 # Enforce that all tests declare their size. 593 if not any(a in _VALID_ANNOTATIONS for a in all_annotations): 594 raise MissingSizeAnnotationError('%s.%s' % (c['class'], m['method'])) 595 596 if (not annotation_filter(all_annotations) 597 or not excluded_annotation_filter(all_annotations)): 598 continue 599 600 filtered_methods.append(m) 601 602 if filtered_methods: 603 filtered_class = dict(c) 604 filtered_class['methods'] = filtered_methods 605 filtered_classes.append(filtered_class) 606 607 return filtered_classes 608 609 def _InflateTests(self, tests): 610 inflated_tests = [] 611 for c in tests: 612 for m in c['methods']: 613 a = dict(c['annotations']) 614 a.update(m['annotations']) 615 inflated_tests.append({ 616 'class': c['class'], 617 'method': m['method'], 618 'annotations': a, 619 }) 620 return inflated_tests 621 622 def _ParametrizeTestsWithFlags(self, tests): 623 new_tests = [] 624 for t in tests: 625 parameters = ParseCommandLineFlagParameters(t['annotations']) 626 if parameters: 627 t['flags'] = parameters[0] 628 for p in parameters[1:]: 629 parameterized_t = copy.copy(t) 630 parameterized_t['flags'] = p 631 new_tests.append(parameterized_t) 632 return tests + new_tests 633 634 def GetDriverEnvironmentVars( 635 self, test_list=None, test_list_file_path=None): 636 env = { 637 _EXTRA_DRIVER_TARGET_PACKAGE: self.test_package, 638 _EXTRA_DRIVER_TARGET_CLASS: self.test_runner, 639 _EXTRA_TIMEOUT_SCALE: self._timeout_scale, 640 } 641 642 if test_list: 643 env[_EXTRA_DRIVER_TEST_LIST] = ','.join(test_list) 644 645 if test_list_file_path: 646 env[_EXTRA_DRIVER_TEST_LIST_FILE] = ( 647 os.path.basename(test_list_file_path)) 648 649 return env 650 651 @staticmethod 652 def ParseAmInstrumentRawOutput(raw_output): 653 return ParseAmInstrumentRawOutput(raw_output) 654 655 @staticmethod 656 def GenerateTestResults( 657 result_code, result_bundle, statuses, start_ms, duration_ms): 658 return GenerateTestResults(result_code, result_bundle, statuses, 659 start_ms, duration_ms) 660 661 #override 662 def TearDown(self): 663 if self._isolate_delegate: 664 self._isolate_delegate.Clear() 665 666