1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.testtype.junit4; 18 19 import static org.junit.Assert.assertTrue; 20 21 import com.android.annotations.VisibleForTesting; 22 import com.android.ddmlib.Log.LogLevel; 23 import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; 24 import com.android.ddmlib.testrunner.TestResult.TestStatus; 25 import com.android.tradefed.build.IBuildInfo; 26 import com.android.tradefed.device.DeviceNotAvailableException; 27 import com.android.tradefed.device.ITestDevice; 28 import com.android.tradefed.invoker.IInvocationContext; 29 import com.android.tradefed.log.LogUtil.CLog; 30 import com.android.tradefed.result.CollectingTestListener; 31 import com.android.tradefed.result.ITestLifeCycleReceiver; 32 import com.android.tradefed.result.TestDescription; 33 import com.android.tradefed.result.TestResult; 34 import com.android.tradefed.result.TestRunResult; 35 import com.android.tradefed.result.ddmlib.DefaultRemoteAndroidTestRunner; 36 import com.android.tradefed.targetprep.TargetSetupError; 37 import com.android.tradefed.targetprep.suite.SuiteApkInstaller; 38 import com.android.tradefed.testtype.IAbi; 39 import com.android.tradefed.testtype.IAbiReceiver; 40 import com.android.tradefed.testtype.IBuildReceiver; 41 import com.android.tradefed.testtype.IDeviceTest; 42 import com.android.tradefed.testtype.IInvocationContextReceiver; 43 import com.android.tradefed.util.ListInstrumentationParser; 44 import com.android.tradefed.util.ListInstrumentationParser.InstrumentationTarget; 45 46 import org.junit.After; 47 import org.junit.Assume; 48 49 import java.util.ArrayList; 50 import java.util.HashMap; 51 import java.util.LinkedHashMap; 52 import java.util.List; 53 import java.util.Map; 54 import java.util.concurrent.TimeUnit; 55 56 /** 57 * Base test class for running host JUnit4 style tests. This class provides support to install, run 58 * and clean up instrumentation tests from the host side. This class is multi-devices compatible. 59 * Should be the single source of truth to run instrumentation tests from host side in order to 60 * avoid duplicated utility and base class. 61 */ 62 public abstract class BaseHostJUnit4Test 63 implements IAbiReceiver, IBuildReceiver, IDeviceTest, IInvocationContextReceiver { 64 65 static final long DEFAULT_TEST_TIMEOUT_MS = 10 * 60 * 1000L; 66 private static final long DEFAULT_MAX_TIMEOUT_TO_OUTPUT_MS = 10 * 60 * 1000L; // 10min 67 private static final Map<String, String> DEFAULT_INSTRUMENTATION_ARGS = new HashMap<>(); 68 69 private ITestDevice mDevice; 70 private IBuildInfo mBuild; 71 private IAbi mAbi; 72 private IInvocationContext mContext; 73 private Map<SuiteApkInstaller, ITestDevice> mInstallers = new LinkedHashMap<>(); 74 private TestRunResult mLatestInstruRes; 75 76 @Override setDevice(ITestDevice device)77 public final void setDevice(ITestDevice device) { 78 mDevice = device; 79 } 80 81 @Override getDevice()82 public final ITestDevice getDevice() { 83 return mDevice; 84 } 85 86 @Override setBuild(IBuildInfo buildInfo)87 public final void setBuild(IBuildInfo buildInfo) { 88 mBuild = buildInfo; 89 } 90 getBuild()91 public final IBuildInfo getBuild() { 92 return mBuild; 93 } 94 95 @Override setAbi(IAbi abi)96 public final void setAbi(IAbi abi) { 97 mAbi = abi; 98 } 99 100 @Override getAbi()101 public final IAbi getAbi() { 102 return mAbi; 103 } 104 105 @Override setInvocationContext(IInvocationContext invocationContext)106 public final void setInvocationContext(IInvocationContext invocationContext) { 107 mContext = invocationContext; 108 } 109 getInvocationContext()110 public final IInvocationContext getInvocationContext() { 111 return mContext; 112 } 113 getListDevices()114 public final List<ITestDevice> getListDevices() { 115 return mContext.getDevices(); 116 } 117 118 /** 119 * Automatic tear down for all the apk installed. This will uninstall all the apk from the 120 * device they where installed on. 121 */ 122 @After autoTearDown()123 public final void autoTearDown() throws DeviceNotAvailableException { 124 mLatestInstruRes = null; 125 for (SuiteApkInstaller installer : mInstallers.keySet()) { 126 ITestDevice device = mInstallers.get(installer); 127 installer.tearDown(device, mContext.getBuildInfo(device), null); 128 } 129 mInstallers.clear(); 130 } 131 132 // ------------------------- Utility APIs provided for tests ------------------------- 133 134 /** 135 * Install an apk given its name on the device. Apk will be auto-cleaned. 136 * 137 * @param apkFileName The name of the apk file. 138 * @param options extra options given to the install command 139 */ installPackage(String apkFileName, String... options)140 public final void installPackage(String apkFileName, String... options) 141 throws DeviceNotAvailableException, TargetSetupError { 142 installPackage(getDevice(), apkFileName, options); 143 } 144 145 /** 146 * Install an apk given its name on a given device. Apk will be auto-cleaned. 147 * 148 * @param device the {@link ITestDevice} on which to install the apk. 149 * @param apkFileName The name of the apk file. 150 * @param options extra options given to the install command 151 */ installPackage(ITestDevice device, String apkFileName, String... options)152 public final void installPackage(ITestDevice device, String apkFileName, String... options) 153 throws DeviceNotAvailableException, TargetSetupError { 154 SuiteApkInstaller installer = createSuiteApkInstaller(); 155 // Force the apk clean up 156 installer.setCleanApk(true); 157 // Store the preparer for cleanup 158 mInstallers.put(installer, device); 159 installer.addTestFileName(apkFileName); 160 installer.setAbi(getAbi()); 161 for (String option : options) { 162 installer.addInstallArg(option); 163 } 164 installer.setUp(device, mContext.getBuildInfo(device)); 165 } 166 167 /** 168 * Install an apk given its name for a specific user. 169 * 170 * @param apkFileName The name of the apk file. 171 * @param grantPermission whether to pass the grant permission flag when installing the apk. 172 * @param userId the user id of the user where to install the apk. 173 * @param options extra options given to the install command 174 */ installPackageAsUser( String apkFileName, boolean grantPermission, int userId, String... options)175 public final void installPackageAsUser( 176 String apkFileName, boolean grantPermission, int userId, String... options) 177 throws DeviceNotAvailableException, TargetSetupError { 178 installPackageAsUser(getDevice(), apkFileName, grantPermission, userId, options); 179 } 180 181 /** 182 * Install an apk given its name for a specific user on a given device. 183 * 184 * @param device the {@link ITestDevice} on which to install the apk. 185 * @param apkFileName The name of the apk file. 186 * @param grantPermission whether to pass the grant permission flag when installing the apk. 187 * @param userId the user id of the user where to install the apk. 188 * @param options extra options given to the install command 189 */ installPackageAsUser( ITestDevice device, String apkFileName, boolean grantPermission, int userId, String... options)190 public final void installPackageAsUser( 191 ITestDevice device, 192 String apkFileName, 193 boolean grantPermission, 194 int userId, 195 String... options) 196 throws DeviceNotAvailableException, TargetSetupError { 197 SuiteApkInstaller installer = createSuiteApkInstaller(); 198 // Force the apk clean up 199 installer.setCleanApk(true); 200 // Store the preparer for cleanup 201 mInstallers.put(installer, device); 202 installer.addTestFileName(apkFileName); 203 installer.setUserId(userId); 204 installer.setShouldGrantPermission(grantPermission); 205 installer.setAbi(getAbi()); 206 for (String option : options) { 207 installer.addInstallArg(option); 208 } 209 installer.setUp(device, mContext.getBuildInfo(device)); 210 } 211 212 /** 213 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 214 * right after to get the details of results. 215 * 216 * @param pkgName the name of the package to run. 217 * @param testClassName the name of the test class to run. 218 * @return True if it succeed without failure. False otherwise. 219 */ runDeviceTests(String pkgName, String testClassName)220 public final boolean runDeviceTests(String pkgName, String testClassName) 221 throws DeviceNotAvailableException { 222 return runDeviceTests(getDevice(), pkgName, testClassName, null); 223 } 224 225 /** 226 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 227 * right after to get the details of results. 228 * 229 * @param pkgName the name of the package to run. 230 * @param testClassName the name of the test class to run. 231 * @param testTimeoutMs the timeout in millisecond to be applied to each test case. 232 * @return True if it succeed without failure. False otherwise. 233 */ runDeviceTests(String pkgName, String testClassName, Long testTimeoutMs)234 public final boolean runDeviceTests(String pkgName, String testClassName, Long testTimeoutMs) 235 throws DeviceNotAvailableException { 236 return runDeviceTests( 237 getDevice(), 238 null, 239 pkgName, 240 testClassName, 241 null, 242 null, 243 testTimeoutMs, 244 DEFAULT_MAX_TIMEOUT_TO_OUTPUT_MS, 245 0L, 246 true, 247 false, 248 DEFAULT_INSTRUMENTATION_ARGS); 249 } 250 251 /** 252 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 253 * right after to get the details of results. 254 * 255 * @param pkgName the name of the package to run. 256 * @param testClassName the name of the test class to run. 257 * @param userId the id of the user to run the test against. can be null. 258 * @param testTimeoutMs the timeout in millisecond to be applied to each test case. 259 * @return True if it succeed without failure. False otherwise. 260 */ runDeviceTests( String pkgName, String testClassName, Integer userId, Long testTimeoutMs)261 public final boolean runDeviceTests( 262 String pkgName, String testClassName, Integer userId, Long testTimeoutMs) 263 throws DeviceNotAvailableException { 264 return runDeviceTests( 265 getDevice(), 266 null, 267 pkgName, 268 testClassName, 269 null, 270 userId, 271 testTimeoutMs, 272 DEFAULT_MAX_TIMEOUT_TO_OUTPUT_MS, 273 0L, 274 true, 275 false, 276 DEFAULT_INSTRUMENTATION_ARGS); 277 } 278 279 /** 280 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 281 * right after to get the details of results. 282 * 283 * @param pkgName the name of the package to run. 284 * @param testClassName the name of the test class to run. 285 * @param testMethodName the name of the test method in the class to be run. 286 * @return True if it succeed without failure. False otherwise. 287 */ runDeviceTests(String pkgName, String testClassName, String testMethodName)288 public final boolean runDeviceTests(String pkgName, String testClassName, String testMethodName) 289 throws DeviceNotAvailableException { 290 return runDeviceTests(getDevice(), pkgName, testClassName, testMethodName); 291 } 292 293 /** 294 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 295 * right after to get the details of results. 296 * 297 * @param runner the instrumentation runner to be used. 298 * @param pkgName the name of the package to run. 299 * @param testClassName the name of the test class to run. 300 * @param testMethodName the name of the test method in the class to be run. 301 * @return True if it succeed without failure. False otherwise. 302 */ runDeviceTests( String runner, String pkgName, String testClassName, String testMethodName)303 public final boolean runDeviceTests( 304 String runner, String pkgName, String testClassName, String testMethodName) 305 throws DeviceNotAvailableException { 306 return runDeviceTests( 307 getDevice(), 308 runner, 309 pkgName, 310 testClassName, 311 testMethodName, 312 null, 313 DEFAULT_TEST_TIMEOUT_MS, 314 DEFAULT_MAX_TIMEOUT_TO_OUTPUT_MS, 315 0L, 316 true, 317 false, 318 DEFAULT_INSTRUMENTATION_ARGS); 319 } 320 321 /** 322 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 323 * right after to get the details of results. 324 * 325 * @param device the device agaisnt which to run the instrumentation. 326 * @param pkgName the name of the package to run. 327 * @param testClassName the name of the test class to run. 328 * @param testMethodName the name of the test method in the class to be run. 329 * @return True if it succeed without failure. False otherwise. 330 */ runDeviceTests( ITestDevice device, String pkgName, String testClassName, String testMethodName)331 public final boolean runDeviceTests( 332 ITestDevice device, String pkgName, String testClassName, String testMethodName) 333 throws DeviceNotAvailableException { 334 return runDeviceTests( 335 device, 336 null, 337 pkgName, 338 testClassName, 339 testMethodName, 340 null, 341 DEFAULT_TEST_TIMEOUT_MS, 342 DEFAULT_MAX_TIMEOUT_TO_OUTPUT_MS, 343 0L, 344 true, 345 false, 346 DEFAULT_INSTRUMENTATION_ARGS); 347 } 348 349 /** 350 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 351 * right after to get the details of results. 352 * 353 * @param device the device agaisnt which to run the instrumentation. 354 * @param pkgName the name of the package to run. 355 * @param testClassName the name of the test class to run. 356 * @param testMethodName the name of the test method in the class to be run. 357 * @param testTimeoutMs the timeout in millisecond to be applied to each test case. 358 * @return True if it succeed without failure. False otherwise. 359 */ runDeviceTests( ITestDevice device, String pkgName, String testClassName, String testMethodName, Long testTimeoutMs)360 public final boolean runDeviceTests( 361 ITestDevice device, 362 String pkgName, 363 String testClassName, 364 String testMethodName, 365 Long testTimeoutMs) 366 throws DeviceNotAvailableException { 367 return runDeviceTests( 368 device, 369 null, 370 pkgName, 371 testClassName, 372 testMethodName, 373 null, 374 testTimeoutMs, 375 DEFAULT_MAX_TIMEOUT_TO_OUTPUT_MS, 376 0L, 377 true, 378 false, 379 DEFAULT_INSTRUMENTATION_ARGS); 380 } 381 382 /** 383 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 384 * right after to get the details of results. 385 * 386 * @param device the device agaisnt which to run the instrumentation. 387 * @param pkgName the name of the package to run. 388 * @param testClassName the name of the test class to run. 389 * @param testMethodName the name of the test method in the class to be run. 390 * @param userId the id of the user to run the test against. can be null. 391 * @param testTimeoutMs the timeout in millisecond to be applied to each test case. 392 * @return True if it succeed without failure. False otherwise. 393 */ runDeviceTests( ITestDevice device, String pkgName, String testClassName, String testMethodName, Integer userId, Long testTimeoutMs)394 public final boolean runDeviceTests( 395 ITestDevice device, 396 String pkgName, 397 String testClassName, 398 String testMethodName, 399 Integer userId, 400 Long testTimeoutMs) 401 throws DeviceNotAvailableException { 402 return runDeviceTests( 403 device, 404 null, 405 pkgName, 406 testClassName, 407 testMethodName, 408 userId, 409 testTimeoutMs, 410 DEFAULT_MAX_TIMEOUT_TO_OUTPUT_MS, 411 0L, 412 true, 413 false, 414 DEFAULT_INSTRUMENTATION_ARGS); 415 } 416 417 /** 418 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 419 * right after to get the details of results. 420 * 421 * @param device the device agaisnt which to run the instrumentation. 422 * @param pkgName the name of the package to run. 423 * @param testClassName the name of the test class to run. 424 * @param testMethodName the name of the test method in the class to be run. 425 * @param testTimeoutMs the timeout in millisecond to be applied to each test case. 426 * @param maxTimeToOutputMs the max timeout the test has to start outputting something. 427 * @param maxInstrumentationTimeoutMs the max timeout the full instrumentation has to complete. 428 * @return True if it succeed without failure. False otherwise. 429 */ runDeviceTests( ITestDevice device, String pkgName, String testClassName, String testMethodName, Long testTimeoutMs, Long maxTimeToOutputMs, Long maxInstrumentationTimeoutMs)430 public final boolean runDeviceTests( 431 ITestDevice device, 432 String pkgName, 433 String testClassName, 434 String testMethodName, 435 Long testTimeoutMs, 436 Long maxTimeToOutputMs, 437 Long maxInstrumentationTimeoutMs) 438 throws DeviceNotAvailableException { 439 return runDeviceTests( 440 device, 441 null, 442 pkgName, 443 testClassName, 444 testMethodName, 445 null, 446 testTimeoutMs, 447 maxTimeToOutputMs, 448 maxInstrumentationTimeoutMs, 449 true, 450 false, 451 DEFAULT_INSTRUMENTATION_ARGS); 452 } 453 454 /** 455 * Runs the instrumentation base on the information in {@link DeviceTestRunOptions}. 456 * 457 * @param options the {@link DeviceTestRunOptions} driving the instrumentation setup. 458 * @return True if it succeeded without failure. False otherwise. 459 * @throws DeviceNotAvailableException 460 */ runDeviceTests(DeviceTestRunOptions options)461 public final boolean runDeviceTests(DeviceTestRunOptions options) 462 throws DeviceNotAvailableException { 463 return runDeviceTests( 464 options.getDevice() == null ? getDevice() : options.getDevice(), 465 options.getRunner(), 466 options.getPackageName(), 467 options.getTestClassName(), 468 options.getTestMethodName(), 469 options.getUserId(), 470 options.getTestTimeoutMs(), 471 options.getMaxTimeToOutputMs(), 472 options.getMaxInstrumentationTimeoutMs(), 473 options.shouldCheckResults(), 474 options.isHiddenApiCheckDisabled(), 475 options.isIsolatedStorageDisabled(), 476 options.getInstrumentationArgs(), 477 options.getExtraListeners()); 478 } 479 480 /** 481 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 482 * right after to get the details of results. 483 * 484 * @param device the device agaisnt which to run the instrumentation. 485 * @param pkgName the name of the package to run. 486 * @param testClassName the name of the test class to run. 487 * @param testMethodName the name of the test method in the class to be run. 488 * @param userId the id of the user to run the test against. can be null. 489 * @param testTimeoutMs the timeout in millisecond to be applied to each test case. 490 * @param maxTimeToOutputMs the max timeout the test has to start outputting something. 491 * @param maxInstrumentationTimeoutMs the max timeout the full instrumentation has to complete. 492 * @param checkResults whether or not the results are checked for crashes. 493 * @param isHiddenApiCheckDisabled whether or not we should disable the hidden api check. 494 * @param instrumentationArgs arguments to pass to the instrumentation. 495 * @return True if it succeeded without failure. False otherwise. 496 */ runDeviceTests( ITestDevice device, String runner, String pkgName, String testClassName, String testMethodName, Integer userId, Long testTimeoutMs, Long maxTimeToOutputMs, Long maxInstrumentationTimeoutMs, boolean checkResults, boolean isHiddenApiCheckDisabled, Map<String, String> instrumentationArgs)497 public final boolean runDeviceTests( 498 ITestDevice device, 499 String runner, 500 String pkgName, 501 String testClassName, 502 String testMethodName, 503 Integer userId, 504 Long testTimeoutMs, 505 Long maxTimeToOutputMs, 506 Long maxInstrumentationTimeoutMs, 507 boolean checkResults, 508 boolean isHiddenApiCheckDisabled, 509 Map<String, String> instrumentationArgs) 510 throws DeviceNotAvailableException { 511 return runDeviceTests( 512 device, 513 runner, 514 pkgName, 515 testClassName, 516 testMethodName, 517 userId, 518 testTimeoutMs, 519 maxTimeToOutputMs, 520 maxInstrumentationTimeoutMs, 521 checkResults, 522 isHiddenApiCheckDisabled, 523 false, 524 instrumentationArgs, 525 new ArrayList<>()); 526 } 527 528 /** 529 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 530 * right after to get the details of results. 531 * 532 * @param device the device agaisnt which to run the instrumentation. 533 * @param pkgName the name of the package to run. 534 * @param testClassName the name of the test class to run. 535 * @param testMethodName the name of the test method in the class to be run. 536 * @param userId the id of the user to run the test against. can be null. 537 * @param testTimeoutMs the timeout in millisecond to be applied to each test case. 538 * @param maxTimeToOutputMs the max timeout the test has to start outputting something. 539 * @param maxInstrumentationTimeoutMs the max timeout the full instrumentation has to complete. 540 * @param checkResults whether or not the results are checked for crashes. 541 * @param isHiddenApiCheckDisabled whether or not we should disable the hidden api check. 542 * @param isIsolatedStorageDisabled whether or not we should disable isolated storage. 543 * @param instrumentationArgs arguments to pass to the instrumentation. 544 * @return True if it succeeded without failure. False otherwise. 545 */ runDeviceTests( ITestDevice device, String runner, String pkgName, String testClassName, String testMethodName, Integer userId, Long testTimeoutMs, Long maxTimeToOutputMs, Long maxInstrumentationTimeoutMs, boolean checkResults, boolean isHiddenApiCheckDisabled, boolean isIsolatedStorageDisabled, Map<String, String> instrumentationArgs, List<ITestLifeCycleReceiver> extraListeners)546 public final boolean runDeviceTests( 547 ITestDevice device, 548 String runner, 549 String pkgName, 550 String testClassName, 551 String testMethodName, 552 Integer userId, 553 Long testTimeoutMs, 554 Long maxTimeToOutputMs, 555 Long maxInstrumentationTimeoutMs, 556 boolean checkResults, 557 boolean isHiddenApiCheckDisabled, 558 boolean isIsolatedStorageDisabled, 559 Map<String, String> instrumentationArgs, 560 List<ITestLifeCycleReceiver> extraListeners) 561 throws DeviceNotAvailableException { 562 TestRunResult runResult = 563 doRunTests( 564 device, 565 runner, 566 pkgName, 567 testClassName, 568 testMethodName, 569 userId, 570 testTimeoutMs, 571 maxTimeToOutputMs, 572 maxInstrumentationTimeoutMs, 573 isHiddenApiCheckDisabled, 574 isIsolatedStorageDisabled, 575 instrumentationArgs, 576 extraListeners); 577 mLatestInstruRes = runResult; 578 printTestResult(runResult); 579 if (checkResults) { 580 if (runResult.isRunFailure()) { 581 throw new AssertionError( 582 "Failed to successfully run device tests for " 583 + runResult.getName() 584 + ": " 585 + runResult.getRunFailureMessage()); 586 } 587 if (runResult.getNumTests() == 0) { 588 throw new AssertionError("No tests were run on the device"); 589 } 590 if (runResult.hasFailedTests()) { 591 // build a meaningful error message 592 StringBuilder errorBuilder = new StringBuilder("on-device tests failed:\n"); 593 for (Map.Entry<TestDescription, TestResult> resultEntry : 594 runResult.getTestResults().entrySet()) { 595 if (!TestStatus.PASSED.equals(resultEntry.getValue().getStatus())) { 596 errorBuilder.append(resultEntry.getKey().toString()); 597 errorBuilder.append(":\n"); 598 errorBuilder.append(resultEntry.getValue().getStackTrace()); 599 } 600 } 601 throw new AssertionError(errorBuilder.toString()); 602 } 603 // Assume not all tests have skipped (and rethrow AssumptionViolatedException if so) 604 Assume.assumeTrue( 605 runResult.getNumTests() 606 != runResult.getNumTestsInState(TestStatus.ASSUMPTION_FAILURE)); 607 } 608 return !runResult.hasFailedTests() && runResult.getNumTests() > 0; 609 } 610 611 /** 612 * Returns the {@link TestRunResult} resulting from the latest runDeviceTests that ran. Or null 613 * if no results available. 614 */ getLastDeviceRunResults()615 public final TestRunResult getLastDeviceRunResults() { 616 return mLatestInstruRes; 617 } 618 619 /** Helper method to run tests and return the listener that collected the results. */ doRunTests( ITestDevice device, String runner, String pkgName, String testClassName, String testMethodName, Integer userId, Long testTimeoutMs, Long maxTimeToOutputMs, Long maxInstrumentationTimeoutMs, boolean isHiddenApiCheckDisabled, boolean isIsolatedStorageDisabled, Map<String, String> instrumentationArgs, List<ITestLifeCycleReceiver> extraListeners)620 private TestRunResult doRunTests( 621 ITestDevice device, 622 String runner, 623 String pkgName, 624 String testClassName, 625 String testMethodName, 626 Integer userId, 627 Long testTimeoutMs, 628 Long maxTimeToOutputMs, 629 Long maxInstrumentationTimeoutMs, 630 boolean isHiddenApiCheckDisabled, 631 boolean isIsolatedStorageDisabled, 632 Map<String, String> instrumentationArgs, 633 List<ITestLifeCycleReceiver> extraListeners) 634 throws DeviceNotAvailableException { 635 RemoteAndroidTestRunner testRunner = createTestRunner(pkgName, runner, device); 636 String runOptions = ""; 637 // hidden-api-checks flag only exists in P and after. 638 if (isHiddenApiCheckDisabled && (device.getApiLevel() >= 28)) { 639 runOptions += "--no-hidden-api-checks "; 640 } 641 // isolated-storage flag only exists in Q and after. 642 if (isIsolatedStorageDisabled && device.checkApiLevelAgainstNextRelease(29)) { 643 runOptions += "--no-isolated-storage "; 644 } 645 if (getAbi() != null) { 646 runOptions += String.format("--abi %s", getAbi().getName()); 647 } 648 // Set the run options if any. 649 if (!runOptions.isEmpty()) { 650 testRunner.setRunOptions(runOptions); 651 } 652 653 if (testClassName != null && testMethodName != null) { 654 testRunner.setMethodName(testClassName, testMethodName); 655 } else if (testClassName != null) { 656 testRunner.setClassName(testClassName); 657 } 658 659 if (testTimeoutMs != null) { 660 testRunner.addInstrumentationArg("timeout_msec", Long.toString(testTimeoutMs)); 661 } else { 662 testRunner.addInstrumentationArg( 663 "timeout_msec", Long.toString(DEFAULT_TEST_TIMEOUT_MS)); 664 } 665 if (maxTimeToOutputMs != null) { 666 testRunner.setMaxTimeToOutputResponse(maxTimeToOutputMs, TimeUnit.MILLISECONDS); 667 } 668 if (maxInstrumentationTimeoutMs != null) { 669 testRunner.setMaxTimeout(maxInstrumentationTimeoutMs, TimeUnit.MILLISECONDS); 670 } 671 // Pass all the instrumentation arguments 672 for (String key : instrumentationArgs.keySet()) { 673 testRunner.addInstrumentationArg(key, instrumentationArgs.get(key)); 674 } 675 676 CollectingTestListener listener = createListener(); 677 List<ITestLifeCycleReceiver> allReceiver = new ArrayList<>(); 678 allReceiver.add(listener); 679 allReceiver.addAll(extraListeners); 680 if (userId == null) { 681 assertTrue(device.runInstrumentationTests(testRunner, allReceiver)); 682 } else { 683 assertTrue(device.runInstrumentationTestsAsUser(testRunner, userId, allReceiver)); 684 } 685 return listener.getCurrentRunResults(); 686 } 687 688 @VisibleForTesting createTestRunner( String packageName, String runnerName, ITestDevice device)689 RemoteAndroidTestRunner createTestRunner( 690 String packageName, String runnerName, ITestDevice device) 691 throws DeviceNotAvailableException { 692 if (runnerName == null) { 693 ListInstrumentationParser parser = getListInstrumentationParser(); 694 device.executeShellCommand("pm list instrumentation", parser); 695 for (InstrumentationTarget target : parser.getInstrumentationTargets()) { 696 if (packageName.equals(target.packageName)) { 697 runnerName = target.runnerName; 698 } 699 } 700 } 701 // If the runner name is still null 702 if (runnerName == null) { 703 throw new RuntimeException("No runner was defined and couldn't dynamically find one."); 704 } 705 return new DefaultRemoteAndroidTestRunner(packageName, runnerName, device.getIDevice()); 706 } 707 708 @VisibleForTesting getListInstrumentationParser()709 ListInstrumentationParser getListInstrumentationParser() { 710 return new ListInstrumentationParser(); 711 } 712 713 @VisibleForTesting createListener()714 CollectingTestListener createListener() { 715 return new CollectingTestListener(); 716 } 717 printTestResult(TestRunResult runResult)718 private void printTestResult(TestRunResult runResult) { 719 for (Map.Entry<TestDescription, TestResult> testEntry : 720 runResult.getTestResults().entrySet()) { 721 TestResult testResult = testEntry.getValue(); 722 TestStatus testStatus = testResult.getStatus(); 723 CLog.logAndDisplay(LogLevel.INFO, "Test " + testEntry.getKey() + ": " + testStatus); 724 if (!TestStatus.PASSED.equals(testStatus) 725 && !TestStatus.ASSUMPTION_FAILURE.equals(testStatus)) { 726 CLog.logAndDisplay(LogLevel.WARN, testResult.getStackTrace()); 727 } 728 } 729 } 730 731 /** 732 * Uninstalls a package on the device. 733 * 734 * @param pkgName the Android package to uninstall 735 * @return a {@link String} with an error code, or <code>null</code> if success 736 */ uninstallPackage(String pkgName)737 public final String uninstallPackage(String pkgName) throws DeviceNotAvailableException { 738 return getDevice().uninstallPackage(pkgName); 739 } 740 741 /** 742 * Uninstalls a package on the device 743 * 744 * @param device the device that should uninstall the package. 745 * @param pkgName the Android package to uninstall 746 * @return a {@link String} with an error code, or <code>null</code> if success 747 */ uninstallPackage(ITestDevice device, String pkgName)748 public final String uninstallPackage(ITestDevice device, String pkgName) 749 throws DeviceNotAvailableException { 750 return device.uninstallPackage(pkgName); 751 } 752 753 /** 754 * Checks if a package of a given name is installed on the device 755 * 756 * @param pkg the name of the package 757 * @return true if the package is found on the device 758 */ isPackageInstalled(String pkg)759 public final boolean isPackageInstalled(String pkg) throws DeviceNotAvailableException { 760 return isPackageInstalled(getDevice(), pkg); 761 } 762 763 /** 764 * Checks if a package of a given name is installed on the device 765 * 766 * @param device the device that should uninstall the package. 767 * @param pkg the name of the package 768 * @return true if the package is found on the device 769 */ isPackageInstalled(ITestDevice device, String pkg)770 public final boolean isPackageInstalled(ITestDevice device, String pkg) 771 throws DeviceNotAvailableException { 772 for (String installedPackage : device.getInstalledPackageNames()) { 773 if (pkg.equals(installedPackage)) { 774 return true; 775 } 776 } 777 return false; 778 } 779 hasDeviceFeature(String feature)780 public boolean hasDeviceFeature(String feature) throws DeviceNotAvailableException { 781 return getDevice().hasFeature("feature:" + feature); 782 } 783 784 @VisibleForTesting createSuiteApkInstaller()785 SuiteApkInstaller createSuiteApkInstaller() { 786 return new SuiteApkInstaller(); 787 } 788 } 789