1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.testtype.junit4; 18 19 import static org.junit.Assert.assertTrue; 20 21 import com.android.annotations.VisibleForTesting; 22 import com.android.ddmlib.Log.LogLevel; 23 import com.android.tradefed.build.IBuildInfo; 24 import com.android.tradefed.device.DeviceNotAvailableException; 25 import com.android.tradefed.device.ITestDevice; 26 import com.android.tradefed.invoker.IInvocationContext; 27 import com.android.tradefed.invoker.TestInformation; 28 import com.android.tradefed.log.LogUtil.CLog; 29 import com.android.tradefed.result.CollectingTestListener; 30 import com.android.tradefed.result.ITestLifeCycleReceiver; 31 import com.android.tradefed.result.TestDescription; 32 import com.android.tradefed.result.TestResult; 33 import com.android.tradefed.result.TestRunResult; 34 import com.android.tradefed.result.TestStatus; 35 import com.android.tradefed.result.ddmlib.DefaultRemoteAndroidTestRunner; 36 import com.android.tradefed.result.ddmlib.RemoteAndroidTestRunner; 37 import com.android.tradefed.targetprep.BuildError; 38 import com.android.tradefed.targetprep.TargetSetupError; 39 import com.android.tradefed.targetprep.suite.SuiteApkInstaller; 40 import com.android.tradefed.testtype.IAbi; 41 import com.android.tradefed.testtype.IAbiReceiver; 42 import com.android.tradefed.testtype.ITestInformationReceiver; 43 import com.android.tradefed.util.ListInstrumentationParser; 44 import com.android.tradefed.util.ListInstrumentationParser.InstrumentationTarget; 45 46 import org.junit.After; 47 import org.junit.Assume; 48 49 import java.util.ArrayList; 50 import java.util.HashMap; 51 import java.util.LinkedHashMap; 52 import java.util.List; 53 import java.util.Map; 54 import java.util.concurrent.TimeUnit; 55 import java.util.stream.Collectors; 56 57 /** 58 * Base test class for running host JUnit4 style tests. This class provides support to install, run 59 * and clean up instrumentation tests from the host side. This class is multi-devices compatible. 60 * Should be the single source of truth to run instrumentation tests from host side in order to 61 * avoid duplicated utility and base class. 62 */ 63 public abstract class BaseHostJUnit4Test implements IAbiReceiver, ITestInformationReceiver { 64 65 static final long DEFAULT_TEST_TIMEOUT_MS = 10 * 60 * 1000L; 66 static final long DEFAULT_MAX_TIMEOUT_TO_OUTPUT_MS = 10 * 60 * 1000L; // 10 min 67 private static final Map<String, String> DEFAULT_INSTRUMENTATION_ARGS = new HashMap<>(); 68 69 private IAbi mAbi; 70 private TestInformation mTestInfo; 71 private Map<SuiteApkInstaller, ITestDevice> mInstallers = new LinkedHashMap<>(); 72 private TestRunResult mLatestInstruRes; 73 getDevice()74 public final ITestDevice getDevice() { 75 return mTestInfo.getDevice(); 76 } 77 getBuild()78 public final IBuildInfo getBuild() { 79 return mTestInfo.getBuildInfo(); 80 } 81 82 @Override setTestInformation(TestInformation testInformation)83 public final void setTestInformation(TestInformation testInformation) { 84 mTestInfo = testInformation; 85 } 86 87 @Override getTestInformation()88 public TestInformation getTestInformation() { 89 return mTestInfo; 90 } 91 92 @Override setAbi(IAbi abi)93 public final void setAbi(IAbi abi) { 94 mAbi = abi; 95 } 96 97 @Override getAbi()98 public final IAbi getAbi() { 99 return mAbi; 100 } 101 getInvocationContext()102 public final IInvocationContext getInvocationContext() { 103 return mTestInfo.getContext(); 104 } 105 getListDevices()106 public final List<ITestDevice> getListDevices() { 107 return mTestInfo.getContext().getDevices(); 108 } 109 110 /** 111 * Automatic tear down for all the apk installed. This will uninstall all the apk from the 112 * device they where installed on. 113 */ 114 @After autoTearDown()115 public final void autoTearDown() throws DeviceNotAvailableException { 116 mLatestInstruRes = null; 117 for (SuiteApkInstaller installer : mInstallers.keySet()) { 118 installer.tearDown(mTestInfo, null); 119 } 120 mInstallers.clear(); 121 } 122 123 // ------------------------- Utility APIs provided for tests ------------------------- 124 125 /** 126 * Install an apk given its name on the device. Apk will be auto-cleaned. 127 * 128 * @param apkFileName The name of the apk file. 129 * @param options extra options given to the install command 130 */ installPackage(String apkFileName, String... options)131 public final void installPackage(String apkFileName, String... options) 132 throws DeviceNotAvailableException, TargetSetupError { 133 final DeviceTestRunOptions installOptions = new DeviceTestRunOptions(null /*unused*/); 134 installOptions.setApkFileName(apkFileName); 135 installOptions.setInstallArgs(options); 136 installPackage(installOptions); 137 } 138 139 /** 140 * Install an apk given its name on a given device. Apk will be auto-cleaned. 141 * 142 * @param device the {@link ITestDevice} on which to install the apk. 143 * @param apkFileName The name of the apk file. 144 * @param options extra options given to the install command 145 */ installPackage(ITestDevice device, String apkFileName, String... options)146 public final void installPackage(ITestDevice device, String apkFileName, String... options) 147 throws DeviceNotAvailableException, TargetSetupError { 148 final DeviceTestRunOptions installOptions = new DeviceTestRunOptions(null /*unused*/); 149 installOptions.setApkFileName(apkFileName); 150 installOptions.setDevice(device); 151 installOptions.setInstallArgs(options); 152 installPackage(installOptions); 153 } 154 155 /** 156 * Install an apk based on the {@link DeviceTestRunOptions} on the device. Apk will be 157 * auto-cleaned. 158 * 159 * @param options The options of the package installation. 160 */ installPackage(DeviceTestRunOptions options)161 public final void installPackage(DeviceTestRunOptions options) 162 throws DeviceNotAvailableException, TargetSetupError { 163 final SuiteApkInstaller installer = createSuiteApkInstaller(); 164 final ITestDevice device = options.getDevice() == null 165 ? getDevice() : options.getDevice(); 166 // Force the apk clean up 167 installer.setCleanApk(true); 168 // Store the preparer for cleanup 169 mInstallers.put(installer, device); 170 installer.addTestFileName(options.getApkFileName()); 171 // If a userId is provided, grantPermission can be set for the apk installation. 172 if (options.getUserId() != null) { 173 installer.setUserId(options.getUserId()); 174 installer.setShouldGrantPermission(options.isGrantPermission()); 175 } 176 installer.setForceQueryable(options.isForceQueryable()); 177 installer.setAbi(getAbi()); 178 for (String option : options.getInstallArgs()) { 179 installer.addInstallArg(option); 180 } 181 try { 182 installer.setUp(mTestInfo); 183 } catch (BuildError e) { 184 // For some reason we forgot the BuildError part of the interface so it's hard to add 185 // it now 186 throw new TargetSetupError( 187 e.getMessage(), e, device.getDeviceDescriptor(), e.getErrorId()); 188 } 189 } 190 191 /** 192 * Install an apk given its name for a specific user. 193 * 194 * @param apkFileName The name of the apk file. 195 * @param grantPermission whether to pass the grant permission flag when installing the apk. 196 * @param userId the user id of the user where to install the apk. 197 * @param options extra options given to the install command 198 */ installPackageAsUser( String apkFileName, boolean grantPermission, int userId, String... options)199 public final void installPackageAsUser( 200 String apkFileName, boolean grantPermission, int userId, String... options) 201 throws DeviceNotAvailableException, TargetSetupError { 202 final DeviceTestRunOptions installOptions = new DeviceTestRunOptions(null /*unused*/); 203 installOptions.setApkFileName(apkFileName); 204 installOptions.setGrantPermission(grantPermission); 205 installOptions.setUserId(userId); 206 installOptions.setInstallArgs(options); 207 installPackage(installOptions); 208 } 209 210 /** 211 * Install an apk given its name for a specific user on a given device. 212 * 213 * @param device the {@link ITestDevice} on which to install the apk. 214 * @param apkFileName The name of the apk file. 215 * @param grantPermission whether to pass the grant permission flag when installing the apk. 216 * @param userId the user id of the user where to install the apk. 217 * @param options extra options given to the install command 218 */ installPackageAsUser( ITestDevice device, String apkFileName, boolean grantPermission, int userId, String... options)219 public final void installPackageAsUser( 220 ITestDevice device, 221 String apkFileName, 222 boolean grantPermission, 223 int userId, 224 String... options) 225 throws DeviceNotAvailableException, TargetSetupError { 226 final DeviceTestRunOptions installOptions = new DeviceTestRunOptions(null /*unused*/); 227 installOptions.setApkFileName(apkFileName); 228 installOptions.setDevice(device); 229 installOptions.setGrantPermission(grantPermission); 230 installOptions.setUserId(userId); 231 installOptions.setInstallArgs(options); 232 installPackage(installOptions); 233 } 234 235 /** 236 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 237 * right after to get the details of results. 238 * 239 * @param pkgName the name of the package to run. 240 * @param testClassName the name of the test class to run. 241 * @return True if it succeed without failure. False otherwise. 242 */ runDeviceTests(String pkgName, String testClassName)243 public final boolean runDeviceTests(String pkgName, String testClassName) 244 throws DeviceNotAvailableException { 245 return runDeviceTests(getDevice(), pkgName, testClassName, null); 246 } 247 248 /** 249 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 250 * right after to get the details of results. 251 * 252 * @param pkgName the name of the package to run. 253 * @param testClassName the name of the test class to run. 254 * @param testTimeoutMs the timeout in millisecond to be applied to each test case. 255 * @return True if it succeed without failure. False otherwise. 256 */ runDeviceTests(String pkgName, String testClassName, Long testTimeoutMs)257 public final boolean runDeviceTests(String pkgName, String testClassName, Long testTimeoutMs) 258 throws DeviceNotAvailableException { 259 return runDeviceTests( 260 getDevice(), 261 null, 262 pkgName, 263 testClassName, 264 null, 265 null, 266 testTimeoutMs, 267 DEFAULT_MAX_TIMEOUT_TO_OUTPUT_MS, 268 0L, 269 true, 270 false, 271 DEFAULT_INSTRUMENTATION_ARGS); 272 } 273 274 /** 275 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 276 * right after to get the details of results. 277 * 278 * @param pkgName the name of the package to run. 279 * @param testClassName the name of the test class to run. 280 * @param userId the id of the user to run the test against. can be null. 281 * @param testTimeoutMs the timeout in millisecond to be applied to each test case. 282 * @return True if it succeed without failure. False otherwise. 283 */ runDeviceTests( String pkgName, String testClassName, Integer userId, Long testTimeoutMs)284 public final boolean runDeviceTests( 285 String pkgName, String testClassName, Integer userId, Long testTimeoutMs) 286 throws DeviceNotAvailableException { 287 return runDeviceTests( 288 getDevice(), 289 null, 290 pkgName, 291 testClassName, 292 null, 293 userId, 294 testTimeoutMs, 295 DEFAULT_MAX_TIMEOUT_TO_OUTPUT_MS, 296 0L, 297 true, 298 false, 299 DEFAULT_INSTRUMENTATION_ARGS); 300 } 301 302 /** 303 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 304 * right after to get the details of results. 305 * 306 * @param pkgName the name of the package to run. 307 * @param testClassName the name of the test class to run. 308 * @param testMethodName the name of the test method in the class to be run. 309 * @return True if it succeed without failure. False otherwise. 310 */ runDeviceTests(String pkgName, String testClassName, String testMethodName)311 public final boolean runDeviceTests(String pkgName, String testClassName, String testMethodName) 312 throws DeviceNotAvailableException { 313 return runDeviceTests(getDevice(), pkgName, testClassName, testMethodName); 314 } 315 316 /** 317 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 318 * right after to get the details of results. 319 * 320 * @param runner the instrumentation runner to be used. 321 * @param pkgName the name of the package to run. 322 * @param testClassName the name of the test class to run. 323 * @param testMethodName the name of the test method in the class to be run. 324 * @return True if it succeed without failure. False otherwise. 325 */ runDeviceTests( String runner, String pkgName, String testClassName, String testMethodName)326 public final boolean runDeviceTests( 327 String runner, String pkgName, String testClassName, String testMethodName) 328 throws DeviceNotAvailableException { 329 return runDeviceTests( 330 getDevice(), 331 runner, 332 pkgName, 333 testClassName, 334 testMethodName, 335 null, 336 DEFAULT_TEST_TIMEOUT_MS, 337 DEFAULT_MAX_TIMEOUT_TO_OUTPUT_MS, 338 0L, 339 true, 340 false, 341 DEFAULT_INSTRUMENTATION_ARGS); 342 } 343 344 /** 345 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 346 * right after to get the details of results. 347 * 348 * @param device the device agaisnt which to run the instrumentation. 349 * @param pkgName the name of the package to run. 350 * @param testClassName the name of the test class to run. 351 * @param testMethodName the name of the test method in the class to be run. 352 * @return True if it succeed without failure. False otherwise. 353 */ runDeviceTests( ITestDevice device, String pkgName, String testClassName, String testMethodName)354 public final boolean runDeviceTests( 355 ITestDevice device, String pkgName, String testClassName, String testMethodName) 356 throws DeviceNotAvailableException { 357 return runDeviceTests( 358 device, 359 null, 360 pkgName, 361 testClassName, 362 testMethodName, 363 null, 364 DEFAULT_TEST_TIMEOUT_MS, 365 DEFAULT_MAX_TIMEOUT_TO_OUTPUT_MS, 366 0L, 367 true, 368 false, 369 DEFAULT_INSTRUMENTATION_ARGS); 370 } 371 372 /** 373 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 374 * right after to get the details of results. 375 * 376 * @param device the device agaisnt which to run the instrumentation. 377 * @param pkgName the name of the package to run. 378 * @param testClassName the name of the test class to run. 379 * @param testMethodName the name of the test method in the class to be run. 380 * @param testTimeoutMs the timeout in millisecond to be applied to each test case. 381 * @return True if it succeed without failure. False otherwise. 382 */ runDeviceTests( ITestDevice device, String pkgName, String testClassName, String testMethodName, Long testTimeoutMs)383 public final boolean runDeviceTests( 384 ITestDevice device, 385 String pkgName, 386 String testClassName, 387 String testMethodName, 388 Long testTimeoutMs) 389 throws DeviceNotAvailableException { 390 return runDeviceTests( 391 device, 392 null, 393 pkgName, 394 testClassName, 395 testMethodName, 396 null, 397 testTimeoutMs, 398 DEFAULT_MAX_TIMEOUT_TO_OUTPUT_MS, 399 0L, 400 true, 401 false, 402 DEFAULT_INSTRUMENTATION_ARGS); 403 } 404 405 /** 406 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 407 * right after to get the details of results. 408 * 409 * @param device the device agaisnt which to run the instrumentation. 410 * @param pkgName the name of the package to run. 411 * @param testClassName the name of the test class to run. 412 * @param testMethodName the name of the test method in the class to be run. 413 * @param userId the id of the user to run the test against. can be null. 414 * @param testTimeoutMs the timeout in millisecond to be applied to each test case. 415 * @return True if it succeed without failure. False otherwise. 416 */ runDeviceTests( ITestDevice device, String pkgName, String testClassName, String testMethodName, Integer userId, Long testTimeoutMs)417 public final boolean runDeviceTests( 418 ITestDevice device, 419 String pkgName, 420 String testClassName, 421 String testMethodName, 422 Integer userId, 423 Long testTimeoutMs) 424 throws DeviceNotAvailableException { 425 return runDeviceTests( 426 device, 427 null, 428 pkgName, 429 testClassName, 430 testMethodName, 431 userId, 432 testTimeoutMs, 433 DEFAULT_MAX_TIMEOUT_TO_OUTPUT_MS, 434 0L, 435 true, 436 false, 437 DEFAULT_INSTRUMENTATION_ARGS); 438 } 439 440 /** 441 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 442 * right after to get the details of results. 443 * 444 * @param device the device agaisnt which to run the instrumentation. 445 * @param pkgName the name of the package to run. 446 * @param testClassName the name of the test class to run. 447 * @param testMethodName the name of the test method in the class to be run. 448 * @param testTimeoutMs the timeout in millisecond to be applied to each test case. 449 * @param maxTimeToOutputMs the max timeout the test has to start outputting something. 450 * @param maxInstrumentationTimeoutMs the max timeout the full instrumentation has to complete. 451 * @return True if it succeed without failure. False otherwise. 452 */ runDeviceTests( ITestDevice device, String pkgName, String testClassName, String testMethodName, Long testTimeoutMs, Long maxTimeToOutputMs, Long maxInstrumentationTimeoutMs)453 public final boolean runDeviceTests( 454 ITestDevice device, 455 String pkgName, 456 String testClassName, 457 String testMethodName, 458 Long testTimeoutMs, 459 Long maxTimeToOutputMs, 460 Long maxInstrumentationTimeoutMs) 461 throws DeviceNotAvailableException { 462 return runDeviceTests( 463 device, 464 null, 465 pkgName, 466 testClassName, 467 testMethodName, 468 null, 469 testTimeoutMs, 470 maxTimeToOutputMs, 471 maxInstrumentationTimeoutMs, 472 true, 473 false, 474 DEFAULT_INSTRUMENTATION_ARGS); 475 } 476 477 /** 478 * Runs the instrumentation base on the information in {@link DeviceTestRunOptions}. 479 * 480 * @param options the {@link DeviceTestRunOptions} driving the instrumentation setup. 481 * @return True if it succeeded without failure. False otherwise. 482 * @throws DeviceNotAvailableException 483 */ runDeviceTests(DeviceTestRunOptions options)484 public final boolean runDeviceTests(DeviceTestRunOptions options) 485 throws DeviceNotAvailableException { 486 return runDeviceTests( 487 options.getDevice() == null ? getDevice() : options.getDevice(), 488 options.getRunner(), 489 options.getPackageName(), 490 options.getTestClassName(), 491 options.getTestMethodName(), 492 options.getUserId(), 493 options.getTestTimeoutMs(), 494 options.getMaxTimeToOutputMs(), 495 options.getMaxInstrumentationTimeoutMs(), 496 options.shouldCheckResults(), 497 options.isHiddenApiCheckDisabled(), 498 options.isTestApiCheckDisabled(), 499 options.isIsolatedStorageDisabled(), 500 options.isWindowAnimationDisabled(), 501 options.isRestartDisabled(), 502 options.getInstrumentationArgs(), 503 options.getExtraListeners()); 504 } 505 506 /** 507 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 508 * right after to get the details of results. 509 * 510 * @param device the device agaisnt which to run the instrumentation. 511 * @param pkgName the name of the package to run. 512 * @param testClassName the name of the test class to run. 513 * @param testMethodName the name of the test method in the class to be run. 514 * @param userId the id of the user to run the test against. can be null. 515 * @param testTimeoutMs the timeout in millisecond to be applied to each test case. 516 * @param maxTimeToOutputMs the max timeout the test has to start outputting something. 517 * @param maxInstrumentationTimeoutMs the max timeout the full instrumentation has to complete. 518 * @param checkResults whether or not the results are checked for crashes. 519 * @param isHiddenApiCheckDisabled whether or not we should disable the hidden api check. 520 * @param instrumentationArgs arguments to pass to the instrumentation. 521 * @return True if it succeeded without failure. False otherwise. 522 */ runDeviceTests( ITestDevice device, String runner, String pkgName, String testClassName, String testMethodName, Integer userId, Long testTimeoutMs, Long maxTimeToOutputMs, Long maxInstrumentationTimeoutMs, boolean checkResults, boolean isHiddenApiCheckDisabled, Map<String, String> instrumentationArgs)523 public final boolean runDeviceTests( 524 ITestDevice device, 525 String runner, 526 String pkgName, 527 String testClassName, 528 String testMethodName, 529 Integer userId, 530 Long testTimeoutMs, 531 Long maxTimeToOutputMs, 532 Long maxInstrumentationTimeoutMs, 533 boolean checkResults, 534 boolean isHiddenApiCheckDisabled, 535 Map<String, String> instrumentationArgs) 536 throws DeviceNotAvailableException { 537 return runDeviceTests( 538 device, 539 runner, 540 pkgName, 541 testClassName, 542 testMethodName, 543 userId, 544 testTimeoutMs, 545 maxTimeToOutputMs, 546 maxInstrumentationTimeoutMs, 547 checkResults, 548 isHiddenApiCheckDisabled, 549 true, 550 false, 551 instrumentationArgs, 552 new ArrayList<>()); 553 } 554 555 /** 556 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 557 * right after to get the details of results. 558 * 559 * @param device the device agaisnt which to run the instrumentation. 560 * @param pkgName the name of the package to run. 561 * @param testClassName the name of the test class to run. 562 * @param testMethodName the name of the test method in the class to be run. 563 * @param userId the id of the user to run the test against. can be null. 564 * @param testTimeoutMs the timeout in millisecond to be applied to each test case. 565 * @param maxTimeToOutputMs the max timeout the test has to start outputting something. 566 * @param maxInstrumentationTimeoutMs the max timeout the full instrumentation has to complete. 567 * @param checkResults whether or not the results are checked for crashes. 568 * @param isHiddenApiCheckDisabled whether or not we should disable the hidden api check. 569 * @param isTestApiCheckDisabled whether or not we should disable the test api check. 570 * @param isIsolatedStorageDisabled whether or not we should disable isolated storage. 571 * @param instrumentationArgs arguments to pass to the instrumentation. 572 * @return True if it succeeded without failure. False otherwise. 573 */ runDeviceTests( ITestDevice device, String runner, String pkgName, String testClassName, String testMethodName, Integer userId, Long testTimeoutMs, Long maxTimeToOutputMs, Long maxInstrumentationTimeoutMs, boolean checkResults, boolean isHiddenApiCheckDisabled, boolean isTestApiCheckDisabled, boolean isIsolatedStorageDisabled, Map<String, String> instrumentationArgs, List<ITestLifeCycleReceiver> extraListeners)574 public final boolean runDeviceTests( 575 ITestDevice device, 576 String runner, 577 String pkgName, 578 String testClassName, 579 String testMethodName, 580 Integer userId, 581 Long testTimeoutMs, 582 Long maxTimeToOutputMs, 583 Long maxInstrumentationTimeoutMs, 584 boolean checkResults, 585 boolean isHiddenApiCheckDisabled, 586 boolean isTestApiCheckDisabled, 587 boolean isIsolatedStorageDisabled, 588 Map<String, String> instrumentationArgs, 589 List<ITestLifeCycleReceiver> extraListeners) 590 throws DeviceNotAvailableException { 591 return runDeviceTests( 592 device, 593 runner, 594 pkgName, 595 testClassName, 596 testMethodName, 597 userId, 598 testTimeoutMs, 599 maxTimeToOutputMs, 600 maxInstrumentationTimeoutMs, 601 checkResults, 602 isHiddenApiCheckDisabled, 603 isTestApiCheckDisabled, 604 isIsolatedStorageDisabled, 605 false, // leave window animations enabled for existing invocations 606 instrumentationArgs, 607 extraListeners); 608 } 609 610 /** 611 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 612 * right after to get the details of results. 613 * 614 * @param device the device agaisnt which to run the instrumentation. 615 * @param pkgName the name of the package to run. 616 * @param testClassName the name of the test class to run. 617 * @param testMethodName the name of the test method in the class to be run. 618 * @param userId the id of the user to run the test against. can be null. 619 * @param testTimeoutMs the timeout in millisecond to be applied to each test case. 620 * @param maxTimeToOutputMs the max timeout the test has to start outputting something. 621 * @param maxInstrumentationTimeoutMs the max timeout the full instrumentation has to complete. 622 * @param checkResults whether or not the results are checked for crashes. 623 * @param isHiddenApiCheckDisabled whether or not we should disable the hidden api check. 624 * @param isTestApiCheckDisabled whether or not we should disable the test api check. 625 * @param isIsolatedStorageDisabled whether or not we should disable isolated storage. 626 * @param isWindowAnimationDisabled whether or not we should disable window animation. 627 * @param instrumentationArgs arguments to pass to the instrumentation. 628 * @return True if it succeeded without failure. False otherwise. 629 */ runDeviceTests( ITestDevice device, String runner, String pkgName, String testClassName, String testMethodName, Integer userId, Long testTimeoutMs, Long maxTimeToOutputMs, Long maxInstrumentationTimeoutMs, boolean checkResults, boolean isHiddenApiCheckDisabled, boolean isTestApiCheckDisabled, boolean isIsolatedStorageDisabled, boolean isWindowAnimationDisabled, Map<String, String> instrumentationArgs, List<ITestLifeCycleReceiver> extraListeners)630 public final boolean runDeviceTests( 631 ITestDevice device, 632 String runner, 633 String pkgName, 634 String testClassName, 635 String testMethodName, 636 Integer userId, 637 Long testTimeoutMs, 638 Long maxTimeToOutputMs, 639 Long maxInstrumentationTimeoutMs, 640 boolean checkResults, 641 boolean isHiddenApiCheckDisabled, 642 boolean isTestApiCheckDisabled, 643 boolean isIsolatedStorageDisabled, 644 boolean isWindowAnimationDisabled, 645 Map<String, String> instrumentationArgs, 646 List<ITestLifeCycleReceiver> extraListeners) 647 throws DeviceNotAvailableException { 648 return runDeviceTests( 649 device, 650 runner, 651 pkgName, 652 testClassName, 653 testMethodName, 654 userId, 655 testTimeoutMs, 656 maxTimeToOutputMs, 657 maxInstrumentationTimeoutMs, 658 checkResults, 659 isHiddenApiCheckDisabled, 660 isTestApiCheckDisabled, 661 isIsolatedStorageDisabled, 662 isWindowAnimationDisabled, 663 false, // leave restart enabled 664 instrumentationArgs, 665 extraListeners); 666 } 667 668 /** 669 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 670 * right after to get the details of results. 671 * 672 * @param device the device agaisnt which to run the instrumentation. 673 * @param pkgName the name of the package to run. 674 * @param testClassName the name of the test class to run. 675 * @param testMethodName the name of the test method in the class to be run. 676 * @param userId the id of the user to run the test against. can be null. 677 * @param testTimeoutMs the timeout in millisecond to be applied to each test case. 678 * @param maxTimeToOutputMs the max timeout the test has to start outputting something. 679 * @param maxInstrumentationTimeoutMs the max timeout the full instrumentation has to complete. 680 * @param checkResults whether or not the results are checked for crashes. 681 * @param isHiddenApiCheckDisabled whether or not we should disable the hidden api check. 682 * @param isTestApiCheckDisabled whether or not we should disable the test api check. 683 * @param isIsolatedStorageDisabled whether or not we should disable isolated storage. 684 * @param isWindowAnimationDisabled whether or not we should disable window animation. 685 * @param instrumentationArgs arguments to pass to the instrumentation. 686 * @return True if it succeeded without failure. False otherwise. 687 */ runDeviceTests( ITestDevice device, String runner, String pkgName, String testClassName, String testMethodName, Integer userId, Long testTimeoutMs, Long maxTimeToOutputMs, Long maxInstrumentationTimeoutMs, boolean checkResults, boolean isHiddenApiCheckDisabled, boolean isTestApiCheckDisabled, boolean isIsolatedStorageDisabled, boolean isWindowAnimationDisabled, boolean isRestartDisabled, Map<String, String> instrumentationArgs, List<ITestLifeCycleReceiver> extraListeners)688 public final boolean runDeviceTests( 689 ITestDevice device, 690 String runner, 691 String pkgName, 692 String testClassName, 693 String testMethodName, 694 Integer userId, 695 Long testTimeoutMs, 696 Long maxTimeToOutputMs, 697 Long maxInstrumentationTimeoutMs, 698 boolean checkResults, 699 boolean isHiddenApiCheckDisabled, 700 boolean isTestApiCheckDisabled, 701 boolean isIsolatedStorageDisabled, 702 boolean isWindowAnimationDisabled, 703 boolean isRestartDisabled, 704 Map<String, String> instrumentationArgs, 705 List<ITestLifeCycleReceiver> extraListeners) 706 throws DeviceNotAvailableException { 707 TestRunResult runResult = 708 doRunTests( 709 device, 710 runner, 711 pkgName, 712 testClassName, 713 testMethodName, 714 userId, 715 testTimeoutMs, 716 maxTimeToOutputMs, 717 maxInstrumentationTimeoutMs, 718 isHiddenApiCheckDisabled, 719 isTestApiCheckDisabled, 720 isIsolatedStorageDisabled, 721 isWindowAnimationDisabled, 722 isRestartDisabled, 723 instrumentationArgs, 724 extraListeners); 725 mLatestInstruRes = runResult; 726 printTestResult(runResult); 727 if (checkResults) { 728 if (runResult.isRunFailure()) { 729 throw new AssertionError( 730 "Failed to successfully run device tests for " 731 + runResult.getName() 732 + ": " 733 + runResult.getRunFailureMessage()); 734 } 735 if (runResult.getNumTests() == 0) { 736 throw new AssertionError("No tests were run on the device"); 737 } 738 if (runResult.hasFailedTests()) { 739 // build a meaningful error message 740 StringBuilder errorBuilder = new StringBuilder("on-device tests failed:\n"); 741 for (Map.Entry<TestDescription, TestResult> resultEntry : 742 runResult.getTestResults().entrySet()) { 743 if (!TestStatus.PASSED.equals(resultEntry.getValue().getResultStatus())) { 744 errorBuilder.append(resultEntry.getKey().toString()); 745 errorBuilder.append(":\n"); 746 errorBuilder.append(resultEntry.getValue().getStackTrace()); 747 } 748 } 749 throw new AssertionError(errorBuilder.toString()); 750 } 751 // Assume not all tests have skipped (and rethrow AssumptionViolatedException if so) 752 List<TestResult> assumpFail = 753 runResult.getTestsResultsInState(TestStatus.ASSUMPTION_FAILURE); 754 List<String> messages = 755 assumpFail.stream().map(r -> r.getStackTrace()).collect(Collectors.toList()); 756 String errors = String.join("\n\n", messages); 757 Assume.assumeTrue( 758 errors, 759 runResult.getNumTests() 760 != runResult.getNumTestsInState(TestStatus.ASSUMPTION_FAILURE)); 761 } 762 return !runResult.hasFailedTests() && runResult.getNumTests() > 0; 763 } 764 765 /** 766 * Returns the {@link TestRunResult} resulting from the latest runDeviceTests that ran. Or null 767 * if no results available. 768 */ getLastDeviceRunResults()769 public final TestRunResult getLastDeviceRunResults() { 770 return mLatestInstruRes; 771 } 772 773 /** Helper method to run tests and return the listener that collected the results. */ doRunTests( ITestDevice device, String runner, String pkgName, String testClassName, String testMethodName, Integer userId, Long testTimeoutMs, Long maxTimeToOutputMs, Long maxInstrumentationTimeoutMs, boolean isHiddenApiCheckDisabled, boolean isTestApiCheckDisabled, boolean isIsolatedStorageDisabled, boolean isWindowAnimationDisabled, boolean isRestartDisabled, Map<String, String> instrumentationArgs, List<ITestLifeCycleReceiver> extraListeners)774 private TestRunResult doRunTests( 775 ITestDevice device, 776 String runner, 777 String pkgName, 778 String testClassName, 779 String testMethodName, 780 Integer userId, 781 Long testTimeoutMs, 782 Long maxTimeToOutputMs, 783 Long maxInstrumentationTimeoutMs, 784 boolean isHiddenApiCheckDisabled, 785 boolean isTestApiCheckDisabled, 786 boolean isIsolatedStorageDisabled, 787 boolean isWindowAnimationDisabled, 788 boolean isRestartDisabled, 789 Map<String, String> instrumentationArgs, 790 List<ITestLifeCycleReceiver> extraListeners) 791 throws DeviceNotAvailableException { 792 RemoteAndroidTestRunner testRunner = createTestRunner(pkgName, runner, device); 793 String runOptions = ""; 794 // hidden-api-checks flag only exists in P and after. 795 // Using a temp variable to consolidate the dynamic checks 796 int apiLevel = isHiddenApiCheckDisabled || isWindowAnimationDisabled 797 ? device.getApiLevel() : 0; 798 if (isHiddenApiCheckDisabled && (apiLevel >= 28)) { 799 runOptions += "--no-hidden-api-checks "; 800 } 801 if (!isHiddenApiCheckDisabled 802 && !isTestApiCheckDisabled 803 && device.checkApiLevelAgainstNextRelease(30)) { 804 runOptions += "--no-test-api-access "; 805 } 806 // isolated-storage flag only exists in Q and after. 807 if (isIsolatedStorageDisabled && device.checkApiLevelAgainstNextRelease(29)) { 808 runOptions += "--no-isolated-storage "; 809 } 810 // window-animation flag only exists in ICS and after 811 if (isWindowAnimationDisabled && apiLevel >= 14) { 812 runOptions += "--no-window-animation "; 813 } 814 // restart flag only exists in S and after. 815 if (isRestartDisabled && device.checkApiLevelAgainstNextRelease(31)) { 816 runOptions += "--no-restart "; 817 } 818 819 if (getAbi() != null) { 820 runOptions += String.format("--abi %s", getAbi().getName()); 821 } 822 // Set the run options if any. 823 if (!runOptions.isEmpty()) { 824 testRunner.setRunOptions(runOptions); 825 } 826 827 if (testClassName != null && testMethodName != null) { 828 testRunner.setMethodName(testClassName, testMethodName); 829 } else if (testClassName != null) { 830 testRunner.setClassName(testClassName); 831 } 832 if (testTimeoutMs != null) { 833 testRunner.addInstrumentationArg("timeout_msec", Long.toString(testTimeoutMs)); 834 } else { 835 testTimeoutMs = DEFAULT_TEST_TIMEOUT_MS; 836 testRunner.addInstrumentationArg( 837 "timeout_msec", Long.toString(DEFAULT_TEST_TIMEOUT_MS)); 838 } 839 if (maxTimeToOutputMs != null && maxTimeToOutputMs < testTimeoutMs) { 840 // Similar logic as InstrumentationTest 841 maxTimeToOutputMs = testTimeoutMs + testTimeoutMs / 10; 842 CLog.w( 843 String.format( 844 "maxTimeToOutputMs should be larger than testtimeout %d; NOTE:" 845 + " extending maxTimeToOutputMs to %d, please consider fixing" 846 + " this!", 847 testTimeoutMs, maxTimeToOutputMs)); 848 } 849 if (maxTimeToOutputMs != null) { 850 testRunner.setMaxTimeToOutputResponse(maxTimeToOutputMs, TimeUnit.MILLISECONDS); 851 } 852 if (maxInstrumentationTimeoutMs != null) { 853 testRunner.setMaxTimeout(maxInstrumentationTimeoutMs, TimeUnit.MILLISECONDS); 854 } 855 // Pass all the instrumentation arguments 856 for (String key : instrumentationArgs.keySet()) { 857 testRunner.addInstrumentationArg(key, instrumentationArgs.get(key)); 858 } 859 860 CollectingTestListener listener = createListener(); 861 List<ITestLifeCycleReceiver> allReceiver = new ArrayList<>(); 862 allReceiver.add(listener); 863 allReceiver.addAll(extraListeners); 864 if (userId == null) { 865 assertTrue(device.runInstrumentationTests(testRunner, allReceiver)); 866 } else { 867 // Ensure userId is correct before starting instrumentation 868 assertTrue(userId >= 0); 869 assertTrue(device.runInstrumentationTestsAsUser(testRunner, userId, allReceiver)); 870 } 871 return listener.getCurrentRunResults(); 872 } 873 874 @VisibleForTesting createTestRunner( String packageName, String runnerName, ITestDevice device)875 RemoteAndroidTestRunner createTestRunner( 876 String packageName, String runnerName, ITestDevice device) 877 throws DeviceNotAvailableException { 878 if (runnerName == null) { 879 ListInstrumentationParser parser = getListInstrumentationParser(); 880 device.executeShellCommand("pm list instrumentation", parser); 881 for (InstrumentationTarget target : parser.getInstrumentationTargets()) { 882 if (packageName.equals(target.packageName)) { 883 runnerName = target.runnerName; 884 } 885 } 886 } 887 // If the runner name is still null 888 if (runnerName == null) { 889 throw new RuntimeException("No runner was defined and couldn't dynamically find one."); 890 } 891 return new DefaultRemoteAndroidTestRunner(packageName, runnerName, device.getIDevice()); 892 } 893 894 @VisibleForTesting getListInstrumentationParser()895 ListInstrumentationParser getListInstrumentationParser() { 896 return new ListInstrumentationParser(); 897 } 898 899 @VisibleForTesting createListener()900 CollectingTestListener createListener() { 901 return new CollectingTestListener(); 902 } 903 printTestResult(TestRunResult runResult)904 private void printTestResult(TestRunResult runResult) { 905 for (Map.Entry<TestDescription, TestResult> testEntry : 906 runResult.getTestResults().entrySet()) { 907 TestResult testResult = testEntry.getValue(); 908 TestStatus testStatus = testResult.getResultStatus(); 909 CLog.logAndDisplay(LogLevel.INFO, "Test " + testEntry.getKey() + ": " + testStatus); 910 if (!TestStatus.PASSED.equals(testStatus) 911 && !TestStatus.ASSUMPTION_FAILURE.equals(testStatus)) { 912 CLog.logAndDisplay(LogLevel.WARN, testResult.getStackTrace()); 913 } 914 } 915 } 916 917 /** 918 * Uninstalls a package on the device. 919 * 920 * @param pkgName the Android package to uninstall 921 * @return a {@link String} with an error code, or <code>null</code> if success 922 */ uninstallPackage(String pkgName)923 public final String uninstallPackage(String pkgName) throws DeviceNotAvailableException { 924 return getDevice().uninstallPackage(pkgName); 925 } 926 927 /** 928 * Uninstalls a package on the device 929 * 930 * @param device the device that should uninstall the package. 931 * @param pkgName the Android package to uninstall 932 * @return a {@link String} with an error code, or <code>null</code> if success 933 */ uninstallPackage(ITestDevice device, String pkgName)934 public final String uninstallPackage(ITestDevice device, String pkgName) 935 throws DeviceNotAvailableException { 936 return device.uninstallPackage(pkgName); 937 } 938 939 /** 940 * Checks if a package of a given name is installed on the device 941 * 942 * @param pkg the name of the package 943 * @return true if the package is found on the device 944 */ isPackageInstalled(String pkg)945 public final boolean isPackageInstalled(String pkg) throws DeviceNotAvailableException { 946 return isPackageInstalled(getDevice(), pkg); 947 } 948 949 /** 950 * Checks if a package of a given name is installed on the device 951 * 952 * @param device the device that should uninstall the package. 953 * @param pkg the name of the package 954 * @return true if the package is found on the device 955 */ isPackageInstalled(ITestDevice device, String pkg)956 public final boolean isPackageInstalled(ITestDevice device, String pkg) 957 throws DeviceNotAvailableException { 958 for (String installedPackage : device.getInstalledPackageNames()) { 959 if (pkg.equals(installedPackage)) { 960 return true; 961 } 962 } 963 return false; 964 } 965 hasDeviceFeature(String feature)966 public boolean hasDeviceFeature(String feature) throws DeviceNotAvailableException { 967 return getDevice().hasFeature("feature:" + feature); 968 } 969 970 @VisibleForTesting createSuiteApkInstaller()971 SuiteApkInstaller createSuiteApkInstaller() { 972 return new SuiteApkInstaller(); 973 } 974 } 975