1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.csuite.core; 18 19 import com.android.csuite.core.DeviceUtils.DeviceTimestamp; 20 import com.android.csuite.core.TestUtils.TestUtilsException; 21 import com.android.tradefed.device.DeviceNotAvailableException; 22 import com.android.tradefed.invoker.TestInformation; 23 import com.android.tradefed.log.LogUtil.CLog; 24 import com.android.tradefed.result.LogDataType; 25 import com.android.tradefed.testtype.DeviceJUnit4ClassRunner.TestLogData; 26 import com.android.tradefed.util.CommandResult; 27 import com.android.tradefed.util.CommandStatus; 28 import com.android.tradefed.util.IRunUtil; 29 import com.android.tradefed.util.RunUtil; 30 import com.android.tradefed.util.ZipUtil; 31 32 import com.google.common.annotations.VisibleForTesting; 33 import com.google.common.base.Preconditions; 34 import com.google.common.io.MoreFiles; 35 36 import org.junit.Assert; 37 38 import java.io.File; 39 import java.io.IOException; 40 import java.nio.file.FileSystem; 41 import java.nio.file.FileSystems; 42 import java.nio.file.Files; 43 import java.nio.file.Path; 44 import java.util.ArrayList; 45 import java.util.Arrays; 46 import java.util.List; 47 import java.util.concurrent.atomic.AtomicReference; 48 import java.util.stream.Collectors; 49 import java.util.stream.Stream; 50 51 import javax.annotation.Nullable; 52 53 /** A tester that interact with an app crawler during testing. */ 54 public final class AppCrawlTester { 55 @VisibleForTesting Path mOutput; 56 private final RunUtilProvider mRunUtilProvider; 57 private final TestUtils mTestUtils; 58 private final String mPackageName; 59 private boolean mRecordScreen = false; 60 private boolean mCollectGmsVersion = false; 61 private boolean mCollectAppVersion = false; 62 private boolean mUiAutomatorMode = false; 63 private int mTimeoutSec; 64 private String mCrawlControllerEndpoint; 65 private Path mApkRoot; 66 private Path mRoboscriptFile; 67 private Path mCrawlGuidanceProtoFile; 68 private Path mLoginConfigDir; 69 private FileSystem mFileSystem; 70 71 /** 72 * Creates an {@link AppCrawlTester} instance. 73 * 74 * @param packageName The package name of the apk files. 75 * @param testInformation The TradeFed test information. 76 * @param testLogData The TradeFed test output receiver. 77 * @return an {@link AppCrawlTester} instance. 78 */ newInstance( String packageName, TestInformation testInformation, TestLogData testLogData)79 public static AppCrawlTester newInstance( 80 String packageName, TestInformation testInformation, TestLogData testLogData) { 81 return new AppCrawlTester( 82 packageName, 83 TestUtils.getInstance(testInformation, testLogData), 84 () -> new RunUtil(), 85 FileSystems.getDefault()); 86 } 87 88 @VisibleForTesting AppCrawlTester( String packageName, TestUtils testUtils, RunUtilProvider runUtilProvider, FileSystem fileSystem)89 AppCrawlTester( 90 String packageName, 91 TestUtils testUtils, 92 RunUtilProvider runUtilProvider, 93 FileSystem fileSystem) { 94 mRunUtilProvider = runUtilProvider; 95 mPackageName = packageName; 96 mTestUtils = testUtils; 97 mFileSystem = fileSystem; 98 } 99 100 /** An exception class representing crawler test failures. */ 101 public static final class CrawlerException extends Exception { 102 /** 103 * Constructs a new {@link CrawlerException} with a meaningful error message. 104 * 105 * @param message A error message describing the cause of the error. 106 */ CrawlerException(String message)107 private CrawlerException(String message) { 108 super(message); 109 } 110 111 /** 112 * Constructs a new {@link CrawlerException} with a meaningful error message, and a cause. 113 * 114 * @param message A detailed error message. 115 * @param cause A {@link Throwable} capturing the original cause of the CrawlerException. 116 */ CrawlerException(String message, Throwable cause)117 private CrawlerException(String message, Throwable cause) { 118 super(message, cause); 119 } 120 121 /** 122 * Constructs a new {@link CrawlerException} with a cause. 123 * 124 * @param cause A {@link Throwable} capturing the original cause of the CrawlerException. 125 */ CrawlerException(Throwable cause)126 private CrawlerException(Throwable cause) { 127 super(cause); 128 } 129 } 130 131 /** 132 * Starts crawling the app and throw AssertionError if app crash is detected. 133 * 134 * @throws DeviceNotAvailableException When device because unavailable. 135 */ startAndAssertAppNoCrash()136 public void startAndAssertAppNoCrash() throws DeviceNotAvailableException { 137 DeviceTimestamp startTime = mTestUtils.getDeviceUtils().currentTimeMillis(); 138 139 CrawlerException crawlerException = null; 140 try { 141 start(); 142 } catch (CrawlerException e) { 143 crawlerException = e; 144 } 145 146 ArrayList<String> failureMessages = new ArrayList<>(); 147 148 try { 149 String dropboxCrashLog = 150 mTestUtils.getDropboxPackageCrashLog(mPackageName, startTime, true); 151 if (dropboxCrashLog != null) { 152 // Put dropbox crash log on the top of the failure messages. 153 failureMessages.add(dropboxCrashLog); 154 } 155 } catch (IOException e) { 156 failureMessages.add("Error while getting dropbox crash log: " + e.getMessage()); 157 } 158 159 if (crawlerException != null) { 160 failureMessages.add(crawlerException.getMessage()); 161 } 162 163 Assert.assertTrue( 164 String.join( 165 "\n============\n", 166 failureMessages.toArray(new String[failureMessages.size()])), 167 failureMessages.isEmpty()); 168 } 169 170 /** 171 * Starts a crawler run on the configured app. 172 * 173 * @throws CrawlerException When the crawler was not set up correctly or the crawler run command 174 * failed. 175 * @throws DeviceNotAvailableException When device because unavailable. 176 */ start()177 public void start() throws CrawlerException, DeviceNotAvailableException { 178 if (!AppCrawlTesterHostPreparer.isReady(mTestUtils.getTestInformation())) { 179 throw new CrawlerException( 180 "The " 181 + AppCrawlTesterHostPreparer.class.getName() 182 + " is not ready. Please check whether " 183 + AppCrawlTesterHostPreparer.class.getName() 184 + " was included in the test plan and completed successfully."); 185 } 186 187 if (mOutput != null) { 188 throw new CrawlerException( 189 "The crawler has already run. Multiple runs in the same " 190 + AppCrawlTester.class.getName() 191 + " instance are not supported."); 192 } 193 194 try { 195 mOutput = Files.createTempDirectory("crawler"); 196 } catch (IOException e) { 197 throw new CrawlerException("Failed to create temp directory for output.", e); 198 } 199 200 IRunUtil runUtil = mRunUtilProvider.get(); 201 AtomicReference<String[]> command = new AtomicReference<>(); 202 AtomicReference<CommandResult> commandResult = new AtomicReference<>(); 203 204 CLog.d("Start to crawl package: %s.", mPackageName); 205 206 Path bin = 207 mFileSystem.getPath( 208 AppCrawlTesterHostPreparer.getCrawlerBinPath( 209 mTestUtils.getTestInformation())); 210 boolean isUtpClient = false; 211 if (Files.exists(bin.resolve("utp-cli-android_deploy.jar"))) { 212 command.set(createUtpCrawlerRunCommand(mTestUtils.getTestInformation())); 213 runUtil.setEnvVariable( 214 "ANDROID_SDK", 215 AppCrawlTesterHostPreparer.getSdkPath(mTestUtils.getTestInformation()) 216 .toString()); 217 isUtpClient = true; 218 } else if (Files.exists(bin.resolve("crawl_launcher_deploy.jar"))) { 219 command.set(createCrawlerRunCommand(mTestUtils.getTestInformation())); 220 runUtil.setEnvVariable( 221 "GOOGLE_APPLICATION_CREDENTIALS", 222 AppCrawlTesterHostPreparer.getCredentialPath(mTestUtils.getTestInformation()) 223 .toString()); 224 } else { 225 throw new CrawlerException( 226 "Crawler executable binaries not found in " + bin.toString()); 227 } 228 229 if (mCollectGmsVersion) { 230 mTestUtils.collectGmsVersion(mPackageName); 231 } 232 233 // Minimum timeout 3 minutes plus crawl test timeout. 234 long commandTimeout = 3 * 60 * 1000 + mTimeoutSec * 1000; 235 236 // TODO(yuexima): When the obb_file option is supported in espresso mode, the timeout need 237 // to be extended. 238 if (mRecordScreen) { 239 mTestUtils.collectScreenRecord( 240 () -> { 241 commandResult.set(runUtil.runTimedCmd(commandTimeout, command.get())); 242 }, 243 mPackageName); 244 } else { 245 commandResult.set(runUtil.runTimedCmd(commandTimeout, command.get())); 246 } 247 248 // Must be done after the crawler run because the app is installed by the crawler. 249 if (mCollectAppVersion) { 250 mTestUtils.collectAppVersion(mPackageName); 251 } 252 253 collectOutputZip(); 254 collectCrawlStepScreenshots(isUtpClient); 255 256 if (!commandResult.get().getStatus().equals(CommandStatus.SUCCESS) 257 || commandResult.get().getStdout().contains("Unknown options:")) { 258 throw new CrawlerException("Crawler command failed: " + commandResult.get()); 259 } 260 261 CLog.i("Completed crawling the package %s. Outputs: %s", mPackageName, commandResult.get()); 262 } 263 264 /** Copys the step screenshots into test outputs for easier access. */ collectCrawlStepScreenshots(boolean isUtpClient)265 private void collectCrawlStepScreenshots(boolean isUtpClient) { 266 if (mOutput == null) { 267 CLog.e("Output directory is not created yet. Skipping collecting step screenshots."); 268 return; 269 } 270 271 Path subDir = 272 isUtpClient 273 ? mOutput.resolve("output").resolve("artifacts") 274 : mOutput.resolve("app_firebase_test_lab"); 275 if (!Files.exists(subDir)) { 276 CLog.e( 277 "The crawler output directory is not complete, skipping collecting step" 278 + " screenshots."); 279 return; 280 } 281 282 try (Stream<Path> files = Files.list(subDir)) { 283 files.filter(path -> path.getFileName().toString().toLowerCase().endsWith(".png")) 284 .forEach( 285 path -> { 286 mTestUtils 287 .getTestArtifactReceiver() 288 .addTestArtifact( 289 mPackageName 290 + "-crawl_step_screenshot_" 291 + path.getFileName(), 292 LogDataType.PNG, 293 path.toFile()); 294 }); 295 } catch (IOException e) { 296 CLog.e(e); 297 } 298 } 299 300 /** Puts the zipped crawler output files into test output. */ collectOutputZip()301 private void collectOutputZip() { 302 if (mOutput == null) { 303 CLog.e("Output directory is not created yet. Skipping collecting output."); 304 return; 305 } 306 307 // Compress the crawler output directory and add it to test outputs. 308 try { 309 File outputZip = ZipUtil.createZip(mOutput.toFile()); 310 mTestUtils 311 .getTestArtifactReceiver() 312 .addTestArtifact(mPackageName + "-crawler_output", LogDataType.ZIP, outputZip); 313 } catch (IOException e) { 314 CLog.e("Failed to zip the output directory: " + e); 315 } 316 } 317 318 @VisibleForTesting createUtpCrawlerRunCommand(TestInformation testInfo)319 String[] createUtpCrawlerRunCommand(TestInformation testInfo) throws CrawlerException { 320 321 Path bin = 322 mFileSystem.getPath( 323 AppCrawlTesterHostPreparer.getCrawlerBinPath( 324 mTestUtils.getTestInformation())); 325 ArrayList<String> cmd = new ArrayList<>(); 326 cmd.addAll( 327 Arrays.asList( 328 "java", 329 "-jar", 330 bin.resolve("utp-cli-android_deploy.jar").toString(), 331 "android", 332 "robo", 333 "--device-id", 334 testInfo.getDevice().getSerialNumber(), 335 "--app-id", 336 mPackageName, 337 "--controller-endpoint", 338 "PROD", 339 "--utp-binaries-dir", 340 bin.toString(), 341 "--key-file", 342 AppCrawlTesterHostPreparer.getCredentialPath( 343 mTestUtils.getTestInformation()) 344 .toString(), 345 "--base-crawler-apk", 346 bin.resolve("crawler_app.apk").toString(), 347 "--stub-crawler-apk", 348 bin.resolve("crawler_stubapp_androidx.apk").toString(), 349 "--tmp-dir", 350 mOutput.toString())); 351 352 if (mTimeoutSec > 0) { 353 cmd.add("--crawler-flag"); 354 cmd.add("crawlDurationSec=" + Integer.toString(mTimeoutSec)); 355 } 356 357 if (mUiAutomatorMode) { 358 cmd.addAll(Arrays.asList("--ui-automator-mode", "--app-installed-on-device")); 359 } else { 360 Preconditions.checkNotNull( 361 mApkRoot, "Apk file path is required when not running in UIAutomator mode"); 362 363 List<Path> apks; 364 try { 365 apks = 366 TestUtils.listApks(mApkRoot).stream() 367 .filter( 368 path -> 369 path.getFileName() 370 .toString() 371 .toLowerCase() 372 .endsWith(".apk")) 373 .collect(Collectors.toList()); 374 } catch (TestUtilsException e) { 375 throw new CrawlerException(e); 376 } 377 378 cmd.add("--apks-to-crawl"); 379 cmd.add(apks.stream().map(Path::toString).collect(Collectors.joining(","))); 380 } 381 382 if (mRoboscriptFile != null) { 383 Assert.assertTrue( 384 "Please provide a valid roboscript file.", 385 Files.isRegularFile(mRoboscriptFile)); 386 cmd.add("--crawler-asset"); 387 cmd.add("robo.script=" + mRoboscriptFile.toString()); 388 } 389 390 if (mCrawlGuidanceProtoFile != null) { 391 Assert.assertTrue( 392 "Please provide a valid CrawlGuidance file.", 393 Files.isRegularFile(mCrawlGuidanceProtoFile)); 394 cmd.add("--crawl-guidance-proto-path"); 395 cmd.add(mCrawlGuidanceProtoFile.toString()); 396 } 397 398 if (mLoginConfigDir != null) { 399 RoboLoginConfigProvider configProvider = new RoboLoginConfigProvider(mLoginConfigDir); 400 cmd.addAll(configProvider.findConfigFor(mPackageName, true).getLoginArgs()); 401 } 402 403 return cmd.toArray(new String[cmd.size()]); 404 } 405 406 @VisibleForTesting createCrawlerRunCommand(TestInformation testInfo)407 String[] createCrawlerRunCommand(TestInformation testInfo) throws CrawlerException { 408 409 Path bin = 410 mFileSystem.getPath( 411 AppCrawlTesterHostPreparer.getCrawlerBinPath( 412 mTestUtils.getTestInformation())); 413 ArrayList<String> cmd = new ArrayList<>(); 414 cmd.addAll( 415 Arrays.asList( 416 "java", 417 "-jar", 418 bin.resolve("crawl_launcher_deploy.jar").toString(), 419 "--android-sdk-path", 420 AppCrawlTesterHostPreparer.getSdkPath(testInfo).toString(), 421 "--device-serial-code", 422 testInfo.getDevice().getSerialNumber(), 423 "--output-dir", 424 mOutput.toString(), 425 "--key-store-file", 426 // Using the publicly known default file name of the debug keystore. 427 bin.resolve("debug.keystore").toString(), 428 "--key-store-password", 429 // Using the publicly known default password of the debug keystore. 430 "android")); 431 432 if (mCrawlControllerEndpoint != null && mCrawlControllerEndpoint.length() > 0) { 433 cmd.addAll(Arrays.asList("--endpoint", mCrawlControllerEndpoint)); 434 } 435 436 if (mUiAutomatorMode) { 437 cmd.addAll(Arrays.asList("--ui-automator-mode", "--app-package-name", mPackageName)); 438 } else { 439 Preconditions.checkNotNull( 440 mApkRoot, "Apk file path is required when not running in UIAutomator mode"); 441 442 List<Path> apks; 443 try { 444 apks = 445 TestUtils.listApks(mApkRoot).stream() 446 .filter( 447 path -> 448 path.getFileName() 449 .toString() 450 .toLowerCase() 451 .endsWith(".apk")) 452 .collect(Collectors.toList()); 453 } catch (TestUtilsException e) { 454 throw new CrawlerException(e); 455 } 456 457 cmd.add("--apk-file"); 458 cmd.add(apks.get(0).toString()); 459 460 for (int i = 1; i < apks.size(); i++) { 461 cmd.add("--split-apk-files"); 462 cmd.add(apks.get(i).toString()); 463 } 464 } 465 466 if (mTimeoutSec > 0) { 467 cmd.add("--timeout-sec"); 468 cmd.add(Integer.toString(mTimeoutSec)); 469 } 470 471 if (mRoboscriptFile != null) { 472 Assert.assertTrue( 473 "Please provide a valid roboscript file.", 474 Files.isRegularFile(mRoboscriptFile)); 475 cmd.addAll(Arrays.asList("--robo-script-file", mRoboscriptFile.toString())); 476 } 477 478 if (mCrawlGuidanceProtoFile != null) { 479 Assert.assertTrue( 480 "Please provide a valid CrawlGuidance file.", 481 Files.isRegularFile(mCrawlGuidanceProtoFile)); 482 cmd.addAll(Arrays.asList("--text-guide-file", mCrawlGuidanceProtoFile.toString())); 483 } 484 485 if (mLoginConfigDir != null) { 486 RoboLoginConfigProvider configProvider = new RoboLoginConfigProvider(mLoginConfigDir); 487 cmd.addAll(configProvider.findConfigFor(mPackageName, false).getLoginArgs()); 488 } 489 490 return cmd.toArray(new String[cmd.size()]); 491 } 492 493 /** Cleans up the crawler output directory. */ cleanUp()494 public void cleanUp() { 495 if (mOutput == null) { 496 return; 497 } 498 499 try { 500 MoreFiles.deleteRecursively(mOutput); 501 } catch (IOException e) { 502 CLog.e("Failed to clean up the crawler output directory: " + e); 503 } 504 } 505 506 /** Sets the option of whether to record the device screen during crawling. */ setRecordScreen(boolean recordScreen)507 public void setRecordScreen(boolean recordScreen) { 508 mRecordScreen = recordScreen; 509 } 510 511 /** Sets the option of whether to collect GMS version in test artifacts. */ setCollectGmsVersion(boolean collectGmsVersion)512 public void setCollectGmsVersion(boolean collectGmsVersion) { 513 mCollectGmsVersion = collectGmsVersion; 514 } 515 516 /** Sets the option of whether to collect the app version in test artifacts. */ setCollectAppVersion(boolean collectAppVersion)517 public void setCollectAppVersion(boolean collectAppVersion) { 518 mCollectAppVersion = collectAppVersion; 519 } 520 521 /** Sets the option of whether to run the crawler with UIAutomator mode. */ setUiAutomatorMode(boolean uiAutomatorMode)522 public void setUiAutomatorMode(boolean uiAutomatorMode) { 523 mUiAutomatorMode = uiAutomatorMode; 524 } 525 526 /** Sets the value of the "timeout-sec" param for the crawler launcher. */ setTimeoutSec(int timeoutSec)527 public void setTimeoutSec(int timeoutSec) { 528 mTimeoutSec = timeoutSec; 529 } 530 531 /** Sets the robo crawler controller endpoint (optional). */ setCrawlControllerEndpoint(String crawlControllerEndpoint)532 public void setCrawlControllerEndpoint(String crawlControllerEndpoint) { 533 mCrawlControllerEndpoint = crawlControllerEndpoint; 534 } 535 536 /** 537 * Sets the apk file path. Required when not running in UIAutomator mode. 538 * 539 * @param apkRoot The root path for an apk or a directory that contains apk files for a package. 540 */ setApkPath(Path apkRoot)541 public void setApkPath(Path apkRoot) { 542 mApkRoot = apkRoot; 543 } 544 545 /** 546 * Sets the option of the Roboscript file to be used by the crawler. Null can be passed to 547 * remove the reference to the file. 548 */ setRoboscriptFile(@ullable Path roboscriptFile)549 public void setRoboscriptFile(@Nullable Path roboscriptFile) { 550 mRoboscriptFile = roboscriptFile; 551 } 552 553 /** 554 * Sets the option of the CrawlGuidance file to be used by the crawler. Null can be passed to 555 * remove the reference to the file. 556 */ setCrawlGuidanceProtoFile(@ullable Path crawlGuidanceProtoFile)557 public void setCrawlGuidanceProtoFile(@Nullable Path crawlGuidanceProtoFile) { 558 mCrawlGuidanceProtoFile = crawlGuidanceProtoFile; 559 } 560 561 /** Sets the option of the directory that contains configuration for login. */ setLoginConfigDir(@ullable Path loginFilesDir)562 public void setLoginConfigDir(@Nullable Path loginFilesDir) { 563 mLoginConfigDir = loginFilesDir; 564 } 565 566 @VisibleForTesting 567 interface RunUtilProvider { get()568 IRunUtil get(); 569 } 570 } 571